From ee2b0c8dda97c45b5f4b44a01e6c6f5bfdfb1763 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 26 Feb 2024 13:42:31 +0100 Subject: [talk-capitoul] Add talk on 2024-02-09 at capitoul.org --- doc/talks/2024-02-29-capitoul/.gitignore | 17 + doc/talks/2024-02-29-capitoul/Makefile | 10 + doc/talks/2024-02-29-capitoul/talk.pdf | Bin 0 -> 2211166 bytes doc/talks/2024-02-29-capitoul/talk.tex | 543 ++++ doc/talks/assets/garage2a.drawio.pdf | Bin 0 -> 33911 bytes doc/talks/assets/garage2b.drawio.pdf | Bin 0 -> 31051 bytes doc/talks/assets/garage_tables.pdf | Bin 0 -> 27273 bytes doc/talks/assets/intro/slide1.png | Bin 0 -> 89059 bytes doc/talks/assets/intro/slide2.png | Bin 0 -> 83364 bytes doc/talks/assets/intro/slide3.png | Bin 0 -> 127275 bytes doc/talks/assets/intro/slideB1.png | Bin 0 -> 86072 bytes doc/talks/assets/intro/slideB2.png | Bin 0 -> 83399 bytes doc/talks/assets/intro/slideB3.png | Bin 0 -> 82581 bytes doc/talks/assets/intro/slides.svg | 4326 ++++++++++++++++++++++++++++++ doc/talks/assets/intro/slidesB.svg | 444 +++ 15 files changed, 5340 insertions(+) create mode 100644 doc/talks/2024-02-29-capitoul/.gitignore create mode 100644 doc/talks/2024-02-29-capitoul/Makefile create mode 100644 doc/talks/2024-02-29-capitoul/talk.pdf create mode 100644 doc/talks/2024-02-29-capitoul/talk.tex create mode 100644 doc/talks/assets/garage2a.drawio.pdf create mode 100644 doc/talks/assets/garage2b.drawio.pdf create mode 100644 doc/talks/assets/garage_tables.pdf create mode 100644 doc/talks/assets/intro/slide1.png create mode 100644 doc/talks/assets/intro/slide2.png create mode 100644 doc/talks/assets/intro/slide3.png create mode 100644 doc/talks/assets/intro/slideB1.png create mode 100644 doc/talks/assets/intro/slideB2.png create mode 100644 doc/talks/assets/intro/slideB3.png create mode 100644 doc/talks/assets/intro/slides.svg create mode 100644 doc/talks/assets/intro/slidesB.svg diff --git a/doc/talks/2024-02-29-capitoul/.gitignore b/doc/talks/2024-02-29-capitoul/.gitignore new file mode 100644 index 00000000..9f1f00e6 --- /dev/null +++ b/doc/talks/2024-02-29-capitoul/.gitignore @@ -0,0 +1,17 @@ +* + +!*.txt +!*.md + +!assets + +!.gitignore +!*.svg +!*.png +!*.jpg +!*.tex +!Makefile +!.gitignore +!assets/*.drawio.pdf + +!talk.pdf diff --git a/doc/talks/2024-02-29-capitoul/Makefile b/doc/talks/2024-02-29-capitoul/Makefile new file mode 100644 index 00000000..aa9c35af --- /dev/null +++ b/doc/talks/2024-02-29-capitoul/Makefile @@ -0,0 +1,10 @@ +ASSETS=../assets/logos/deuxfleurs.pdf + +talk.pdf: talk.tex $(ASSETS) + pdflatex talk.tex + +%.pdf: %.svg + inkscape -D -z --file=$^ --export-pdf=$@ + +%.pdf_tex: %.svg + inkscape -D -z --file=$^ --export-pdf=$@ --export-latex diff --git a/doc/talks/2024-02-29-capitoul/talk.pdf b/doc/talks/2024-02-29-capitoul/talk.pdf new file mode 100644 index 00000000..33a63e23 Binary files /dev/null and b/doc/talks/2024-02-29-capitoul/talk.pdf differ diff --git a/doc/talks/2024-02-29-capitoul/talk.tex b/doc/talks/2024-02-29-capitoul/talk.tex new file mode 100644 index 00000000..95122b24 --- /dev/null +++ b/doc/talks/2024-02-29-capitoul/talk.tex @@ -0,0 +1,543 @@ +\nonstopmode +\documentclass[aspectratio=169,xcolor={svgnames}]{beamer} +\usepackage[utf8]{inputenc} +% \usepackage[frenchb]{babel} +\usepackage{amsmath} +\usepackage{mathtools} +\usepackage{breqn} +\usepackage{multirow} +\usetheme{boxes} +\usepackage{graphicx} +\usepackage{import} +\usepackage{adjustbox} +\usepackage[absolute,overlay]{textpos} +%\useoutertheme[footline=authortitle,subsection=false]{miniframes} +%\useoutertheme[footline=authorinstitute,subsection=false]{miniframes} +\useoutertheme{infolines} +\setbeamertemplate{headline}{} + +\beamertemplatenavigationsymbolsempty + +\definecolor{TitleOrange}{RGB}{255,137,0} +\setbeamercolor{title}{fg=TitleOrange} +\setbeamercolor{frametitle}{fg=TitleOrange} + +\definecolor{ListOrange}{RGB}{255,145,5} +\setbeamertemplate{itemize item}{\color{ListOrange}$\blacktriangleright$} + +\definecolor{verygrey}{RGB}{70,70,70} +\setbeamercolor{normal text}{fg=verygrey} + + +\usepackage{tabu} +\usepackage{multicol} +\usepackage{vwcol} +\usepackage{stmaryrd} +\usepackage{graphicx} + +\usepackage[normalem]{ulem} + +\AtBeginSection[]{ + \begin{frame} + \vfill + \centering + \begin{beamercolorbox}[sep=8pt,center,shadow=true,rounded=true]{title} + \usebeamerfont{title}\insertsectionhead\par% + \end{beamercolorbox} + \vfill + \end{frame} +} + +\title{Garage} +\author{Alex Auvolat, Deuxfleurs} +\date{Capitoul, 2024-02-29} + +\begin{document} + +\begin{frame} + \centering + \includegraphics[width=.3\linewidth]{../../sticker/Garage.png} + \vspace{1em} + + {\large\bf Alex Auvolat, Deuxfleurs Association} + \vspace{1em} + + \url{https://garagehq.deuxfleurs.fr/} + + Matrix channel: \texttt{\#garage:deuxfleurs.fr} +\end{frame} + +\begin{frame} + \frametitle{Who I am} + \begin{columns}[t] + \begin{column}{.2\textwidth} + \centering + \adjincludegraphics[width=.4\linewidth, valign=t]{../assets/alex.jpg} + \end{column} + \begin{column}{.6\textwidth} + \textbf{Alex Auvolat}\\ + PhD; co-founder of Deuxfleurs + \end{column} + \begin{column}{.2\textwidth} + ~ + \end{column} + \end{columns} + \vspace{2em} + + \begin{columns}[t] + \begin{column}{.2\textwidth} + \centering + \adjincludegraphics[width=.5\linewidth, valign=t]{../assets/logos/deuxfleurs.pdf} + \end{column} + \begin{column}{.6\textwidth} + \textbf{Deuxfleurs}\\ + A non-profit self-hosting collective,\\ + member of the CHATONS network + \end{column} + \begin{column}{.2\textwidth} + \centering + \adjincludegraphics[width=.7\linewidth, valign=t]{../assets/logos/logo_chatons.png} + \end{column} + \end{columns} + +\end{frame} + +\begin{frame} + \frametitle{Our objective at Deuxfleurs} + + \begin{center} + \textbf{Promote self-hosting and small-scale hosting\\ + as an alternative to large cloud providers} + \end{center} + \vspace{2em} + \visible<2->{ + Why is it hard? + \vspace{2em} + \begin{center} + \textbf{\underline{Resilience}}\\ + {\footnotesize we want good uptime/availability with low supervision} + \end{center} + } +\end{frame} + +\begin{frame} + \frametitle{Our very low-tech infrastructure} + + \only<1,3-6>{ + \begin{itemize} + \item \textcolor<4->{gray}{Commodity hardware (e.g. old desktop PCs)\\ + \vspace{.5em} + \visible<3->{{\footnotesize (can die at any time)}}} + \vspace{1.5em} + \item<4-> \textcolor<6->{gray}{Regular Internet (e.g. FTTB, FTTH) and power grid connections\\ + \vspace{.5em} + \visible<5->{{\footnotesize (can be unavailable randomly)}}} + \vspace{1.5em} + \item<6-> \textbf{Geographical redundancy} (multi-site replication) + \end{itemize} + } + \only<2>{ + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/neptune.jpg} + \end{center} + } + \only<7>{ + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/inframap_jdll2023.pdf} + \end{center} + } +\end{frame} + +\begin{frame} + \frametitle{How to make this happen} + \begin{center} + \only<1>{\includegraphics[width=.8\linewidth]{../assets/intro/slide1.png}}% + \only<2>{\includegraphics[width=.8\linewidth]{../assets/intro/slide2.png}}% + \only<3>{\includegraphics[width=.8\linewidth]{../assets/intro/slide3.png}}% + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Distributed file systems are slow} + File systems are complex, for example: + \vspace{1em} + \begin{itemize} + \item Concurrent modification by several processes + \vspace{1em} + \item Folder hierarchies + \vspace{1em} + \item Other requirements of the POSIX spec (e.g.~locks) + \end{itemize} + \vspace{1em} + Coordination in a distributed system is costly + + \vspace{1em} + Costs explode with commodity hardware / Internet connections\\ + {\small (we experienced this!)} +\end{frame} + +\begin{frame} + \frametitle{A simpler solution: object storage} + Only two operations: + \vspace{1em} + \begin{itemize} + \item Put an object at a key + \vspace{1em} + \item Retrieve an object from its key + \end{itemize} + \vspace{1em} + {\footnotesize (and a few others)} + + \vspace{1em} + Sufficient for many applications! +\end{frame} + +\begin{frame} + \frametitle{A simpler solution: object storage} + \begin{center} + \includegraphics[height=6em]{../assets/logos/Amazon-S3.jpg} + \hspace{3em} + \visible<2->{\includegraphics[height=5em]{../assets/logos/minio.png}} + \hspace{3em} + \visible<3>{\includegraphics[height=6em]{../../logo/garage_hires_crop.png}} + \end{center} + \vspace{1em} + S3: a de-facto standard, many compatible applications + + \vspace{1em} + \visible<2->{MinIO is self-hostable but not suited for geo-distributed deployments} + + \vspace{1em} + \visible<3->{\textbf{Garage is a self-hosted drop-in replacement for the Amazon S3 object store}} +\end{frame} + +% --------- BASED ON CRDTS ---------- + +\section{Principle 1: based on CRDTs} + +\begin{frame} + \frametitle{CRDTs / weak consistency instead of consensus} + + \underline{Internally, Garage uses only CRDTs} (conflict-free replicated data types) + + \vspace{2em} + Why not Raft, Paxos, ...? Issues of consensus algorithms: + + \vspace{1em} + \begin{itemize} + \item<2-> \textbf{Software complexity} + \vspace{1em} + \item<3-> \textbf{Performance issues:} + \vspace{.5em} + \begin{itemize} + \item<4-> The leader is a \textbf{bottleneck} for all requests\\ + \vspace{.5em} + \item<5-> \textbf{Sensitive to higher latency} between nodes + \vspace{.5em} + \item<6-> \textbf{Takes time to reconverge} when disrupted (e.g. node going down) + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{The data model of object storage} + Object storage is basically a \textbf{key-value store}: + \vspace{.5em} + + {\scriptsize + \begin{center} + \begin{tabular}{|l|p{7cm}|} + \hline + \textbf{Key: file path + name} & \textbf{Value: file data + metadata} \\ + \hline + \hline + \texttt{index.html} & + \texttt{Content-Type: text/html; charset=utf-8} \newline + \texttt{Content-Length: 24929} \newline + \texttt{} \\ + \hline + \texttt{img/logo.svg} & + \texttt{Content-Type: text/svg+xml} \newline + \texttt{Content-Length: 13429} \newline + \texttt{} \\ + \hline + \texttt{download/index.html} & + \texttt{Content-Type: text/html; charset=utf-8} \newline + \texttt{Content-Length: 26563} \newline + \texttt{} \\ + \hline + \end{tabular} + \end{center} + } + + \vspace{.5em} + \begin{itemize} + \item<2-> Maps well to CRDT data types + \item<3> Read-after-write consistency with quorums + \end{itemize} +\end{frame} + + +\begin{frame} + \frametitle{Performance gains in practice} + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/perf/endpoint_latency_0.7_0.8_minio.png} + \end{center} +\end{frame} + +% --------- GEO-DISTRIBUTED MODEL ---------- + +\section{Principle 2: geo-distributed data model} + +\begin{frame} + \frametitle{Key-value stores, upgraded: the Dynamo model} + \textbf{Two keys:} + \begin{itemize} + \item Partition key: used to divide data into partitions {\small (a.k.a.~shards)} + \item Sort key: used to identify items inside a partition + \end{itemize} + + \vspace{1em} + + \begin{center} + \begin{tabular}{|l|l|p{3cm}|} + \hline + \textbf{Partition key: bucket} & \textbf{Sort key: filename} & \textbf{Value} \\ + \hline + \hline + \texttt{website} & \texttt{index.html} & (file data) \\ + \hline + \texttt{website} & \texttt{img/logo.svg} & (file data) \\ + \hline + \texttt{website} & \texttt{download/index.html} & (file data) \\ + \hline + \hline + \texttt{backup} & \texttt{borg/index.2822} & (file data) \\ + \hline + \texttt{backup} & \texttt{borg/data/2/2329} & (file data) \\ + \hline + \texttt{backup} & \texttt{borg/data/2/2680} & (file data) \\ + \hline + \hline + \texttt{private} & \texttt{qq3a2nbe1qjq0ebbvo6ocsp6co} & (file data) \\ + \hline + \end{tabular} + \end{center} +\end{frame} + + +\begin{frame} + \frametitle{Layout computation} + \begin{overprint} + \onslide<1> + \begin{center} + \includegraphics[width=\linewidth, trim=0 0 0 -4cm]{../assets/screenshots/garage_status_0.9_prod_zonehl.png} + \end{center} + \onslide<2> + \begin{center} + \includegraphics[width=.7\linewidth]{../assets/map.png} + \end{center} + \end{overprint} + \vspace{1em} + Garage stores replicas on different zones when possible +\end{frame} + +\begin{frame} + \frametitle{What a "layout" is} + \textbf{A layout is a precomputed index table:} + \vspace{1em} + + {\footnotesize + \begin{center} + \begin{tabular}{|l|l|l|l|} + \hline + \textbf{Partition} & \textbf{Node 1} & \textbf{Node 2} & \textbf{Node 3} \\ + \hline + \hline + Partition 0 & df-ymk (bespin) & Abricot (scorpio) & Courgette (neptune) \\ + \hline + Partition 1 & Ananas (scorpio) & Courgette (neptune) & df-ykl (bespin) \\ + \hline + Partition 2 & df-ymf (bespin) & Celeri (neptune) & Abricot (scorpio) \\ + \hline + \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ \\ + \hline + Partition 255 & Concombre (neptune) & df-ykl (bespin) & Abricot (scorpio) \\ + \hline + \end{tabular} + \end{center} + } + + \vspace{2em} + \visible<2->{ + The index table is built centrally using an optimal algorithm,\\ + then propagated to all nodes + } + + \vspace{1em} + \visible<3->{ + \footnotesize + Oulamara, M., \& Auvolat, A. (2023). \emph{An algorithm for geo-distributed and redundant storage in Garage}.\\ arXiv preprint arXiv:2302.13798. + } +\end{frame} + + +\begin{frame} + \frametitle{The relationship between \emph{partition} and \emph{partition key}} + \begin{center} + \begin{tabular}{|l|l|l|l|} + \hline + \textbf{Partition key} & \textbf{Partition} & \textbf{Sort key} & \textbf{Value} \\ + \hline + \hline + \texttt{website} & Partition 12 & \texttt{index.html} & (file data) \\ + \hline + \texttt{website} & Partition 12 & \texttt{img/logo.svg} & (file data) \\ + \hline + \texttt{website} & Partition 12 &\texttt{download/index.html} & (file data) \\ + \hline + \hline + \texttt{backup} & Partition 42 & \texttt{borg/index.2822} & (file data) \\ + \hline + \texttt{backup} & Partition 42 & \texttt{borg/data/2/2329} & (file data) \\ + \hline + \texttt{backup} & Partition 42 & \texttt{borg/data/2/2680} & (file data) \\ + \hline + \hline + \texttt{private} & Partition 42 & \texttt{qq3a2nbe1qjq0ebbvo6ocsp6co} & (file data) \\ + \hline + \end{tabular} + \end{center} + \vspace{1em} + \textbf{To read or write an item:} hash partition key + \\ \hspace{5cm} $\to$ determine partition number (first 8 bits) + \\ \hspace{5cm} $\to$ find associated nodes +\end{frame} + +\begin{frame} + \frametitle{Garage's internal data structures} + \centering + \includegraphics[width=.75\columnwidth]{../assets/garage_tables.pdf} +\end{frame} + +% ---------- OPERATING GARAGE --------- + +\section{Operating Garage clusters} + +\begin{frame} + \frametitle{Operating Garage} + \begin{center} + \only<1-2>{ + \includegraphics[width=.9\linewidth]{../assets/screenshots/garage_status_0.10.png} + \\\vspace{1em} + \visible<2>{\includegraphics[width=.9\linewidth]{../assets/screenshots/garage_status_unhealthy_0.10.png}} + } + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Background synchronization} + \begin{center} + \includegraphics[width=.6\linewidth]{../assets/garage_sync.drawio.pdf} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Digging deeper} + \begin{center} + \only<1>{\includegraphics[width=.9\linewidth]{../assets/screenshots/garage_stats_0.10.png}} + \only<2>{\includegraphics[width=.5\linewidth]{../assets/screenshots/garage_worker_list_0.10.png}} + \only<3>{\includegraphics[width=.6\linewidth]{../assets/screenshots/garage_worker_param_0.10.png}} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Monitoring with Prometheus + Grafana} + \begin{center} + \includegraphics[width=.9\linewidth]{../assets/screenshots/grafana_dashboard.png} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Debugging with traces} + \begin{center} + \includegraphics[width=.8\linewidth]{../assets/screenshots/jaeger_listobjects.png} + \end{center} +\end{frame} + +% ---------- SCALING GARAGE --------- + +\section{Scaling Garage clusters} + +\begin{frame} + \frametitle{Potential limitations and bottlenecks} + \begin{itemize} + \item Global: + \begin{itemize} + \item Max. $\sim$100 nodes per cluster (excluding gateways) + \end{itemize} + \vspace{1em} + \item Metadata: + \begin{itemize} + \item One big bucket = bottleneck, object list on 3 nodes only + \end{itemize} + \vspace{1em} + \item Block manager: + \begin{itemize} + \item Lots of small files on disk + \item Processing the resync queue can be slow + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Deployment advice for very large clusters} + \begin{itemize} + \item Metadata storage: + \begin{itemize} + \item ZFS mirror (x2) on fast NVMe + \item Use LMDB storage engine + \end{itemize} + \vspace{.5em} + \item Data block storage: + \begin{itemize} + \item Use Garage's native multi-HDD support + \item XFS on individual drives + \item Increase block size (1MB $\to$ 10MB, requires more RAM and good networking) + \item Tune \texttt{resync-tranquility} and \texttt{resync-worker-count} dynamically + \end{itemize} + \vspace{.5em} + \item Other : + \begin{itemize} + \item Split data over several buckets + \item Use less than 100 storage nodes + \item Use gateway nodes + \end{itemize} + \vspace{.5em} + \end{itemize} + Our deployments: $< 10$ TB. Some people have done more! +\end{frame} + + +% ======================================== END +% ======================================== END +% ======================================== END + +\begin{frame} + \frametitle{Where to find us} + \begin{center} + \includegraphics[width=.25\linewidth]{../../logo/garage_hires.png}\\ + \vspace{-1em} + \url{https://garagehq.deuxfleurs.fr/}\\ + \url{mailto:garagehq@deuxfleurs.fr}\\ + \texttt{\#garage:deuxfleurs.fr} on Matrix + + \vspace{1.5em} + \includegraphics[width=.06\linewidth]{../assets/logos/rust_logo.png} + \includegraphics[width=.13\linewidth]{../assets/logos/AGPLv3_Logo.png} + \end{center} +\end{frame} + +\end{document} + +%% vim: set ts=4 sw=4 tw=0 noet spelllang=en : diff --git a/doc/talks/assets/garage2a.drawio.pdf b/doc/talks/assets/garage2a.drawio.pdf new file mode 100644 index 00000000..422c9343 Binary files /dev/null and b/doc/talks/assets/garage2a.drawio.pdf differ diff --git a/doc/talks/assets/garage2b.drawio.pdf b/doc/talks/assets/garage2b.drawio.pdf new file mode 100644 index 00000000..05a9710e Binary files /dev/null and b/doc/talks/assets/garage2b.drawio.pdf differ diff --git a/doc/talks/assets/garage_tables.pdf b/doc/talks/assets/garage_tables.pdf new file mode 100644 index 00000000..3c54cdce Binary files /dev/null and b/doc/talks/assets/garage_tables.pdf differ diff --git a/doc/talks/assets/intro/slide1.png b/doc/talks/assets/intro/slide1.png new file mode 100644 index 00000000..eb2e67a0 Binary files /dev/null and b/doc/talks/assets/intro/slide1.png differ diff --git a/doc/talks/assets/intro/slide2.png b/doc/talks/assets/intro/slide2.png new file mode 100644 index 00000000..126a39b8 Binary files /dev/null and b/doc/talks/assets/intro/slide2.png differ diff --git a/doc/talks/assets/intro/slide3.png b/doc/talks/assets/intro/slide3.png new file mode 100644 index 00000000..a39f96bf Binary files /dev/null and b/doc/talks/assets/intro/slide3.png differ diff --git a/doc/talks/assets/intro/slideB1.png b/doc/talks/assets/intro/slideB1.png new file mode 100644 index 00000000..b14b6070 Binary files /dev/null and b/doc/talks/assets/intro/slideB1.png differ diff --git a/doc/talks/assets/intro/slideB2.png b/doc/talks/assets/intro/slideB2.png new file mode 100644 index 00000000..a881a796 Binary files /dev/null and b/doc/talks/assets/intro/slideB2.png differ diff --git a/doc/talks/assets/intro/slideB3.png b/doc/talks/assets/intro/slideB3.png new file mode 100644 index 00000000..830709d2 Binary files /dev/null and b/doc/talks/assets/intro/slideB3.png differ diff --git a/doc/talks/assets/intro/slides.svg b/doc/talks/assets/intro/slides.svg new file mode 100644 index 00000000..9946c6fb --- /dev/null +++ b/doc/talks/assets/intro/slides.svg @@ -0,0 +1,4326 @@ + + + + + + + + + + + + + + + + + + + + + + User-facing application + Database + Filesystem + + + + + + + + + + diff --git a/doc/talks/assets/intro/slidesB.svg b/doc/talks/assets/intro/slidesB.svg new file mode 100644 index 00000000..c0a6e97c --- /dev/null +++ b/doc/talks/assets/intro/slidesB.svg @@ -0,0 +1,444 @@ + + + +User-facing applicationDatabase*K2VObject storage*(not really a database)Database -- cgit v1.2.3 From a8cb8e8a8b0507a9035083d64eb46cde7d39005d Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 27 Feb 2024 17:56:57 +0100 Subject: [fix-presigned] split presigned/normal signature verification --- src/api/s3/post_object.rs | 26 +- src/api/signature/payload.rs | 560 +++++++++++++++++----------- src/garage/tests/common/custom_requester.rs | 85 +++-- src/garage/tests/s3/streaming_signature.rs | 2 +- 4 files changed, 411 insertions(+), 262 deletions(-) diff --git a/src/api/s3/post_object.rs b/src/api/s3/post_object.rs index bca8d6c6..b542cc1a 100644 --- a/src/api/s3/post_object.rs +++ b/src/api/s3/post_object.rs @@ -21,7 +21,7 @@ use crate::s3::cors::*; use crate::s3::error::*; use crate::s3::put::{get_headers, save_stream}; use crate::s3::xml as s3_xml; -use crate::signature::payload::{parse_date, verify_v4}; +use crate::signature::payload::{verify_v4, Authorization}; pub async fn handle_post_object( garage: Arc, @@ -88,22 +88,11 @@ pub async fn handle_post_object( .get("key") .ok_or_bad_request("No key was provided")? .to_str()?; - let credential = params - .get("x-amz-credential") - .ok_or_else(|| Error::forbidden("Garage does not support anonymous access yet"))? - .to_str()?; let policy = params .get("policy") .ok_or_bad_request("No policy was provided")? .to_str()?; - let signature = params - .get("x-amz-signature") - .ok_or_bad_request("No signature was provided")? - .to_str()?; - let date = params - .get("x-amz-date") - .ok_or_bad_request("No date was provided")? - .to_str()?; + let authorization = Authorization::parse_form(¶ms)?; let key = if key.contains("${filename}") { // if no filename is provided, don't replace. This matches the behavior of AWS. @@ -116,16 +105,7 @@ pub async fn handle_post_object( key.to_owned() }; - let date = parse_date(date)?; - let api_key = verify_v4( - &garage, - "s3", - credential, - &date, - signature, - policy.as_bytes(), - ) - .await?; + let api_key = verify_v4(&garage, "s3", &authorization, policy.as_bytes()).await?; let bucket_id = garage .bucket_helper() diff --git a/src/api/signature/payload.rs b/src/api/signature/payload.rs index 423aad93..29ed7081 100644 --- a/src/api/signature/payload.rs +++ b/src/api/signature/payload.rs @@ -1,7 +1,9 @@ use std::collections::HashMap; +use std::convert::TryFrom; use chrono::{DateTime, Duration, NaiveDateTime, TimeZone, Utc}; use hmac::Mac; +use hyper::header::{HeaderMap, HeaderName, AUTHORIZATION, CONTENT_TYPE, HOST}; use hyper::{body::Incoming as IncomingBody, Method, Request}; use sha2::{Digest, Sha256}; @@ -17,66 +19,87 @@ use super::{compute_scope, signing_hmac}; use crate::encoding::uri_encode; use crate::signature::error::*; +pub const X_AMZ_ALGORITHM: HeaderName = HeaderName::from_static("x-amz-algorithm"); +pub const X_AMZ_CREDENTIAL: HeaderName = HeaderName::from_static("x-amz-credential"); +pub const X_AMZ_DATE: HeaderName = HeaderName::from_static("x-amz-date"); +pub const X_AMZ_EXPIRES: HeaderName = HeaderName::from_static("x-amz-expires"); +pub const X_AMZ_SIGNEDHEADERS: HeaderName = HeaderName::from_static("x-amz-signedheaders"); +pub const X_AMZ_SIGNATURE: HeaderName = HeaderName::from_static("x-amz-signature"); +pub const X_AMZ_CONTENT_SH256: HeaderName = HeaderName::from_static("x-amz-content-sha256"); + +pub const AWS4_HMAC_SHA256: &str = "AWS4-HMAC-SHA256"; +pub const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD"; +pub const STREAMING_AWS4_HMAC_SHA256_PAYLOAD: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"; + +pub type QueryMap = HashMap; + pub async fn check_payload_signature( garage: &Garage, service: &'static str, request: &Request, ) -> Result<(Option, Option), Error> { - let mut headers = HashMap::new(); - for (key, val) in request.headers() { - headers.insert(key.to_string(), val.to_str()?.to_string()); - } - if let Some(query) = request.uri().query() { - let query_pairs = url::form_urlencoded::parse(query.as_bytes()); - for (key, val) in query_pairs { - headers.insert(key.to_lowercase(), val.to_string()); - } - } + let query = parse_query_map(request.uri())?; - let authorization = if let Some(authorization) = headers.get("authorization") { - parse_authorization(authorization, &headers)? - } else if let Some(algorithm) = headers.get("x-amz-algorithm") { - parse_query_authorization(algorithm, &headers)? + let res = if query.contains_key(X_AMZ_ALGORITHM.as_str()) { + check_presigned_signature(garage, service, request, query).await } else { - let content_sha256 = headers.get("x-amz-content-sha256"); - if let Some(content_sha256) = content_sha256.filter(|c| "UNSIGNED-PAYLOAD" != c.as_str()) { - let sha256 = hex::decode(content_sha256) - .ok() - .and_then(|bytes| Hash::try_from(&bytes)) - .ok_or_bad_request("Invalid content sha256 hash")?; - return Ok((None, Some(sha256))); - } else { - return Ok((None, None)); - } + check_standard_signature(garage, service, request, query).await }; + if let Err(e) = &res { + error!("ERROR IN SIGNATURE\n{:?}\n{}", request, e); + } + + res +} + +async fn check_standard_signature( + garage: &Garage, + service: &'static str, + request: &Request, + query: QueryMap, +) -> Result<(Option, Option), Error> { + let authorization = Authorization::parse(request.headers())?; + + // Verify that all necessary request headers are signed + let signed_headers = split_signed_headers(&authorization)?; + for (name, _) in request.headers().iter() { + if name.as_str().starts_with("x-amz-") || name == CONTENT_TYPE { + if !signed_headers.contains(name) { + return Err(Error::bad_request(format!( + "Header `{}` should be signed", + name + ))); + } + } + } + if !signed_headers.contains(&HOST) { + return Err(Error::bad_request("Header `Host` should be signed")); + } + let canonical_request = canonical_request( service, request.method(), - request.uri(), - &headers, - &authorization.signed_headers, + request.uri().path(), + &query, + request.headers(), + signed_headers, &authorization.content_sha256, + )?; + let string_to_sign = string_to_sign( + &authorization.date, + &authorization.scope, + &canonical_request, ); - let (_, scope) = parse_credential(&authorization.credential)?; - let string_to_sign = string_to_sign(&authorization.date, &scope, &canonical_request); trace!("canonical request:\n{}", canonical_request); trace!("string to sign:\n{}", string_to_sign); - let key = verify_v4( - garage, - service, - &authorization.credential, - &authorization.date, - &authorization.signature, - string_to_sign.as_bytes(), - ) - .await?; + let key = verify_v4(garage, service, &authorization, string_to_sign.as_bytes()).await?; - let content_sha256 = if authorization.content_sha256 == "UNSIGNED-PAYLOAD" { + let content_sha256 = if authorization.content_sha256 == UNSIGNED_PAYLOAD { None - } else if authorization.content_sha256 == "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" { + } else if authorization.content_sha256 == STREAMING_AWS4_HMAC_SHA256_PAYLOAD { let bytes = hex::decode(authorization.signature).ok_or_bad_request("Invalid signature")?; Some(Hash::try_from(&bytes).ok_or_bad_request("Invalid signature")?) } else { @@ -88,142 +111,86 @@ pub async fn check_payload_signature( Ok((Some(key), content_sha256)) } -struct Authorization { - credential: String, - signed_headers: String, - signature: String, - content_sha256: String, - date: DateTime, -} - -fn parse_authorization( - authorization: &str, - headers: &HashMap, -) -> Result { - let first_space = authorization - .find(' ') - .ok_or_bad_request("Authorization field to short")?; - let (auth_kind, rest) = authorization.split_at(first_space); - - if auth_kind != "AWS4-HMAC-SHA256" { - return Err(Error::bad_request("Unsupported authorization method")); +async fn check_presigned_signature( + garage: &Garage, + service: &'static str, + request: &Request, + mut query: QueryMap, +) -> Result<(Option, Option), Error> { + let algorithm = query.get(X_AMZ_ALGORITHM.as_str()).unwrap(); + let authorization = Authorization::parse_presigned(algorithm, &query)?; + + // Check that all mandatory signed headers are included + let signed_headers = split_signed_headers(&authorization)?; + for (name, _) in request.headers().iter() { + if name.as_str().starts_with("x-amz-") { + if !signed_headers.contains(name) { + return Err(Error::bad_request(format!( + "Header `{}` should be signed", + name + ))); + } + } } - - let mut auth_params = HashMap::new(); - for auth_part in rest.split(',') { - let auth_part = auth_part.trim(); - let eq = auth_part - .find('=') - .ok_or_bad_request("Field without value in authorization header")?; - let (key, value) = auth_part.split_at(eq); - auth_params.insert(key.to_string(), value.trim_start_matches('=').to_string()); + if !signed_headers.contains(&HOST) { + return Err(Error::bad_request("Header `Host` should be signed")); } - let cred = auth_params - .get("Credential") - .ok_or_bad_request("Could not find Credential in Authorization field")?; - - let content_sha256 = headers - .get("x-amz-content-sha256") - .ok_or_bad_request("Missing X-Amz-Content-Sha256 field")?; + query.remove(X_AMZ_SIGNATURE.as_str()); + let canonical_request = canonical_request( + service, + request.method(), + request.uri().path(), + &query, + request.headers(), + signed_headers, + &authorization.content_sha256, + )?; + let string_to_sign = string_to_sign( + &authorization.date, + &authorization.scope, + &canonical_request, + ); - let date = headers - .get("x-amz-date") - .ok_or_bad_request("Missing X-Amz-Date field") - .map_err(Error::from) - .and_then(|d| parse_date(d))?; + trace!("canonical request:\n{}", canonical_request); + trace!("string to sign:\n{}", string_to_sign); - if Utc::now() - date > Duration::hours(24) { - return Err(Error::bad_request("Date is too old".to_string())); - } + let key = verify_v4(garage, service, &authorization, string_to_sign.as_bytes()).await?; - let auth = Authorization { - credential: cred.to_string(), - signed_headers: auth_params - .get("SignedHeaders") - .ok_or_bad_request("Could not find SignedHeaders in Authorization field")? - .to_string(), - signature: auth_params - .get("Signature") - .ok_or_bad_request("Could not find Signature in Authorization field")? - .to_string(), - content_sha256: content_sha256.to_string(), - date, - }; - Ok(auth) + Ok((Some(key), None)) } -fn parse_query_authorization( - algorithm: &str, - headers: &HashMap, -) -> Result { - if algorithm != "AWS4-HMAC-SHA256" { - return Err(Error::bad_request( - "Unsupported authorization method".to_string(), - )); - } - - let cred = headers - .get("x-amz-credential") - .ok_or_bad_request("X-Amz-Credential not found in query parameters")?; - let signed_headers = headers - .get("x-amz-signedheaders") - .ok_or_bad_request("X-Amz-SignedHeaders not found in query parameters")?; - let signature = headers - .get("x-amz-signature") - .ok_or_bad_request("X-Amz-Signature not found in query parameters")?; - let content_sha256 = headers - .get("x-amz-content-sha256") - .map(|x| x.as_str()) - .unwrap_or("UNSIGNED-PAYLOAD"); - - let duration = headers - .get("x-amz-expires") - .ok_or_bad_request("X-Amz-Expires not found in query parameters")? - .parse() - .map_err(|_| Error::bad_request("X-Amz-Expires is not a number".to_string()))?; - - if duration > 7 * 24 * 3600 { - return Err(Error::bad_request( - "X-Amz-Expires may not exceed a week".to_string(), - )); - } - - let date = headers - .get("x-amz-date") - .ok_or_bad_request("Missing X-Amz-Date field") - .map_err(Error::from) - .and_then(|d| parse_date(d))?; - - if Utc::now() - date > Duration::seconds(duration) { - return Err(Error::bad_request("Date is too old".to_string())); +pub fn parse_query_map(uri: &http::uri::Uri) -> Result { + let mut query = QueryMap::new(); + if let Some(query_str) = uri.query() { + let query_pairs = url::form_urlencoded::parse(query_str.as_bytes()); + for (key, val) in query_pairs { + if query.insert(key.to_string(), val.into_owned()).is_some() { + return Err(Error::bad_request(format!( + "duplicate query parameter: `{}`", + key + ))); + } + } } - - Ok(Authorization { - credential: cred.to_string(), - signed_headers: signed_headers.to_string(), - signature: signature.to_string(), - content_sha256: content_sha256.to_string(), - date, - }) + Ok(query) } -fn parse_credential(cred: &str) -> Result<(String, String), Error> { - let first_slash = cred - .find('/') - .ok_or_bad_request("Credentials does not contain '/' in authorization field")?; - let (key_id, scope) = cred.split_at(first_slash); - Ok(( - key_id.to_string(), - scope.trim_start_matches('/').to_string(), - )) +fn split_signed_headers(authorization: &Authorization) -> Result, Error> { + let signed_headers = authorization + .signed_headers + .split(';') + .map(HeaderName::try_from) + .collect::, _>>() + .ok_or_bad_request("invalid header name")?; + Ok(signed_headers) } pub fn string_to_sign(datetime: &DateTime, scope_string: &str, canonical_req: &str) -> String { let mut hasher = Sha256::default(); hasher.update(canonical_req.as_bytes()); [ - "AWS4-HMAC-SHA256", + AWS4_HMAC_SHA256, &datetime.format(LONG_DATETIME).to_string(), scope_string, &hex::encode(hasher.finalize().as_slice()), @@ -234,11 +201,12 @@ pub fn string_to_sign(datetime: &DateTime, scope_string: &str, canonical_re pub fn canonical_request( service: &'static str, method: &Method, - uri: &hyper::Uri, - headers: &HashMap, - signed_headers: &str, + canonical_uri: &str, + query: &QueryMap, + headers: &HeaderMap, + mut signed_headers: Vec, content_sha256: &str, -) -> String { +) -> Result { // There seems to be evidence that in AWSv4 signatures, the path component is url-encoded // a second time when building the canonical request, as specified in this documentation page: // -> https://docs.aws.amazon.com/rolesanywhere/latest/userguide/authentication-sign-process.html @@ -268,49 +236,51 @@ pub fn canonical_request( // it mentions it in the comments (same link to the souce code as above). // We make the explicit choice of NOT normalizing paths in the K2V API because doing so // would make non-normalized paths invalid K2V partition keys, and we don't want that. - let path: std::borrow::Cow = if service != "s3" { - uri_encode(uri.path(), false).into() + let canonical_uri: std::borrow::Cow = if service != "s3" { + uri_encode(canonical_uri, false).into() } else { - uri.path().into() + canonical_uri.into() }; - [ - method.as_str(), - &path, - &canonical_query_string(uri), - &canonical_header_string(headers, signed_headers), - "", - signed_headers, - content_sha256, - ] - .join("\n") -} - -fn canonical_header_string(headers: &HashMap, signed_headers: &str) -> String { - let signed_headers_vec = signed_headers.split(';').collect::>(); - let mut items = headers - .iter() - .filter(|(key, _)| signed_headers_vec.contains(&key.as_str())) - .collect::>(); - items.sort_by(|(k1, _), (k2, _)| k1.cmp(k2)); - items - .iter() - .map(|(key, value)| key.to_lowercase() + ":" + value.trim()) - .collect::>() - .join("\n") -} -fn canonical_query_string(uri: &hyper::Uri) -> String { - if let Some(query) = uri.query() { - let query_pairs = url::form_urlencoded::parse(query.as_bytes()); - let mut items = query_pairs - .filter(|(key, _)| key != "X-Amz-Signature") - .map(|(key, value)| uri_encode(&key, true) + "=" + &uri_encode(&value, true)) - .collect::>(); + // Canonical query string from passed HeaderMap + let canonical_query_string = { + let mut items = Vec::with_capacity(query.len()); + for (key, value) in query.iter() { + items.push(uri_encode(&key, true) + "=" + &uri_encode(&value, true)); + } items.sort(); items.join("&") - } else { - "".to_string() - } + }; + + // Canonical header string calculated from signed headers + signed_headers.sort_by(|h1, h2| h1.as_str().cmp(h2.as_str())); + let canonical_header_string = { + let mut items = Vec::with_capacity(signed_headers.len()); + for name in signed_headers.iter() { + let value = headers + .get(name) + .ok_or_bad_request(format!("signed header `{}` is not present", name))? + .to_str()?; + items.push((name, value)); + } + items + .iter() + .map(|(key, value)| format!("{}:{}", key.as_str(), value.trim())) + .collect::>() + .join("\n") + }; + let signed_headers = signed_headers.join(";"); + + let list = [ + method.as_str(), + &canonical_uri, + &canonical_query_string, + &canonical_header_string, + "", + &signed_headers, + content_sha256, + ]; + Ok(list.join("\n")) } pub fn parse_date(date: &str) -> Result, Error> { @@ -322,28 +292,24 @@ pub fn parse_date(date: &str) -> Result, Error> { pub async fn verify_v4( garage: &Garage, service: &str, - credential: &str, - date: &DateTime, - signature: &str, + auth: &Authorization, payload: &[u8], ) -> Result { - let (key_id, scope) = parse_credential(credential)?; - - let scope_expected = compute_scope(date, &garage.config.s3_api.s3_region, service); - if scope != scope_expected { - return Err(Error::AuthorizationHeaderMalformed(scope.to_string())); + let scope_expected = compute_scope(&auth.date, &garage.config.s3_api.s3_region, service); + if auth.scope != scope_expected { + return Err(Error::AuthorizationHeaderMalformed(auth.scope.to_string())); } let key = garage .key_table - .get(&EmptyKey, &key_id) + .get(&EmptyKey, &auth.key_id) .await? .filter(|k| !k.state.is_deleted()) - .ok_or_else(|| Error::forbidden(format!("No such key: {}", &key_id)))?; + .ok_or_else(|| Error::forbidden(format!("No such key: {}", &auth.key_id)))?; let key_p = key.params().unwrap(); let mut hmac = signing_hmac( - date, + &auth.date, &key_p.secret_key, &garage.config.s3_api.s3_region, service, @@ -351,9 +317,177 @@ pub async fn verify_v4( .ok_or_internal_error("Unable to build signing HMAC")?; hmac.update(payload); let our_signature = hex::encode(hmac.finalize().into_bytes()); - if signature != our_signature { + if auth.signature != our_signature { return Err(Error::forbidden("Invalid signature".to_string())); } Ok(key) } + +// ============ Authorization header, or X-Amz-* query params ========= + +pub struct Authorization { + key_id: String, + scope: String, + signed_headers: String, + signature: String, + content_sha256: String, + date: DateTime, +} + +impl Authorization { + fn parse(headers: &HeaderMap) -> Result { + let authorization = headers + .get(AUTHORIZATION) + .ok_or_bad_request("Missing authorization header")? + .to_str()?; + + let (auth_kind, rest) = authorization + .split_once(' ') + .ok_or_bad_request("Authorization field to short")?; + + if auth_kind != AWS4_HMAC_SHA256 { + return Err(Error::bad_request("Unsupported authorization method")); + } + + let mut auth_params = HashMap::new(); + for auth_part in rest.split(',') { + let auth_part = auth_part.trim(); + let eq = auth_part + .find('=') + .ok_or_bad_request("Field without value in authorization header")?; + let (key, value) = auth_part.split_at(eq); + auth_params.insert(key.to_string(), value.trim_start_matches('=').to_string()); + } + + let cred = auth_params + .get("Credential") + .ok_or_bad_request("Could not find Credential in Authorization field")?; + let signed_headers = auth_params + .get("SignedHeaders") + .ok_or_bad_request("Could not find SignedHeaders in Authorization field")? + .to_string(); + let signature = auth_params + .get("Signature") + .ok_or_bad_request("Could not find Signature in Authorization field")? + .to_string(); + + let content_sha256 = headers + .get(X_AMZ_CONTENT_SH256) + .ok_or_bad_request("Missing X-Amz-Content-Sha256 field")?; + + let date = headers + .get(X_AMZ_DATE) + .ok_or_bad_request("Missing X-Amz-Date field") + .map_err(Error::from)? + .to_str()?; + let date = parse_date(date)?; + + if Utc::now() - date > Duration::hours(24) { + return Err(Error::bad_request("Date is too old".to_string())); + } + + let (key_id, scope) = parse_credential(cred)?; + let auth = Authorization { + key_id, + scope, + signed_headers, + signature, + content_sha256: content_sha256.to_str()?.to_string(), + date, + }; + Ok(auth) + } + + fn parse_presigned(algorithm: &str, query: &QueryMap) -> Result { + if algorithm != AWS4_HMAC_SHA256 { + return Err(Error::bad_request( + "Unsupported authorization method".to_string(), + )); + } + + let cred = query + .get(X_AMZ_CREDENTIAL.as_str()) + .ok_or_bad_request("X-Amz-Credential not found in query parameters")?; + let signed_headers = query + .get(X_AMZ_SIGNEDHEADERS.as_str()) + .ok_or_bad_request("X-Amz-SignedHeaders not found in query parameters")?; + let signature = query + .get(X_AMZ_SIGNATURE.as_str()) + .ok_or_bad_request("X-Amz-Signature not found in query parameters")?; + + let duration = query + .get(X_AMZ_EXPIRES.as_str()) + .ok_or_bad_request("X-Amz-Expires not found in query parameters")? + .parse() + .map_err(|_| Error::bad_request("X-Amz-Expires is not a number".to_string()))?; + + if duration > 7 * 24 * 3600 { + return Err(Error::bad_request( + "X-Amz-Expires may not exceed a week".to_string(), + )); + } + + let date = query + .get(X_AMZ_DATE.as_str()) + .ok_or_bad_request("Missing X-Amz-Date field")?; + let date = parse_date(date)?; + + if Utc::now() - date > Duration::seconds(duration) { + return Err(Error::bad_request("Date is too old".to_string())); + } + + let (key_id, scope) = parse_credential(cred)?; + Ok(Authorization { + key_id, + scope, + signed_headers: signed_headers.to_string(), + signature: signature.to_string(), + content_sha256: UNSIGNED_PAYLOAD.to_string(), + date, + }) + } + + pub(crate) fn parse_form(params: &HeaderMap) -> Result { + let credential = params + .get(X_AMZ_CREDENTIAL) + .ok_or_else(|| Error::forbidden("Garage does not support anonymous access yet"))? + .to_str()?; + let signature = params + .get(X_AMZ_SIGNATURE) + .ok_or_bad_request("No signature was provided")? + .to_str()? + .to_string(); + let date = params + .get(X_AMZ_DATE) + .ok_or_bad_request("No date was provided")? + .to_str()?; + let date = parse_date(date)?; + + if Utc::now() - date > Duration::hours(24) { + return Err(Error::bad_request("Date is too old".to_string())); + } + + let (key_id, scope) = parse_credential(credential)?; + let auth = Authorization { + key_id, + scope, + signed_headers: "".to_string(), + signature, + content_sha256: UNSIGNED_PAYLOAD.to_string(), + date, + }; + Ok(auth) + } +} + +fn parse_credential(cred: &str) -> Result<(String, String), Error> { + let first_slash = cred + .find('/') + .ok_or_bad_request("Credentials does not contain '/' in authorization field")?; + let (key_id, scope) = cred.split_at(first_slash); + Ok(( + key_id.to_string(), + scope.trim_start_matches('/').to_string(), + )) +} diff --git a/src/garage/tests/common/custom_requester.rs b/src/garage/tests/common/custom_requester.rs index e5f4cca1..f311418c 100644 --- a/src/garage/tests/common/custom_requester.rs +++ b/src/garage/tests/common/custom_requester.rs @@ -1,12 +1,15 @@ #![allow(dead_code)] use std::collections::HashMap; -use std::convert::TryFrom; +use std::convert::{TryFrom, TryInto}; use chrono::{offset::Utc, DateTime}; use hmac::{Hmac, Mac}; use http_body_util::BodyExt; use http_body_util::Full as FullBody; +use hyper::header::{ + HeaderMap, HeaderName, HeaderValue, AUTHORIZATION, CONTENT_ENCODING, CONTENT_LENGTH, HOST, +}; use hyper::{Method, Request, Response, Uri}; use hyper_util::client::legacy::{connect::HttpConnector, Client}; use hyper_util::rt::TokioExecutor; @@ -173,54 +176,85 @@ impl<'a> RequestBuilder<'a> { .unwrap(); let streaming_signer = signer.clone(); - let mut all_headers = self.signed_headers.clone(); + let mut all_headers = self + .signed_headers + .iter() + .map(|(k, v)| { + ( + HeaderName::try_from(k).expect("invalid header name"), + HeaderValue::try_from(v).expect("invalid header value"), + ) + }) + .collect::(); let date = now.format(signature::LONG_DATETIME).to_string(); - all_headers.insert("x-amz-date".to_owned(), date); - all_headers.insert("host".to_owned(), host); + all_headers.insert( + signature::payload::X_AMZ_DATE, + HeaderValue::from_str(&date).unwrap(), + ); + all_headers.insert(HOST, HeaderValue::from_str(&host).unwrap()); let body_sha = match self.body_signature { BodySignature::Unsigned => "UNSIGNED-PAYLOAD".to_owned(), BodySignature::Classic => hex::encode(garage_util::data::sha256sum(&self.body)), BodySignature::Streaming(size) => { - all_headers.insert("content-encoding".to_owned(), "aws-chunked".to_owned()); all_headers.insert( - "x-amz-decoded-content-length".to_owned(), - self.body.len().to_string(), + CONTENT_ENCODING, + HeaderValue::from_str("aws-chunked").unwrap(), + ); + all_headers.insert( + HeaderName::from_static("x-amz-decoded-content-length"), + HeaderValue::from_str(&self.body.len().to_string()).unwrap(), ); // Get lenght of body by doing the conversion to a streaming body with an // invalid signature (we don't know the seed) just to get its length. This // is a pretty lazy and inefficient way to do it, but it's enought for test // code. all_headers.insert( - "content-length".to_owned(), + CONTENT_LENGTH, to_streaming_body(&self.body, size, String::new(), signer.clone(), now, "") .len() - .to_string(), + .to_string() + .try_into() + .unwrap(), ); "STREAMING-AWS4-HMAC-SHA256-PAYLOAD".to_owned() } }; - all_headers.insert("x-amz-content-sha256".to_owned(), body_sha.clone()); + all_headers.insert( + signature::payload::X_AMZ_CONTENT_SH256, + HeaderValue::from_str(&body_sha).unwrap(), + ); + + let mut signed_headers = all_headers.keys().cloned().collect::>(); + signed_headers.sort_by(|h1, h2| h1.as_str().cmp(h2.as_str())); + let signed_headers_str = signed_headers + .iter() + .map(ToString::to_string) + .collect::>() + .join(";"); - let mut signed_headers = all_headers - .keys() - .map(|k| k.as_ref()) - .collect::>(); - signed_headers.sort(); - let signed_headers = signed_headers.join(";"); + all_headers.extend(self.unsigned_headers.iter().map(|(k, v)| { + ( + HeaderName::try_from(k).expect("invalid header name"), + HeaderValue::try_from(v).expect("invalid header value"), + ) + })); - all_headers.extend(self.unsigned_headers.clone()); + let uri = Uri::try_from(&uri).unwrap(); + let query = signature::payload::parse_query_map(&uri).unwrap(); let canonical_request = signature::payload::canonical_request( self.service, &self.method, - &Uri::try_from(&uri).unwrap(), + uri.path(), + &query, &all_headers, - &signed_headers, + signed_headers, &body_sha, - ); + ) + .unwrap(); let string_to_sign = signature::payload::string_to_sign(&now, &scope, &canonical_request); @@ -228,14 +262,15 @@ impl<'a> RequestBuilder<'a> { let signature = hex::encode(signer.finalize().into_bytes()); let authorization = format!( "AWS4-HMAC-SHA256 Credential={}/{},SignedHeaders={},Signature={}", - self.requester.key.id, scope, signed_headers, signature + self.requester.key.id, scope, signed_headers_str, signature + ); + all_headers.insert( + AUTHORIZATION, + HeaderValue::from_str(&authorization).unwrap(), ); - all_headers.insert("authorization".to_owned(), authorization); let mut request = Request::builder(); - for (k, v) in all_headers { - request = request.header(k, v); - } + *request.headers_mut().unwrap() = all_headers; let body = if let BodySignature::Streaming(size) = self.body_signature { to_streaming_body(&self.body, size, signature, streaming_signer, now, &scope) diff --git a/src/garage/tests/s3/streaming_signature.rs b/src/garage/tests/s3/streaming_signature.rs index 224b9ed5..351aa422 100644 --- a/src/garage/tests/s3/streaming_signature.rs +++ b/src/garage/tests/s3/streaming_signature.rs @@ -26,7 +26,7 @@ async fn test_putobject_streaming() { .builder(bucket.clone()) .method(Method::PUT) .path(STD_KEY.to_owned()) - .unsigned_headers(headers) + .signed_headers(headers) .vhost_style(true) .body(vec![]) .body_signature(BodySignature::Streaming(10)) -- cgit v1.2.3 From 2efa9c5a1a568e28e41af790750f224d334d4e3d Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 27 Feb 2024 22:59:30 +0100 Subject: [fix-presigned] PostObject: verify X-Amz-Algorithm --- src/api/signature/payload.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/api/signature/payload.rs b/src/api/signature/payload.rs index 29ed7081..8841a5e5 100644 --- a/src/api/signature/payload.rs +++ b/src/api/signature/payload.rs @@ -449,6 +449,16 @@ impl Authorization { } pub(crate) fn parse_form(params: &HeaderMap) -> Result { + let algorithm = params + .get(X_AMZ_ALGORITHM) + .ok_or_bad_request("Missing X-Amz-Algorithm header")? + .to_str()?; + if algorithm != AWS4_HMAC_SHA256 { + return Err(Error::bad_request( + "Unsupported authorization method".to_string(), + )); + } + let credential = params .get(X_AMZ_CREDENTIAL) .ok_or_else(|| Error::forbidden("Garage does not support anonymous access yet"))? -- cgit v1.2.3 From 4c1d42cc5fcaa69818ec177f19577ad57952a117 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 27 Feb 2024 23:33:26 +0100 Subject: [fix-presigned] add back anonymous request code path + refactoring --- src/api/signature/payload.rs | 57 +++++++++++++++++++++++------------------- src/api/signature/streaming.rs | 13 +++++++--- 2 files changed, 40 insertions(+), 30 deletions(-) diff --git a/src/api/signature/payload.rs b/src/api/signature/payload.rs index 8841a5e5..c88bb144 100644 --- a/src/api/signature/payload.rs +++ b/src/api/signature/payload.rs @@ -40,17 +40,26 @@ pub async fn check_payload_signature( ) -> Result<(Option, Option), Error> { let query = parse_query_map(request.uri())?; - let res = if query.contains_key(X_AMZ_ALGORITHM.as_str()) { + if query.contains_key(X_AMZ_ALGORITHM.as_str()) { check_presigned_signature(garage, service, request, query).await - } else { + } else if request.headers().contains_key(AUTHORIZATION) { check_standard_signature(garage, service, request, query).await - }; - - if let Err(e) = &res { - error!("ERROR IN SIGNATURE\n{:?}\n{}", request, e); + } else { + // Unsigned (anonymous) request + let content_sha256 = request + .headers() + .get("x-amz-content-sha256") + .filter(|c| c.as_bytes() != UNSIGNED_PAYLOAD.as_bytes()); + if let Some(content_sha256) = content_sha256 { + let sha256 = hex::decode(content_sha256) + .ok() + .and_then(|bytes| Hash::try_from(&bytes)) + .ok_or_bad_request("Invalid content sha256 hash")?; + Ok((None, Some(sha256))) + } else { + Ok((None, None)) + } } - - res } async fn check_standard_signature( @@ -63,8 +72,11 @@ async fn check_standard_signature( // Verify that all necessary request headers are signed let signed_headers = split_signed_headers(&authorization)?; + if !signed_headers.contains(&HOST) { + return Err(Error::bad_request("Header `Host` should be signed")); + } for (name, _) in request.headers().iter() { - if name.as_str().starts_with("x-amz-") || name == CONTENT_TYPE { + if name == CONTENT_TYPE || name.as_str().starts_with("x-amz-") { if !signed_headers.contains(name) { return Err(Error::bad_request(format!( "Header `{}` should be signed", @@ -73,9 +85,6 @@ async fn check_standard_signature( } } } - if !signed_headers.contains(&HOST) { - return Err(Error::bad_request("Header `Host` should be signed")); - } let canonical_request = canonical_request( service, @@ -122,6 +131,9 @@ async fn check_presigned_signature( // Check that all mandatory signed headers are included let signed_headers = split_signed_headers(&authorization)?; + if !signed_headers.contains(&HOST) { + return Err(Error::bad_request("Header `Host` should be signed")); + } for (name, _) in request.headers().iter() { if name.as_str().starts_with("x-amz-") { if !signed_headers.contains(name) { @@ -132,9 +144,6 @@ async fn check_presigned_signature( } } } - if !signed_headers.contains(&HOST) { - return Err(Error::bad_request("Header `Host` should be signed")); - } query.remove(X_AMZ_SIGNATURE.as_str()); let canonical_request = canonical_request( @@ -254,21 +263,17 @@ pub fn canonical_request( // Canonical header string calculated from signed headers signed_headers.sort_by(|h1, h2| h1.as_str().cmp(h2.as_str())); - let canonical_header_string = { - let mut items = Vec::with_capacity(signed_headers.len()); - for name in signed_headers.iter() { + let canonical_header_string = signed_headers + .iter() + .map(|name| { let value = headers .get(name) .ok_or_bad_request(format!("signed header `{}` is not present", name))? .to_str()?; - items.push((name, value)); - } - items - .iter() - .map(|(key, value)| format!("{}:{}", key.as_str(), value.trim())) - .collect::>() - .join("\n") - }; + Ok(format!("{}:{}", name.as_str(), value.trim())) + }) + .collect::, Error>>()? + .join("\n"); let signed_headers = signed_headers.join(";"); let list = [ diff --git a/src/api/signature/streaming.rs b/src/api/signature/streaming.rs index a2a71f6b..e223d1b1 100644 --- a/src/api/signature/streaming.rs +++ b/src/api/signature/streaming.rs @@ -15,6 +15,11 @@ use super::{compute_scope, sha256sum, HmacSha256, LONG_DATETIME}; use crate::helpers::*; use crate::signature::error::*; +use crate::signature::payload::{ + STREAMING_AWS4_HMAC_SHA256_PAYLOAD, X_AMZ_CONTENT_SH256, X_AMZ_DATE, +}; + +pub const AWS4_HMAC_SHA256_PAYLOAD: &str = "AWS4-HMAC-SHA256-PAYLOAD"; pub type ReqBody = BoxBody; @@ -25,8 +30,8 @@ pub fn parse_streaming_body( region: &str, service: &str, ) -> Result, Error> { - match req.headers().get("x-amz-content-sha256") { - Some(header) if header == "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" => { + match req.headers().get(X_AMZ_CONTENT_SH256) { + Some(header) if header == STREAMING_AWS4_HMAC_SHA256_PAYLOAD => { let signature = content_sha256 .take() .ok_or_bad_request("No signature provided")?; @@ -39,7 +44,7 @@ pub fn parse_streaming_body( let date = req .headers() - .get("x-amz-date") + .get(X_AMZ_DATE) .ok_or_bad_request("Missing X-Amz-Date field")? .to_str()?; let date: NaiveDateTime = NaiveDateTime::parse_from_str(date, LONG_DATETIME) @@ -75,7 +80,7 @@ fn compute_streaming_payload_signature( content_sha256: Hash, ) -> Result { let string_to_sign = [ - "AWS4-HMAC-SHA256-PAYLOAD", + AWS4_HMAC_SHA256_PAYLOAD, &date.format(LONG_DATETIME).to_string(), scope, &hex::encode(previous_signature), -- cgit v1.2.3 From a5e4bfeae9d705e0c8a56dfd8268e1309999c5cd Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 27 Feb 2024 23:46:49 +0100 Subject: [fix-presigned] write comments --- src/api/signature/payload.rs | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/api/signature/payload.rs b/src/api/signature/payload.rs index c88bb144..8a7a4341 100644 --- a/src/api/signature/payload.rs +++ b/src/api/signature/payload.rs @@ -40,10 +40,10 @@ pub async fn check_payload_signature( ) -> Result<(Option, Option), Error> { let query = parse_query_map(request.uri())?; - if query.contains_key(X_AMZ_ALGORITHM.as_str()) { - check_presigned_signature(garage, service, request, query).await - } else if request.headers().contains_key(AUTHORIZATION) { + if request.headers().contains_key(AUTHORIZATION) { check_standard_signature(garage, service, request, query).await + } else if query.contains_key(X_AMZ_ALGORITHM.as_str()) { + check_presigned_signature(garage, service, request, query).await } else { // Unsigned (anonymous) request let content_sha256 = request @@ -70,7 +70,11 @@ async fn check_standard_signature( ) -> Result<(Option, Option), Error> { let authorization = Authorization::parse(request.headers())?; - // Verify that all necessary request headers are signed + // Verify that all necessary request headers are included in signed_headers + // For standard AWSv4 signatures, the following must be incldued: + // - the Host header (in all cases) + // - the Content-Type header, if it is used in the request + // - all x-amz-* headers used in the request let signed_headers = split_signed_headers(&authorization)?; if !signed_headers.contains(&HOST) { return Err(Error::bad_request("Header `Host` should be signed")); @@ -129,7 +133,10 @@ async fn check_presigned_signature( let algorithm = query.get(X_AMZ_ALGORITHM.as_str()).unwrap(); let authorization = Authorization::parse_presigned(algorithm, &query)?; - // Check that all mandatory signed headers are included + // Verify that all necessary request headers are included in signed_headers + // For AWSv4 pre-signed URLs, the following must be incldued: + // - the Host header (in all cases) + // - all x-amz-* headers used in the request let signed_headers = split_signed_headers(&authorization)?; if !signed_headers.contains(&HOST) { return Err(Error::bad_request("Header `Host` should be signed")); @@ -145,6 +152,10 @@ async fn check_presigned_signature( } } + // The X-Amz-Signature value is passed as a query parameter, + // but the signature cannot be computed from a string that contains itself. + // AWS specifies that all query params except X-Amz-Signature are included + // in the canonical request. query.remove(X_AMZ_SIGNATURE.as_str()); let canonical_request = canonical_request( service, -- cgit v1.2.3 From e9f759d4cb9be28584ab511a0a2dc78b579475c8 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 28 Feb 2024 00:27:54 +0100 Subject: [fix-presigned] presigned requests: allow x-amz-* query parameters to stand in for equivalent headers --- src/api/k2v/api_server.rs | 5 ++-- src/api/s3/api_server.rs | 5 ++-- src/api/signature/payload.rs | 45 ++++++++++++++++++++++++----- src/garage/tests/common/custom_requester.rs | 2 +- 4 files changed, 44 insertions(+), 13 deletions(-) diff --git a/src/api/k2v/api_server.rs b/src/api/k2v/api_server.rs index e97da2af..5ed7e286 100644 --- a/src/api/k2v/api_server.rs +++ b/src/api/k2v/api_server.rs @@ -69,7 +69,7 @@ impl ApiHandler for K2VApiServer { async fn handle( &self, - req: Request, + mut req: Request, endpoint: K2VApiEndpoint, ) -> Result, Error> { let K2VApiEndpoint { @@ -86,7 +86,8 @@ impl ApiHandler for K2VApiServer { return Ok(options_res.map(|_empty_body: EmptyBody| empty_body())); } - let (api_key, mut content_sha256) = check_payload_signature(&garage, "k2v", &req).await?; + let (api_key, mut content_sha256) = + check_payload_signature(&garage, "k2v", &mut req).await?; let api_key = api_key .ok_or_else(|| Error::forbidden("Garage does not support anonymous access yet"))?; diff --git a/src/api/s3/api_server.rs b/src/api/s3/api_server.rs index 08405923..fdfaf0a4 100644 --- a/src/api/s3/api_server.rs +++ b/src/api/s3/api_server.rs @@ -107,7 +107,7 @@ impl ApiHandler for S3ApiServer { async fn handle( &self, - req: Request, + mut req: Request, endpoint: S3ApiEndpoint, ) -> Result, Error> { let S3ApiEndpoint { @@ -125,7 +125,8 @@ impl ApiHandler for S3ApiServer { return Ok(options_res.map(|_empty_body: EmptyBody| empty_body())); } - let (api_key, mut content_sha256) = check_payload_signature(&garage, "s3", &req).await?; + let (api_key, mut content_sha256) = + check_payload_signature(&garage, "s3", &mut req).await?; let api_key = api_key .ok_or_else(|| Error::forbidden("Garage does not support anonymous access yet"))?; diff --git a/src/api/signature/payload.rs b/src/api/signature/payload.rs index 8a7a4341..435b4206 100644 --- a/src/api/signature/payload.rs +++ b/src/api/signature/payload.rs @@ -3,7 +3,7 @@ use std::convert::TryFrom; use chrono::{DateTime, Duration, NaiveDateTime, TimeZone, Utc}; use hmac::Mac; -use hyper::header::{HeaderMap, HeaderName, AUTHORIZATION, CONTENT_TYPE, HOST}; +use hyper::header::{HeaderMap, HeaderName, HeaderValue, AUTHORIZATION, CONTENT_TYPE, HOST}; use hyper::{body::Incoming as IncomingBody, Method, Request}; use sha2::{Digest, Sha256}; @@ -36,7 +36,7 @@ pub type QueryMap = HashMap; pub async fn check_payload_signature( garage: &Garage, service: &'static str, - request: &Request, + request: &mut Request, ) -> Result<(Option, Option), Error> { let query = parse_query_map(request.uri())?; @@ -96,7 +96,7 @@ async fn check_standard_signature( request.uri().path(), &query, request.headers(), - signed_headers, + &signed_headers, &authorization.content_sha256, )?; let string_to_sign = string_to_sign( @@ -127,7 +127,7 @@ async fn check_standard_signature( async fn check_presigned_signature( garage: &Garage, service: &'static str, - request: &Request, + request: &mut Request, mut query: QueryMap, ) -> Result<(Option, Option), Error> { let algorithm = query.get(X_AMZ_ALGORITHM.as_str()).unwrap(); @@ -163,7 +163,7 @@ async fn check_presigned_signature( request.uri().path(), &query, request.headers(), - signed_headers, + &signed_headers, &authorization.content_sha256, )?; let string_to_sign = string_to_sign( @@ -177,6 +177,35 @@ async fn check_presigned_signature( let key = verify_v4(garage, service, &authorization, string_to_sign.as_bytes()).await?; + // AWS specifies that if a signed query parameter and a signed header of the same name + // have different values, then an InvalidRequest error is raised. + let headers_mut = request.headers_mut(); + for (name, value) in query.iter() { + let name = + HeaderName::from_bytes(name.as_bytes()).ok_or_bad_request("Invalid header name")?; + if let Some(existing) = headers_mut.get(&name) { + if signed_headers.contains(&name) && existing.as_bytes() != value.as_bytes() { + return Err(Error::bad_request(format!( + "Conflicting values for `{}` in query parameters and request headers", + name + ))); + } + } + if name.as_str().starts_with("x-amz-") { + // Query parameters that start by x-amz- are actually intended to stand in for + // headers that can't be added at the time the request is made. + // What we do is just add them to the Request object as regular headers, + // that will be handled downstream as if they were included like in a normal request. + // (Here we allow such query parameters to override headers with the same name + // if they are not signed, however there is not much reason that this would happen) + headers_mut.insert( + name, + HeaderValue::from_bytes(value.as_bytes()) + .ok_or_bad_request("invalid query parameter value")?, + ); + } + } + Ok((Some(key), None)) } @@ -197,12 +226,13 @@ pub fn parse_query_map(uri: &http::uri::Uri) -> Result { } fn split_signed_headers(authorization: &Authorization) -> Result, Error> { - let signed_headers = authorization + let mut signed_headers = authorization .signed_headers .split(';') .map(HeaderName::try_from) .collect::, _>>() .ok_or_bad_request("invalid header name")?; + signed_headers.sort_by(|h1, h2| h1.as_str().cmp(h2.as_str())); Ok(signed_headers) } @@ -224,7 +254,7 @@ pub fn canonical_request( canonical_uri: &str, query: &QueryMap, headers: &HeaderMap, - mut signed_headers: Vec, + signed_headers: &[HeaderName], content_sha256: &str, ) -> Result { // There seems to be evidence that in AWSv4 signatures, the path component is url-encoded @@ -273,7 +303,6 @@ pub fn canonical_request( }; // Canonical header string calculated from signed headers - signed_headers.sort_by(|h1, h2| h1.as_str().cmp(h2.as_str())); let canonical_header_string = signed_headers .iter() .map(|name| { diff --git a/src/garage/tests/common/custom_requester.rs b/src/garage/tests/common/custom_requester.rs index f311418c..2cac5cd5 100644 --- a/src/garage/tests/common/custom_requester.rs +++ b/src/garage/tests/common/custom_requester.rs @@ -251,7 +251,7 @@ impl<'a> RequestBuilder<'a> { uri.path(), &query, &all_headers, - signed_headers, + &signed_headers, &body_sha, ) .unwrap(); -- cgit v1.2.3 From 90cab5b8f2b5212668975bf445a1e86f638fe1c7 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 28 Feb 2024 10:51:08 +0100 Subject: [fix-presigned] add comments and reorganize --- src/api/k2v/api_server.rs | 18 ++------ src/api/s3/api_server.rs | 18 ++------ src/api/signature/mod.rs | 29 +++++++++++- src/api/signature/payload.rs | 105 ++++++++++++++++++++++--------------------- 4 files changed, 87 insertions(+), 83 deletions(-) diff --git a/src/api/k2v/api_server.rs b/src/api/k2v/api_server.rs index 5ed7e286..fdb5db4c 100644 --- a/src/api/k2v/api_server.rs +++ b/src/api/k2v/api_server.rs @@ -15,8 +15,7 @@ use garage_model::garage::Garage; use crate::generic_server::*; use crate::k2v::error::*; -use crate::signature::payload::check_payload_signature; -use crate::signature::streaming::*; +use crate::signature::verify_request; use crate::helpers::*; use crate::k2v::batch::*; @@ -69,7 +68,7 @@ impl ApiHandler for K2VApiServer { async fn handle( &self, - mut req: Request, + req: Request, endpoint: K2VApiEndpoint, ) -> Result, Error> { let K2VApiEndpoint { @@ -86,18 +85,7 @@ impl ApiHandler for K2VApiServer { return Ok(options_res.map(|_empty_body: EmptyBody| empty_body())); } - let (api_key, mut content_sha256) = - check_payload_signature(&garage, "k2v", &mut req).await?; - let api_key = api_key - .ok_or_else(|| Error::forbidden("Garage does not support anonymous access yet"))?; - - let req = parse_streaming_body( - &api_key, - req, - &mut content_sha256, - &garage.config.s3_api.s3_region, - "k2v", - )?; + let (req, api_key, _content_sha256) = verify_request(&garage, req, "k2v").await?; let bucket_id = garage .bucket_helper() diff --git a/src/api/s3/api_server.rs b/src/api/s3/api_server.rs index fdfaf0a4..51f19554 100644 --- a/src/api/s3/api_server.rs +++ b/src/api/s3/api_server.rs @@ -17,8 +17,7 @@ use garage_model::key_table::Key; use crate::generic_server::*; use crate::s3::error::*; -use crate::signature::payload::check_payload_signature; -use crate::signature::streaming::*; +use crate::signature::verify_request; use crate::helpers::*; use crate::s3::bucket::*; @@ -107,7 +106,7 @@ impl ApiHandler for S3ApiServer { async fn handle( &self, - mut req: Request, + req: Request, endpoint: S3ApiEndpoint, ) -> Result, Error> { let S3ApiEndpoint { @@ -125,18 +124,7 @@ impl ApiHandler for S3ApiServer { return Ok(options_res.map(|_empty_body: EmptyBody| empty_body())); } - let (api_key, mut content_sha256) = - check_payload_signature(&garage, "s3", &mut req).await?; - let api_key = api_key - .ok_or_else(|| Error::forbidden("Garage does not support anonymous access yet"))?; - - let req = parse_streaming_body( - &api_key, - req, - &mut content_sha256, - &garage.config.s3_api.s3_region, - "s3", - )?; + let (req, api_key, content_sha256) = verify_request(&garage, req, "s3").await?; let bucket_name = match bucket_name { None => { diff --git a/src/api/signature/mod.rs b/src/api/signature/mod.rs index 4b8b990f..6514da43 100644 --- a/src/api/signature/mod.rs +++ b/src/api/signature/mod.rs @@ -2,19 +2,44 @@ use chrono::{DateTime, Utc}; use hmac::{Hmac, Mac}; use sha2::Sha256; +use hyper::{body::Incoming as IncomingBody, Request}; + +use garage_model::garage::Garage; +use garage_model::key_table::Key; use garage_util::data::{sha256sum, Hash}; +use error::*; + pub mod error; pub mod payload; pub mod streaming; -use error::*; - pub const SHORT_DATE: &str = "%Y%m%d"; pub const LONG_DATETIME: &str = "%Y%m%dT%H%M%SZ"; type HmacSha256 = Hmac; +pub async fn verify_request( + garage: &Garage, + mut req: Request, + service: &'static str, +) -> Result<(Request, Key, Option), Error> { + let (api_key, mut content_sha256) = + payload::check_payload_signature(&garage, &mut req, service).await?; + let api_key = + api_key.ok_or_else(|| Error::forbidden("Garage does not support anonymous access yet"))?; + + let req = streaming::parse_streaming_body( + &api_key, + req, + &mut content_sha256, + &garage.config.s3_api.s3_region, + service, + )?; + + Ok((req, api_key, content_sha256)) +} + pub fn verify_signed_content(expected_sha256: Hash, body: &[u8]) -> Result<(), Error> { if expected_sha256 != sha256sum(body) { return Err(Error::bad_request( diff --git a/src/api/signature/payload.rs b/src/api/signature/payload.rs index 435b4206..949da601 100644 --- a/src/api/signature/payload.rs +++ b/src/api/signature/payload.rs @@ -35,15 +35,18 @@ pub type QueryMap = HashMap; pub async fn check_payload_signature( garage: &Garage, - service: &'static str, request: &mut Request, + service: &'static str, ) -> Result<(Option, Option), Error> { let query = parse_query_map(request.uri())?; - if request.headers().contains_key(AUTHORIZATION) { - check_standard_signature(garage, service, request, query).await - } else if query.contains_key(X_AMZ_ALGORITHM.as_str()) { + if query.contains_key(X_AMZ_ALGORITHM.as_str()) { + // We check for presigned-URL-style authentification first, because + // the browser or someting else could inject an Authorization header + // that is totally unrelated to AWS signatures. check_presigned_signature(garage, service, request, query).await + } else if request.headers().contains_key(AUTHORIZATION) { + check_standard_signature(garage, service, request, query).await } else { // Unsigned (anonymous) request let content_sha256 = request @@ -68,27 +71,15 @@ async fn check_standard_signature( request: &Request, query: QueryMap, ) -> Result<(Option, Option), Error> { - let authorization = Authorization::parse(request.headers())?; + let authorization = Authorization::parse_header(request.headers())?; // Verify that all necessary request headers are included in signed_headers - // For standard AWSv4 signatures, the following must be incldued: - // - the Host header (in all cases) + // For standard AWSv4 signatures, the following must be included: + // - the Host header (mandatory) // - the Content-Type header, if it is used in the request // - all x-amz-* headers used in the request let signed_headers = split_signed_headers(&authorization)?; - if !signed_headers.contains(&HOST) { - return Err(Error::bad_request("Header `Host` should be signed")); - } - for (name, _) in request.headers().iter() { - if name == CONTENT_TYPE || name.as_str().starts_with("x-amz-") { - if !signed_headers.contains(name) { - return Err(Error::bad_request(format!( - "Header `{}` should be signed", - name - ))); - } - } - } + verify_signed_headers(request.headers(), &signed_headers, &[CONTENT_TYPE])?; let canonical_request = canonical_request( service, @@ -135,22 +126,10 @@ async fn check_presigned_signature( // Verify that all necessary request headers are included in signed_headers // For AWSv4 pre-signed URLs, the following must be incldued: - // - the Host header (in all cases) + // - the Host header (mandatory) // - all x-amz-* headers used in the request let signed_headers = split_signed_headers(&authorization)?; - if !signed_headers.contains(&HOST) { - return Err(Error::bad_request("Header `Host` should be signed")); - } - for (name, _) in request.headers().iter() { - if name.as_str().starts_with("x-amz-") { - if !signed_headers.contains(name) { - return Err(Error::bad_request(format!( - "Header `{}` should be signed", - name - ))); - } - } - } + verify_signed_headers(request.headers(), &signed_headers, &[])?; // The X-Amz-Signature value is passed as a query parameter, // but the signature cannot be computed from a string that contains itself. @@ -172,13 +151,14 @@ async fn check_presigned_signature( &canonical_request, ); - trace!("canonical request:\n{}", canonical_request); - trace!("string to sign:\n{}", string_to_sign); + trace!("canonical request (presigned url):\n{}", canonical_request); + trace!("string to sign (presigned url):\n{}", string_to_sign); let key = verify_v4(garage, service, &authorization, string_to_sign.as_bytes()).await?; - // AWS specifies that if a signed query parameter and a signed header of the same name - // have different values, then an InvalidRequest error is raised. + // In the page on presigned URLs, AWS specifies that if a signed query + // parameter and a signed header of the same name have different values, + // then an InvalidRequest error is raised. let headers_mut = request.headers_mut(); for (name, value) in query.iter() { let name = @@ -197,7 +177,7 @@ async fn check_presigned_signature( // What we do is just add them to the Request object as regular headers, // that will be handled downstream as if they were included like in a normal request. // (Here we allow such query parameters to override headers with the same name - // if they are not signed, however there is not much reason that this would happen) + // that are not signed, however there is not much reason that this would happen) headers_mut.insert( name, HeaderValue::from_bytes(value.as_bytes()) @@ -206,6 +186,8 @@ async fn check_presigned_signature( } } + // Presigned URLs always use UNSIGNED-PAYLOAD, + // so there is no sha256 hash to return. Ok((Some(key), None)) } @@ -225,6 +207,17 @@ pub fn parse_query_map(uri: &http::uri::Uri) -> Result { Ok(query) } +fn parse_credential(cred: &str) -> Result<(String, String), Error> { + let first_slash = cred + .find('/') + .ok_or_bad_request("Credentials does not contain '/' in authorization field")?; + let (key_id, scope) = cred.split_at(first_slash); + Ok(( + key_id.to_string(), + scope.trim_start_matches('/').to_string(), + )) +} + fn split_signed_headers(authorization: &Authorization) -> Result, Error> { let mut signed_headers = authorization .signed_headers @@ -236,6 +229,27 @@ fn split_signed_headers(authorization: &Authorization) -> Result Ok(signed_headers) } +fn verify_signed_headers( + headers: &HeaderMap, + signed_headers: &[HeaderName], + extra_headers: &[HeaderName], +) -> Result<(), Error> { + if !signed_headers.contains(&HOST) { + return Err(Error::bad_request("Header `Host` should be signed")); + } + for (name, _) in headers.iter() { + if name.as_str().starts_with("x-amz-") || extra_headers.contains(name) { + if !signed_headers.contains(name) { + return Err(Error::bad_request(format!( + "Header `{}` should be signed", + name + ))); + } + } + } + Ok(()) +} + pub fn string_to_sign(datetime: &DateTime, scope_string: &str, canonical_req: &str) -> String { let mut hasher = Sha256::default(); hasher.update(canonical_req.as_bytes()); @@ -381,7 +395,7 @@ pub struct Authorization { } impl Authorization { - fn parse(headers: &HeaderMap) -> Result { + fn parse_header(headers: &HeaderMap) -> Result { let authorization = headers .get(AUTHORIZATION) .ok_or_bad_request("Missing authorization header")? @@ -535,14 +549,3 @@ impl Authorization { Ok(auth) } } - -fn parse_credential(cred: &str) -> Result<(String, String), Error> { - let first_slash = cred - .find('/') - .ok_or_bad_request("Credentials does not contain '/' in authorization field")?; - let (key_id, scope) = cred.split_at(first_slash); - Ok(( - key_id.to_string(), - scope.trim_start_matches('/').to_string(), - )) -} -- cgit v1.2.3 From dc995059aa63b06013dd482854a0019fa21e48c3 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 28 Feb 2024 13:43:30 +0100 Subject: [doc-dedup] mention deduplication and compression in features page --- doc/book/reference-manual/features.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/book/reference-manual/features.md b/doc/book/reference-manual/features.md index e8ba9510..433efd44 100644 --- a/doc/book/reference-manual/features.md +++ b/doc/book/reference-manual/features.md @@ -55,6 +55,14 @@ and with various levels of consistency, in order to adapt to a variety of usage Read our reference page on [supported replication modes](@/documentation/reference-manual/configuration.md#replication_mode) to select the replication mode best suited to your use case (hint: in most cases, `replication_mode = "3"` is what you want). +### Compression and deduplication + +All data stored in Garage is deduplicated, and optionnally compressed using +Zstd. Objects uploaded to Garage are chunked in blocks of constant sizes (see +[`block_size`](@/documentation/reference-manual/configuration.md#block_size)), +and the hashes of individual blocks are used to dispatch them to storage nodes +and to deduplicate them. + ### Web server for static websites A storage bucket can easily be configured to be served directly by Garage as a static web site. -- cgit v1.2.3 From 947973982d3e9876e1bee8534e63a8ffcd6c0d8d Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 28 Feb 2024 13:45:30 +0100 Subject: [doc-dedup] fix #rpc_bind_outgoing anchor in config page --- doc/book/reference-manual/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/book/reference-manual/configuration.md b/doc/book/reference-manual/configuration.md index f1474613..af66c4b8 100644 --- a/doc/book/reference-manual/configuration.md +++ b/doc/book/reference-manual/configuration.md @@ -417,7 +417,7 @@ the node, even in the case of a NAT: the NAT should be configured to forward the port number to the same internal port nubmer. This means that if you have several nodes running behind a NAT, they should each use a different RPC port number. -#### `rpc_bind_outgoing` {#rpc_bind_outgoing} (since v0.9.2) +#### `rpc_bind_outgoing`(since v0.9.2) {#rpc_bind_outgoing} If enabled, pre-bind all sockets for outgoing connections to the same IP address used for listening (the IP address specified in `rpc_bind_addr`) before -- cgit v1.2.3 From a294dd9473fb47e142d0dbac7028481ff00f921e Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 28 Feb 2024 13:48:45 +0100 Subject: [doc-dedup] reorder features, move no-RAFT down --- doc/book/reference-manual/features.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/doc/book/reference-manual/features.md b/doc/book/reference-manual/features.md index 433efd44..f7014b26 100644 --- a/doc/book/reference-manual/features.md +++ b/doc/book/reference-manual/features.md @@ -37,17 +37,6 @@ A Garage cluster can very easily evolve over time, as storage nodes are added or Garage will automatically rebalance data between nodes as needed to ensure the desired number of copies. Read about cluster layout management [here](@/documentation/operations/layout.md). -### No RAFT slowing you down - -It might seem strange to tout the absence of something as a desirable feature, -but this is in fact a very important point! Garage does not use RAFT or another -consensus algorithm internally to order incoming requests: this means that all requests -directed to a Garage cluster can be handled independently of one another instead -of going through a central bottleneck (the leader node). -As a consequence, requests can be handled much faster, even in cases where latency -between cluster nodes is important (see our [benchmarks](@/documentation/design/benchmarks/index.md) for data on this). -This is particularly usefull when nodes are far from one another and talk to one other through standard Internet connections. - ### Several replication modes Garage supports a variety of replication modes, with 1 copy, 2 copies or 3 copies of your data, @@ -63,6 +52,17 @@ Zstd. Objects uploaded to Garage are chunked in blocks of constant sizes (see and the hashes of individual blocks are used to dispatch them to storage nodes and to deduplicate them. +### No RAFT slowing you down + +It might seem strange to tout the absence of something as a desirable feature, +but this is in fact a very important point! Garage does not use RAFT or another +consensus algorithm internally to order incoming requests: this means that all requests +directed to a Garage cluster can be handled independently of one another instead +of going through a central bottleneck (the leader node). +As a consequence, requests can be handled much faster, even in cases where latency +between cluster nodes is important (see our [benchmarks](@/documentation/design/benchmarks/index.md) for data on this). +This is particularly usefull when nodes are far from one another and talk to one other through standard Internet connections. + ### Web server for static websites A storage bucket can easily be configured to be served directly by Garage as a static web site. -- cgit v1.2.3 From eaac4924ef5c18eb40eabfbf246f5ae9c894889a Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 29 Feb 2024 10:57:07 +0100 Subject: [fix-auth-ct-eq] use argon2 hashing and verification for admin/metrics token checking --- Cargo.lock | 24 +++++++++++++++++++ Cargo.nix | 39 +++++++++++++++++++++++++++++- Cargo.toml | 1 + src/api/Cargo.toml | 1 + src/api/admin/api_server.rs | 58 ++++++++++++++++++++++++++++++++------------- 5 files changed, 105 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 38cc5e1f..baa2f8a3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -120,6 +120,18 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" +[[package]] +name = "argon2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072" +dependencies = [ + "base64ct", + "blake2", + "cpufeatures", + "password-hash", +] + [[package]] name = "arrayvec" version = "0.5.2" @@ -1321,6 +1333,7 @@ dependencies = [ name = "garage_api" version = "0.9.1" dependencies = [ + "argon2", "async-trait", "base64 0.21.7", "bytes", @@ -2799,6 +2812,17 @@ dependencies = [ "regex", ] +[[package]] +name = "password-hash" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" +dependencies = [ + "base64ct", + "rand_core", + "subtle", +] + [[package]] name = "paste" version = "1.0.14" diff --git a/Cargo.nix b/Cargo.nix index dfe01bfe..059cc744 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -34,7 +34,7 @@ args@{ ignoreLockHash, }: let - nixifiedLockHash = "9377d18da3b48658f9d8b2070db135db2d9ac6d9c692d6656948b765348498cc"; + nixifiedLockHash = "69c86fff0acd6c7a9a19dc6966b4cbd48e8a50c5a9fb40b3090ad71aaa5b55d0"; workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc; currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock); lockHashIgnored = if ignoreLockHash @@ -235,6 +235,25 @@ in src = fetchCratesIo { inherit name version; sha256 = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6"; }; }); + "registry+https://github.com/rust-lang/crates.io-index".argon2."0.5.3" = overridableMkRustCrate (profileName: rec { + name = "argon2"; + version = "0.5.3"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072"; }; + features = builtins.concatLists [ + [ "alloc" ] + [ "default" ] + [ "password-hash" ] + [ "rand" ] + ]; + dependencies = { + base64ct = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64ct."1.6.0" { inherit profileName; }).out; + blake2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".blake2."0.10.6" { inherit profileName; }).out; + ${ if hostPlatform.parsed.cpu.name == "i686" || hostPlatform.parsed.cpu.name == "x86_64" then "cpufeatures" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cpufeatures."0.2.12" { inherit profileName; }).out; + password_hash = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".password-hash."0.5.0" { inherit profileName; }).out; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".arrayvec."0.5.2" = overridableMkRustCrate (profileName: rec { name = "arrayvec"; version = "0.5.2"; @@ -1939,6 +1958,7 @@ in (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/prometheus") "prometheus") ]; dependencies = { + argon2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".argon2."0.5.3" { inherit profileName; }).out; async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.77" { profileName = "__noProfile"; }).out; base64 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.21.7" { inherit profileName; }).out; bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.5.0" { inherit profileName; }).out; @@ -3989,6 +4009,23 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".password-hash."0.5.0" = overridableMkRustCrate (profileName: rec { + name = "password-hash"; + version = "0.5.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166"; }; + features = builtins.concatLists [ + [ "alloc" ] + [ "default" ] + [ "rand_core" ] + ]; + dependencies = { + base64ct = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64ct."1.6.0" { inherit profileName; }).out; + rand_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand_core."0.6.4" { inherit profileName; }).out; + subtle = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".subtle."2.5.0" { inherit profileName; }).out; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".paste."1.0.14" = overridableMkRustCrate (profileName: rec { name = "paste"; version = "1.0.14"; diff --git a/Cargo.toml b/Cargo.toml index b7ffcfc5..07e06440 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,7 @@ k2v-client = { version = "0.0.4", path = "src/k2v-client" } # External crates from crates.io arc-swap = "1.0" +argon2 = "0.5" async-trait = "0.1.7" backtrace = "0.3" base64 = "0.21" diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index bc6b6aa7..cb87d9e1 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -21,6 +21,7 @@ garage_net.workspace = true garage_util.workspace = true garage_rpc.workspace = true +argon2.workspace = true async-trait.workspace = true base64.workspace = true bytes.workspace = true diff --git a/src/api/admin/api_server.rs b/src/api/admin/api_server.rs index 50813d11..265639c4 100644 --- a/src/api/admin/api_server.rs +++ b/src/api/admin/api_server.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use std::sync::Arc; +use argon2::password_hash::PasswordHash; use async_trait::async_trait; use http::header::{ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_ORIGIN, ALLOW}; @@ -45,14 +46,8 @@ impl AdminApiServer { #[cfg(feature = "metrics")] exporter: PrometheusExporter, ) -> Self { let cfg = &garage.config.admin; - let metrics_token = cfg - .metrics_token - .as_ref() - .map(|tok| format!("Bearer {}", tok)); - let admin_token = cfg - .admin_token - .as_ref() - .map(|tok| format!("Bearer {}", tok)); + let metrics_token = cfg.metrics_token.as_deref().map(hash_bearer_token); + let admin_token = cfg.admin_token.as_deref().map(hash_bearer_token); Self { garage, #[cfg(feature = "metrics")] @@ -248,11 +243,11 @@ impl ApiHandler for AdminApiServer { req: Request, endpoint: Endpoint, ) -> Result, Error> { - let expected_auth_header = + let required_auth_hash = match endpoint.authorization_type() { Authorization::None => None, - Authorization::MetricsToken => self.metrics_token.as_ref(), - Authorization::AdminToken => match &self.admin_token { + Authorization::MetricsToken => self.metrics_token.as_deref(), + Authorization::AdminToken => match self.admin_token.as_deref() { None => return Err(Error::forbidden( "Admin token isn't configured, admin API access is disabled for security.", )), @@ -260,14 +255,11 @@ impl ApiHandler for AdminApiServer { }, }; - if let Some(h) = expected_auth_header { + if let Some(password_hash) = required_auth_hash { match req.headers().get("Authorization") { None => return Err(Error::forbidden("Authorization token must be provided")), - Some(v) => { - let authorized = v.to_str().map(|hv| hv.trim() == h).unwrap_or(false); - if !authorized { - return Err(Error::forbidden("Invalid authorization token provided")); - } + Some(authorization) => { + verify_bearer_token(&authorization, password_hash)?; } } } @@ -342,3 +334,35 @@ impl ApiEndpoint for Endpoint { fn add_span_attributes(&self, _span: SpanRef<'_>) {} } + +fn hash_bearer_token(token: &str) -> String { + use argon2::{ + password_hash::{rand_core::OsRng, PasswordHasher, SaltString}, + Argon2, + }; + + let salt = SaltString::generate(&mut OsRng); + let argon2 = Argon2::default(); + argon2 + .hash_password(token.trim().as_bytes(), &salt) + .expect("could not hash API token") + .to_string() +} + +fn verify_bearer_token(token: &hyper::http::HeaderValue, password_hash: &str) -> Result<(), Error> { + use argon2::{password_hash::PasswordVerifier, Argon2}; + + let parsed_hash = PasswordHash::new(&password_hash).unwrap(); + + token + .to_str()? + .strip_prefix("Bearer ") + .and_then(|token| { + Argon2::default() + .verify_password(token.trim().as_bytes(), &parsed_hash) + .ok() + }) + .ok_or_else(|| Error::forbidden("Invalid authorization token"))?; + + Ok(()) +} -- cgit v1.2.3 From 6d33e721c41bdb0fe7da6404e6d6d32509eed6be Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 29 Feb 2024 12:43:25 +0100 Subject: [fix-auth-ct-eq] use consant time comparison for awsv4 signature verification --- src/api/signature/payload.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/api/signature/payload.rs b/src/api/signature/payload.rs index 949da601..a9e7d34d 100644 --- a/src/api/signature/payload.rs +++ b/src/api/signature/payload.rs @@ -375,9 +375,10 @@ pub async fn verify_v4( ) .ok_or_internal_error("Unable to build signing HMAC")?; hmac.update(payload); - let our_signature = hex::encode(hmac.finalize().into_bytes()); - if auth.signature != our_signature { - return Err(Error::forbidden("Invalid signature".to_string())); + let signature = + hex::decode(&auth.signature).map_err(|_| Error::forbidden("Invalid signature"))?; + if hmac.verify_slice(&signature).is_err() { + return Err(Error::forbidden("Invalid signature")); } Ok(key) -- cgit v1.2.3 From 73b11eb17cfdb0578f38a18d6c5258f6f5ca3d79 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 29 Feb 2024 13:17:36 +0100 Subject: [doc-default-token] add default metrics_token in quick start + uniformize use of base64 --- doc/book/quick-start/_index.md | 3 ++- doc/book/reference-manual/configuration.md | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/doc/book/quick-start/_index.md b/doc/book/quick-start/_index.md index acfefb07..f359843d 100644 --- a/doc/book/quick-start/_index.md +++ b/doc/book/quick-start/_index.md @@ -79,8 +79,9 @@ index = "index.html" api_bind_addr = "[::]:3904" [admin] -api_bind_addr = "0.0.0.0:3903" +api_bind_addr = "[::]:3903" admin_token = "$(openssl rand -base64 32)" +metrics_token = "$(openssl rand -base64 32)" EOF ``` diff --git a/doc/book/reference-manual/configuration.md b/doc/book/reference-manual/configuration.md index f1474613..556b5eed 100644 --- a/doc/book/reference-manual/configuration.md +++ b/doc/book/reference-manual/configuration.md @@ -69,8 +69,8 @@ root_domain = ".web.garage" [admin] api_bind_addr = "0.0.0.0:3903" -metrics_token = "cacce0b2de4bc2d9f5b5fdff551e01ac1496055aed248202d415398987e35f81" -admin_token = "ae8cb40ea7368bbdbb6430af11cca7da833d3458a5f52086f4e805a570fb5c2a" +metrics_token = "BCAdFjoa9G0KJR0WXnHHm7fs1ZAbfpI8iIZ+Z/a2NgI=" +admin_token = "UkLeGWEvHnXBqnueR3ISEMWpOnm40jH2tM2HnnL/0F4=" trace_sink = "http://localhost:4317" ``` @@ -612,7 +612,7 @@ the socket will have 0220 mode. Make sure to set user and group permissions acco The token for accessing the Metrics endpoint. If this token is not set, the Metrics endpoint can be accessed without access control. -You can use any random string for this value. We recommend generating a random token with `openssl rand -hex 32`. +You can use any random string for this value. We recommend generating a random token with `openssl rand -base64 32`. `metrics_token` was introduced in Garage `v0.7.2`. `metrics_token_file` and the `GARAGE_METRICS_TOKEN` environment variable are supported since Garage `v0.8.2`. @@ -624,7 +624,7 @@ You can use any random string for this value. We recommend generating a random t The token for accessing all of the other administration endpoints. If this token is not set, access to these endpoints is disabled entirely. -You can use any random string for this value. We recommend generating a random token with `openssl rand -hex 32`. +You can use any random string for this value. We recommend generating a random token with `openssl rand -base64 32`. `admin_token` was introduced in Garage `v0.7.2`. `admin_token_file` and the `GARAGE_ADMIN_TOKEN` environment variable are supported since Garage `v0.8.2`. -- cgit v1.2.3 From a36248a1695de02cc19b25ba127810bd32b6d350 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 1 Mar 2024 13:11:41 +0100 Subject: [fix-signed-headers] aws signatures v4: don't actually check Content-Type is signed This page of the AWS docs indicate that Content-Type should be part of the CanonicalHeaders (and therefore SignedHeaders) strings in signature calculation: https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html However, testing with Minio Client revealed that it did not sign the Content-Type header, and therefore we broke CI by expecting it to be signed. With this commit, we don't mandate Content-Type to be signed anymore, for better compatibility with the ecosystem. Testing against the official behavior of S3 on AWS has not been done. --- script/test-smoke.sh | 4 +--- src/api/signature/payload.rs | 19 ++++++++----------- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/script/test-smoke.sh b/script/test-smoke.sh index 6965c0f3..9f9ea50c 100755 --- a/script/test-smoke.sh +++ b/script/test-smoke.sh @@ -81,11 +81,9 @@ if [ -z "$SKIP_AWS" ]; then echo "Invalid multipart upload" exit 1 fi + aws s3api delete-object --bucket eprouvette --key upload fi -echo "OK!!" -exit 0 - # S3CMD if [ -z "$SKIP_S3CMD" ]; then echo "🛠️ Testing with s3cmd" diff --git a/src/api/signature/payload.rs b/src/api/signature/payload.rs index a9e7d34d..0029716a 100644 --- a/src/api/signature/payload.rs +++ b/src/api/signature/payload.rs @@ -3,7 +3,7 @@ use std::convert::TryFrom; use chrono::{DateTime, Duration, NaiveDateTime, TimeZone, Utc}; use hmac::Mac; -use hyper::header::{HeaderMap, HeaderName, HeaderValue, AUTHORIZATION, CONTENT_TYPE, HOST}; +use hyper::header::{HeaderMap, HeaderName, HeaderValue, AUTHORIZATION, HOST}; use hyper::{body::Incoming as IncomingBody, Method, Request}; use sha2::{Digest, Sha256}; @@ -74,12 +74,13 @@ async fn check_standard_signature( let authorization = Authorization::parse_header(request.headers())?; // Verify that all necessary request headers are included in signed_headers - // For standard AWSv4 signatures, the following must be included: + // The following must be included for all signatures: // - the Host header (mandatory) - // - the Content-Type header, if it is used in the request // - all x-amz-* headers used in the request + // AWS also indicates that the Content-Type header should be signed if + // it is used, but Minio client doesn't sign it so we don't check it for compatibility. let signed_headers = split_signed_headers(&authorization)?; - verify_signed_headers(request.headers(), &signed_headers, &[CONTENT_TYPE])?; + verify_signed_headers(request.headers(), &signed_headers)?; let canonical_request = canonical_request( service, @@ -129,7 +130,7 @@ async fn check_presigned_signature( // - the Host header (mandatory) // - all x-amz-* headers used in the request let signed_headers = split_signed_headers(&authorization)?; - verify_signed_headers(request.headers(), &signed_headers, &[])?; + verify_signed_headers(request.headers(), &signed_headers)?; // The X-Amz-Signature value is passed as a query parameter, // but the signature cannot be computed from a string that contains itself. @@ -229,16 +230,12 @@ fn split_signed_headers(authorization: &Authorization) -> Result Ok(signed_headers) } -fn verify_signed_headers( - headers: &HeaderMap, - signed_headers: &[HeaderName], - extra_headers: &[HeaderName], -) -> Result<(), Error> { +fn verify_signed_headers(headers: &HeaderMap, signed_headers: &[HeaderName]) -> Result<(), Error> { if !signed_headers.contains(&HOST) { return Err(Error::bad_request("Header `Host` should be signed")); } for (name, _) in headers.iter() { - if name.as_str().starts_with("x-amz-") || extra_headers.contains(name) { + if name.as_str().starts_with("x-amz-") { if !signed_headers.contains(name) { return Err(Error::bad_request(format!( "Header `{}` should be signed", -- cgit v1.2.3 From 6a7623e90d19c670b4fbb3ae6bc51c4c291765f3 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 1 Mar 2024 15:14:06 +0100 Subject: [rel-0.9.2] Bump version to v0.9.2 --- Cargo.lock | 20 +++--- Cargo.nix | 138 ++++++++++++++++++++-------------------- Cargo.toml | 18 +++--- doc/book/cookbook/real-world.md | 10 +-- script/helm/garage/Chart.yaml | 2 +- src/api/Cargo.toml | 2 +- src/block/Cargo.toml | 2 +- src/db/Cargo.toml | 2 +- src/garage/Cargo.toml | 2 +- src/model/Cargo.toml | 2 +- src/net/Cargo.toml | 4 +- src/rpc/Cargo.toml | 2 +- src/table/Cargo.toml | 2 +- src/util/Cargo.toml | 2 +- src/web/Cargo.toml | 2 +- 15 files changed, 105 insertions(+), 105 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index baa2f8a3..b2956559 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1277,7 +1277,7 @@ dependencies = [ [[package]] name = "garage" -version = "0.9.1" +version = "0.9.2" dependencies = [ "assert-json-diff", "async-trait", @@ -1331,7 +1331,7 @@ dependencies = [ [[package]] name = "garage_api" -version = "0.9.1" +version = "0.9.2" dependencies = [ "argon2", "async-trait", @@ -1380,7 +1380,7 @@ dependencies = [ [[package]] name = "garage_block" -version = "0.9.1" +version = "0.9.2" dependencies = [ "arc-swap", "async-compression", @@ -1407,7 +1407,7 @@ dependencies = [ [[package]] name = "garage_db" -version = "0.9.1" +version = "0.9.2" dependencies = [ "err-derive", "heed", @@ -1420,7 +1420,7 @@ dependencies = [ [[package]] name = "garage_model" -version = "0.9.1" +version = "0.9.2" dependencies = [ "arc-swap", "async-trait", @@ -1448,7 +1448,7 @@ dependencies = [ [[package]] name = "garage_net" -version = "0.9.1" +version = "0.9.2" dependencies = [ "arc-swap", "async-trait", @@ -1474,7 +1474,7 @@ dependencies = [ [[package]] name = "garage_rpc" -version = "0.9.1" +version = "0.9.2" dependencies = [ "arc-swap", "async-trait", @@ -1509,7 +1509,7 @@ dependencies = [ [[package]] name = "garage_table" -version = "0.9.1" +version = "0.9.2" dependencies = [ "arc-swap", "async-trait", @@ -1531,7 +1531,7 @@ dependencies = [ [[package]] name = "garage_util" -version = "0.9.1" +version = "0.9.2" dependencies = [ "arc-swap", "async-trait", @@ -1565,7 +1565,7 @@ dependencies = [ [[package]] name = "garage_web" -version = "0.9.1" +version = "0.9.2" dependencies = [ "err-derive", "futures", diff --git a/Cargo.nix b/Cargo.nix index 059cc744..f976d2e6 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -34,7 +34,7 @@ args@{ ignoreLockHash, }: let - nixifiedLockHash = "69c86fff0acd6c7a9a19dc6966b4cbd48e8a50c5a9fb40b3090ad71aaa5b55d0"; + nixifiedLockHash = "3e1e0730302ee7d1f4185a13ad0086392562eb7cb962b1212801847887beaa47"; workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc; currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock); lockHashIgnored = if ignoreLockHash @@ -58,17 +58,17 @@ in { cargo2nixVersion = "0.11.0"; workspace = { - garage_db = rustPackages.unknown.garage_db."0.9.1"; - garage_util = rustPackages.unknown.garage_util."0.9.1"; - garage_net = rustPackages.unknown.garage_net."0.9.1"; - garage_rpc = rustPackages.unknown.garage_rpc."0.9.1"; + garage_db = rustPackages.unknown.garage_db."0.9.2"; + garage_util = rustPackages.unknown.garage_util."0.9.2"; + garage_net = rustPackages.unknown.garage_net."0.9.2"; + garage_rpc = rustPackages.unknown.garage_rpc."0.9.2"; format_table = rustPackages.unknown.format_table."0.1.1"; - garage_table = rustPackages.unknown.garage_table."0.9.1"; - garage_block = rustPackages.unknown.garage_block."0.9.1"; - garage_model = rustPackages.unknown.garage_model."0.9.1"; - garage_api = rustPackages.unknown.garage_api."0.9.1"; - garage_web = rustPackages.unknown.garage_web."0.9.1"; - garage = rustPackages.unknown.garage."0.9.1"; + garage_table = rustPackages.unknown.garage_table."0.9.2"; + garage_block = rustPackages.unknown.garage_block."0.9.2"; + garage_model = rustPackages.unknown.garage_model."0.9.2"; + garage_api = rustPackages.unknown.garage_api."0.9.2"; + garage_web = rustPackages.unknown.garage_web."0.9.2"; + garage = rustPackages.unknown.garage."0.9.2"; k2v-client = rustPackages.unknown.k2v-client."0.0.4"; }; "registry+https://github.com/rust-lang/crates.io-index".addr2line."0.21.0" = overridableMkRustCrate (profileName: rec { @@ -1871,9 +1871,9 @@ in }; }); - "unknown".garage."0.9.1" = overridableMkRustCrate (profileName: rec { + "unknown".garage."0.9.2" = overridableMkRustCrate (profileName: rec { name = "garage"; - version = "0.9.1"; + version = "0.9.2"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/garage"); features = builtins.concatLists [ @@ -1900,15 +1900,15 @@ in format_table = (rustPackages."unknown".format_table."0.1.1" { inherit profileName; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" { inherit profileName; }).out; futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.30" { inherit profileName; }).out; - garage_api = (rustPackages."unknown".garage_api."0.9.1" { inherit profileName; }).out; - garage_block = (rustPackages."unknown".garage_block."0.9.1" { inherit profileName; }).out; - garage_db = (rustPackages."unknown".garage_db."0.9.1" { inherit profileName; }).out; - garage_model = (rustPackages."unknown".garage_model."0.9.1" { inherit profileName; }).out; - garage_net = (rustPackages."unknown".garage_net."0.9.1" { inherit profileName; }).out; - garage_rpc = (rustPackages."unknown".garage_rpc."0.9.1" { inherit profileName; }).out; - garage_table = (rustPackages."unknown".garage_table."0.9.1" { inherit profileName; }).out; - garage_util = (rustPackages."unknown".garage_util."0.9.1" { inherit profileName; }).out; - garage_web = (rustPackages."unknown".garage_web."0.9.1" { inherit profileName; }).out; + garage_api = (rustPackages."unknown".garage_api."0.9.2" { inherit profileName; }).out; + garage_block = (rustPackages."unknown".garage_block."0.9.2" { inherit profileName; }).out; + garage_db = (rustPackages."unknown".garage_db."0.9.2" { inherit profileName; }).out; + garage_model = (rustPackages."unknown".garage_model."0.9.2" { inherit profileName; }).out; + garage_net = (rustPackages."unknown".garage_net."0.9.2" { inherit profileName; }).out; + garage_rpc = (rustPackages."unknown".garage_rpc."0.9.2" { inherit profileName; }).out; + garage_table = (rustPackages."unknown".garage_table."0.9.2" { inherit profileName; }).out; + garage_util = (rustPackages."unknown".garage_util."0.9.2" { inherit profileName; }).out; + garage_web = (rustPackages."unknown".garage_web."0.9.2" { inherit profileName; }).out; git_version = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.9" { inherit profileName; }).out; hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; sodiumoxide = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }).out; @@ -1946,9 +1946,9 @@ in }; }); - "unknown".garage_api."0.9.1" = overridableMkRustCrate (profileName: rec { + "unknown".garage_api."0.9.2" = overridableMkRustCrate (profileName: rec { name = "garage_api"; - version = "0.9.1"; + version = "0.9.2"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/api"); features = builtins.concatLists [ @@ -1968,12 +1968,12 @@ in form_urlencoded = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.2.1" { inherit profileName; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" { inherit profileName; }).out; futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.30" { inherit profileName; }).out; - garage_block = (rustPackages."unknown".garage_block."0.9.1" { inherit profileName; }).out; - garage_model = (rustPackages."unknown".garage_model."0.9.1" { inherit profileName; }).out; - garage_net = (rustPackages."unknown".garage_net."0.9.1" { inherit profileName; }).out; - garage_rpc = (rustPackages."unknown".garage_rpc."0.9.1" { inherit profileName; }).out; - garage_table = (rustPackages."unknown".garage_table."0.9.1" { inherit profileName; }).out; - garage_util = (rustPackages."unknown".garage_util."0.9.1" { inherit profileName; }).out; + garage_block = (rustPackages."unknown".garage_block."0.9.2" { inherit profileName; }).out; + garage_model = (rustPackages."unknown".garage_model."0.9.2" { inherit profileName; }).out; + garage_net = (rustPackages."unknown".garage_net."0.9.2" { inherit profileName; }).out; + garage_rpc = (rustPackages."unknown".garage_rpc."0.9.2" { inherit profileName; }).out; + garage_table = (rustPackages."unknown".garage_table."0.9.2" { inherit profileName; }).out; + garage_util = (rustPackages."unknown".garage_util."0.9.2" { inherit profileName; }).out; hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; hmac = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hmac."0.12.1" { inherit profileName; }).out; http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."1.0.0" { inherit profileName; }).out; @@ -2004,9 +2004,9 @@ in }; }); - "unknown".garage_block."0.9.1" = overridableMkRustCrate (profileName: rec { + "unknown".garage_block."0.9.2" = overridableMkRustCrate (profileName: rec { name = "garage_block"; - version = "0.9.1"; + version = "0.9.2"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/block"); features = builtins.concatLists [ @@ -2020,11 +2020,11 @@ in bytesize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.3.0" { inherit profileName; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" { inherit profileName; }).out; futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.30" { inherit profileName; }).out; - garage_db = (rustPackages."unknown".garage_db."0.9.1" { inherit profileName; }).out; - garage_net = (rustPackages."unknown".garage_net."0.9.1" { inherit profileName; }).out; - garage_rpc = (rustPackages."unknown".garage_rpc."0.9.1" { inherit profileName; }).out; - garage_table = (rustPackages."unknown".garage_table."0.9.1" { inherit profileName; }).out; - garage_util = (rustPackages."unknown".garage_util."0.9.1" { inherit profileName; }).out; + garage_db = (rustPackages."unknown".garage_db."0.9.2" { inherit profileName; }).out; + garage_net = (rustPackages."unknown".garage_net."0.9.2" { inherit profileName; }).out; + garage_rpc = (rustPackages."unknown".garage_rpc."0.9.2" { inherit profileName; }).out; + garage_table = (rustPackages."unknown".garage_table."0.9.2" { inherit profileName; }).out; + garage_util = (rustPackages."unknown".garage_util."0.9.2" { inherit profileName; }).out; hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out; @@ -2037,9 +2037,9 @@ in }; }); - "unknown".garage_db."0.9.1" = overridableMkRustCrate (profileName: rec { + "unknown".garage_db."0.9.2" = overridableMkRustCrate (profileName: rec { name = "garage_db"; - version = "0.9.1"; + version = "0.9.2"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/db"); features = builtins.concatLists [ @@ -2064,9 +2064,9 @@ in }; }); - "unknown".garage_model."0.9.1" = overridableMkRustCrate (profileName: rec { + "unknown".garage_model."0.9.2" = overridableMkRustCrate (profileName: rec { name = "garage_model"; - version = "0.9.1"; + version = "0.9.2"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/model"); features = builtins.concatLists [ @@ -2085,12 +2085,12 @@ in err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" { inherit profileName; }).out; futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.30" { inherit profileName; }).out; - garage_block = (rustPackages."unknown".garage_block."0.9.1" { inherit profileName; }).out; - garage_db = (rustPackages."unknown".garage_db."0.9.1" { inherit profileName; }).out; - garage_net = (rustPackages."unknown".garage_net."0.9.1" { inherit profileName; }).out; - garage_rpc = (rustPackages."unknown".garage_rpc."0.9.1" { inherit profileName; }).out; - garage_table = (rustPackages."unknown".garage_table."0.9.1" { inherit profileName; }).out; - garage_util = (rustPackages."unknown".garage_util."0.9.1" { inherit profileName; }).out; + garage_block = (rustPackages."unknown".garage_block."0.9.2" { inherit profileName; }).out; + garage_db = (rustPackages."unknown".garage_db."0.9.2" { inherit profileName; }).out; + garage_net = (rustPackages."unknown".garage_net."0.9.2" { inherit profileName; }).out; + garage_rpc = (rustPackages."unknown".garage_rpc."0.9.2" { inherit profileName; }).out; + garage_table = (rustPackages."unknown".garage_table."0.9.2" { inherit profileName; }).out; + garage_util = (rustPackages."unknown".garage_util."0.9.2" { inherit profileName; }).out; hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out; @@ -2102,9 +2102,9 @@ in }; }); - "unknown".garage_net."0.9.1" = overridableMkRustCrate (profileName: rec { + "unknown".garage_net."0.9.2" = overridableMkRustCrate (profileName: rec { name = "garage_net"; - version = "0.9.1"; + version = "0.9.2"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/net"); features = builtins.concatLists [ @@ -2139,9 +2139,9 @@ in }; }); - "unknown".garage_rpc."0.9.1" = overridableMkRustCrate (profileName: rec { + "unknown".garage_rpc."0.9.2" = overridableMkRustCrate (profileName: rec { name = "garage_rpc"; - version = "0.9.1"; + version = "0.9.2"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/rpc"); features = builtins.concatLists [ @@ -2163,9 +2163,9 @@ in format_table = (rustPackages."unknown".format_table."0.1.1" { inherit profileName; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" { inherit profileName; }).out; futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.30" { inherit profileName; }).out; - garage_db = (rustPackages."unknown".garage_db."0.9.1" { inherit profileName; }).out; - garage_net = (rustPackages."unknown".garage_net."0.9.1" { inherit profileName; }).out; - garage_util = (rustPackages."unknown".garage_util."0.9.1" { inherit profileName; }).out; + garage_db = (rustPackages."unknown".garage_db."0.9.2" { inherit profileName; }).out; + garage_net = (rustPackages."unknown".garage_net."0.9.2" { inherit profileName; }).out; + garage_util = (rustPackages."unknown".garage_util."0.9.2" { inherit profileName; }).out; gethostname = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.4.3" { inherit profileName; }).out; hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; itertools = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".itertools."0.12.1" { inherit profileName; }).out; @@ -2187,9 +2187,9 @@ in }; }); - "unknown".garage_table."0.9.1" = overridableMkRustCrate (profileName: rec { + "unknown".garage_table."0.9.2" = overridableMkRustCrate (profileName: rec { name = "garage_table"; - version = "0.9.1"; + version = "0.9.2"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/table"); dependencies = { @@ -2198,9 +2198,9 @@ in bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.5.0" { inherit profileName; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" { inherit profileName; }).out; futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.30" { inherit profileName; }).out; - garage_db = (rustPackages."unknown".garage_db."0.9.1" { inherit profileName; }).out; - garage_rpc = (rustPackages."unknown".garage_rpc."0.9.1" { inherit profileName; }).out; - garage_util = (rustPackages."unknown".garage_util."0.9.1" { inherit profileName; }).out; + garage_db = (rustPackages."unknown".garage_db."0.9.2" { inherit profileName; }).out; + garage_rpc = (rustPackages."unknown".garage_rpc."0.9.2" { inherit profileName; }).out; + garage_util = (rustPackages."unknown".garage_util."0.9.2" { inherit profileName; }).out; hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out; opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; @@ -2212,9 +2212,9 @@ in }; }); - "unknown".garage_util."0.9.1" = overridableMkRustCrate (profileName: rec { + "unknown".garage_util."0.9.2" = overridableMkRustCrate (profileName: rec { name = "garage_util"; - version = "0.9.1"; + version = "0.9.2"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/util"); features = builtins.concatLists [ @@ -2230,8 +2230,8 @@ in digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.10.7" { inherit profileName; }).out; err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" { inherit profileName; }).out; - garage_db = (rustPackages."unknown".garage_db."0.9.1" { inherit profileName; }).out; - garage_net = (rustPackages."unknown".garage_net."0.9.1" { inherit profileName; }).out; + garage_db = (rustPackages."unknown".garage_db."0.9.2" { inherit profileName; }).out; + garage_net = (rustPackages."unknown".garage_net."0.9.2" { inherit profileName; }).out; hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out; http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."1.0.0" { inherit profileName; }).out; @@ -2256,18 +2256,18 @@ in }; }); - "unknown".garage_web."0.9.1" = overridableMkRustCrate (profileName: rec { + "unknown".garage_web."0.9.2" = overridableMkRustCrate (profileName: rec { name = "garage_web"; - version = "0.9.1"; + version = "0.9.2"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/web"); dependencies = { err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" { inherit profileName; }).out; - garage_api = (rustPackages."unknown".garage_api."0.9.1" { inherit profileName; }).out; - garage_model = (rustPackages."unknown".garage_model."0.9.1" { inherit profileName; }).out; - garage_table = (rustPackages."unknown".garage_table."0.9.1" { inherit profileName; }).out; - garage_util = (rustPackages."unknown".garage_util."0.9.1" { inherit profileName; }).out; + garage_api = (rustPackages."unknown".garage_api."0.9.2" { inherit profileName; }).out; + garage_model = (rustPackages."unknown".garage_model."0.9.2" { inherit profileName; }).out; + garage_table = (rustPackages."unknown".garage_table."0.9.2" { inherit profileName; }).out; + garage_util = (rustPackages."unknown".garage_util."0.9.2" { inherit profileName; }).out; http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."1.0.0" { inherit profileName; }).out; http_body_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body-util."0.1.0" { inherit profileName; }).out; hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."1.1.0" { inherit profileName; }).out; diff --git a/Cargo.toml b/Cargo.toml index 07e06440..59d1c26b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,15 +21,15 @@ default-members = ["src/garage"] # Internal Garage crates format_table = { version = "0.1.1", path = "src/format-table" } -garage_api = { version = "0.9.1", path = "src/api" } -garage_block = { version = "0.9.1", path = "src/block" } -garage_db = { version = "0.9.1", path = "src/db", default-features = false } -garage_model = { version = "0.9.1", path = "src/model", default-features = false } -garage_net = { version = "0.9.1", path = "src/net" } -garage_rpc = { version = "0.9.1", path = "src/rpc" } -garage_table = { version = "0.9.1", path = "src/table" } -garage_util = { version = "0.9.1", path = "src/util" } -garage_web = { version = "0.9.1", path = "src/web" } +garage_api = { version = "0.9.2", path = "src/api" } +garage_block = { version = "0.9.2", path = "src/block" } +garage_db = { version = "0.9.2", path = "src/db", default-features = false } +garage_model = { version = "0.9.2", path = "src/model", default-features = false } +garage_net = { version = "0.9.2", path = "src/net" } +garage_rpc = { version = "0.9.2", path = "src/rpc" } +garage_table = { version = "0.9.2", path = "src/table" } +garage_util = { version = "0.9.2", path = "src/util" } +garage_web = { version = "0.9.2", path = "src/web" } k2v-client = { version = "0.0.4", path = "src/k2v-client" } # External crates from crates.io diff --git a/doc/book/cookbook/real-world.md b/doc/book/cookbook/real-world.md index ce0abddd..5c2dd0f1 100644 --- a/doc/book/cookbook/real-world.md +++ b/doc/book/cookbook/real-world.md @@ -85,14 +85,14 @@ to store 2 TB of data in total. ## Get a Docker image Our docker image is currently named `dxflrs/garage` and is stored on the [Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated). -We encourage you to use a fixed tag (eg. `v0.9.1`) and not the `latest` tag. -For this example, we will use the latest published version at the time of the writing which is `v0.9.1` but it's up to you +We encourage you to use a fixed tag (eg. `v0.9.2`) and not the `latest` tag. +For this example, we will use the latest published version at the time of the writing which is `v0.9.2` but it's up to you to check [the most recent versions on the Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated). For example: ``` -sudo docker pull dxflrs/garage:v0.9.1 +sudo docker pull dxflrs/garage:v0.9.2 ``` ## Deploying and configuring Garage @@ -157,7 +157,7 @@ docker run \ -v /etc/garage.toml:/etc/garage.toml \ -v /var/lib/garage/meta:/var/lib/garage/meta \ -v /var/lib/garage/data:/var/lib/garage/data \ - dxflrs/garage:v0.9.1 + dxflrs/garage:v0.9.2 ``` With this command line, Garage should be started automatically at each boot. @@ -171,7 +171,7 @@ If you want to use `docker-compose`, you may use the following `docker-compose.y version: "3" services: garage: - image: dxflrs/garage:v0.9.1 + image: dxflrs/garage:v0.9.2 network_mode: "host" restart: unless-stopped volumes: diff --git a/script/helm/garage/Chart.yaml b/script/helm/garage/Chart.yaml index 31b75c1f..9a21aa8e 100644 --- a/script/helm/garage/Chart.yaml +++ b/script/helm/garage/Chart.yaml @@ -21,4 +21,4 @@ version: 0.4.1 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "v0.9.1" +appVersion: "v0.9.2" diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index cb87d9e1..40fab769 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_api" -version = "0.9.1" +version = "0.9.2" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" diff --git a/src/block/Cargo.toml b/src/block/Cargo.toml index d2666b10..54093e4e 100644 --- a/src/block/Cargo.toml +++ b/src/block/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_block" -version = "0.9.1" +version = "0.9.2" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" diff --git a/src/db/Cargo.toml b/src/db/Cargo.toml index 9a925136..bb72d5cd 100644 --- a/src/db/Cargo.toml +++ b/src/db/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_db" -version = "0.9.1" +version = "0.9.2" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index 65135530..2c8ea3f9 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage" -version = "0.9.1" +version = "0.9.2" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" diff --git a/src/model/Cargo.toml b/src/model/Cargo.toml index ce0ccff0..3060c133 100644 --- a/src/model/Cargo.toml +++ b/src/model/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_model" -version = "0.9.1" +version = "0.9.2" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" diff --git a/src/net/Cargo.toml b/src/net/Cargo.toml index a2674498..af7b4cbe 100644 --- a/src/net/Cargo.toml +++ b/src/net/Cargo.toml @@ -1,9 +1,9 @@ [package] name = "garage_net" -version = "0.9.1" +version = "0.9.2" authors = ["Alex Auvolat "] edition = "2018" -license-file = "AGPL-3.0" +license = "AGPL-3.0" description = "Networking library for Garage RPC communication, forked from Netapp" repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage" readme = "../../README.md" diff --git a/src/rpc/Cargo.toml b/src/rpc/Cargo.toml index 5d5750c4..de0e9e17 100644 --- a/src/rpc/Cargo.toml +++ b/src/rpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_rpc" -version = "0.9.1" +version = "0.9.2" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" diff --git a/src/table/Cargo.toml b/src/table/Cargo.toml index 4f2aed7a..4cf21f9f 100644 --- a/src/table/Cargo.toml +++ b/src/table/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_table" -version = "0.9.1" +version = "0.9.2" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" diff --git a/src/util/Cargo.toml b/src/util/Cargo.toml index b4d8477c..fb7b632d 100644 --- a/src/util/Cargo.toml +++ b/src/util/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_util" -version = "0.9.1" +version = "0.9.2" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" diff --git a/src/web/Cargo.toml b/src/web/Cargo.toml index 3add5200..12bec5a7 100644 --- a/src/web/Cargo.toml +++ b/src/web/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_web" -version = "0.9.1" +version = "0.9.2" authors = ["Alex Auvolat ", "Quentin Dufour "] edition = "2018" license = "AGPL-3.0" -- cgit v1.2.3 From 16e17375c5eeefb06cbe99e1ff36c1f13336c0cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arma=C3=ABl=20Gu=C3=A9neau?= Date: Sun, 3 Mar 2024 13:02:56 +0100 Subject: doc: fix typo in connect/backup.md --- doc/book/connect/backup.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/book/connect/backup.md b/doc/book/connect/backup.md index 585ec469..f39cc3b6 100644 --- a/doc/book/connect/backup.md +++ b/doc/book/connect/backup.md @@ -55,8 +55,8 @@ Create your key and bucket: ```bash garage key create my-key -garage bucket create backup -garage bucket allow backup --read --write --key my-key +garage bucket create backups +garage bucket allow backups --read --write --key my-key ``` Then register your Key ID and Secret key in your environment: -- cgit v1.2.3 From b6a91e549ba8fa9dad8f90fa8b98f282dc211551 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 4 Mar 2024 12:52:33 +0100 Subject: [test-presigned] Add API test for presigned requests --- src/garage/tests/common/custom_requester.rs | 4 ++ src/garage/tests/s3/mod.rs | 1 + src/garage/tests/s3/presigned.rs | 72 +++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+) create mode 100644 src/garage/tests/s3/presigned.rs diff --git a/src/garage/tests/common/custom_requester.rs b/src/garage/tests/common/custom_requester.rs index 2cac5cd5..8e1eaa56 100644 --- a/src/garage/tests/common/custom_requester.rs +++ b/src/garage/tests/common/custom_requester.rs @@ -64,6 +64,10 @@ impl CustomRequester { vhost_style: false, } } + + pub fn client(&self) -> &Client { + &self.client + } } pub struct RequestBuilder<'a> { diff --git a/src/garage/tests/s3/mod.rs b/src/garage/tests/s3/mod.rs index 623eb665..4ebc4914 100644 --- a/src/garage/tests/s3/mod.rs +++ b/src/garage/tests/s3/mod.rs @@ -1,6 +1,7 @@ mod list; mod multipart; mod objects; +mod presigned; mod simple; mod streaming_signature; mod website; diff --git a/src/garage/tests/s3/presigned.rs b/src/garage/tests/s3/presigned.rs new file mode 100644 index 00000000..15270361 --- /dev/null +++ b/src/garage/tests/s3/presigned.rs @@ -0,0 +1,72 @@ +use std::time::{Duration, SystemTime}; + +use crate::common; +use aws_sdk_s3::presigning::PresigningConfig; +use bytes::Bytes; +use http_body_util::{BodyExt, Full}; +use hyper::Request; + +const STD_KEY: &str = "hello world"; +const BODY: &[u8; 62] = b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + +#[tokio::test] +async fn test_presigned_url() { + let ctx = common::context(); + let bucket = ctx.create_bucket("presigned"); + + let etag = "\"46cf18a9b447991b450cad3facf5937e\""; + let body = Bytes::from(BODY.to_vec()); + + let psc = PresigningConfig::builder() + .start_time(SystemTime::now() - Duration::from_secs(60)) + .expires_in(Duration::from_secs(3600)) + .build() + .unwrap(); + + { + // PutObject + let req = ctx + .client + .put_object() + .bucket(&bucket) + .key(STD_KEY) + .presigned(psc.clone()) + .await + .unwrap(); + + let client = ctx.custom_request.client(); + let req = Request::builder() + .method("PUT") + .uri(req.uri()) + .body(Full::new(body.clone())) + .unwrap(); + let res = client.request(req).await.unwrap(); + assert_eq!(res.status(), 200); + assert_eq!(res.headers().get("etag").unwrap(), etag); + } + + { + // GetObject + let req = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .presigned(psc) + .await + .unwrap(); + + let client = ctx.custom_request.client(); + let req = Request::builder() + .method("GET") + .uri(req.uri()) + .body(Full::new(Bytes::new())) + .unwrap(); + let res = client.request(req).await.unwrap(); + assert_eq!(res.status(), 200); + assert_eq!(res.headers().get("etag").unwrap(), etag); + + let body2 = BodyExt::collect(res.into_body()).await.unwrap().to_bytes(); + assert_eq!(body, body2); + } +} -- cgit v1.2.3 From 7c4f3473afa3e5501a21bf6e9466da52726bac8e Mon Sep 17 00:00:00 2001 From: asonix Date: Sun, 3 Mar 2024 14:35:01 -0600 Subject: Lowercase query parameter keys when parsing --- src/api/signature/payload.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/signature/payload.rs b/src/api/signature/payload.rs index 0029716a..250e007e 100644 --- a/src/api/signature/payload.rs +++ b/src/api/signature/payload.rs @@ -197,7 +197,7 @@ pub fn parse_query_map(uri: &http::uri::Uri) -> Result { if let Some(query_str) = uri.query() { let query_pairs = url::form_urlencoded::parse(query_str.as_bytes()); for (key, val) in query_pairs { - if query.insert(key.to_string(), val.into_owned()).is_some() { + if query.insert(key.to_lowercase().to_string(), val.into_owned()).is_some() { return Err(Error::bad_request(format!( "duplicate query parameter: `{}`", key -- cgit v1.2.3 From c94bf45cbab61ad6471d43d8a7aabb8a32eaad38 Mon Sep 17 00:00:00 2001 From: asonix Date: Sun, 3 Mar 2024 16:26:57 -0600 Subject: Store original-cased query keys alongside query values --- src/api/signature/payload.rs | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/src/api/signature/payload.rs b/src/api/signature/payload.rs index 250e007e..515c0f91 100644 --- a/src/api/signature/payload.rs +++ b/src/api/signature/payload.rs @@ -31,7 +31,12 @@ pub const AWS4_HMAC_SHA256: &str = "AWS4-HMAC-SHA256"; pub const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD"; pub const STREAMING_AWS4_HMAC_SHA256_PAYLOAD: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"; -pub type QueryMap = HashMap; +pub type QueryMap = HashMap; + +pub struct QueryValue { + key: String, + value: String, +} pub async fn check_payload_signature( garage: &Garage, @@ -123,7 +128,7 @@ async fn check_presigned_signature( mut query: QueryMap, ) -> Result<(Option, Option), Error> { let algorithm = query.get(X_AMZ_ALGORITHM.as_str()).unwrap(); - let authorization = Authorization::parse_presigned(algorithm, &query)?; + let authorization = Authorization::parse_presigned(&algorithm.value, &query)?; // Verify that all necessary request headers are included in signed_headers // For AWSv4 pre-signed URLs, the following must be incldued: @@ -165,7 +170,7 @@ async fn check_presigned_signature( let name = HeaderName::from_bytes(name.as_bytes()).ok_or_bad_request("Invalid header name")?; if let Some(existing) = headers_mut.get(&name) { - if signed_headers.contains(&name) && existing.as_bytes() != value.as_bytes() { + if signed_headers.contains(&name) && existing.as_bytes() != value.value.as_bytes() { return Err(Error::bad_request(format!( "Conflicting values for `{}` in query parameters and request headers", name @@ -181,7 +186,7 @@ async fn check_presigned_signature( // that are not signed, however there is not much reason that this would happen) headers_mut.insert( name, - HeaderValue::from_bytes(value.as_bytes()) + HeaderValue::from_bytes(value.value.as_bytes()) .ok_or_bad_request("invalid query parameter value")?, ); } @@ -197,7 +202,14 @@ pub fn parse_query_map(uri: &http::uri::Uri) -> Result { if let Some(query_str) = uri.query() { let query_pairs = url::form_urlencoded::parse(query_str.as_bytes()); for (key, val) in query_pairs { - if query.insert(key.to_lowercase().to_string(), val.into_owned()).is_some() { + let key = key.into_owned(); + + let value = QueryValue { + key: key.clone(), + value: val.into_owned(), + }; + + if query.insert(key.to_lowercase(), value).is_some() { return Err(Error::bad_request(format!( "duplicate query parameter: `{}`", key @@ -306,7 +318,7 @@ pub fn canonical_request( // Canonical query string from passed HeaderMap let canonical_query_string = { let mut items = Vec::with_capacity(query.len()); - for (key, value) in query.iter() { + for (_, QueryValue { key, value }) in query.iter() { items.push(uri_encode(&key, true) + "=" + &uri_encode(&value, true)); } items.sort(); @@ -476,6 +488,7 @@ impl Authorization { let duration = query .get(X_AMZ_EXPIRES.as_str()) .ok_or_bad_request("X-Amz-Expires not found in query parameters")? + .value .parse() .map_err(|_| Error::bad_request("X-Amz-Expires is not a number".to_string()))?; @@ -488,18 +501,18 @@ impl Authorization { let date = query .get(X_AMZ_DATE.as_str()) .ok_or_bad_request("Missing X-Amz-Date field")?; - let date = parse_date(date)?; + let date = parse_date(&date.value)?; if Utc::now() - date > Duration::seconds(duration) { return Err(Error::bad_request("Date is too old".to_string())); } - let (key_id, scope) = parse_credential(cred)?; + let (key_id, scope) = parse_credential(&cred.value)?; Ok(Authorization { key_id, scope, - signed_headers: signed_headers.to_string(), - signature: signature.to_string(), + signed_headers: signed_headers.value.clone(), + signature: signature.value.clone(), content_sha256: UNSIGNED_PAYLOAD.to_string(), date, }) -- cgit v1.2.3 From fb55682c66092921f766f82c16eb9e046f1bbb41 Mon Sep 17 00:00:00 2001 From: Yureka Date: Sun, 3 Mar 2024 14:56:52 +0100 Subject: add request context helper --- src/api/helpers.rs | 14 +++ src/api/k2v/api_server.rs | 45 +++++----- src/api/k2v/batch.rs | 51 +++++------ src/api/k2v/index.rs | 10 +-- src/api/k2v/item.rs | 38 +++++---- src/api/s3/api_server.rs | 212 ++++++++++++++++++---------------------------- src/api/s3/bucket.rs | 48 ++++++----- src/api/s3/copy.rs | 35 ++++---- src/api/s3/cors.rs | 65 +++++++------- src/api/s3/delete.rs | 29 +++---- src/api/s3/get.rs | 21 +++++ src/api/s3/lifecycle.rs | 53 ++++++------ src/api/s3/list.rs | 14 +-- src/api/s3/multipart.rs | 68 ++++++++------- src/api/s3/post_object.rs | 28 ++++-- src/api/s3/put.rs | 68 ++++++++------- src/api/s3/website.rs | 54 ++++++------ src/model/bucket_table.rs | 7 ++ src/web/web_server.rs | 29 ++++--- 19 files changed, 458 insertions(+), 431 deletions(-) diff --git a/src/api/helpers.rs b/src/api/helpers.rs index 5f488912..cf60005d 100644 --- a/src/api/helpers.rs +++ b/src/api/helpers.rs @@ -1,4 +1,5 @@ use std::convert::Infallible; +use std::sync::Arc; use futures::{Stream, StreamExt, TryStreamExt}; @@ -10,6 +11,10 @@ use hyper::{ use idna::domain_to_unicode; use serde::{Deserialize, Serialize}; +use garage_model::bucket_table::BucketParams; +use garage_model::garage::Garage; +use garage_model::key_table::Key; +use garage_util::data::Uuid; use garage_util::error::Error as GarageError; use crate::common_error::{CommonError as Error, *}; @@ -27,6 +32,15 @@ pub enum Authorization { Owner, } +/// The values which are known for each request related to a bucket +pub struct ReqCtx { + pub garage: Arc, + pub bucket_id: Uuid, + pub bucket_name: String, + pub bucket_params: BucketParams, + pub api_key: Key, +} + /// Host to bucket /// /// Convert a host, like "bucket.garage-site.tld" to the corresponding bucket "bucket", diff --git a/src/api/k2v/api_server.rs b/src/api/k2v/api_server.rs index fdb5db4c..658cfcc8 100644 --- a/src/api/k2v/api_server.rs +++ b/src/api/k2v/api_server.rs @@ -95,6 +95,7 @@ impl ApiHandler for K2VApiServer { .bucket_helper() .get_existing_bucket(bucket_id) .await?; + let bucket_params = bucket.state.into_option().unwrap(); let allowed = match endpoint.authorization_type() { Authorization::Read => api_key.allow_read(&bucket_id), @@ -112,40 +113,42 @@ impl ApiHandler for K2VApiServer { // are always preflighted, i.e. the browser should make // an OPTIONS call before to check it is allowed let matching_cors_rule = match *req.method() { - Method::GET | Method::HEAD | Method::POST => find_matching_cors_rule(&bucket, &req) - .ok_or_internal_error("Error looking up CORS rule")?, + Method::GET | Method::HEAD | Method::POST => { + find_matching_cors_rule(&bucket_params, &req) + .ok_or_internal_error("Error looking up CORS rule")? + .cloned() + } _ => None, }; + let ctx = ReqCtx { + garage, + bucket_id, + bucket_name, + bucket_params, + api_key, + }; + let resp = match endpoint { Endpoint::DeleteItem { partition_key, sort_key, - } => handle_delete_item(garage, req, bucket_id, &partition_key, &sort_key).await, + } => handle_delete_item(ctx, req, &partition_key, &sort_key).await, Endpoint::InsertItem { partition_key, sort_key, - } => handle_insert_item(garage, req, bucket_id, &partition_key, &sort_key).await, + } => handle_insert_item(ctx, req, &partition_key, &sort_key).await, Endpoint::ReadItem { partition_key, sort_key, - } => handle_read_item(garage, &req, bucket_id, &partition_key, &sort_key).await, + } => handle_read_item(ctx, &req, &partition_key, &sort_key).await, Endpoint::PollItem { partition_key, sort_key, causality_token, timeout, } => { - handle_poll_item( - garage, - &req, - bucket_id, - partition_key, - sort_key, - causality_token, - timeout, - ) - .await + handle_poll_item(ctx, &req, partition_key, sort_key, causality_token, timeout).await } Endpoint::ReadIndex { prefix, @@ -153,12 +156,12 @@ impl ApiHandler for K2VApiServer { end, limit, reverse, - } => handle_read_index(garage, bucket_id, prefix, start, end, limit, reverse).await, - Endpoint::InsertBatch {} => handle_insert_batch(garage, bucket_id, req).await, - Endpoint::ReadBatch {} => handle_read_batch(garage, bucket_id, req).await, - Endpoint::DeleteBatch {} => handle_delete_batch(garage, bucket_id, req).await, + } => handle_read_index(ctx, prefix, start, end, limit, reverse).await, + Endpoint::InsertBatch {} => handle_insert_batch(ctx, req).await, + Endpoint::ReadBatch {} => handle_read_batch(ctx, req).await, + Endpoint::DeleteBatch {} => handle_delete_batch(ctx, req).await, Endpoint::PollRange { partition_key } => { - handle_poll_range(garage, bucket_id, &partition_key, req).await + handle_poll_range(ctx, &partition_key, req).await } Endpoint::Options => unreachable!(), }; @@ -167,7 +170,7 @@ impl ApiHandler for K2VApiServer { // add the corresponding CORS headers to the response let mut resp_ok = resp?; if let Some(rule) = matching_cors_rule { - add_cors_headers(&mut resp_ok, rule) + add_cors_headers(&mut resp_ok, &rule) .ok_or_internal_error("Invalid bucket CORS configuration")?; } diff --git a/src/api/k2v/batch.rs b/src/api/k2v/batch.rs index ae2778b1..02b7ae8b 100644 --- a/src/api/k2v/batch.rs +++ b/src/api/k2v/batch.rs @@ -1,14 +1,9 @@ -use std::sync::Arc; - use base64::prelude::*; use hyper::{Request, Response, StatusCode}; use serde::{Deserialize, Serialize}; -use garage_util::data::*; - use garage_table::{EnumerationOrder, TableSchema}; -use garage_model::garage::Garage; use garage_model::k2v::causality::*; use garage_model::k2v::item_table::*; @@ -18,10 +13,12 @@ use crate::k2v::error::*; use crate::k2v::range::read_range; pub async fn handle_insert_batch( - garage: Arc, - bucket_id: Uuid, + ctx: ReqCtx, req: Request, ) -> Result, Error> { + let ReqCtx { + garage, bucket_id, .. + } = &ctx; let items = parse_json_body::, _, Error>(req).await?; let mut items2 = vec![]; @@ -38,7 +35,7 @@ pub async fn handle_insert_batch( items2.push((it.pk, it.sk, ct, v)); } - garage.k2v.rpc.insert_batch(bucket_id, items2).await?; + garage.k2v.rpc.insert_batch(*bucket_id, items2).await?; Ok(Response::builder() .status(StatusCode::NO_CONTENT) @@ -46,8 +43,7 @@ pub async fn handle_insert_batch( } pub async fn handle_read_batch( - garage: Arc, - bucket_id: Uuid, + ctx: ReqCtx, req: Request, ) -> Result, Error> { let queries = parse_json_body::, _, Error>(req).await?; @@ -55,7 +51,7 @@ pub async fn handle_read_batch( let resp_results = futures::future::join_all( queries .into_iter() - .map(|q| handle_read_batch_query(&garage, bucket_id, q)), + .map(|q| handle_read_batch_query(&ctx, q)), ) .await; @@ -68,12 +64,15 @@ pub async fn handle_read_batch( } async fn handle_read_batch_query( - garage: &Arc, - bucket_id: Uuid, + ctx: &ReqCtx, query: ReadBatchQuery, ) -> Result { + let ReqCtx { + garage, bucket_id, .. + } = ctx; + let partition = K2VItemPartition { - bucket_id, + bucket_id: *bucket_id, partition_key: query.partition_key.clone(), }; @@ -138,8 +137,7 @@ async fn handle_read_batch_query( } pub async fn handle_delete_batch( - garage: Arc, - bucket_id: Uuid, + ctx: ReqCtx, req: Request, ) -> Result, Error> { let queries = parse_json_body::, _, Error>(req).await?; @@ -147,7 +145,7 @@ pub async fn handle_delete_batch( let resp_results = futures::future::join_all( queries .into_iter() - .map(|q| handle_delete_batch_query(&garage, bucket_id, q)), + .map(|q| handle_delete_batch_query(&ctx, q)), ) .await; @@ -160,12 +158,15 @@ pub async fn handle_delete_batch( } async fn handle_delete_batch_query( - garage: &Arc, - bucket_id: Uuid, + ctx: &ReqCtx, query: DeleteBatchQuery, ) -> Result { + let ReqCtx { + garage, bucket_id, .. + } = &ctx; + let partition = K2VItemPartition { - bucket_id, + bucket_id: *bucket_id, partition_key: query.partition_key.clone(), }; @@ -195,7 +196,7 @@ async fn handle_delete_batch_query( .k2v .rpc .insert( - bucket_id, + *bucket_id, i.partition.partition_key, i.sort_key, Some(cc), @@ -235,7 +236,7 @@ async fn handle_delete_batch_query( .collect::>(); let n = items.len(); - garage.k2v.rpc.insert_batch(bucket_id, items).await?; + garage.k2v.rpc.insert_batch(*bucket_id, items).await?; n }; @@ -251,11 +252,13 @@ async fn handle_delete_batch_query( } pub(crate) async fn handle_poll_range( - garage: Arc, - bucket_id: Uuid, + ctx: ReqCtx, partition_key: &str, req: Request, ) -> Result, Error> { + let ReqCtx { + garage, bucket_id, .. + } = ctx; use garage_model::k2v::sub::PollRange; let query = parse_json_body::(req).await?; diff --git a/src/api/k2v/index.rs b/src/api/k2v/index.rs index 1baec1db..822bec44 100644 --- a/src/api/k2v/index.rs +++ b/src/api/k2v/index.rs @@ -3,12 +3,9 @@ use std::sync::Arc; use hyper::Response; use serde::Serialize; -use garage_util::data::*; - use garage_rpc::ring::Ring; use garage_table::util::*; -use garage_model::garage::Garage; use garage_model::k2v::item_table::{BYTES, CONFLICTS, ENTRIES, VALUES}; use crate::helpers::*; @@ -17,14 +14,17 @@ use crate::k2v::error::*; use crate::k2v::range::read_range; pub async fn handle_read_index( - garage: Arc, - bucket_id: Uuid, + ctx: ReqCtx, prefix: Option, start: Option, end: Option, limit: Option, reverse: Option, ) -> Result, Error> { + let ReqCtx { + garage, bucket_id, .. + } = &ctx; + let reverse = reverse.unwrap_or(false); let ring: Arc = garage.system.ring.borrow().clone(); diff --git a/src/api/k2v/item.rs b/src/api/k2v/item.rs index 0c5931a1..af3af4e4 100644 --- a/src/api/k2v/item.rs +++ b/src/api/k2v/item.rs @@ -1,13 +1,8 @@ -use std::sync::Arc; - use base64::prelude::*; use http::header; use hyper::{Request, Response, StatusCode}; -use garage_util::data::*; - -use garage_model::garage::Garage; use garage_model::k2v::causality::*; use garage_model::k2v::item_table::*; @@ -100,12 +95,15 @@ impl ReturnFormat { /// Handle ReadItem request #[allow(clippy::ptr_arg)] pub async fn handle_read_item( - garage: Arc, + ctx: ReqCtx, req: &Request, - bucket_id: Uuid, partition_key: &str, sort_key: &String, ) -> Result, Error> { + let ReqCtx { + garage, bucket_id, .. + } = &ctx; + let format = ReturnFormat::from(req)?; let item = garage @@ -113,7 +111,7 @@ pub async fn handle_read_item( .item_table .get( &K2VItemPartition { - bucket_id, + bucket_id: *bucket_id, partition_key: partition_key.to_string(), }, sort_key, @@ -125,12 +123,14 @@ pub async fn handle_read_item( } pub async fn handle_insert_item( - garage: Arc, + ctx: ReqCtx, req: Request, - bucket_id: Uuid, partition_key: &str, sort_key: &str, ) -> Result, Error> { + let ReqCtx { + garage, bucket_id, .. + } = &ctx; let causal_context = req .headers() .get(X_GARAGE_CAUSALITY_TOKEN) @@ -149,7 +149,7 @@ pub async fn handle_insert_item( .k2v .rpc .insert( - bucket_id, + *bucket_id, partition_key.to_string(), sort_key.to_string(), causal_context, @@ -163,12 +163,14 @@ pub async fn handle_insert_item( } pub async fn handle_delete_item( - garage: Arc, + ctx: ReqCtx, req: Request, - bucket_id: Uuid, partition_key: &str, sort_key: &str, ) -> Result, Error> { + let ReqCtx { + garage, bucket_id, .. + } = &ctx; let causal_context = req .headers() .get(X_GARAGE_CAUSALITY_TOKEN) @@ -183,7 +185,7 @@ pub async fn handle_delete_item( .k2v .rpc .insert( - bucket_id, + *bucket_id, partition_key.to_string(), sort_key.to_string(), causal_context, @@ -199,14 +201,16 @@ pub async fn handle_delete_item( /// Handle ReadItem request #[allow(clippy::ptr_arg)] pub async fn handle_poll_item( - garage: Arc, + ctx: ReqCtx, req: &Request, - bucket_id: Uuid, partition_key: String, sort_key: String, causality_token: String, timeout_secs: Option, ) -> Result, Error> { + let ReqCtx { + garage, bucket_id, .. + } = &ctx; let format = ReturnFormat::from(req)?; let causal_context = @@ -218,7 +222,7 @@ pub async fn handle_poll_item( .k2v .rpc .poll_item( - bucket_id, + *bucket_id, partition_key, sort_key, causal_context, diff --git a/src/api/s3/api_server.rs b/src/api/s3/api_server.rs index 51f19554..1ed30996 100644 --- a/src/api/s3/api_server.rs +++ b/src/api/s3/api_server.rs @@ -155,6 +155,7 @@ impl ApiHandler for S3ApiServer { .bucket_helper() .get_existing_bucket(bucket_id) .await?; + let bucket_params = bucket.state.into_option().unwrap(); let allowed = match endpoint.authorization_type() { Authorization::Read => api_key.allow_read(&bucket_id), @@ -167,12 +168,20 @@ impl ApiHandler for S3ApiServer { return Err(Error::forbidden("Operation is not allowed for this key.")); } - let matching_cors_rule = find_matching_cors_rule(&bucket, &req)?; + let matching_cors_rule = find_matching_cors_rule(&bucket_params, &req)?.cloned(); + + let ctx = ReqCtx { + garage, + bucket_id, + bucket_name, + bucket_params, + api_key, + }; let resp = match endpoint { Endpoint::HeadObject { key, part_number, .. - } => handle_head(garage, &req, bucket_id, &key, part_number).await, + } => handle_head(ctx, &req, &key, part_number).await, Endpoint::GetObject { key, part_number, @@ -192,74 +201,37 @@ impl ApiHandler for S3ApiServer { response_content_type, response_expires, }; - handle_get(garage, &req, bucket_id, &key, part_number, overrides).await + handle_get(ctx, &req, &key, part_number, overrides).await } Endpoint::UploadPart { key, part_number, upload_id, - } => { - handle_put_part( - garage, - req, - bucket_id, - &key, - part_number, - &upload_id, - content_sha256, - ) - .await - } - Endpoint::CopyObject { key } => { - handle_copy(garage, &api_key, &req, bucket_id, &key).await - } + } => handle_put_part(ctx, req, &key, part_number, &upload_id, content_sha256).await, + Endpoint::CopyObject { key } => handle_copy(ctx, &req, &key).await, Endpoint::UploadPartCopy { key, part_number, upload_id, - } => { - handle_upload_part_copy( - garage, - &api_key, - &req, - bucket_id, - &key, - part_number, - &upload_id, - ) - .await - } - Endpoint::PutObject { key } => { - handle_put(garage, req, &bucket, &key, content_sha256).await - } + } => handle_upload_part_copy(ctx, &req, &key, part_number, &upload_id).await, + Endpoint::PutObject { key } => handle_put(ctx, req, &key, content_sha256).await, Endpoint::AbortMultipartUpload { key, upload_id } => { - handle_abort_multipart_upload(garage, bucket_id, &key, &upload_id).await + handle_abort_multipart_upload(ctx, &key, &upload_id).await } - Endpoint::DeleteObject { key, .. } => handle_delete(garage, bucket_id, &key).await, + Endpoint::DeleteObject { key, .. } => handle_delete(ctx, &key).await, Endpoint::CreateMultipartUpload { key } => { - handle_create_multipart_upload(garage, &req, &bucket_name, bucket_id, &key).await + handle_create_multipart_upload(ctx, &req, &key).await } Endpoint::CompleteMultipartUpload { key, upload_id } => { - handle_complete_multipart_upload( - garage, - req, - &bucket_name, - &bucket, - &key, - &upload_id, - content_sha256, - ) - .await + handle_complete_multipart_upload(ctx, req, &key, &upload_id, content_sha256).await } Endpoint::CreateBucket {} => unreachable!(), Endpoint::HeadBucket {} => { let response = Response::builder().body(empty_body()).unwrap(); Ok(response) } - Endpoint::DeleteBucket {} => { - handle_delete_bucket(&garage, bucket_id, bucket_name, &api_key.key_id).await - } - Endpoint::GetBucketLocation {} => handle_get_bucket_location(garage), + Endpoint::DeleteBucket {} => handle_delete_bucket(ctx).await, + Endpoint::GetBucketLocation {} => handle_get_bucket_location(ctx), Endpoint::GetBucketVersioning {} => handle_get_bucket_versioning(), Endpoint::ListObjects { delimiter, @@ -268,24 +240,21 @@ impl ApiHandler for S3ApiServer { max_keys, prefix, } => { - handle_list( - garage, - &ListObjectsQuery { - common: ListQueryCommon { - bucket_name, - bucket_id, - delimiter, - page_size: max_keys.unwrap_or(1000).clamp(1, 1000), - prefix: prefix.unwrap_or_default(), - urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false), - }, - is_v2: false, - marker, - continuation_token: None, - start_after: None, + let query = ListObjectsQuery { + common: ListQueryCommon { + bucket_name: ctx.bucket_name.clone(), + bucket_id, + delimiter, + page_size: max_keys.unwrap_or(1000).clamp(1, 1000), + prefix: prefix.unwrap_or_default(), + urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false), }, - ) - .await + is_v2: false, + marker, + continuation_token: None, + start_after: None, + }; + handle_list(ctx, &query).await } Endpoint::ListObjectsV2 { delimiter, @@ -298,24 +267,21 @@ impl ApiHandler for S3ApiServer { .. } => { if list_type == "2" { - handle_list( - garage, - &ListObjectsQuery { - common: ListQueryCommon { - bucket_name, - bucket_id, - delimiter, - page_size: max_keys.unwrap_or(1000).clamp(1, 1000), - urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false), - prefix: prefix.unwrap_or_default(), - }, - is_v2: true, - marker: None, - continuation_token, - start_after, + let query = ListObjectsQuery { + common: ListQueryCommon { + bucket_name: ctx.bucket_name.clone(), + bucket_id, + delimiter, + page_size: max_keys.unwrap_or(1000).clamp(1, 1000), + urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false), + prefix: prefix.unwrap_or_default(), }, - ) - .await + is_v2: true, + marker: None, + continuation_token, + start_after, + }; + handle_list(ctx, &query).await } else { Err(Error::bad_request(format!( "Invalid endpoint: list-type={}", @@ -331,22 +297,19 @@ impl ApiHandler for S3ApiServer { prefix, upload_id_marker, } => { - handle_list_multipart_upload( - garage, - &ListMultipartUploadsQuery { - common: ListQueryCommon { - bucket_name, - bucket_id, - delimiter, - page_size: max_uploads.unwrap_or(1000).clamp(1, 1000), - prefix: prefix.unwrap_or_default(), - urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false), - }, - key_marker, - upload_id_marker, + let query = ListMultipartUploadsQuery { + common: ListQueryCommon { + bucket_name: ctx.bucket_name.clone(), + bucket_id, + delimiter, + page_size: max_uploads.unwrap_or(1000).clamp(1, 1000), + prefix: prefix.unwrap_or_default(), + urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false), }, - ) - .await + key_marker, + upload_id_marker, + }; + handle_list_multipart_upload(ctx, &query).await } Endpoint::ListParts { key, @@ -354,39 +317,28 @@ impl ApiHandler for S3ApiServer { part_number_marker, upload_id, } => { - handle_list_parts( - garage, - &ListPartsQuery { - bucket_name, - bucket_id, - key, - upload_id, - part_number_marker: part_number_marker.map(|p| p.min(10000)), - max_parts: max_parts.unwrap_or(1000).clamp(1, 1000), - }, - ) - .await - } - Endpoint::DeleteObjects {} => { - handle_delete_objects(garage, bucket_id, req, content_sha256).await - } - Endpoint::GetBucketWebsite {} => handle_get_website(&bucket).await, - Endpoint::PutBucketWebsite {} => { - handle_put_website(garage, bucket.clone(), req, content_sha256).await - } - Endpoint::DeleteBucketWebsite {} => handle_delete_website(garage, bucket.clone()).await, - Endpoint::GetBucketCors {} => handle_get_cors(&bucket).await, - Endpoint::PutBucketCors {} => { - handle_put_cors(garage, bucket.clone(), req, content_sha256).await + let query = ListPartsQuery { + bucket_name: ctx.bucket_name.clone(), + bucket_id, + key, + upload_id, + part_number_marker: part_number_marker.map(|p| p.min(10000)), + max_parts: max_parts.unwrap_or(1000).clamp(1, 1000), + }; + handle_list_parts(ctx, &query).await } - Endpoint::DeleteBucketCors {} => handle_delete_cors(garage, bucket.clone()).await, - Endpoint::GetBucketLifecycleConfiguration {} => handle_get_lifecycle(&bucket).await, + Endpoint::DeleteObjects {} => handle_delete_objects(ctx, req, content_sha256).await, + Endpoint::GetBucketWebsite {} => handle_get_website(ctx).await, + Endpoint::PutBucketWebsite {} => handle_put_website(ctx, req, content_sha256).await, + Endpoint::DeleteBucketWebsite {} => handle_delete_website(ctx).await, + Endpoint::GetBucketCors {} => handle_get_cors(ctx).await, + Endpoint::PutBucketCors {} => handle_put_cors(ctx, req, content_sha256).await, + Endpoint::DeleteBucketCors {} => handle_delete_cors(ctx).await, + Endpoint::GetBucketLifecycleConfiguration {} => handle_get_lifecycle(ctx).await, Endpoint::PutBucketLifecycleConfiguration {} => { - handle_put_lifecycle(garage, bucket.clone(), req, content_sha256).await - } - Endpoint::DeleteBucketLifecycle {} => { - handle_delete_lifecycle(garage, bucket.clone()).await + handle_put_lifecycle(ctx, req, content_sha256).await } + Endpoint::DeleteBucketLifecycle {} => handle_delete_lifecycle(ctx).await, endpoint => Err(Error::NotImplemented(endpoint.name().to_owned())), }; @@ -394,7 +346,7 @@ impl ApiHandler for S3ApiServer { // add the corresponding CORS headers to the response let mut resp_ok = resp?; if let Some(rule) = matching_cors_rule { - add_cors_headers(&mut resp_ok, rule) + add_cors_headers(&mut resp_ok, &rule) .ok_or_internal_error("Invalid bucket CORS configuration")?; } diff --git a/src/api/s3/bucket.rs b/src/api/s3/bucket.rs index fa337566..6a12aa9c 100644 --- a/src/api/s3/bucket.rs +++ b/src/api/s3/bucket.rs @@ -1,5 +1,4 @@ use std::collections::HashMap; -use std::sync::Arc; use http_body_util::BodyExt; use hyper::{Request, Response, StatusCode}; @@ -21,7 +20,8 @@ use crate::s3::error::*; use crate::s3::xml as s3_xml; use crate::signature::verify_signed_content; -pub fn handle_get_bucket_location(garage: Arc) -> Result, Error> { +pub fn handle_get_bucket_location(ctx: ReqCtx) -> Result, Error> { + let ReqCtx { garage, .. } = ctx; let loc = s3_xml::LocationConstraint { xmlns: (), region: garage.config.s3_api.s3_region.to_string(), @@ -204,21 +204,20 @@ pub async fn handle_create_bucket( .unwrap()) } -pub async fn handle_delete_bucket( - garage: &Garage, - bucket_id: Uuid, - bucket_name: String, - api_key_id: &String, -) -> Result, Error> { +pub async fn handle_delete_bucket(ctx: ReqCtx) -> Result, Error> { + let ReqCtx { + garage, + bucket_id, + bucket_name, + bucket_params: bucket_state, + api_key, + .. + } = &ctx; let helper = garage.locked_helper().await; - let api_key = helper.key().get_existing_key(api_key_id).await?; let key_params = api_key.params().unwrap(); - let is_local_alias = matches!(key_params.local_aliases.get(&bucket_name), Some(Some(_))); - - let mut bucket = helper.bucket().get_existing_bucket(bucket_id).await?; - let bucket_state = bucket.state.as_option().unwrap(); + let is_local_alias = matches!(key_params.local_aliases.get(bucket_name), Some(Some(_))); // If the bucket has no other aliases, this is a true deletion. // Otherwise, it is just an alias removal. @@ -228,20 +227,20 @@ pub async fn handle_delete_bucket( .items() .iter() .filter(|(_, _, active)| *active) - .any(|(n, _, _)| is_local_alias || (*n != bucket_name)); + .any(|(n, _, _)| is_local_alias || (*n != *bucket_name)); let has_other_local_aliases = bucket_state .local_aliases .items() .iter() .filter(|(_, _, active)| *active) - .any(|((k, n), _, _)| !is_local_alias || *n != bucket_name || *k != api_key.key_id); + .any(|((k, n), _, _)| !is_local_alias || *n != *bucket_name || *k != api_key.key_id); if !has_other_global_aliases && !has_other_local_aliases { // Delete bucket // Check bucket is empty - if !helper.bucket().is_bucket_empty(bucket_id).await? { + if !helper.bucket().is_bucket_empty(*bucket_id).await? { return Err(CommonError::BucketNotEmpty.into()); } @@ -249,33 +248,36 @@ pub async fn handle_delete_bucket( // 1. delete bucket alias if is_local_alias { helper - .unset_local_bucket_alias(bucket_id, &api_key.key_id, &bucket_name) + .unset_local_bucket_alias(*bucket_id, &api_key.key_id, bucket_name) .await?; } else { helper - .unset_global_bucket_alias(bucket_id, &bucket_name) + .unset_global_bucket_alias(*bucket_id, bucket_name) .await?; } // 2. delete authorization from keys that had access - for (key_id, _) in bucket.authorized_keys() { + for (key_id, _) in bucket_state.authorized_keys.items() { helper - .set_bucket_key_permissions(bucket.id, key_id, BucketKeyPerm::NO_PERMISSIONS) + .set_bucket_key_permissions(*bucket_id, key_id, BucketKeyPerm::NO_PERMISSIONS) .await?; } + let bucket = Bucket { + id: *bucket_id, + state: Deletable::delete(), + }; // 3. delete bucket - bucket.state = Deletable::delete(); garage.bucket_table.insert(&bucket).await?; } else if is_local_alias { // Just unalias helper - .unset_local_bucket_alias(bucket_id, &api_key.key_id, &bucket_name) + .unset_local_bucket_alias(*bucket_id, &api_key.key_id, bucket_name) .await?; } else { // Just unalias (but from global namespace) helper - .unset_global_bucket_alias(bucket_id, &bucket_name) + .unset_global_bucket_alias(*bucket_id, bucket_name) .await?; } diff --git a/src/api/s3/copy.rs b/src/api/s3/copy.rs index 880ce5f4..3c2bd483 100644 --- a/src/api/s3/copy.rs +++ b/src/api/s3/copy.rs @@ -1,5 +1,4 @@ use std::pin::Pin; -use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use futures::{stream, stream::Stream, StreamExt}; @@ -15,8 +14,6 @@ use garage_table::*; use garage_util::data::*; use garage_util::time::*; -use garage_model::garage::Garage; -use garage_model::key_table::Key; use garage_model::s3::block_ref_table::*; use garage_model::s3::mpu_table::*; use garage_model::s3::object_table::*; @@ -30,15 +27,19 @@ use crate::s3::put::get_headers; use crate::s3::xml::{self as s3_xml, xmlns_tag}; pub async fn handle_copy( - garage: Arc, - api_key: &Key, + ctx: ReqCtx, req: &Request, - dest_bucket_id: Uuid, dest_key: &str, ) -> Result, Error> { let copy_precondition = CopyPreconditionHeaders::parse(req)?; - let source_object = get_copy_source(&garage, api_key, req).await?; + let source_object = get_copy_source(&ctx, req).await?; + + let ReqCtx { + garage, + bucket_id: dest_bucket_id, + .. + } = ctx; let (source_version, source_version_data, source_version_meta) = extract_source_info(&source_object)?; @@ -181,10 +182,8 @@ pub async fn handle_copy( } pub async fn handle_upload_part_copy( - garage: Arc, - api_key: &Key, + ctx: ReqCtx, req: &Request, - dest_bucket_id: Uuid, dest_key: &str, part_number: u64, upload_id: &str, @@ -195,10 +194,12 @@ pub async fn handle_upload_part_copy( let dest_key = dest_key.to_string(); let (source_object, (_, _, mut dest_mpu)) = futures::try_join!( - get_copy_source(&garage, api_key, req), - multipart::get_upload(&garage, &dest_bucket_id, &dest_key, &dest_upload_id) + get_copy_source(&ctx, req), + multipart::get_upload(&ctx, &dest_key, &dest_upload_id) )?; + let ReqCtx { garage, .. } = ctx; + let (source_object_version, source_version_data, source_version_meta) = extract_source_info(&source_object)?; @@ -439,11 +440,11 @@ pub async fn handle_upload_part_copy( .body(string_body(resp_xml))?) } -async fn get_copy_source( - garage: &Garage, - api_key: &Key, - req: &Request, -) -> Result { +async fn get_copy_source(ctx: &ReqCtx, req: &Request) -> Result { + let ReqCtx { + garage, api_key, .. + } = ctx; + let copy_source = req.headers().get("x-amz-copy-source").unwrap().to_str()?; let copy_source = percent_encoding::percent_decode_str(copy_source).decode_utf8()?; diff --git a/src/api/s3/cors.rs b/src/api/s3/cors.rs index e069cae4..173b7ffe 100644 --- a/src/api/s3/cors.rs +++ b/src/api/s3/cors.rs @@ -21,16 +21,13 @@ use crate::s3::error::*; use crate::s3::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; use crate::signature::verify_signed_content; -use garage_model::bucket_table::{Bucket, CorsRule as GarageCorsRule}; +use garage_model::bucket_table::{Bucket, BucketParams, CorsRule as GarageCorsRule}; use garage_model::garage::Garage; use garage_util::data::*; -pub async fn handle_get_cors(bucket: &Bucket) -> Result, Error> { - let param = bucket - .params() - .ok_or_internal_error("Bucket should not be deleted at this point")?; - - if let Some(cors) = param.cors_config.get() { +pub async fn handle_get_cors(ctx: ReqCtx) -> Result, Error> { + let ReqCtx { bucket_params, .. } = ctx; + if let Some(cors) = bucket_params.cors_config.get() { let wc = CorsConfiguration { xmlns: (), cors_rules: cors @@ -50,16 +47,18 @@ pub async fn handle_get_cors(bucket: &Bucket) -> Result, Error } } -pub async fn handle_delete_cors( - garage: Arc, - mut bucket: Bucket, -) -> Result, Error> { - let param = bucket - .params_mut() - .ok_or_internal_error("Bucket should not be deleted at this point")?; - - param.cors_config.update(None); - garage.bucket_table.insert(&bucket).await?; +pub async fn handle_delete_cors(ctx: ReqCtx) -> Result, Error> { + let ReqCtx { + garage, + bucket_id, + mut bucket_params, + .. + } = ctx; + bucket_params.cors_config.update(None); + garage + .bucket_table + .insert(&Bucket::present(bucket_id, bucket_params)) + .await?; Ok(Response::builder() .status(StatusCode::NO_CONTENT) @@ -67,28 +66,33 @@ pub async fn handle_delete_cors( } pub async fn handle_put_cors( - garage: Arc, - mut bucket: Bucket, + ctx: ReqCtx, req: Request, content_sha256: Option, ) -> Result, Error> { + let ReqCtx { + garage, + bucket_id, + mut bucket_params, + .. + } = ctx; + let body = BodyExt::collect(req.into_body()).await?.to_bytes(); if let Some(content_sha256) = content_sha256 { verify_signed_content(content_sha256, &body[..])?; } - let param = bucket - .params_mut() - .ok_or_internal_error("Bucket should not be deleted at this point")?; - let conf: CorsConfiguration = from_reader(&body as &[u8])?; conf.validate()?; - param + bucket_params .cors_config .update(Some(conf.into_garage_cors_config()?)); - garage.bucket_table.insert(&bucket).await?; + garage + .bucket_table + .insert(&Bucket::present(bucket_id, bucket_params)) + .await?; Ok(Response::builder() .status(StatusCode::OK) @@ -115,7 +119,8 @@ pub async fn handle_options_api( let bucket_id = helper.resolve_global_bucket_name(&bn).await?; if let Some(id) = bucket_id { let bucket = garage.bucket_helper().get_existing_bucket(id).await?; - handle_options_for_bucket(req, &bucket) + let bucket_params = bucket.state.into_option().unwrap(); + handle_options_for_bucket(req, &bucket_params) } else { // If there is a bucket name in the request, but that name // does not correspond to a global alias for a bucket, @@ -145,7 +150,7 @@ pub async fn handle_options_api( pub fn handle_options_for_bucket( req: &Request, - bucket: &Bucket, + bucket_params: &BucketParams, ) -> Result, CommonError> { let origin = req .headers() @@ -162,7 +167,7 @@ pub fn handle_options_for_bucket( None => vec![], }; - if let Some(cors_config) = bucket.params().unwrap().cors_config.get() { + if let Some(cors_config) = bucket_params.cors_config.get() { let matching_rule = cors_config .iter() .find(|rule| cors_rule_matches(rule, origin, request_method, request_headers.iter())); @@ -181,10 +186,10 @@ pub fn handle_options_for_bucket( } pub fn find_matching_cors_rule<'a>( - bucket: &'a Bucket, + bucket_params: &'a BucketParams, req: &Request, ) -> Result, Error> { - if let Some(cors_config) = bucket.params().unwrap().cors_config.get() { + if let Some(cors_config) = bucket_params.cors_config.get() { if let Some(origin) = req.headers().get("Origin") { let origin = origin.to_str()?; let request_headers = match req.headers().get(ACCESS_CONTROL_REQUEST_HEADERS) { diff --git a/src/api/s3/delete.rs b/src/api/s3/delete.rs index 3fb39147..57f6f948 100644 --- a/src/api/s3/delete.rs +++ b/src/api/s3/delete.rs @@ -1,11 +1,8 @@ -use std::sync::Arc; - use http_body_util::BodyExt; use hyper::{Request, Response, StatusCode}; use garage_util::data::*; -use garage_model::garage::Garage; use garage_model::s3::object_table::*; use crate::helpers::*; @@ -15,14 +12,13 @@ use crate::s3::put::next_timestamp; use crate::s3::xml as s3_xml; use crate::signature::verify_signed_content; -async fn handle_delete_internal( - garage: &Garage, - bucket_id: Uuid, - key: &str, -) -> Result<(Uuid, Uuid), Error> { +async fn handle_delete_internal(ctx: &ReqCtx, key: &str) -> Result<(Uuid, Uuid), Error> { + let ReqCtx { + garage, bucket_id, .. + } = ctx; let object = garage .object_table - .get(&bucket_id, &key.to_string()) + .get(bucket_id, &key.to_string()) .await? .ok_or(Error::NoSuchKey)?; // No need to delete @@ -44,7 +40,7 @@ async fn handle_delete_internal( }; let object = Object::new( - bucket_id, + *bucket_id, key.into(), vec![ObjectVersion { uuid: del_uuid, @@ -58,12 +54,8 @@ async fn handle_delete_internal( Ok((deleted_version, del_uuid)) } -pub async fn handle_delete( - garage: Arc, - bucket_id: Uuid, - key: &str, -) -> Result, Error> { - match handle_delete_internal(&garage, bucket_id, key).await { +pub async fn handle_delete(ctx: ReqCtx, key: &str) -> Result, Error> { + match handle_delete_internal(&ctx, key).await { Ok(_) | Err(Error::NoSuchKey) => Ok(Response::builder() .status(StatusCode::NO_CONTENT) .body(empty_body()) @@ -73,8 +65,7 @@ pub async fn handle_delete( } pub async fn handle_delete_objects( - garage: Arc, - bucket_id: Uuid, + ctx: ReqCtx, req: Request, content_sha256: Option, ) -> Result, Error> { @@ -91,7 +82,7 @@ pub async fn handle_delete_objects( let mut ret_errors = Vec::new(); for obj in cmd.objects.iter() { - match handle_delete_internal(&garage, bucket_id, &obj.key).await { + match handle_delete_internal(&ctx, &obj.key).await { Ok((deleted_version, delete_marker_version)) => { if cmd.quiet { continue; diff --git a/src/api/s3/get.rs b/src/api/s3/get.rs index 0d18e775..ed996fb1 100644 --- a/src/api/s3/get.rs +++ b/src/api/s3/get.rs @@ -131,6 +131,16 @@ fn try_answer_cached( /// Handle HEAD request pub async fn handle_head( + ctx: ReqCtx, + req: &Request, + key: &str, + part_number: Option, +) -> Result, Error> { + handle_head_without_ctx(ctx.garage, req, ctx.bucket_id, key, part_number).await +} + +/// Handle HEAD request for website +pub async fn handle_head_without_ctx( garage: Arc, req: &Request, bucket_id: Uuid, @@ -218,6 +228,17 @@ pub async fn handle_head( /// Handle GET request pub async fn handle_get( + ctx: ReqCtx, + req: &Request, + key: &str, + part_number: Option, + overrides: GetObjectOverrides, +) -> Result, Error> { + handle_get_without_ctx(ctx.garage, req, ctx.bucket_id, key, part_number, overrides).await +} + +/// Handle GET request +pub async fn handle_get_without_ctx( garage: Arc, req: &Request, bucket_id: Uuid, diff --git a/src/api/s3/lifecycle.rs b/src/api/s3/lifecycle.rs index 35757e8c..7eb1c2cb 100644 --- a/src/api/s3/lifecycle.rs +++ b/src/api/s3/lifecycle.rs @@ -1,5 +1,4 @@ use quick_xml::de::from_reader; -use std::sync::Arc; use http_body_util::BodyExt; use hyper::{Request, Response, StatusCode}; @@ -16,15 +15,12 @@ use garage_model::bucket_table::{ parse_lifecycle_date, Bucket, LifecycleExpiration as GarageLifecycleExpiration, LifecycleFilter as GarageLifecycleFilter, LifecycleRule as GarageLifecycleRule, }; -use garage_model::garage::Garage; use garage_util::data::*; -pub async fn handle_get_lifecycle(bucket: &Bucket) -> Result, Error> { - let param = bucket - .params() - .ok_or_internal_error("Bucket should not be deleted at this point")?; +pub async fn handle_get_lifecycle(ctx: ReqCtx) -> Result, Error> { + let ReqCtx { bucket_params, .. } = ctx; - if let Some(lifecycle) = param.lifecycle_config.get() { + if let Some(lifecycle) = bucket_params.lifecycle_config.get() { let wc = LifecycleConfiguration::from_garage_lifecycle_config(lifecycle); let xml = to_xml_with_header(&wc)?; Ok(Response::builder() @@ -38,16 +34,18 @@ pub async fn handle_get_lifecycle(bucket: &Bucket) -> Result, } } -pub async fn handle_delete_lifecycle( - garage: Arc, - mut bucket: Bucket, -) -> Result, Error> { - let param = bucket - .params_mut() - .ok_or_internal_error("Bucket should not be deleted at this point")?; - - param.lifecycle_config.update(None); - garage.bucket_table.insert(&bucket).await?; +pub async fn handle_delete_lifecycle(ctx: ReqCtx) -> Result, Error> { + let ReqCtx { + garage, + bucket_id, + mut bucket_params, + .. + } = ctx; + bucket_params.lifecycle_config.update(None); + garage + .bucket_table + .insert(&Bucket::present(bucket_id, bucket_params)) + .await?; Ok(Response::builder() .status(StatusCode::NO_CONTENT) @@ -55,28 +53,33 @@ pub async fn handle_delete_lifecycle( } pub async fn handle_put_lifecycle( - garage: Arc, - mut bucket: Bucket, + ctx: ReqCtx, req: Request, content_sha256: Option, ) -> Result, Error> { + let ReqCtx { + garage, + bucket_id, + mut bucket_params, + .. + } = ctx; + let body = BodyExt::collect(req.into_body()).await?.to_bytes(); if let Some(content_sha256) = content_sha256 { verify_signed_content(content_sha256, &body[..])?; } - let param = bucket - .params_mut() - .ok_or_internal_error("Bucket should not be deleted at this point")?; - let conf: LifecycleConfiguration = from_reader(&body as &[u8])?; let config = conf .validate_into_garage_lifecycle_config() .ok_or_bad_request("Invalid lifecycle configuration")?; - param.lifecycle_config.update(Some(config)); - garage.bucket_table.insert(&bucket).await?; + bucket_params.lifecycle_config.update(Some(config)); + garage + .bucket_table + .insert(&Bucket::present(bucket_id, bucket_params)) + .await?; Ok(Response::builder() .status(StatusCode::OK) diff --git a/src/api/s3/list.rs b/src/api/s3/list.rs index b832a4f4..302c03f4 100644 --- a/src/api/s3/list.rs +++ b/src/api/s3/list.rs @@ -1,6 +1,5 @@ use std::collections::{BTreeMap, BTreeSet}; use std::iter::{Iterator, Peekable}; -use std::sync::Arc; use base64::prelude::*; use hyper::Response; @@ -9,7 +8,6 @@ use garage_util::data::*; use garage_util::error::Error as GarageError; use garage_util::time::*; -use garage_model::garage::Garage; use garage_model::s3::mpu_table::*; use garage_model::s3::object_table::*; @@ -62,9 +60,10 @@ pub struct ListPartsQuery { } pub async fn handle_list( - garage: Arc, + ctx: ReqCtx, query: &ListObjectsQuery, ) -> Result, Error> { + let ReqCtx { garage, .. } = &ctx; let io = |bucket, key, count| { let t = &garage.object_table; async move { @@ -167,9 +166,11 @@ pub async fn handle_list( } pub async fn handle_list_multipart_upload( - garage: Arc, + ctx: ReqCtx, query: &ListMultipartUploadsQuery, ) -> Result, Error> { + let ReqCtx { garage, .. } = &ctx; + let io = |bucket, key, count| { let t = &garage.object_table; async move { @@ -269,15 +270,14 @@ pub async fn handle_list_multipart_upload( } pub async fn handle_list_parts( - garage: Arc, + ctx: ReqCtx, query: &ListPartsQuery, ) -> Result, Error> { debug!("ListParts {:?}", query); let upload_id = s3_multipart::decode_upload_id(&query.upload_id)?; - let (_, _, mpu) = - s3_multipart::get_upload(&garage, &query.bucket_id, &query.key, &upload_id).await?; + let (_, _, mpu) = s3_multipart::get_upload(&ctx, &query.key, &upload_id).await?; let (info, next) = fetch_part_info(query, &mpu)?; diff --git a/src/api/s3/multipart.rs b/src/api/s3/multipart.rs index 5959bcd6..1d5aeb26 100644 --- a/src/api/s3/multipart.rs +++ b/src/api/s3/multipart.rs @@ -8,7 +8,6 @@ use md5::{Digest as Md5Digest, Md5}; use garage_table::*; use garage_util::data::*; -use garage_model::bucket_table::Bucket; use garage_model::garage::Garage; use garage_model::s3::block_ref_table::*; use garage_model::s3::mpu_table::*; @@ -25,12 +24,16 @@ use crate::signature::verify_signed_content; // ---- pub async fn handle_create_multipart_upload( - garage: Arc, + ctx: ReqCtx, req: &Request, - bucket_name: &str, - bucket_id: Uuid, key: &String, ) -> Result, Error> { + let ReqCtx { + garage, + bucket_id, + bucket_name, + .. + } = &ctx; let existing_object = garage.object_table.get(&bucket_id, &key).await?; let upload_id = gen_uuid(); @@ -47,13 +50,13 @@ pub async fn handle_create_multipart_upload( headers, }, }; - let object = Object::new(bucket_id, key.to_string(), vec![object_version]); + let object = Object::new(*bucket_id, key.to_string(), vec![object_version]); garage.object_table.insert(&object).await?; // Create multipart upload in mpu table // This multipart upload will hold references to uploaded parts // (which are entries in the Version table) - let mpu = MultipartUpload::new(upload_id, timestamp, bucket_id, key.into(), false); + let mpu = MultipartUpload::new(upload_id, timestamp, *bucket_id, key.into(), false); garage.mpu_table.insert(&mpu).await?; // Send success response @@ -69,14 +72,15 @@ pub async fn handle_create_multipart_upload( } pub async fn handle_put_part( - garage: Arc, + ctx: ReqCtx, req: Request, - bucket_id: Uuid, key: &str, part_number: u64, upload_id: &str, content_sha256: Option, ) -> Result, Error> { + let ReqCtx { garage, .. } = &ctx; + let upload_id = decode_upload_id(upload_id)?; let content_md5 = match req.headers().get("content-md5") { @@ -90,10 +94,8 @@ pub async fn handle_put_part( let stream = body_stream(req.into_body()); let mut chunker = StreamChunker::new(stream, garage.config.block_size); - let ((_, _, mut mpu), first_block) = futures::try_join!( - get_upload(&garage, &bucket_id, &key, &upload_id), - chunker.next(), - )?; + let ((_, _, mut mpu), first_block) = + futures::try_join!(get_upload(&ctx, &key, &upload_id), chunker.next(),)?; // Check object is valid and part can be accepted let first_block = first_block.ok_or_bad_request("Empty body")?; @@ -135,7 +137,7 @@ pub async fn handle_put_part( // Copy data to version let (total_size, data_md5sum, data_sha256sum, _) = - read_and_put_blocks(&garage, &version, part_number, first_block, &mut chunker).await?; + read_and_put_blocks(&ctx, &version, part_number, first_block, &mut chunker).await?; // Verify that checksums map ensure_checksum_matches( @@ -200,14 +202,19 @@ impl Drop for InterruptedCleanup { } pub async fn handle_complete_multipart_upload( - garage: Arc, + ctx: ReqCtx, req: Request, - bucket_name: &str, - bucket: &Bucket, key: &str, upload_id: &str, content_sha256: Option, ) -> Result, Error> { + let ReqCtx { + garage, + bucket_id, + bucket_name, + .. + } = &ctx; + let body = http_body_util::BodyExt::collect(req.into_body()) .await? .to_bytes(); @@ -228,8 +235,7 @@ pub async fn handle_complete_multipart_upload( // Get object and multipart upload let key = key.to_string(); - let (object, mut object_version, mpu) = - get_upload(&garage, &bucket.id, &key, &upload_id).await?; + let (object, mut object_version, mpu) = get_upload(&ctx, &key, &upload_id).await?; if mpu.parts.is_empty() { return Err(Error::bad_request("No data was uploaded")); @@ -283,7 +289,7 @@ pub async fn handle_complete_multipart_upload( let mut final_version = Version::new( upload_id, VersionBacklink::Object { - bucket_id: bucket.id, + bucket_id: *bucket_id, key: key.to_string(), }, false, @@ -327,9 +333,9 @@ pub async fn handle_complete_multipart_upload( // Calculate total size of final object let total_size = parts.iter().map(|x| x.size.unwrap()).sum(); - if let Err(e) = check_quotas(&garage, bucket, total_size, Some(&object)).await { + if let Err(e) = check_quotas(&ctx, total_size, Some(&object)).await { object_version.state = ObjectVersionState::Aborted; - let final_object = Object::new(bucket.id, key.clone(), vec![object_version]); + let final_object = Object::new(*bucket_id, key.clone(), vec![object_version]); garage.object_table.insert(&final_object).await?; return Err(e); @@ -345,7 +351,7 @@ pub async fn handle_complete_multipart_upload( final_version.blocks.items()[0].1.hash, )); - let final_object = Object::new(bucket.id, key.clone(), vec![object_version]); + let final_object = Object::new(*bucket_id, key.clone(), vec![object_version]); garage.object_table.insert(&final_object).await?; // Send response saying ok we're done @@ -362,18 +368,20 @@ pub async fn handle_complete_multipart_upload( } pub async fn handle_abort_multipart_upload( - garage: Arc, - bucket_id: Uuid, + ctx: ReqCtx, key: &str, upload_id: &str, ) -> Result, Error> { + let ReqCtx { + garage, bucket_id, .. + } = &ctx; + let upload_id = decode_upload_id(upload_id)?; - let (_, mut object_version, _) = - get_upload(&garage, &bucket_id, &key.to_string(), &upload_id).await?; + let (_, mut object_version, _) = get_upload(&ctx, &key.to_string(), &upload_id).await?; object_version.state = ObjectVersionState::Aborted; - let final_object = Object::new(bucket_id, key.to_string(), vec![object_version]); + let final_object = Object::new(*bucket_id, key.to_string(), vec![object_version]); garage.object_table.insert(&final_object).await?; Ok(Response::new(empty_body())) @@ -383,11 +391,13 @@ pub async fn handle_abort_multipart_upload( #[allow(clippy::ptr_arg)] pub(crate) async fn get_upload( - garage: &Garage, - bucket_id: &Uuid, + ctx: &ReqCtx, key: &String, upload_id: &Uuid, ) -> Result<(Object, ObjectVersion, MultipartUpload), Error> { + let ReqCtx { + garage, bucket_id, .. + } = ctx; let (object, mpu) = futures::try_join!( garage.object_table.get(bucket_id, key).map_err(Error::from), garage diff --git a/src/api/s3/post_object.rs b/src/api/s3/post_object.rs index b542cc1a..66f8174c 100644 --- a/src/api/s3/post_object.rs +++ b/src/api/s3/post_object.rs @@ -120,6 +120,12 @@ pub async fn handle_post_object( .bucket_helper() .get_existing_bucket(bucket_id) .await?; + let bucket_params = bucket.state.into_option().unwrap(); + let matching_cors_rule = find_matching_cors_rule( + &bucket_params, + &Request::from_parts(head.clone(), empty_body::()), + )? + .cloned(); let decoded_policy = BASE64_STANDARD .decode(policy) @@ -213,11 +219,19 @@ pub async fn handle_post_object( let headers = get_headers(¶ms)?; let stream = field.map(|r| r.map_err(Into::into)); - let (_, md5) = save_stream( + + let ctx = ReqCtx { garage, + bucket_id, + bucket_name, + bucket_params, + api_key, + }; + + let (_, md5) = save_stream( + &ctx, headers, StreamLimiter::new(stream, conditions.content_length), - &bucket, &key, None, None, @@ -234,7 +248,7 @@ pub async fn handle_post_object( { target .query_pairs_mut() - .append_pair("bucket", &bucket_name) + .append_pair("bucket", &ctx.bucket_name) .append_pair("key", &key) .append_pair("etag", &etag); let target = target.to_string(); @@ -278,7 +292,7 @@ pub async fn handle_post_object( let xml = s3_xml::PostObject { xmlns: (), location: s3_xml::Value(location), - bucket: s3_xml::Value(bucket_name), + bucket: s3_xml::Value(ctx.bucket_name), key: s3_xml::Value(key), etag: s3_xml::Value(etag), }; @@ -291,12 +305,8 @@ pub async fn handle_post_object( } }; - let matching_cors_rule = find_matching_cors_rule( - &bucket, - &Request::from_parts(head, empty_body::()), - )?; if let Some(rule) = matching_cors_rule { - add_cors_headers(&mut resp, rule) + add_cors_headers(&mut resp, &rule) .ok_or_internal_error("Invalid bucket CORS configuration")?; } diff --git a/src/api/s3/put.rs b/src/api/s3/put.rs index 489f1136..36523b30 100644 --- a/src/api/s3/put.rs +++ b/src/api/s3/put.rs @@ -28,7 +28,6 @@ use garage_util::error::Error as GarageError; use garage_util::time::*; use garage_block::manager::INLINE_THRESHOLD; -use garage_model::bucket_table::Bucket; use garage_model::garage::Garage; use garage_model::index_counter::CountedItem; use garage_model::s3::block_ref_table::*; @@ -42,9 +41,8 @@ use crate::s3::error::*; const PUT_BLOCKS_MAX_PARALLEL: usize = 3; pub async fn handle_put( - garage: Arc, + ctx: ReqCtx, req: Request, - bucket: &Bucket, key: &String, content_sha256: Option, ) -> Result, Error> { @@ -59,35 +57,27 @@ pub async fn handle_put( let stream = body_stream(req.into_body()); - save_stream( - garage, - headers, - stream, - bucket, - key, - content_md5, - content_sha256, - ) - .await - .map(|(uuid, md5)| put_response(uuid, md5)) + save_stream(&ctx, headers, stream, key, content_md5, content_sha256) + .await + .map(|(uuid, md5)| put_response(uuid, md5)) } pub(crate) async fn save_stream> + Unpin>( - garage: Arc, + ctx: &ReqCtx, headers: ObjectVersionHeaders, body: S, - bucket: &Bucket, key: &String, content_md5: Option, content_sha256: Option, ) -> Result<(Uuid, String), Error> { + let ReqCtx { + garage, bucket_id, .. + } = ctx; + let mut chunker = StreamChunker::new(body, garage.config.block_size); let (first_block_opt, existing_object) = try_join!( chunker.next(), - garage - .object_table - .get(&bucket.id, key) - .map_err(Error::from), + garage.object_table.get(bucket_id, key).map_err(Error::from), )?; let first_block = first_block_opt.unwrap_or_default(); @@ -114,7 +104,7 @@ pub(crate) async fn save_stream> + Unpin>( content_sha256, )?; - check_quotas(&garage, bucket, size, existing_object.as_ref()).await?; + check_quotas(ctx, size, existing_object.as_ref()).await?; let object_version = ObjectVersion { uuid: version_uuid, @@ -129,7 +119,7 @@ pub(crate) async fn save_stream> + Unpin>( )), }; - let object = Object::new(bucket.id, key.into(), vec![object_version]); + let object = Object::new(*bucket_id, key.into(), vec![object_version]); garage.object_table.insert(&object).await?; return Ok((version_uuid, data_md5sum_hex)); @@ -140,7 +130,7 @@ pub(crate) async fn save_stream> + Unpin>( // before everything is finished (cleanup is done using the Drop trait). let mut interrupted_cleanup = InterruptedCleanup(Some(InterruptedCleanupInner { garage: garage.clone(), - bucket_id: bucket.id, + bucket_id: *bucket_id, key: key.into(), version_uuid, version_timestamp, @@ -156,7 +146,7 @@ pub(crate) async fn save_stream> + Unpin>( multipart: false, }, }; - let object = Object::new(bucket.id, key.into(), vec![object_version.clone()]); + let object = Object::new(*bucket_id, key.into(), vec![object_version.clone()]); garage.object_table.insert(&object).await?; // Initialize corresponding entry in version table @@ -166,7 +156,7 @@ pub(crate) async fn save_stream> + Unpin>( let version = Version::new( version_uuid, VersionBacklink::Object { - bucket_id: bucket.id, + bucket_id: *bucket_id, key: key.into(), }, false, @@ -175,7 +165,7 @@ pub(crate) async fn save_stream> + Unpin>( // Transfer data and verify checksum let (total_size, data_md5sum, data_sha256sum, first_block_hash) = - read_and_put_blocks(&garage, &version, 1, first_block, &mut chunker).await?; + read_and_put_blocks(ctx, &version, 1, first_block, &mut chunker).await?; ensure_checksum_matches( data_md5sum.as_slice(), @@ -184,7 +174,7 @@ pub(crate) async fn save_stream> + Unpin>( content_sha256, )?; - check_quotas(&garage, bucket, total_size, existing_object.as_ref()).await?; + check_quotas(ctx, total_size, existing_object.as_ref()).await?; // Save final object state, marked as Complete let md5sum_hex = hex::encode(data_md5sum); @@ -196,7 +186,7 @@ pub(crate) async fn save_stream> + Unpin>( }, first_block_hash, )); - let object = Object::new(bucket.id, key.into(), vec![object_version]); + let object = Object::new(*bucket_id, key.into(), vec![object_version]); garage.object_table.insert(&object).await?; // We were not interrupted, everything went fine. @@ -235,12 +225,18 @@ pub(crate) fn ensure_checksum_matches( /// Check that inserting this object with this size doesn't exceed bucket quotas pub(crate) async fn check_quotas( - garage: &Arc, - bucket: &Bucket, + ctx: &ReqCtx, size: u64, prev_object: Option<&Object>, ) -> Result<(), Error> { - let quotas = bucket.state.as_option().unwrap().quotas.get(); + let ReqCtx { + garage, + bucket_id, + bucket_params, + .. + } = ctx; + + let quotas = bucket_params.quotas.get(); if quotas.max_objects.is_none() && quotas.max_size.is_none() { return Ok(()); }; @@ -248,7 +244,7 @@ pub(crate) async fn check_quotas( let counters = garage .object_counter_table .table - .get(&bucket.id, &EmptyKey) + .get(bucket_id, &EmptyKey) .await?; let counters = counters @@ -292,7 +288,7 @@ pub(crate) async fn check_quotas( } pub(crate) async fn read_and_put_blocks> + Unpin>( - garage: &Garage, + ctx: &ReqCtx, version: &Version, part_number: u64, first_block: Bytes, @@ -417,7 +413,7 @@ pub(crate) async fn read_and_put_blocks> + let offset = written_bytes; written_bytes += block.len() as u64; write_futs.push_back(put_block_and_meta( - garage, + ctx, version, part_number, offset, @@ -447,7 +443,7 @@ pub(crate) async fn read_and_put_blocks> + } async fn put_block_and_meta( - garage: &Garage, + ctx: &ReqCtx, version: &Version, part_number: u64, offset: u64, @@ -455,6 +451,8 @@ async fn put_block_and_meta( block: Bytes, order_tag: OrderTag, ) -> Result<(), GarageError> { + let ReqCtx { garage, .. } = ctx; + let mut version = version.clone(); version.blocks.put( VersionBlockKey { diff --git a/src/api/s3/website.rs b/src/api/s3/website.rs index 1c1dbf20..6af55677 100644 --- a/src/api/s3/website.rs +++ b/src/api/s3/website.rs @@ -1,5 +1,4 @@ use quick_xml::de::from_reader; -use std::sync::Arc; use http_body_util::BodyExt; use hyper::{Request, Response, StatusCode}; @@ -12,15 +11,11 @@ use crate::s3::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; use crate::signature::verify_signed_content; use garage_model::bucket_table::*; -use garage_model::garage::Garage; use garage_util::data::*; -pub async fn handle_get_website(bucket: &Bucket) -> Result, Error> { - let param = bucket - .params() - .ok_or_internal_error("Bucket should not be deleted at this point")?; - - if let Some(website) = param.website_config.get() { +pub async fn handle_get_website(ctx: ReqCtx) -> Result, Error> { + let ReqCtx { bucket_params, .. } = ctx; + if let Some(website) = bucket_params.website_config.get() { let wc = WebsiteConfiguration { xmlns: (), error_document: website.error_document.as_ref().map(|v| Key { @@ -44,16 +39,18 @@ pub async fn handle_get_website(bucket: &Bucket) -> Result, Er } } -pub async fn handle_delete_website( - garage: Arc, - mut bucket: Bucket, -) -> Result, Error> { - let param = bucket - .params_mut() - .ok_or_internal_error("Bucket should not be deleted at this point")?; - - param.website_config.update(None); - garage.bucket_table.insert(&bucket).await?; +pub async fn handle_delete_website(ctx: ReqCtx) -> Result, Error> { + let ReqCtx { + garage, + bucket_id, + mut bucket_params, + .. + } = ctx; + bucket_params.website_config.update(None); + garage + .bucket_table + .insert(&Bucket::present(bucket_id, bucket_params)) + .await?; Ok(Response::builder() .status(StatusCode::NO_CONTENT) @@ -61,28 +58,33 @@ pub async fn handle_delete_website( } pub async fn handle_put_website( - garage: Arc, - mut bucket: Bucket, + ctx: ReqCtx, req: Request, content_sha256: Option, ) -> Result, Error> { + let ReqCtx { + garage, + bucket_id, + mut bucket_params, + .. + } = ctx; + let body = BodyExt::collect(req.into_body()).await?.to_bytes(); if let Some(content_sha256) = content_sha256 { verify_signed_content(content_sha256, &body[..])?; } - let param = bucket - .params_mut() - .ok_or_internal_error("Bucket should not be deleted at this point")?; - let conf: WebsiteConfiguration = from_reader(&body as &[u8])?; conf.validate()?; - param + bucket_params .website_config .update(Some(conf.into_garage_website_config()?)); - garage.bucket_table.insert(&bucket).await?; + garage + .bucket_table + .insert(&Bucket::present(bucket_id, bucket_params)) + .await?; Ok(Response::builder() .status(StatusCode::OK) diff --git a/src/model/bucket_table.rs b/src/model/bucket_table.rs index 4c48a76f..1dbdfac2 100644 --- a/src/model/bucket_table.rs +++ b/src/model/bucket_table.rs @@ -191,6 +191,13 @@ impl Bucket { } } + pub fn present(id: Uuid, params: BucketParams) -> Self { + Bucket { + id, + state: crdt::Deletable::present(params), + } + } + /// Returns true if this represents a deleted bucket pub fn is_deleted(&self) -> bool { self.state.is_deleted() diff --git a/src/web/web_server.rs b/src/web/web_server.rs index 0f9b5dc8..69939f65 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -26,7 +26,7 @@ use garage_api::s3::cors::{add_cors_headers, find_matching_cors_rule, handle_opt use garage_api::s3::error::{ CommonErrorDerivative, Error as ApiError, OkOrBadRequest, OkOrInternalError, }; -use garage_api::s3::get::{handle_get, handle_head}; +use garage_api::s3::get::{handle_get_without_ctx, handle_head_without_ctx}; use garage_model::garage::Garage; @@ -219,14 +219,13 @@ impl WebServer { // Check bucket isn't deleted and has website access enabled let bucket = self .garage - .bucket_table - .get(&EmptyKey, &bucket_id) - .await? - .ok_or(Error::NotFound)?; + .bucket_helper() + .get_existing_bucket(bucket_id) + .await + .map_err(|_| Error::NotFound)?; + let bucket_params = bucket.state.into_option().unwrap(); - let website_config = bucket - .params() - .ok_or(Error::NotFound)? + let website_config = bucket_params .website_config .get() .as_ref() @@ -243,14 +242,16 @@ impl WebServer { ); let ret_doc = match *req.method() { - Method::OPTIONS => handle_options_for_bucket(req, &bucket) + Method::OPTIONS => handle_options_for_bucket(req, &bucket_params) .map_err(ApiError::from) .map(|res| res.map(|_empty_body: EmptyBody| empty_body())), - Method::HEAD => handle_head(self.garage.clone(), &req, bucket_id, &key, None).await, + Method::HEAD => { + handle_head_without_ctx(self.garage.clone(), req, bucket_id, &key, None).await + } Method::GET => { - handle_get( + handle_get_without_ctx( self.garage.clone(), - &req, + req, bucket_id, &key, None, @@ -301,7 +302,7 @@ impl WebServer { .body(empty_body::()) .unwrap(); - match handle_get( + match handle_get_without_ctx( self.garage.clone(), &req2, bucket_id, @@ -344,7 +345,7 @@ impl WebServer { } Ok(mut resp) => { // Maybe add CORS headers - if let Some(rule) = find_matching_cors_rule(&bucket, req)? { + if let Some(rule) = find_matching_cors_rule(&bucket_params, req)? { add_cors_headers(&mut resp, rule) .ok_or_internal_error("Invalid bucket CORS configuration")?; } -- cgit v1.2.3 From c8e416aaa5203347a35407a929ac5dfcb1f2a6bc Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 4 Mar 2024 13:28:54 +0100 Subject: [test-presigned] Use a HeaderMap type for QueryMap --- src/api/signature/payload.rs | 46 ++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/src/api/signature/payload.rs b/src/api/signature/payload.rs index 515c0f91..d72736bb 100644 --- a/src/api/signature/payload.rs +++ b/src/api/signature/payload.rs @@ -31,11 +31,12 @@ pub const AWS4_HMAC_SHA256: &str = "AWS4-HMAC-SHA256"; pub const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD"; pub const STREAMING_AWS4_HMAC_SHA256_PAYLOAD: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"; -pub type QueryMap = HashMap; - +pub type QueryMap = HeaderMap; pub struct QueryValue { - key: String, - value: String, + /// Original key with potential uppercase characters, + /// for use in signature calculation + key: String, + value: String, } pub async fn check_payload_signature( @@ -45,7 +46,7 @@ pub async fn check_payload_signature( ) -> Result<(Option, Option), Error> { let query = parse_query_map(request.uri())?; - if query.contains_key(X_AMZ_ALGORITHM.as_str()) { + if query.contains_key(&X_AMZ_ALGORITHM) { // We check for presigned-URL-style authentification first, because // the browser or someting else could inject an Authorization header // that is totally unrelated to AWS signatures. @@ -127,7 +128,7 @@ async fn check_presigned_signature( request: &mut Request, mut query: QueryMap, ) -> Result<(Option, Option), Error> { - let algorithm = query.get(X_AMZ_ALGORITHM.as_str()).unwrap(); + let algorithm = query.get(&X_AMZ_ALGORITHM).unwrap(); let authorization = Authorization::parse_presigned(&algorithm.value, &query)?; // Verify that all necessary request headers are included in signed_headers @@ -141,7 +142,7 @@ async fn check_presigned_signature( // but the signature cannot be computed from a string that contains itself. // AWS specifies that all query params except X-Amz-Signature are included // in the canonical request. - query.remove(X_AMZ_SIGNATURE.as_str()); + query.remove(&X_AMZ_SIGNATURE); let canonical_request = canonical_request( service, request.method(), @@ -167,9 +168,7 @@ async fn check_presigned_signature( // then an InvalidRequest error is raised. let headers_mut = request.headers_mut(); for (name, value) in query.iter() { - let name = - HeaderName::from_bytes(name.as_bytes()).ok_or_bad_request("Invalid header name")?; - if let Some(existing) = headers_mut.get(&name) { + if let Some(existing) = headers_mut.get(name) { if signed_headers.contains(&name) && existing.as_bytes() != value.value.as_bytes() { return Err(Error::bad_request(format!( "Conflicting values for `{}` in query parameters and request headers", @@ -198,18 +197,19 @@ async fn check_presigned_signature( } pub fn parse_query_map(uri: &http::uri::Uri) -> Result { - let mut query = QueryMap::new(); + let mut query = QueryMap::with_capacity(0); if let Some(query_str) = uri.query() { let query_pairs = url::form_urlencoded::parse(query_str.as_bytes()); for (key, val) in query_pairs { - let key = key.into_owned(); + let name = + HeaderName::from_bytes(key.as_bytes()).ok_or_bad_request("Invalid header name")?; - let value = QueryValue { - key: key.clone(), - value: val.into_owned(), - }; + let value = QueryValue { + key: key.to_string(), + value: val.into_owned(), + }; - if query.insert(key.to_lowercase(), value).is_some() { + if query.insert(name, value).is_some() { return Err(Error::bad_request(format!( "duplicate query parameter: `{}`", key @@ -476,19 +476,19 @@ impl Authorization { } let cred = query - .get(X_AMZ_CREDENTIAL.as_str()) + .get(&X_AMZ_CREDENTIAL) .ok_or_bad_request("X-Amz-Credential not found in query parameters")?; let signed_headers = query - .get(X_AMZ_SIGNEDHEADERS.as_str()) + .get(&X_AMZ_SIGNEDHEADERS) .ok_or_bad_request("X-Amz-SignedHeaders not found in query parameters")?; let signature = query - .get(X_AMZ_SIGNATURE.as_str()) + .get(&X_AMZ_SIGNATURE) .ok_or_bad_request("X-Amz-Signature not found in query parameters")?; let duration = query - .get(X_AMZ_EXPIRES.as_str()) + .get(&X_AMZ_EXPIRES) .ok_or_bad_request("X-Amz-Expires not found in query parameters")? - .value + .value .parse() .map_err(|_| Error::bad_request("X-Amz-Expires is not a number".to_string()))?; @@ -499,7 +499,7 @@ impl Authorization { } let date = query - .get(X_AMZ_DATE.as_str()) + .get(&X_AMZ_DATE) .ok_or_bad_request("Missing X-Amz-Date field")?; let date = parse_date(&date.value)?; -- cgit v1.2.3 From 8670140358353542b0c92b7b72a1d223ea2fe068 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 4 Mar 2024 14:00:55 +0100 Subject: [rel-0.9.3] Bump version to 0.9.3 --- Cargo.lock | 20 +++--- Cargo.nix | 138 ++++++++++++++++++++-------------------- Cargo.toml | 18 +++--- doc/book/cookbook/real-world.md | 10 +-- script/helm/garage/Chart.yaml | 2 +- src/api/Cargo.toml | 2 +- src/block/Cargo.toml | 2 +- src/db/Cargo.toml | 2 +- src/garage/Cargo.toml | 2 +- src/model/Cargo.toml | 2 +- src/net/Cargo.toml | 2 +- src/rpc/Cargo.toml | 2 +- src/table/Cargo.toml | 2 +- src/util/Cargo.toml | 2 +- src/web/Cargo.toml | 2 +- 15 files changed, 104 insertions(+), 104 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b2956559..f9bf0c0a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1277,7 +1277,7 @@ dependencies = [ [[package]] name = "garage" -version = "0.9.2" +version = "0.9.3" dependencies = [ "assert-json-diff", "async-trait", @@ -1331,7 +1331,7 @@ dependencies = [ [[package]] name = "garage_api" -version = "0.9.2" +version = "0.9.3" dependencies = [ "argon2", "async-trait", @@ -1380,7 +1380,7 @@ dependencies = [ [[package]] name = "garage_block" -version = "0.9.2" +version = "0.9.3" dependencies = [ "arc-swap", "async-compression", @@ -1407,7 +1407,7 @@ dependencies = [ [[package]] name = "garage_db" -version = "0.9.2" +version = "0.9.3" dependencies = [ "err-derive", "heed", @@ -1420,7 +1420,7 @@ dependencies = [ [[package]] name = "garage_model" -version = "0.9.2" +version = "0.9.3" dependencies = [ "arc-swap", "async-trait", @@ -1448,7 +1448,7 @@ dependencies = [ [[package]] name = "garage_net" -version = "0.9.2" +version = "0.9.3" dependencies = [ "arc-swap", "async-trait", @@ -1474,7 +1474,7 @@ dependencies = [ [[package]] name = "garage_rpc" -version = "0.9.2" +version = "0.9.3" dependencies = [ "arc-swap", "async-trait", @@ -1509,7 +1509,7 @@ dependencies = [ [[package]] name = "garage_table" -version = "0.9.2" +version = "0.9.3" dependencies = [ "arc-swap", "async-trait", @@ -1531,7 +1531,7 @@ dependencies = [ [[package]] name = "garage_util" -version = "0.9.2" +version = "0.9.3" dependencies = [ "arc-swap", "async-trait", @@ -1565,7 +1565,7 @@ dependencies = [ [[package]] name = "garage_web" -version = "0.9.2" +version = "0.9.3" dependencies = [ "err-derive", "futures", diff --git a/Cargo.nix b/Cargo.nix index f976d2e6..0a4ca99a 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -34,7 +34,7 @@ args@{ ignoreLockHash, }: let - nixifiedLockHash = "3e1e0730302ee7d1f4185a13ad0086392562eb7cb962b1212801847887beaa47"; + nixifiedLockHash = "8112e20b0e356bed77a9769600c2b2952662ec8af9548eecf8a2d46fe8433189"; workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc; currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock); lockHashIgnored = if ignoreLockHash @@ -58,17 +58,17 @@ in { cargo2nixVersion = "0.11.0"; workspace = { - garage_db = rustPackages.unknown.garage_db."0.9.2"; - garage_util = rustPackages.unknown.garage_util."0.9.2"; - garage_net = rustPackages.unknown.garage_net."0.9.2"; - garage_rpc = rustPackages.unknown.garage_rpc."0.9.2"; + garage_db = rustPackages.unknown.garage_db."0.9.3"; + garage_util = rustPackages.unknown.garage_util."0.9.3"; + garage_net = rustPackages.unknown.garage_net."0.9.3"; + garage_rpc = rustPackages.unknown.garage_rpc."0.9.3"; format_table = rustPackages.unknown.format_table."0.1.1"; - garage_table = rustPackages.unknown.garage_table."0.9.2"; - garage_block = rustPackages.unknown.garage_block."0.9.2"; - garage_model = rustPackages.unknown.garage_model."0.9.2"; - garage_api = rustPackages.unknown.garage_api."0.9.2"; - garage_web = rustPackages.unknown.garage_web."0.9.2"; - garage = rustPackages.unknown.garage."0.9.2"; + garage_table = rustPackages.unknown.garage_table."0.9.3"; + garage_block = rustPackages.unknown.garage_block."0.9.3"; + garage_model = rustPackages.unknown.garage_model."0.9.3"; + garage_api = rustPackages.unknown.garage_api."0.9.3"; + garage_web = rustPackages.unknown.garage_web."0.9.3"; + garage = rustPackages.unknown.garage."0.9.3"; k2v-client = rustPackages.unknown.k2v-client."0.0.4"; }; "registry+https://github.com/rust-lang/crates.io-index".addr2line."0.21.0" = overridableMkRustCrate (profileName: rec { @@ -1871,9 +1871,9 @@ in }; }); - "unknown".garage."0.9.2" = overridableMkRustCrate (profileName: rec { + "unknown".garage."0.9.3" = overridableMkRustCrate (profileName: rec { name = "garage"; - version = "0.9.2"; + version = "0.9.3"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/garage"); features = builtins.concatLists [ @@ -1900,15 +1900,15 @@ in format_table = (rustPackages."unknown".format_table."0.1.1" { inherit profileName; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" { inherit profileName; }).out; futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.30" { inherit profileName; }).out; - garage_api = (rustPackages."unknown".garage_api."0.9.2" { inherit profileName; }).out; - garage_block = (rustPackages."unknown".garage_block."0.9.2" { inherit profileName; }).out; - garage_db = (rustPackages."unknown".garage_db."0.9.2" { inherit profileName; }).out; - garage_model = (rustPackages."unknown".garage_model."0.9.2" { inherit profileName; }).out; - garage_net = (rustPackages."unknown".garage_net."0.9.2" { inherit profileName; }).out; - garage_rpc = (rustPackages."unknown".garage_rpc."0.9.2" { inherit profileName; }).out; - garage_table = (rustPackages."unknown".garage_table."0.9.2" { inherit profileName; }).out; - garage_util = (rustPackages."unknown".garage_util."0.9.2" { inherit profileName; }).out; - garage_web = (rustPackages."unknown".garage_web."0.9.2" { inherit profileName; }).out; + garage_api = (rustPackages."unknown".garage_api."0.9.3" { inherit profileName; }).out; + garage_block = (rustPackages."unknown".garage_block."0.9.3" { inherit profileName; }).out; + garage_db = (rustPackages."unknown".garage_db."0.9.3" { inherit profileName; }).out; + garage_model = (rustPackages."unknown".garage_model."0.9.3" { inherit profileName; }).out; + garage_net = (rustPackages."unknown".garage_net."0.9.3" { inherit profileName; }).out; + garage_rpc = (rustPackages."unknown".garage_rpc."0.9.3" { inherit profileName; }).out; + garage_table = (rustPackages."unknown".garage_table."0.9.3" { inherit profileName; }).out; + garage_util = (rustPackages."unknown".garage_util."0.9.3" { inherit profileName; }).out; + garage_web = (rustPackages."unknown".garage_web."0.9.3" { inherit profileName; }).out; git_version = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.9" { inherit profileName; }).out; hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; sodiumoxide = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }).out; @@ -1946,9 +1946,9 @@ in }; }); - "unknown".garage_api."0.9.2" = overridableMkRustCrate (profileName: rec { + "unknown".garage_api."0.9.3" = overridableMkRustCrate (profileName: rec { name = "garage_api"; - version = "0.9.2"; + version = "0.9.3"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/api"); features = builtins.concatLists [ @@ -1968,12 +1968,12 @@ in form_urlencoded = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.2.1" { inherit profileName; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" { inherit profileName; }).out; futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.30" { inherit profileName; }).out; - garage_block = (rustPackages."unknown".garage_block."0.9.2" { inherit profileName; }).out; - garage_model = (rustPackages."unknown".garage_model."0.9.2" { inherit profileName; }).out; - garage_net = (rustPackages."unknown".garage_net."0.9.2" { inherit profileName; }).out; - garage_rpc = (rustPackages."unknown".garage_rpc."0.9.2" { inherit profileName; }).out; - garage_table = (rustPackages."unknown".garage_table."0.9.2" { inherit profileName; }).out; - garage_util = (rustPackages."unknown".garage_util."0.9.2" { inherit profileName; }).out; + garage_block = (rustPackages."unknown".garage_block."0.9.3" { inherit profileName; }).out; + garage_model = (rustPackages."unknown".garage_model."0.9.3" { inherit profileName; }).out; + garage_net = (rustPackages."unknown".garage_net."0.9.3" { inherit profileName; }).out; + garage_rpc = (rustPackages."unknown".garage_rpc."0.9.3" { inherit profileName; }).out; + garage_table = (rustPackages."unknown".garage_table."0.9.3" { inherit profileName; }).out; + garage_util = (rustPackages."unknown".garage_util."0.9.3" { inherit profileName; }).out; hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; hmac = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hmac."0.12.1" { inherit profileName; }).out; http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."1.0.0" { inherit profileName; }).out; @@ -2004,9 +2004,9 @@ in }; }); - "unknown".garage_block."0.9.2" = overridableMkRustCrate (profileName: rec { + "unknown".garage_block."0.9.3" = overridableMkRustCrate (profileName: rec { name = "garage_block"; - version = "0.9.2"; + version = "0.9.3"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/block"); features = builtins.concatLists [ @@ -2020,11 +2020,11 @@ in bytesize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.3.0" { inherit profileName; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" { inherit profileName; }).out; futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.30" { inherit profileName; }).out; - garage_db = (rustPackages."unknown".garage_db."0.9.2" { inherit profileName; }).out; - garage_net = (rustPackages."unknown".garage_net."0.9.2" { inherit profileName; }).out; - garage_rpc = (rustPackages."unknown".garage_rpc."0.9.2" { inherit profileName; }).out; - garage_table = (rustPackages."unknown".garage_table."0.9.2" { inherit profileName; }).out; - garage_util = (rustPackages."unknown".garage_util."0.9.2" { inherit profileName; }).out; + garage_db = (rustPackages."unknown".garage_db."0.9.3" { inherit profileName; }).out; + garage_net = (rustPackages."unknown".garage_net."0.9.3" { inherit profileName; }).out; + garage_rpc = (rustPackages."unknown".garage_rpc."0.9.3" { inherit profileName; }).out; + garage_table = (rustPackages."unknown".garage_table."0.9.3" { inherit profileName; }).out; + garage_util = (rustPackages."unknown".garage_util."0.9.3" { inherit profileName; }).out; hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out; @@ -2037,9 +2037,9 @@ in }; }); - "unknown".garage_db."0.9.2" = overridableMkRustCrate (profileName: rec { + "unknown".garage_db."0.9.3" = overridableMkRustCrate (profileName: rec { name = "garage_db"; - version = "0.9.2"; + version = "0.9.3"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/db"); features = builtins.concatLists [ @@ -2064,9 +2064,9 @@ in }; }); - "unknown".garage_model."0.9.2" = overridableMkRustCrate (profileName: rec { + "unknown".garage_model."0.9.3" = overridableMkRustCrate (profileName: rec { name = "garage_model"; - version = "0.9.2"; + version = "0.9.3"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/model"); features = builtins.concatLists [ @@ -2085,12 +2085,12 @@ in err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" { inherit profileName; }).out; futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.30" { inherit profileName; }).out; - garage_block = (rustPackages."unknown".garage_block."0.9.2" { inherit profileName; }).out; - garage_db = (rustPackages."unknown".garage_db."0.9.2" { inherit profileName; }).out; - garage_net = (rustPackages."unknown".garage_net."0.9.2" { inherit profileName; }).out; - garage_rpc = (rustPackages."unknown".garage_rpc."0.9.2" { inherit profileName; }).out; - garage_table = (rustPackages."unknown".garage_table."0.9.2" { inherit profileName; }).out; - garage_util = (rustPackages."unknown".garage_util."0.9.2" { inherit profileName; }).out; + garage_block = (rustPackages."unknown".garage_block."0.9.3" { inherit profileName; }).out; + garage_db = (rustPackages."unknown".garage_db."0.9.3" { inherit profileName; }).out; + garage_net = (rustPackages."unknown".garage_net."0.9.3" { inherit profileName; }).out; + garage_rpc = (rustPackages."unknown".garage_rpc."0.9.3" { inherit profileName; }).out; + garage_table = (rustPackages."unknown".garage_table."0.9.3" { inherit profileName; }).out; + garage_util = (rustPackages."unknown".garage_util."0.9.3" { inherit profileName; }).out; hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out; @@ -2102,9 +2102,9 @@ in }; }); - "unknown".garage_net."0.9.2" = overridableMkRustCrate (profileName: rec { + "unknown".garage_net."0.9.3" = overridableMkRustCrate (profileName: rec { name = "garage_net"; - version = "0.9.2"; + version = "0.9.3"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/net"); features = builtins.concatLists [ @@ -2139,9 +2139,9 @@ in }; }); - "unknown".garage_rpc."0.9.2" = overridableMkRustCrate (profileName: rec { + "unknown".garage_rpc."0.9.3" = overridableMkRustCrate (profileName: rec { name = "garage_rpc"; - version = "0.9.2"; + version = "0.9.3"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/rpc"); features = builtins.concatLists [ @@ -2163,9 +2163,9 @@ in format_table = (rustPackages."unknown".format_table."0.1.1" { inherit profileName; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" { inherit profileName; }).out; futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.30" { inherit profileName; }).out; - garage_db = (rustPackages."unknown".garage_db."0.9.2" { inherit profileName; }).out; - garage_net = (rustPackages."unknown".garage_net."0.9.2" { inherit profileName; }).out; - garage_util = (rustPackages."unknown".garage_util."0.9.2" { inherit profileName; }).out; + garage_db = (rustPackages."unknown".garage_db."0.9.3" { inherit profileName; }).out; + garage_net = (rustPackages."unknown".garage_net."0.9.3" { inherit profileName; }).out; + garage_util = (rustPackages."unknown".garage_util."0.9.3" { inherit profileName; }).out; gethostname = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.4.3" { inherit profileName; }).out; hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; itertools = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".itertools."0.12.1" { inherit profileName; }).out; @@ -2187,9 +2187,9 @@ in }; }); - "unknown".garage_table."0.9.2" = overridableMkRustCrate (profileName: rec { + "unknown".garage_table."0.9.3" = overridableMkRustCrate (profileName: rec { name = "garage_table"; - version = "0.9.2"; + version = "0.9.3"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/table"); dependencies = { @@ -2198,9 +2198,9 @@ in bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.5.0" { inherit profileName; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" { inherit profileName; }).out; futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.30" { inherit profileName; }).out; - garage_db = (rustPackages."unknown".garage_db."0.9.2" { inherit profileName; }).out; - garage_rpc = (rustPackages."unknown".garage_rpc."0.9.2" { inherit profileName; }).out; - garage_util = (rustPackages."unknown".garage_util."0.9.2" { inherit profileName; }).out; + garage_db = (rustPackages."unknown".garage_db."0.9.3" { inherit profileName; }).out; + garage_rpc = (rustPackages."unknown".garage_rpc."0.9.3" { inherit profileName; }).out; + garage_util = (rustPackages."unknown".garage_util."0.9.3" { inherit profileName; }).out; hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out; opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; @@ -2212,9 +2212,9 @@ in }; }); - "unknown".garage_util."0.9.2" = overridableMkRustCrate (profileName: rec { + "unknown".garage_util."0.9.3" = overridableMkRustCrate (profileName: rec { name = "garage_util"; - version = "0.9.2"; + version = "0.9.3"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/util"); features = builtins.concatLists [ @@ -2230,8 +2230,8 @@ in digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.10.7" { inherit profileName; }).out; err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" { inherit profileName; }).out; - garage_db = (rustPackages."unknown".garage_db."0.9.2" { inherit profileName; }).out; - garage_net = (rustPackages."unknown".garage_net."0.9.2" { inherit profileName; }).out; + garage_db = (rustPackages."unknown".garage_db."0.9.3" { inherit profileName; }).out; + garage_net = (rustPackages."unknown".garage_net."0.9.3" { inherit profileName; }).out; hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out; http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."1.0.0" { inherit profileName; }).out; @@ -2256,18 +2256,18 @@ in }; }); - "unknown".garage_web."0.9.2" = overridableMkRustCrate (profileName: rec { + "unknown".garage_web."0.9.3" = overridableMkRustCrate (profileName: rec { name = "garage_web"; - version = "0.9.2"; + version = "0.9.3"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/web"); dependencies = { err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" { inherit profileName; }).out; - garage_api = (rustPackages."unknown".garage_api."0.9.2" { inherit profileName; }).out; - garage_model = (rustPackages."unknown".garage_model."0.9.2" { inherit profileName; }).out; - garage_table = (rustPackages."unknown".garage_table."0.9.2" { inherit profileName; }).out; - garage_util = (rustPackages."unknown".garage_util."0.9.2" { inherit profileName; }).out; + garage_api = (rustPackages."unknown".garage_api."0.9.3" { inherit profileName; }).out; + garage_model = (rustPackages."unknown".garage_model."0.9.3" { inherit profileName; }).out; + garage_table = (rustPackages."unknown".garage_table."0.9.3" { inherit profileName; }).out; + garage_util = (rustPackages."unknown".garage_util."0.9.3" { inherit profileName; }).out; http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."1.0.0" { inherit profileName; }).out; http_body_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body-util."0.1.0" { inherit profileName; }).out; hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."1.1.0" { inherit profileName; }).out; diff --git a/Cargo.toml b/Cargo.toml index 59d1c26b..568f4e51 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,15 +21,15 @@ default-members = ["src/garage"] # Internal Garage crates format_table = { version = "0.1.1", path = "src/format-table" } -garage_api = { version = "0.9.2", path = "src/api" } -garage_block = { version = "0.9.2", path = "src/block" } -garage_db = { version = "0.9.2", path = "src/db", default-features = false } -garage_model = { version = "0.9.2", path = "src/model", default-features = false } -garage_net = { version = "0.9.2", path = "src/net" } -garage_rpc = { version = "0.9.2", path = "src/rpc" } -garage_table = { version = "0.9.2", path = "src/table" } -garage_util = { version = "0.9.2", path = "src/util" } -garage_web = { version = "0.9.2", path = "src/web" } +garage_api = { version = "0.9.3", path = "src/api" } +garage_block = { version = "0.9.3", path = "src/block" } +garage_db = { version = "0.9.3", path = "src/db", default-features = false } +garage_model = { version = "0.9.3", path = "src/model", default-features = false } +garage_net = { version = "0.9.3", path = "src/net" } +garage_rpc = { version = "0.9.3", path = "src/rpc" } +garage_table = { version = "0.9.3", path = "src/table" } +garage_util = { version = "0.9.3", path = "src/util" } +garage_web = { version = "0.9.3", path = "src/web" } k2v-client = { version = "0.0.4", path = "src/k2v-client" } # External crates from crates.io diff --git a/doc/book/cookbook/real-world.md b/doc/book/cookbook/real-world.md index 5c2dd0f1..c15ea384 100644 --- a/doc/book/cookbook/real-world.md +++ b/doc/book/cookbook/real-world.md @@ -85,14 +85,14 @@ to store 2 TB of data in total. ## Get a Docker image Our docker image is currently named `dxflrs/garage` and is stored on the [Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated). -We encourage you to use a fixed tag (eg. `v0.9.2`) and not the `latest` tag. -For this example, we will use the latest published version at the time of the writing which is `v0.9.2` but it's up to you +We encourage you to use a fixed tag (eg. `v0.9.3`) and not the `latest` tag. +For this example, we will use the latest published version at the time of the writing which is `v0.9.3` but it's up to you to check [the most recent versions on the Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated). For example: ``` -sudo docker pull dxflrs/garage:v0.9.2 +sudo docker pull dxflrs/garage:v0.9.3 ``` ## Deploying and configuring Garage @@ -157,7 +157,7 @@ docker run \ -v /etc/garage.toml:/etc/garage.toml \ -v /var/lib/garage/meta:/var/lib/garage/meta \ -v /var/lib/garage/data:/var/lib/garage/data \ - dxflrs/garage:v0.9.2 + dxflrs/garage:v0.9.3 ``` With this command line, Garage should be started automatically at each boot. @@ -171,7 +171,7 @@ If you want to use `docker-compose`, you may use the following `docker-compose.y version: "3" services: garage: - image: dxflrs/garage:v0.9.2 + image: dxflrs/garage:v0.9.3 network_mode: "host" restart: unless-stopped volumes: diff --git a/script/helm/garage/Chart.yaml b/script/helm/garage/Chart.yaml index 9a21aa8e..71906cfb 100644 --- a/script/helm/garage/Chart.yaml +++ b/script/helm/garage/Chart.yaml @@ -21,4 +21,4 @@ version: 0.4.1 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "v0.9.2" +appVersion: "v0.9.3" diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index 40fab769..317031a7 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_api" -version = "0.9.2" +version = "0.9.3" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" diff --git a/src/block/Cargo.toml b/src/block/Cargo.toml index 54093e4e..0f6fb5bb 100644 --- a/src/block/Cargo.toml +++ b/src/block/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_block" -version = "0.9.2" +version = "0.9.3" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" diff --git a/src/db/Cargo.toml b/src/db/Cargo.toml index bb72d5cd..d7c89620 100644 --- a/src/db/Cargo.toml +++ b/src/db/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_db" -version = "0.9.2" +version = "0.9.3" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index 2c8ea3f9..b022049c 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage" -version = "0.9.2" +version = "0.9.3" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" diff --git a/src/model/Cargo.toml b/src/model/Cargo.toml index 3060c133..2e5b047d 100644 --- a/src/model/Cargo.toml +++ b/src/model/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_model" -version = "0.9.2" +version = "0.9.3" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" diff --git a/src/net/Cargo.toml b/src/net/Cargo.toml index af7b4cbe..5168c939 100644 --- a/src/net/Cargo.toml +++ b/src/net/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_net" -version = "0.9.2" +version = "0.9.3" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" diff --git a/src/rpc/Cargo.toml b/src/rpc/Cargo.toml index de0e9e17..823fab99 100644 --- a/src/rpc/Cargo.toml +++ b/src/rpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_rpc" -version = "0.9.2" +version = "0.9.3" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" diff --git a/src/table/Cargo.toml b/src/table/Cargo.toml index 4cf21f9f..61b8a82e 100644 --- a/src/table/Cargo.toml +++ b/src/table/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_table" -version = "0.9.2" +version = "0.9.3" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" diff --git a/src/util/Cargo.toml b/src/util/Cargo.toml index fb7b632d..72581c16 100644 --- a/src/util/Cargo.toml +++ b/src/util/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_util" -version = "0.9.2" +version = "0.9.3" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" diff --git a/src/web/Cargo.toml b/src/web/Cargo.toml index 12bec5a7..7bbaca56 100644 --- a/src/web/Cargo.toml +++ b/src/web/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_web" -version = "0.9.2" +version = "0.9.3" authors = ["Alex Auvolat ", "Quentin Dufour "] edition = "2018" license = "AGPL-3.0" -- cgit v1.2.3