From be3b1d8f91b7e9a42dee9083ee0e9ebf52bbc57f Mon Sep 17 00:00:00 2001 From: LUXEY Adrien Date: Fri, 30 Apr 2021 10:21:15 +0200 Subject: created doc/talks subfolder and added my talk and the previous one done at the wide team --- doc/talks/2021-04-28_spirals-team/.gitignore | 10 + doc/talks/2021-04-28_spirals-team/Makefile | 33 + doc/talks/2021-04-28_spirals-team/README.md | 1 + doc/talks/2021-04-28_spirals-team/bibliography.bib | 27 + doc/talks/2021-04-28_spirals-team/conclusion.tex | 7 + .../2021-04-28_spirals-team/escaping_the_cloud.tex | 124 +++ doc/talks/2021-04-28_spirals-team/figures/c1.pdf | Bin 0 -> 30105 bytes doc/talks/2021-04-28_spirals-team/figures/c2.pdf | Bin 0 -> 30238 bytes doc/talks/2021-04-28_spirals-team/figures/c3.pdf | Bin 0 -> 30362 bytes doc/talks/2021-04-28_spirals-team/figures/c4.pdf | Bin 0 -> 30663 bytes .../figures/conflict_acid.pdf | Bin 0 -> 13207 bytes .../figures/conflict_base.pdf | Bin 0 -> 26396 bytes .../figures/conflict_crdt.pdf | Bin 0 -> 25331 bytes .../figures/conflict_problem.pdf | Bin 0 -> 13145 bytes .../2021-04-28_spirals-team/figures/dag_crdt.png | Bin 0 -> 150909 bytes .../figures/garage_distributed.png | Bin 0 -> 34956 bytes .../figures/garage_tables.pdf | Bin 0 -> 25375 bytes .../figures/m_proxy_gray.pdf | Bin 0 -> 11898 bytes .../2021-04-28_spirals-team/figures/minio_edge.png | Bin 0 -> 106219 bytes .../figures/rustacean-flat-happy.png | Bin 0 -> 33061 bytes .../figures/svg/conflict.svg | 930 +++++++++++++++++++++ .../2021-04-28_spirals-team/figures/svg/crdt.svg | 227 +++++ doc/talks/2021-04-28_spirals-team/garage.tex | 147 ++++ doc/talks/2021-04-28_spirals-team/header.tex | 134 +++ doc/talks/2021-04-28_spirals-team/introduction.tex | 99 +++ doc/talks/2021-04-28_spirals-team/main.tex | 70 ++ doc/talks/2021-04-28_spirals-team/sota.tex | 323 +++++++ doc/talks/2021-04-28_spirals-team/talk.pdf | Bin 0 -> 976942 bytes 28 files changed, 2132 insertions(+) create mode 100644 doc/talks/2021-04-28_spirals-team/.gitignore create mode 100644 doc/talks/2021-04-28_spirals-team/Makefile create mode 100644 doc/talks/2021-04-28_spirals-team/README.md create mode 100644 doc/talks/2021-04-28_spirals-team/bibliography.bib create mode 100644 doc/talks/2021-04-28_spirals-team/conclusion.tex create mode 100644 doc/talks/2021-04-28_spirals-team/escaping_the_cloud.tex create mode 100644 doc/talks/2021-04-28_spirals-team/figures/c1.pdf create mode 100644 doc/talks/2021-04-28_spirals-team/figures/c2.pdf create mode 100644 doc/talks/2021-04-28_spirals-team/figures/c3.pdf create mode 100644 doc/talks/2021-04-28_spirals-team/figures/c4.pdf create mode 100644 doc/talks/2021-04-28_spirals-team/figures/conflict_acid.pdf create mode 100644 doc/talks/2021-04-28_spirals-team/figures/conflict_base.pdf create mode 100644 doc/talks/2021-04-28_spirals-team/figures/conflict_crdt.pdf create mode 100644 doc/talks/2021-04-28_spirals-team/figures/conflict_problem.pdf create mode 100644 doc/talks/2021-04-28_spirals-team/figures/dag_crdt.png create mode 100644 doc/talks/2021-04-28_spirals-team/figures/garage_distributed.png create mode 100644 doc/talks/2021-04-28_spirals-team/figures/garage_tables.pdf create mode 100644 doc/talks/2021-04-28_spirals-team/figures/m_proxy_gray.pdf create mode 100644 doc/talks/2021-04-28_spirals-team/figures/minio_edge.png create mode 100644 doc/talks/2021-04-28_spirals-team/figures/rustacean-flat-happy.png create mode 100644 doc/talks/2021-04-28_spirals-team/figures/svg/conflict.svg create mode 100644 doc/talks/2021-04-28_spirals-team/figures/svg/crdt.svg create mode 100644 doc/talks/2021-04-28_spirals-team/garage.tex create mode 100644 doc/talks/2021-04-28_spirals-team/header.tex create mode 100644 doc/talks/2021-04-28_spirals-team/introduction.tex create mode 100644 doc/talks/2021-04-28_spirals-team/main.tex create mode 100644 doc/talks/2021-04-28_spirals-team/sota.tex create mode 100644 doc/talks/2021-04-28_spirals-team/talk.pdf (limited to 'doc/talks/2021-04-28_spirals-team') diff --git a/doc/talks/2021-04-28_spirals-team/.gitignore b/doc/talks/2021-04-28_spirals-team/.gitignore new file mode 100644 index 00000000..450d4638 --- /dev/null +++ b/doc/talks/2021-04-28_spirals-team/.gitignore @@ -0,0 +1,10 @@ +*.bbl +*.aux +*.bcf +*.blg +*.log +*.run.xml +*.synctex.gz +build/ +main.pdf +figures/crdt.pdf diff --git a/doc/talks/2021-04-28_spirals-team/Makefile b/doc/talks/2021-04-28_spirals-team/Makefile new file mode 100644 index 00000000..1db0c37a --- /dev/null +++ b/doc/talks/2021-04-28_spirals-team/Makefile @@ -0,0 +1,33 @@ +.PHONY: all viewpdf pdf clean + +TARGET = main +SOURCE_FILES = $(shell find . -type f -name "*.tex" -print) +CLASS_FILES = $(shell find . -type f -name "*.cls" -print) +BIB_FILES = $(shell find . -type f -name "*.bib" -print) +FIGURES = $(shell find . -path "./figures/*" -type f -print) +BUILD_PATH = build +BUILD_FILES = $(shell find $(BUILD_PATH) -type f -print) + +BIB_PROCESSOR := biber + +.PHONY: all pdf clean figures + +all: pdf + +pdf: $(TARGET).pdf + +clean: + @rm $(TARGET).pdf $(BUILD_FILES) > /dev/null 2>&1 || exit 0 + +figures: figures/crdt.pdf + +$(TARGET).pdf: figures $(FIGURES) $(SOURCE_FILES) $(BIB_FILES) $(CLASS_FILES) + @mkdir -p $(BUILD_PATH) > /dev/null 2>&1 || exit 0 + @pdflatex -interaction=nonstopmode -jobname=$(TARGET) -output-directory $(BUILD_PATH) $(TARGET).tex + @$(BIB_PROCESSOR) --output-directory $(BUILD_PATH) $(TARGET) + @pdflatex -interaction=nonstopmode -jobname=$(TARGET) -output-directory $(BUILD_PATH) $(TARGET).tex # For biber + @pdflatex -interaction=nonstopmode -jobname=$(TARGET) -output-directory $(BUILD_PATH) $(TARGET).tex # For biber + @ln -fs $(BUILD_PATH)/$(TARGET).pdf $(TARGET).pdf + +figures/crdt.pdf: figures/svg/crdt.svg + @inkscape -C --file=$< --export-pdf=$@ \ No newline at end of file diff --git a/doc/talks/2021-04-28_spirals-team/README.md b/doc/talks/2021-04-28_spirals-team/README.md new file mode 100644 index 00000000..9fb940c4 --- /dev/null +++ b/doc/talks/2021-04-28_spirals-team/README.md @@ -0,0 +1 @@ +Presentation of Garage by Adrien on April, 28th 2021, for his research team [Spirals](https://team.inria.fr/spirals/). \ No newline at end of file diff --git a/doc/talks/2021-04-28_spirals-team/bibliography.bib b/doc/talks/2021-04-28_spirals-team/bibliography.bib new file mode 100644 index 00000000..d145d2c3 --- /dev/null +++ b/doc/talks/2021-04-28_spirals-team/bibliography.bib @@ -0,0 +1,27 @@ + +@inproceedings{brewer_towards_2000, + title = {Towards {{Robust Distributed Systems}}}, + booktitle = {{{ACM PODC}}}, + author = {Brewer, Eric}, + year = {2000} +} + + + +@incollection{defago_conflict-free_2011, + title = {Conflict-{{Free Replicated Data Types}}}, + booktitle = {Stabilization, {{Safety}}, and {{Security}} of {{Distributed Systems}}}, + author = {Shapiro, Marc and Pregui{\c c}a, Nuno and Baquero, Carlos and Zawirski, Marek}, + year = {2011}, + address = {{Berlin, Heidelberg}}, +} + + +@inproceedings{decandia_dynamo:_2007, + title = {Dynamo: {{Amazon}}'s {{Highly Available Key}}-Value {{Store}}}, + booktitle = {{ACM SOSP}}, + author = {DeCandia, Giuseppe and Hastorun, Deniz and Jampani, Madan and Kakulapati, Gunavardhan and Lakshman, Avinash and Pilchin, Alex and Sivasubramanian, Swaminathan and Vosshall, Peter and Vogels, Werner}, + year = {2007}, + address = {{New York, USA}}, +} + diff --git a/doc/talks/2021-04-28_spirals-team/conclusion.tex b/doc/talks/2021-04-28_spirals-team/conclusion.tex new file mode 100644 index 00000000..ddbcf0b4 --- /dev/null +++ b/doc/talks/2021-04-28_spirals-team/conclusion.tex @@ -0,0 +1,7 @@ +\section{Conclusion} + +\begin{frame}{The future is cooler when we bend it our way} + +Contributions welcome! :D + +\end{frame} \ No newline at end of file diff --git a/doc/talks/2021-04-28_spirals-team/escaping_the_cloud.tex b/doc/talks/2021-04-28_spirals-team/escaping_the_cloud.tex new file mode 100644 index 00000000..9f1f13ba --- /dev/null +++ b/doc/talks/2021-04-28_spirals-team/escaping_the_cloud.tex @@ -0,0 +1,124 @@ +\section{Escaping the cloud} + +% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% \begin{frame}{Down to Earth with home-hosting} + + +% \todo{Stanley Parabole reference?} + +% \end{frame} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\begin{frame}{Why?} + +\begin{itemize} + \item \textbf{Privacy}: no prying eyes besides your ISP + \item \textbf{Control} of your infrastructure + \item \textbf{Ecology}: reuse old hardware +\end{itemize} + +\vfill +\begin{block}{\emph{Tim Berners-Lee} (1994)} +``Now, if someone tries to monopolize the Web, for example pushes proprietary variations on network protocols, then that would make me unhappy.'' +\end{block} + +\begin{itemize} + \item Make Tim Berners-Lee happy +\end{itemize} + +\end{frame} + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\begin{frame}{What?} + +\centering\Large +A data store for commodity hardware on heterogenous household connections. + + +\vfill\raggedright\normalsize + + +\begin{block}{Targetting user-facing services} +\begin{itemize} + \item Static sites + \item E-mails + \item Instant communication + %\item Video streaming % No need for a data store + \item Collaboration +\end{itemize} +\end{block} +\vfill + +Nothing fancy like sensors data streams, AI or IoT. + +\end{frame} + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\begin{frame}{What?} + + +\begin{block}{Requirements} +\begin{itemize} + \item \textbf{No single point of failure} / flat hierarchy: + + Any node can die for extended periods of time. + \item \textbf{Multi-site}: cluster spans regions/countries. + \item \textbf{Acceptable performance}. + \item \textbf{Lightweight}: targets legacy hardware. + \item \textbf{Conceptually simple}: built for low-tech organisations. + + Adding/maintaining cluster nodes should be easy. +\end{itemize} +\end{block} +\vfill + +\begin{block}{Non-goals} +\begin{itemize} + \item \textbf{Super badass performance}. + \item \textbf{NAT traversal} etc.: we require full-mesh connectivity. +\end{itemize} +\end{block} +\end{frame} + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\begin{frame}{How?} + +\begin{itemize} + \item Theoretically possible with object storage \& CRDTs. + \vfill + \item Household uplinks are getting decent (optical fibers). +\end{itemize} + + + +\end{frame} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\begin{frame}{Research Questions} + +\begin{itemize} + \item Decent performance despite bad inter-node connectivity. + \vfill + + \item Tailoring workloads as a function of nodes' capabilities: + + \begin{itemize} + \item Make use of low-end nodes (e.g. Raspberry Pis), + \item Avoid impeding global performance because of low-end nodes. + \end{itemize} + \vfill + + \item Building CRDTs for target use-cases: + + \begin{itemize} + \item Software engineering: DSL or native code? + \item Provide APIs to data store users? Risky? + \end{itemize} + \vfill + + \item Cluster management: effortless UX, low perf. overhead. +\end{itemize} +\end{frame} \ No newline at end of file diff --git a/doc/talks/2021-04-28_spirals-team/figures/c1.pdf b/doc/talks/2021-04-28_spirals-team/figures/c1.pdf new file mode 100644 index 00000000..4568b702 Binary files /dev/null and b/doc/talks/2021-04-28_spirals-team/figures/c1.pdf differ diff --git a/doc/talks/2021-04-28_spirals-team/figures/c2.pdf b/doc/talks/2021-04-28_spirals-team/figures/c2.pdf new file mode 100644 index 00000000..63be7d3c Binary files /dev/null and b/doc/talks/2021-04-28_spirals-team/figures/c2.pdf differ diff --git a/doc/talks/2021-04-28_spirals-team/figures/c3.pdf b/doc/talks/2021-04-28_spirals-team/figures/c3.pdf new file mode 100644 index 00000000..514e66f4 Binary files /dev/null and b/doc/talks/2021-04-28_spirals-team/figures/c3.pdf differ diff --git a/doc/talks/2021-04-28_spirals-team/figures/c4.pdf b/doc/talks/2021-04-28_spirals-team/figures/c4.pdf new file mode 100644 index 00000000..348e900e Binary files /dev/null and b/doc/talks/2021-04-28_spirals-team/figures/c4.pdf differ diff --git a/doc/talks/2021-04-28_spirals-team/figures/conflict_acid.pdf b/doc/talks/2021-04-28_spirals-team/figures/conflict_acid.pdf new file mode 100644 index 00000000..4175e14a Binary files /dev/null and b/doc/talks/2021-04-28_spirals-team/figures/conflict_acid.pdf differ diff --git a/doc/talks/2021-04-28_spirals-team/figures/conflict_base.pdf b/doc/talks/2021-04-28_spirals-team/figures/conflict_base.pdf new file mode 100644 index 00000000..95fe0544 Binary files /dev/null and b/doc/talks/2021-04-28_spirals-team/figures/conflict_base.pdf differ diff --git a/doc/talks/2021-04-28_spirals-team/figures/conflict_crdt.pdf b/doc/talks/2021-04-28_spirals-team/figures/conflict_crdt.pdf new file mode 100644 index 00000000..bf4354ce Binary files /dev/null and b/doc/talks/2021-04-28_spirals-team/figures/conflict_crdt.pdf differ diff --git a/doc/talks/2021-04-28_spirals-team/figures/conflict_problem.pdf b/doc/talks/2021-04-28_spirals-team/figures/conflict_problem.pdf new file mode 100644 index 00000000..b07ad3b4 Binary files /dev/null and b/doc/talks/2021-04-28_spirals-team/figures/conflict_problem.pdf differ diff --git a/doc/talks/2021-04-28_spirals-team/figures/dag_crdt.png b/doc/talks/2021-04-28_spirals-team/figures/dag_crdt.png new file mode 100644 index 00000000..f7abe7fd Binary files /dev/null and b/doc/talks/2021-04-28_spirals-team/figures/dag_crdt.png differ diff --git a/doc/talks/2021-04-28_spirals-team/figures/garage_distributed.png b/doc/talks/2021-04-28_spirals-team/figures/garage_distributed.png new file mode 100644 index 00000000..da658f3b Binary files /dev/null and b/doc/talks/2021-04-28_spirals-team/figures/garage_distributed.png differ diff --git a/doc/talks/2021-04-28_spirals-team/figures/garage_tables.pdf b/doc/talks/2021-04-28_spirals-team/figures/garage_tables.pdf new file mode 100644 index 00000000..a7dd13b3 Binary files /dev/null and b/doc/talks/2021-04-28_spirals-team/figures/garage_tables.pdf differ diff --git a/doc/talks/2021-04-28_spirals-team/figures/m_proxy_gray.pdf b/doc/talks/2021-04-28_spirals-team/figures/m_proxy_gray.pdf new file mode 100644 index 00000000..fc96e3ab Binary files /dev/null and b/doc/talks/2021-04-28_spirals-team/figures/m_proxy_gray.pdf differ diff --git a/doc/talks/2021-04-28_spirals-team/figures/minio_edge.png b/doc/talks/2021-04-28_spirals-team/figures/minio_edge.png new file mode 100644 index 00000000..0efce4b7 Binary files /dev/null and b/doc/talks/2021-04-28_spirals-team/figures/minio_edge.png differ diff --git a/doc/talks/2021-04-28_spirals-team/figures/rustacean-flat-happy.png b/doc/talks/2021-04-28_spirals-team/figures/rustacean-flat-happy.png new file mode 100644 index 00000000..ebce1a14 Binary files /dev/null and b/doc/talks/2021-04-28_spirals-team/figures/rustacean-flat-happy.png differ diff --git a/doc/talks/2021-04-28_spirals-team/figures/svg/conflict.svg b/doc/talks/2021-04-28_spirals-team/figures/svg/conflict.svg new file mode 100644 index 00000000..1c39af9f --- /dev/null +++ b/doc/talks/2021-04-28_spirals-team/figures/svg/conflict.svg @@ -0,0 +1,930 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + # vaccines + Paris + Lille + + + + + + + + W + + + + W + + incr(1000) + incr(500) + + T1: (5000, {(a, 1000), (b, 500)}) + + + R + + 6500 + T0: (5000, ∅) + + diff --git a/doc/talks/2021-04-28_spirals-team/figures/svg/crdt.svg b/doc/talks/2021-04-28_spirals-team/figures/svg/crdt.svg new file mode 100644 index 00000000..6104116b --- /dev/null +++ b/doc/talks/2021-04-28_spirals-team/figures/svg/crdt.svg @@ -0,0 +1,227 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + App + + + CRDT + + + Datastore + + + diff --git a/doc/talks/2021-04-28_spirals-team/garage.tex b/doc/talks/2021-04-28_spirals-team/garage.tex new file mode 100644 index 00000000..9f953602 --- /dev/null +++ b/doc/talks/2021-04-28_spirals-team/garage.tex @@ -0,0 +1,147 @@ +\section{Introducing Garage} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\begin{frame}{Brought to you by the Deuxfleurs association} + +\begin{block}{\textbf{deuxfleurs.fr} -- a libre hosting association with a vision} +``Shifting the current structure of the Internet from a world of a few very large service providers, to a world where services are hosted by a variety of smaller organisations.'' +\end{block} + + +\begin{block}{Our goals} +\begin{itemize} + \item To propose performant \& reliable libre services for the masses + \item To host and administer our infrastructure ourselves + \item To allow members to contribute storage/compute nodes + \item Resilience: for availability \& the sysadmins' sleep + \item Conceptual simplicity to ease onboarding \& demistify hosting +\end{itemize} +\end{block} + + +\end{frame} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\begin{frame}{The lacking state of the practice} + +\begin{block}{Object storage fitted our needs} +\begin{itemize} + \item Distributed by design + \item Objects are replicated + \item Conceptually simple +\end{itemize} +\end{block} +\vfill + +\begin{block}{Existing object stores did not} + \begin{itemize} + \item Too specific / complex + \item Resource hungry + \item Hidden constraints + \end{itemize} +\end{block} +\vfill + +We developed Garage, an object store with minimal functionality. + +It works, and serves our static sites and media. + +\end{frame} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\begin{frame}{Introducing Garage} + +\centering +\url{garagehq.deuxfleurs.fr} + +\url{git.deuxfleurs.fr/Deuxfleurs/garage} + +\includegraphics[width=.4\columnwidth]{figures/garage_distributed.png} +\vfill + +\raggedright +\begin{itemize} + \item Distributed data store + \item Based on DynamoDB object store (P2P!) + \item Modular data types/protocols with CRDTs: + \begin{itemize} + \item Done: objects (media, static sites, backups...) via S3 API + \item To do: e-mails via IMAP protocol, and more + \end{itemize} +\end{itemize} + +\end{frame} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\begin{frame}[t]{The \textbf{RING}} + +\centering +\fullcite{decandia_dynamo:_2007} +\vspace{3ex} + +\only<1>{\includegraphics[width=.5\columnwidth]{figures/c1.pdf}}% +\only<2>{\includegraphics[width=.5\columnwidth]{figures/c2.pdf}}% +\only<3>{\includegraphics[width=.5\columnwidth]{figures/c3.pdf}}% +\only<4>{\includegraphics[width=.5\columnwidth]{figures/c4.pdf}}% +\vspace{5ex} + +%\raggedright +\only<1>{Each node is assigned a unique ID on the circular address space.}% +\only<2-3>{When a new object is added to the store...}% +\only<3>{\\ It is assigned a unique ID (its \emph{key}) on the address space.}% +\only<4>{The $R$ nodes after the object are in charge of replicating it.}% + +\end{frame} + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\begin{frame}{Distributed metadata} + +\centering +\includegraphics[width=.8\columnwidth]{figures/garage_tables.pdf} +\vfill + +The objects, versions and blocks are all stored in the ring. + +\end{frame} + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\begin{frame}{Written in Rust} + +\begin{columns} +\column{.65\columnwidth} +Entirely written in Rust! +\column{.35\columnwidth} +\centering +\includegraphics[width=.85\columnwidth]{figures/rustacean-flat-happy.png} +\end{columns} +\vfill + +\begin{columns} +\column[t]{.6\columnwidth} + \textbf{Pros:} + \begin{itemize} + \item Compiled and fast + \item Features prevent usual mistakes: + + strongly typed, immutable by default, ownership instead of GC... + + \item Best of several paradigms: + + imperative, OO, functional + + \item Good libraries for network programmings: + + serialization, http, async/await... + \end{itemize} +\column[t]{.4\columnwidth} + \textbf{Cons}: + \begin{itemize} + \item Steep learning curve + \item Long compilation times + \item Compiler rage + \end{itemize} +\end{columns} + +\end{frame} \ No newline at end of file diff --git a/doc/talks/2021-04-28_spirals-team/header.tex b/doc/talks/2021-04-28_spirals-team/header.tex new file mode 100644 index 00000000..42eaa25a --- /dev/null +++ b/doc/talks/2021-04-28_spirals-team/header.tex @@ -0,0 +1,134 @@ +%% Imports %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\usepackage{graphicx} +\usepackage[utf8]{inputenc} +\usepackage{csquotes} % Elegant quotes (must be loaded before babel) +\usepackage[english]{babel} +\usepackage[T1]{fontenc} +% \usepackage{amsfonts} +% \usepackage{amssymb} +%\usepackage{lmodern} +\usepackage{iwona} +\usepackage{color} +\usepackage{xspace} +\usepackage{amsmath} +\usepackage{hanging} +\usepackage{listings} +\lstset{basicstyle=\scriptsize} +\usepackage{tikz} +\usepackage{tikzsymbols} % For emojis +%\usepackage{setspace} % Activation disables footnoes +\usepackage{perpage} % Reset footnote counter every page +\MakePerPage{footnote} +\usepackage{nameref} % For printing the sections' name +\usepackage{hyperref} + +%% Biblio %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\usepackage[style=alphabetic,giveninits=true,sorting=none,hyperref,backend=biber,maxnames=3]{biblatex} +\addbibresource{bibliography.bib} + +%% Templating %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\usepackage{pgf} +\usepackage{xifthen, tikz} + +\usetheme{default} + +\makeatletter + + +%%% colors +\definecolor{bleuroi}{RGB}{0, 8, 88} +\definecolor{vertemeraude}{RGB}{0, 64, 44} +\definecolor{lightgray}{RGB}{245,245,245} +\definecolor{encre_de_chine}{RGB}{9,36,53} +\definecolor{im_lost_in_your_eyes}{RGB}{110,138,159} +\definecolor{metallic_grey}{RGB}{209,219,221} +\definecolor{metallic_white}{RGB}{229,239,241} +\definecolor{boss}{RGB}{229,72,27} +\definecolor{toon_eyes}{RGB}{3,4,6} + +\setbeamercolor{frametitle}{bg=bleuroi,fg=white} +\setbeamercolor*{normal text}{fg=toon_eyes,bg=white} +\setbeamercolor*{block title}{fg=white,bg=vertemeraude} +\setbeamercolor*{block body}{fg=toon_eyes,bg=vertemeraude!5} +\setbeamercolor{alerted text}{fg=vertemeraude} +\setbeamercolor*{example text}{fg=toon_eyes,bg=white} +\setbeamercolor*{structure}{fg=bleuroi,bg=white} +% sectionnavigation: +\setbeamercolor*{section in head/foot}{fg=bleuroi,bg=white} + +% \let\oldtextbf\textbf +% \renewcommand{\textbf}[1]{\textcolor{bleuroi}{\oldtextbf{#1}}} + +%%% Rounded boxes with no shading +\pgfdeclareverticalshading[lower.bg,upper.bg]{bmb@transition}{200cm}{% + color(0pt)=(upper.bg); color(2pt)=(upper.bg); color(4pt)=(upper.bg)} +\setbeamertemplate{blocks}[rounded][shadow=false] + +%%% Who on Earth uses navigation bars? +\setbeamertemplate{navigation symbols}{} + + +%%% Headline with sections +\setbeamertemplate{headline} +{% + \begin{beamercolorbox}[wd=\paperwidth,dp=.5ex,ht=2ex]{section in head/foot} + \insertsectionnavigationhorizontal{\paperwidth}{\hskip0pt plus1fill}{\hskip0pt plus1fill}\par % Centered + %\insertsectionnavigationhorizontal{\paperwidth}{}{\hfill\hfill} % Left aligned + \end{beamercolorbox}% +} +% Set the colors of the section bar +\usesectionheadtemplate + {\colorbox{fg}{\color{bg} \insertsectionhead}} + {\color{fg!40!bg} \insertsectionhead} + +% \newlength\lpg@linewd +% \setbox0=\hbox{\strut} +% \newlength\strutht \strutht\ht0 +% \newlength\strutdp \strutdp\dp0 + +\setbeamertemplate{frametitle} +{ + \begin{beamercolorbox}[wd=\paperwidth,dp=1ex,ht=2.6ex,leftskip=.7cm,rightskip=.7cm]{frametitle} + %\vskip-1ex + %[wd=\lpg@linewd, ht=\strutht, dp=\strutdp]{frametitle} + %\vskip-.8ex\hskip0.7cm + %\vskip-.8ex + \usebeamerfont{frametitle}\strut\insertframetitle + \ifx\insertframesubtitle\@empty + \else + \hfill \usebeamerfont{framesubtitle}\strut\insertframesubtitle + \fi% + \end{beamercolorbox} +} + + +%%% Page numbering in footline +\setbeamertemplate{footline}{ +\hbox{\begin{beamercolorbox}[wd=1\paperwidth,ht=2.5ex,dp=1ex,right]{framenumber in head/foot}% + \usebeamerfont{framenumber in head/foot}% + %\insertframenumber\space/\space\inserttotalframenumber\hspace{0.3em} + \insertframenumber\hspace{0.5em} +\end{beamercolorbox}}} + +%%% Reduce foot citation size +\renewcommand{\footnotesize}{\tiny} + +%%% Display footnotes (not working, still no footnotes) +% \setbeamertemplate{footnote}{% +% \hangpara{2em}{1}% +% \makebox[2em][l]{\insertfootnotemark}\footnotesize\insertfootnotetext\par% +% } + + +\makeatother + +%% Commentaires %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\newcommand{\annote}[3]{{ + \colorbox{#3}{\bfseries\sffamily\footnotesize\textcolor{white}{#2}} + \color{#3} + $\blacktriangleright$\textit{#1}$\blacktriangleleft$} +} +%% Uncomment for final version (removes notes) +%\renewcommand{\annote}[3]{} + +\newcommand{\todo}[1]{\annote{#1}{TODO}{green}} diff --git a/doc/talks/2021-04-28_spirals-team/introduction.tex b/doc/talks/2021-04-28_spirals-team/introduction.tex new file mode 100644 index 00000000..a977c09b --- /dev/null +++ b/doc/talks/2021-04-28_spirals-team/introduction.tex @@ -0,0 +1,99 @@ +\section{Introduction} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\begin{frame}{A very casual motivation} + + +\begin{center} +\Large +I want to host \textbf{resilient web services} with \textbf{acceptable performance} on commodity hardware behind \textbf{household networks}. +\end{center} +\vfill + +\begin{block}{Keywords} +\begin{columns} +\column{.5\columnwidth} + \begin{itemize} + \item Decentralised networks + \item Edge computing + \end{itemize} +\column{.5\columnwidth} + \begin{itemize} + \item Distributed storage + \item Privacy + \end{itemize} +\end{columns} +\end{block} + +\end{frame} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\begin{frame}{Context} + +\textbf{Resilience}: Ability to recover quickly from failures and changes. +\vspace{1ex} + +Only achievable through distribution of the hosted applications across several physical locations. +\vfill + + +\begin{block}{Application = \textbf{computations} on \textbf{data}} +\begin{itemize} + \item \textbf{Computation}: Stateless; easy to distribute \& orchestrate. + % where it is performed does not matter as long as the application's state is accessible R/W + %Computation units are "easy" to distribute and orchestrate. + + \item \textbf{Data}: Stateful; hard to distribute \& full of trade-offs. +\end{itemize} +\end{block} + +\end{frame} + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\begin{frame}{Concurrent writes example}{How to lose vaccines} + +\centering + +\includegraphics[width=.5\columnwidth]{figures/conflict_problem.pdf} +\end{frame} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\begin{frame}{The problem} + +% \textbf{Path dependency}: existing data stores are built for data centres. + +% $\implies$ They assume good inter-node connectivity. +% \vfill + +\begin{center} +\Large +Can we design an available data store tailored for adverse network conditions? +\end{center} + +\end{frame} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% Maybe more framing of the context. What kind of data storage? Object vs Block vs what? +% \begin{frame}{``Stateless'', ``serverless'', and the elephant in the room} + +% It seems easy to deploy \& administer web services nowadays ... + +% Because the inherent complexity is shadowed by proprietary ``cloud'' solutions. + +% The IT crowd can gloss over ``statelessness'' \emph{ad nauseam} ... + +% But storing \emph{state} remains an open research problem. + +% Data storage is either: + +% \begin{itemize} +% \item A single point of failure; +% \item Delegated to proprietary solutions; +% \item Pain. +% \end{itemize} + +% Today, we will review networked storage's history, and discuss open research questions. + +% \end{frame} \ No newline at end of file diff --git a/doc/talks/2021-04-28_spirals-team/main.tex b/doc/talks/2021-04-28_spirals-team/main.tex new file mode 100644 index 00000000..5cd9dc0c --- /dev/null +++ b/doc/talks/2021-04-28_spirals-team/main.tex @@ -0,0 +1,70 @@ +\documentclass[compress]{beamer} + +\input{header.tex} + +%% Metadata %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\title{Distributed object storage is centralised} +\subtitle{A quest for autonomy in the modern hosting ecology} +\author{Adrien Luxey} +\date{Wednesday, 28th April, 2021} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Begin document %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\begin{document} + +%% Title page %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +{ +\setbeamertemplate{footline}{} +\setbeamertemplate{headline}{} +\begin{frame} +\titlepage +\end{frame} +} + +%% Contents %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\input{introduction.tex} +\input{sota.tex} +\input{escaping_the_cloud.tex} +\input{garage.tex} +\input{conclusion.tex} + + +%% Bibliography %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +% \begin{frame} +% \large +% \raggedright +% Références + +% \printbibliography +% \end{frame} + +%% Ending %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +{ +\setbeamertemplate{footline}{} +\setbeamertemplate{headline}{} +\begin{frame} +\centering + +\vspace{1cm} +\Large Thank you for your attention. + +\vspace{2cm} +\large Now let's chat! + +\vspace{1cm} +\raggedleft +\includegraphics[width=.2\textwidth]{figures/m_proxy_gray.pdf} +\end{frame} +} + +%% Appendices %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +% \appendix +% \input{sprinkler_appendix.tex} +% \input{cascade_appendix.tex} +% \input{spores_appendix.tex} + + +\end{document} \ No newline at end of file diff --git a/doc/talks/2021-04-28_spirals-team/sota.tex b/doc/talks/2021-04-28_spirals-team/sota.tex new file mode 100644 index 00000000..23b9087a --- /dev/null +++ b/doc/talks/2021-04-28_spirals-team/sota.tex @@ -0,0 +1,323 @@ +\section{State of the art} + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\begin{frame}{The CAP theorem}{Consistency vs. Availability} + +\begin{block}{Eric Brewer's theorem} +``A shared-state system can have \textbf{at most two} of the following properties at any given time: + +\begin{itemize} + \item \textbf{C}onsistency + \item \textbf{A}vailability + \item \textbf{P}artition tolerance'' +\end{itemize} +\end{block} + + +\begin{center} +\Large +Under network partitions, a distributed data store has to sacrifice either availability or consistency. +\end{center} +\vfill + +\begin{itemize} + \item \textbf{Consistency-first}: Abort incoming queries; + \item \textbf{Availability-first}: Return possibly stale data. +\end{itemize} + +\end{frame} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\begin{frame}{Consistency-first: the ACID model}{Consistency vs. Availability} + +\textbf{Transaction}: unit of work within an ACID data store. +%Comprises multiple operations. +%E.g. bank transfer. +%E.g. a bank transfer from A to B is a transaction involving two operations: withdraw money from A & credit B with the same money amount. +\vfill + +\begin{itemize} + \item \textbf{\underline{A}tomicity}: Transactions either complete entirely or fail. + + No transaction ever seen as in-progress. + + \item \textbf{\underline{C}onsistency}: Transactions always generate a valid state. + + The database maintains its invariants across transactions. + + \item \textbf{\underline{I}solation}: Concurrent transactions are seen as sequential. + + Transactions are serializable, or sequentially consistent. + + \item \textbf{\underline{D}urability}: Committed transactions are never forgotten. +\end{itemize} +\vfill\centering + +Reads are fast, writes are slow. + +\vfill\raggedright + +Example: relational databases. +\end{frame} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\begin{frame}[fragile]{Concurrent writes in ACID}{Consistency vs. Availability} + + +\begin{columns} +\column{.5\columnwidth} + \begin{block}{} + \begin{lstlisting} +transaction AcqDoses(y): + x <- SELECT #vaccines; + UPDATE #vaccines = (x + y); + \end{lstlisting} + \end{block} + \vspace{5ex} + + Supports compound operations. +\column{.5\columnwidth} +\centering +\includegraphics[width=\columnwidth]{figures/conflict_acid.pdf} +\end{columns} + +\end{frame} + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\begin{frame}{Availability-first: the BASE model}{Consistency vs. Availability} + + +Some apps prefer availability, e.g. Amazon products' reviews. +\vfill + +The BASE model trades Consistency \& Isolation for Availability. + + +%Some applications do not care about strong consistency and prefer being highly available (e.g. Amazon's product reviews). + +%In order to achieve higher availability, the BASE model relaxes consistency constraints of the ACID model: "eventual consistency". +\vfill + +\begin{itemize} + \item \textbf{\underline{B}asic \underline{A}vailability}: + The data store thrives to be available. + + \item \textbf{\underline{S}oft-state}: + Replicas can disagree on the valid state. + + \item \textbf{\underline{E}ventual consistency}: + In the absence of write queries, + the data store will eventually converge to a single valid state. +\end{itemize} +\vfill\centering + +Writes are fast, reads are slow. + +\vfill\raggedright + +Examples: key-value \& object stores. + +\end{frame} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\begin{frame}{Concurrent writes in BASE}{Consistency vs. Availability} + +\begin{columns} +\column{.5\columnwidth} + \begin{block}{Object} + \begin{itemize} + \item Unique key + \item Arbitrary value + \item Metadata + \end{itemize} + \end{block} + \vspace{5ex} + + Conflict resolution = client's job! + \vspace{5ex} + + No compound operations. +\column{.5\columnwidth} + \centering + \includegraphics[width=\columnwidth]{figures/conflict_base.pdf} +\end{columns} + +% KV storage is another example, distinction is minor here + +% Object = unique key, arbitrary value, metadata. + +% Object storage only provides semantics to investigate causal order of queries *for individual objects*. No compound operations, no transactions. + +% Much easier to distribute, and "scale-out". + +% Write is fast, read is slow (gotta collect all object versions). + +% \todo{vaccines example with BASE model} + +\end{frame} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\begin{frame}{Strong Eventual Consistency w/ CRDTs}{Consistency vs. Availability} + +\centering\small + +\fullcite{defago_conflict-free_2011} + +\vfill\raggedright\normalsize + +\begin{block}{Strong Eventual Consistency (SEC)} + \begin{itemize} + \item CRDTs specify distributed operations + \item Conflicts will be solved according to specification + \item Proven \& bound eventual convergence + \end{itemize} +\end{block} + +\vfill\centering +\includegraphics[width=.5\columnwidth]{figures/crdt.pdf} + +\end{frame} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\begin{frame}[fragile]{Concurrent writes with CRDTs}{Consistency vs. Availability} + +\begin{columns} +\column{.5\columnwidth} + \begin{block}{} + \begin{lstlisting} +CRDT Counter(x0): + history = {} + op. incr(y): + history U= {(UUID(), y)} + op. decr(y): + history U= {(UUID(), -y)} + op. read(): + x = x0 + for (_, y) in history: + x += y + return x + \end{lstlisting} + \end{block} + \vspace{2ex} + + Operations commute? + + $\implies$ screw total order! +\column{.5\columnwidth} + \centering + \includegraphics[width=\columnwidth]{figures/conflict_crdt.pdf} +\end{columns} + +\end{frame} + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\begin{frame}{A complex CRDT: the DAG}{Consistency vs. Availability} + +\centering +\only<1>{\includegraphics[height=\textheight]{figures/dag_crdt.png}}% +\only<2>{ + Just to say I swept a lot under the rug. + \vfill + + For details, go read: + + \fullcite{defago_conflict-free_2011} + \vfill + + For an implementation, check \textbf{AntidoteDB}. +} + +\end{frame} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\begin{frame}{State of the practice}{Path dependency to the ``cloud''} + +\begin{block}{The BASE model is fashionable because} +\centering + +``\emph{High-performance} object storage for \emph{AI analytics} with PBs of \emph{IoT data streams} at the \emph{edge}, using \emph{5G}.'' + % \begin{itemize} + % \item Highest performance + % \item IoT data streams are inherently distributed + % \end{itemize} +\end{block} + +\vfill\centering + +\includegraphics[width=.9\columnwidth]{figures/minio_edge.png} + +\vfill\raggedright + + +%\begin{block}{} +\begin{itemize} + \item Always backed by cloud: high performance network links. + \item Edge nodes always seen as clients or data sources, not peers. +\end{itemize} +%\end{block} + +% There is \textbf{always a central cloud cluster} in these use-cases. + +% Hidden constraint: \textbf{high performance inter-node connectivity}. + + + +\end{frame} + + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% \begin{frame}{A brief history of storage} + +% We keep it short because we'll follow chronological order in the next section too. + +% \end{frame} + + +% \begin{frame}{In the beginning, there were \emph{monoliths}} + +% \includegraphics[width=.5\columnwidth]{figures/stonehenge.jpg} + +% Web applications used to be monolithic: + +% \begin{itemize} +% \item One or two servers; +% \item Availability was not an obsession; +% \item Latency was acceptable. +% \end{itemize} + +% Relational databases were queens. + +% \end{frame} + + +% \begin{frame}{Then came \emph{expectations}} +% Then, the whole world went online, and suddenly: expectations! + +% \begin{itemize} +% \item ``Milliseconds matter.'' (Algolia slogan) +% \item Critical networked services (healthcare, logistics) need 100\% availability +% \end{itemize} + +% $\implies$ Microservices \& horizontal scalability. + +% \todo{Develop on the `herd not sheep' paradigm a bit.} + +% \end{frame} + + +% \begin{frame}{Distributing state/storage: the remaining unknown} + +% The microservices orchestration game works well for \emph{stateless} services. + +% However, any application requires \emph{state}, persistent data. + +% And this is tough. As we will now see. + +% (Not that it's not well studied: distributed storage has always been fashionable.) + +% \end{frame} \ No newline at end of file diff --git a/doc/talks/2021-04-28_spirals-team/talk.pdf b/doc/talks/2021-04-28_spirals-team/talk.pdf new file mode 100644 index 00000000..051d8f6c Binary files /dev/null and b/doc/talks/2021-04-28_spirals-team/talk.pdf differ -- cgit v1.2.3