Slides for talk at IMT Atlantique / STACK on 2022-06-23 #333
BIN
doc/talks/2022-06-23-stack/assets/aerogramme.png
Normal file
After Width: | Height: | Size: 115 KiB |
445
doc/talks/2022-06-23-stack/assets/aerogramme.svg
Normal file
After Width: | Height: | Size: 286 KiB |
After Width: | Height: | Size: 97 KiB |
BIN
doc/talks/2022-06-23-stack/assets/quentin.jpg
Normal file
After Width: | Height: | Size: 38 KiB |
BIN
doc/talks/2022-06-23-stack/assets/slideB1.png
Normal file
After Width: | Height: | Size: 85 KiB |
BIN
doc/talks/2022-06-23-stack/assets/slideB2.png
Normal file
After Width: | Height: | Size: 81 KiB |
BIN
doc/talks/2022-06-23-stack/assets/slideB3.png
Normal file
After Width: | Height: | Size: 81 KiB |
445
doc/talks/2022-06-23-stack/assets/slidesB.svg
Normal file
After Width: | Height: | Size: 286 KiB |
BIN
doc/talks/2022-06-23-stack/talk.pdf
(Stored with Git LFS)
|
@ -9,6 +9,9 @@
|
||||||
\usetheme{boxes}
|
\usetheme{boxes}
|
||||||
\usepackage{graphicx}
|
\usepackage{graphicx}
|
||||||
%\useoutertheme[footline=authortitle,subsection=false]{miniframes}
|
%\useoutertheme[footline=authortitle,subsection=false]{miniframes}
|
||||||
|
%\useoutertheme[footline=authorinstitute,subsection=false]{miniframes}
|
||||||
|
\useoutertheme{infolines}
|
||||||
|
\setbeamertemplate{headline}{}
|
||||||
|
|
||||||
\beamertemplatenavigationsymbolsempty
|
\beamertemplatenavigationsymbolsempty
|
||||||
|
|
||||||
|
@ -72,9 +75,9 @@
|
||||||
\end{frame}
|
\end{frame}
|
||||||
|
|
||||||
\begin{frame}
|
\begin{frame}
|
||||||
\frametitle{How to be resilient (the hard way)}
|
\frametitle{How to make a \underline{stable} system}
|
||||||
|
|
||||||
Entreprise-grade systems typically employ:
|
Enterprise-grade systems typically employ:
|
||||||
\vspace{1em}
|
\vspace{1em}
|
||||||
\begin{itemize}
|
\begin{itemize}
|
||||||
\item RAID
|
\item RAID
|
||||||
|
@ -88,7 +91,7 @@
|
||||||
\end{frame}
|
\end{frame}
|
||||||
|
|
||||||
\begin{frame}
|
\begin{frame}
|
||||||
\frametitle{How to be resilient (the \underline{\textbf{cheap}} way)}
|
\frametitle{How to make a \underline{resilient} system}
|
||||||
|
|
||||||
\only<1,4-5>{
|
\only<1,4-5>{
|
||||||
Instead, we use:
|
Instead, we use:
|
||||||
|
@ -189,9 +192,108 @@
|
||||||
\end{frame}
|
\end{frame}
|
||||||
|
|
||||||
\begin{frame}
|
\begin{frame}
|
||||||
\frametitle{What makes Garage different?}
|
\frametitle{Garage is \emph{location-aware}}
|
||||||
\textbf{Coordination-free:}
|
\begin{center}
|
||||||
\vspace{2em}
|
\includegraphics[width=\linewidth]{assets/location-aware.png}
|
||||||
|
\end{center}
|
||||||
|
\vspace{2em}
|
||||||
|
Garage replicates data on different zones when possible
|
||||||
|
\end{frame}
|
||||||
|
|
||||||
|
\begin{frame}
|
||||||
|
\frametitle{Garage is \emph{location-aware}}
|
||||||
|
\begin{center}
|
||||||
|
\includegraphics[width=.8\linewidth]{assets/map.png}
|
||||||
|
\end{center}
|
||||||
|
\end{frame}
|
||||||
|
|
||||||
|
\begin{frame}
|
||||||
|
\frametitle{How to spread files over different cluster nodes?}
|
||||||
|
\textbf{Consistent hashing (DynamoDB):}
|
||||||
|
\vspace{1em}
|
||||||
|
|
||||||
|
\begin{center}
|
||||||
|
\only<1>{\includegraphics[width=.45\columnwidth]{assets/consistent_hashing_1.pdf}}%
|
||||||
|
\only<2>{\includegraphics[width=.45\columnwidth]{assets/consistent_hashing_2.pdf}}%
|
||||||
|
\only<3>{\includegraphics[width=.45\columnwidth]{assets/consistent_hashing_3.pdf}}%
|
||||||
|
\only<4>{\includegraphics[width=.45\columnwidth]{assets/consistent_hashing_4.pdf}}%
|
||||||
|
\end{center}
|
||||||
|
\end{frame}
|
||||||
|
|
||||||
|
\begin{frame}
|
||||||
|
\frametitle{How to spread files over different cluster nodes?}
|
||||||
|
\textbf{Issues with consistent hashing:}
|
||||||
|
\vspace{1em}
|
||||||
|
\begin{itemize}
|
||||||
|
\item Doesn't dispatch data based on geographical location of nodes
|
||||||
|
\vspace{1em}
|
||||||
|
\item<2-> Geographically aware adaptation, try 1:\\
|
||||||
|
data quantities not well balanced between nodes
|
||||||
|
\vspace{1em}
|
||||||
|
\item<3-> Geographically aware adaptation, try 2:\\
|
||||||
|
too many reshuffles when adding/removing nodes
|
||||||
|
\end{itemize}
|
||||||
|
\end{frame}
|
||||||
|
|
||||||
|
\begin{frame}
|
||||||
|
\frametitle{How to spread files over different cluster nodes?}
|
||||||
|
\textbf{Garage's method: build an index table}
|
||||||
|
\vspace{1em}
|
||||||
|
|
||||||
|
Realization: we can actually precompute an optimal solution
|
||||||
|
\vspace{1em}
|
||||||
|
|
||||||
|
\visible<2->{
|
||||||
|
\begin{center}
|
||||||
|
\begin{tabular}{|l|l|l|l|}
|
||||||
|
\hline
|
||||||
|
\textbf{Partition} & \textbf{Node 1} & \textbf{Node 2} & \textbf{Node 3} \\
|
||||||
|
\hline
|
||||||
|
\hline
|
||||||
|
Partition 0 & Io (jupiter) & Drosera (atuin) & Courgette (neptune) \\
|
||||||
|
\hline
|
||||||
|
Partition 1 & Datura (atuin) & Courgette (neptune) & Io (jupiter) \\
|
||||||
|
\hline
|
||||||
|
Partition 2 & Io(jupiter) & Celeri (neptune) & Drosera (atuin) \\
|
||||||
|
\hline
|
||||||
|
\hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ \\
|
||||||
|
\hline
|
||||||
|
Partition 255 & Concombre (neptune) & Io (jupiter) & Drosera (atuin) \\
|
||||||
|
\hline
|
||||||
|
\end{tabular}
|
||||||
|
\end{center}
|
||||||
|
}
|
||||||
|
\vspace{1em}
|
||||||
|
\visible<3->{
|
||||||
|
The index table is built centrally using an optimal* algorithm,\\
|
||||||
|
then propagated to all nodes\\
|
||||||
|
\hfill\footnotesize *not yet optimal but will be soon
|
||||||
|
}
|
||||||
|
\end{frame}
|
||||||
|
|
||||||
|
\begin{frame}
|
||||||
|
\frametitle{Storing and retrieving files}
|
||||||
|
\begin{center}
|
||||||
|
\only<1>{\includegraphics[width=.45\linewidth]{assets/garage2a.drawio.pdf}}%
|
||||||
|
\only<2>{\includegraphics[width=.45\linewidth]{assets/garage2b.drawio.pdf}}%
|
||||||
|
\end{center}
|
||||||
|
\end{frame}
|
||||||
|
|
||||||
|
\begin{frame}
|
||||||
|
\frametitle{Garage's internal data structures}
|
||||||
|
\centering
|
||||||
|
\includegraphics[width=.75\columnwidth]{assets/garage_tables.pdf}
|
||||||
|
\end{frame}
|
||||||
|
|
||||||
|
\begin{frame}
|
||||||
|
\frametitle{Garage's architecture}
|
||||||
|
\begin{center}
|
||||||
|
\includegraphics[width=.35\linewidth]{assets/garage.drawio.pdf}
|
||||||
|
\end{center}
|
||||||
|
\end{frame}
|
||||||
|
|
||||||
|
\begin{frame}
|
||||||
|
\frametitle{Garage is \emph{coordination-free}:}
|
||||||
\begin{itemize}
|
\begin{itemize}
|
||||||
\item No Raft or Paxos
|
\item No Raft or Paxos
|
||||||
\vspace{1em}
|
\vspace{1em}
|
||||||
|
@ -204,24 +306,14 @@
|
||||||
\end{frame}
|
\end{frame}
|
||||||
|
|
||||||
\begin{frame}
|
\begin{frame}
|
||||||
\frametitle{Storing and retrieving files}
|
\frametitle{Impact on performances}
|
||||||
\begin{center}
|
|
||||||
\only<1>{\includegraphics[width=.5\linewidth]{assets/garage2a.drawio.pdf}}%
|
|
||||||
\only<2>{\includegraphics[width=.5\linewidth]{assets/garage2b.drawio.pdf}}%
|
|
||||||
\end{center}
|
|
||||||
\end{frame}
|
|
||||||
|
|
||||||
\begin{frame}
|
|
||||||
\frametitle{What makes Garage different?}
|
|
||||||
\begin{center}
|
\begin{center}
|
||||||
\includegraphics[width=.9\linewidth]{assets/endpoint-latency-dc.png}
|
\includegraphics[width=.8\linewidth]{assets/endpoint-latency-dc.png}
|
||||||
\end{center}
|
\end{center}
|
||||||
\end{frame}
|
\end{frame}
|
||||||
|
|
||||||
\begin{frame}
|
\begin{frame}
|
||||||
\frametitle{What makes Garage different?}
|
\frametitle{Consistency model}
|
||||||
\textbf{Consistency model:}
|
|
||||||
\vspace{2em}
|
|
||||||
\begin{itemize}
|
\begin{itemize}
|
||||||
\item Not ACID (not required by S3 spec) / not linearizable
|
\item Not ACID (not required by S3 spec) / not linearizable
|
||||||
\vspace{1em}
|
\vspace{1em}
|
||||||
|
@ -230,50 +322,6 @@
|
||||||
\end{itemize}
|
\end{itemize}
|
||||||
\end{frame}
|
\end{frame}
|
||||||
|
|
||||||
\begin{frame}
|
|
||||||
\frametitle{What makes Garage different?}
|
|
||||||
\textbf{Location-aware:}
|
|
||||||
\vspace{2em}
|
|
||||||
\begin{center}
|
|
||||||
\includegraphics[width=\linewidth]{assets/location-aware.png}
|
|
||||||
\end{center}
|
|
||||||
\vspace{2em}
|
|
||||||
Garage replicates data on different zones when possible
|
|
||||||
\end{frame}
|
|
||||||
|
|
||||||
\begin{frame}
|
|
||||||
\frametitle{What makes Garage different?}
|
|
||||||
\begin{center}
|
|
||||||
\includegraphics[width=.8\linewidth]{assets/map.png}
|
|
||||||
\end{center}
|
|
||||||
\end{frame}
|
|
||||||
|
|
||||||
|
|
||||||
\begin{frame}
|
|
||||||
\frametitle{Garage's architecture}
|
|
||||||
\begin{center}
|
|
||||||
\includegraphics[width=.5\linewidth]{assets/garage.drawio.pdf}
|
|
||||||
\end{center}
|
|
||||||
\end{frame}
|
|
||||||
|
|
||||||
\begin{frame}
|
|
||||||
\frametitle{Consistent Hashing (DynamoDB)}
|
|
||||||
\textbf{How to spred files over different cluster nodes?}
|
|
||||||
\vspace{1em}
|
|
||||||
|
|
||||||
\centering
|
|
||||||
|
|
||||||
\only<1>{\includegraphics[width=.55\columnwidth]{assets/consistent_hashing_1.pdf}}%
|
|
||||||
\only<2>{\includegraphics[width=.55\columnwidth]{assets/consistent_hashing_2.pdf}}%
|
|
||||||
\only<3>{\includegraphics[width=.55\columnwidth]{assets/consistent_hashing_3.pdf}}%
|
|
||||||
\only<4>{\includegraphics[width=.55\columnwidth]{assets/consistent_hashing_4.pdf}}%
|
|
||||||
\end{frame}
|
|
||||||
|
|
||||||
\begin{frame}
|
|
||||||
\frametitle{Garage data structures: 3 levels of consistent hashing}
|
|
||||||
\centering
|
|
||||||
\includegraphics[width=.85\columnwidth]{assets/garage_tables.pdf}
|
|
||||||
\end{frame}
|
|
||||||
|
|
||||||
\begin{frame}
|
\begin{frame}
|
||||||
\frametitle{An ever-increasing compatibility list}
|
\frametitle{An ever-increasing compatibility list}
|
||||||
|
@ -282,6 +330,36 @@
|
||||||
\end{center}
|
\end{center}
|
||||||
\end{frame}
|
\end{frame}
|
||||||
|
|
||||||
|
\begin{frame}
|
||||||
|
\frametitle{Further plans for Garage}
|
||||||
|
\begin{center}
|
||||||
|
\only<1>{\includegraphics[width=.8\linewidth]{assets/slideB1.png}}%
|
||||||
|
\only<2>{\includegraphics[width=.8\linewidth]{assets/slideB2.png}}%
|
||||||
|
\only<3>{\includegraphics[width=.8\linewidth]{assets/slideB3.png}}%
|
||||||
|
\end{center}
|
||||||
|
\end{frame}
|
||||||
|
|
||||||
|
\begin{frame}
|
||||||
|
\frametitle{K2V Design}
|
||||||
|
\begin{itemize}
|
||||||
|
\item A new, custom, minimal API
|
||||||
|
\vspace{1em}
|
||||||
|
\item<2-> Exposes the partitoning mechanism of Garage\\
|
||||||
|
K2V = partition key / sort key / value (like Dynamo)
|
||||||
|
\vspace{1em}
|
||||||
|
\item<3-> Coordination-free, CRDT-friendly (inspired by Riak)\\
|
||||||
|
\vspace{1em}
|
||||||
|
\item<4-> Cryptography-friendly: values are binary blobs
|
||||||
|
\end{itemize}
|
||||||
|
\end{frame}
|
||||||
|
|
||||||
|
\begin{frame}
|
||||||
|
\frametitle{Application: an e-mail storage server}
|
||||||
|
\begin{center}
|
||||||
|
\only<1>{\includegraphics[width=.9\linewidth]{assets/aerogramme.png}}%
|
||||||
|
\end{center}
|
||||||
|
\end{frame}
|
||||||
|
|
||||||
\begin{frame}
|
\begin{frame}
|
||||||
\frametitle{Get Garage now!}
|
\frametitle{Get Garage now!}
|
||||||
\begin{center}
|
\begin{center}
|
||||||
|
@ -296,10 +374,6 @@
|
||||||
\end{center}
|
\end{center}
|
||||||
\end{frame}
|
\end{frame}
|
||||||
|
|
||||||
\begin{frame}
|
|
||||||
\frametitle{Demo time!}
|
|
||||||
\end{frame}
|
|
||||||
|
|
||||||
\end{document}
|
\end{document}
|
||||||
|
|
||||||
%% vim: set ts=4 sw=4 tw=0 noet spelllang=fr :
|
%% vim: set ts=4 sw=4 tw=0 noet spelllang=en :
|
||||||
|
|