diff --git a/doc/talks/2022-06-23-stack/.gitignore b/doc/talks/2022-06-23-stack/.gitignore new file mode 100644 index 00000000..121caa92 --- /dev/null +++ b/doc/talks/2022-06-23-stack/.gitignore @@ -0,0 +1,14 @@ +* + +!assets + +!.gitignore +!*.svg +!*.png +!*.jpg +!*.tex +!Makefile +!.gitignore +!assets/*.drawio.pdf + +!talk.pdf diff --git a/doc/talks/2022-06-23-stack/Makefile b/doc/talks/2022-06-23-stack/Makefile new file mode 100644 index 00000000..abdbe38f --- /dev/null +++ b/doc/talks/2022-06-23-stack/Makefile @@ -0,0 +1,5 @@ +talk.pdf: talk.tex assets/consistent_hashing_1.pdf assets/consistent_hashing_2.pdf assets/consistent_hashing_3.pdf assets/consistent_hashing_4.pdf assets/garage_tables.pdf + pdflatex talk.tex + +assets/%.pdf: assets/%.svg + inkscape -D -z --file=$^ --export-pdf=$@ diff --git a/doc/talks/2022-06-23-stack/assets/AGPLv3_Logo.png b/doc/talks/2022-06-23-stack/assets/AGPLv3_Logo.png new file mode 100644 index 00000000..445284a3 Binary files /dev/null and b/doc/talks/2022-06-23-stack/assets/AGPLv3_Logo.png differ diff --git a/doc/talks/2022-06-23-stack/assets/atuin.jpg b/doc/talks/2022-06-23-stack/assets/atuin.jpg new file mode 100644 index 00000000..f2fbd61d Binary files /dev/null and b/doc/talks/2022-06-23-stack/assets/atuin.jpg differ diff --git a/doc/talks/2022-06-23-stack/assets/compatibility.png b/doc/talks/2022-06-23-stack/assets/compatibility.png new file mode 100644 index 00000000..ce364a9b Binary files /dev/null and b/doc/talks/2022-06-23-stack/assets/compatibility.png differ diff --git a/doc/talks/2022-06-23-stack/assets/consistent_hashing_1.svg b/doc/talks/2022-06-23-stack/assets/consistent_hashing_1.svg new file mode 100644 index 00000000..f8d24fd8 --- /dev/null +++ b/doc/talks/2022-06-23-stack/assets/consistent_hashing_1.svg @@ -0,0 +1,301 @@ + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + 1 + + + + 2 + + + + 3 + + + + 4 + + + + 5 + + + + 6 + + + + diff --git a/doc/talks/2022-06-23-stack/assets/consistent_hashing_2.svg b/doc/talks/2022-06-23-stack/assets/consistent_hashing_2.svg new file mode 100644 index 00000000..5ac8faf6 --- /dev/null +++ b/doc/talks/2022-06-23-stack/assets/consistent_hashing_2.svg @@ -0,0 +1,334 @@ + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + 1 + + + + 2 + + + + 3 + + + + 4 + + + + 5 + + + + 6 + + + + + + + + + + + + diff --git a/doc/talks/2022-06-23-stack/assets/consistent_hashing_3.svg b/doc/talks/2022-06-23-stack/assets/consistent_hashing_3.svg new file mode 100644 index 00000000..fdfd3efc --- /dev/null +++ b/doc/talks/2022-06-23-stack/assets/consistent_hashing_3.svg @@ -0,0 +1,358 @@ + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + 1 + + + + 2 + + + + 3 + + + + 4 + + + + 5 + + + + 6 + + + + + + + + + + + + + + diff --git a/doc/talks/2022-06-23-stack/assets/consistent_hashing_4.svg b/doc/talks/2022-06-23-stack/assets/consistent_hashing_4.svg new file mode 100644 index 00000000..95ed0e02 --- /dev/null +++ b/doc/talks/2022-06-23-stack/assets/consistent_hashing_4.svg @@ -0,0 +1,377 @@ + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + 1 + + + + 2 + + + + 3 + + + + 4 + + + + 5 + + + + 6 + + + + + + + + + + + + + + + + + + + diff --git a/doc/talks/2022-06-23-stack/assets/endpoint-latency-dc.png b/doc/talks/2022-06-23-stack/assets/endpoint-latency-dc.png new file mode 100644 index 00000000..7c7411cd Binary files /dev/null and b/doc/talks/2022-06-23-stack/assets/endpoint-latency-dc.png differ diff --git a/doc/talks/2022-06-23-stack/assets/garage.drawio.pdf b/doc/talks/2022-06-23-stack/assets/garage.drawio.pdf new file mode 100644 index 00000000..2f7a5dcb --- /dev/null +++ b/doc/talks/2022-06-23-stack/assets/garage.drawio.pdf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d9ca1a04f943664e14d0e1edbde596789e4798f8ad5e1cff68bc9dd0cc1334f +size 26098 diff --git a/doc/talks/2022-06-23-stack/assets/garage.drawio.png b/doc/talks/2022-06-23-stack/assets/garage.drawio.png new file mode 100644 index 00000000..386dd862 Binary files /dev/null and b/doc/talks/2022-06-23-stack/assets/garage.drawio.png differ diff --git a/doc/talks/2022-06-23-stack/assets/garage2.drawio.png b/doc/talks/2022-06-23-stack/assets/garage2.drawio.png new file mode 100644 index 00000000..8562fbcf Binary files /dev/null and b/doc/talks/2022-06-23-stack/assets/garage2.drawio.png differ diff --git a/doc/talks/2022-06-23-stack/assets/garage2a.drawio.pdf b/doc/talks/2022-06-23-stack/assets/garage2a.drawio.pdf new file mode 100644 index 00000000..c4f5e0b7 --- /dev/null +++ b/doc/talks/2022-06-23-stack/assets/garage2a.drawio.pdf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bffa5d4a72ad25e0b18b43022258a2e4471ce94ff5397aab892fe43ec9d4d7d5 +size 33911 diff --git a/doc/talks/2022-06-23-stack/assets/garage2b.drawio.pdf b/doc/talks/2022-06-23-stack/assets/garage2b.drawio.pdf new file mode 100644 index 00000000..1a12a0d3 --- /dev/null +++ b/doc/talks/2022-06-23-stack/assets/garage2b.drawio.pdf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c4913b52e84b5af3f8effbcb7e2060845a8c6f43d5b38072b9ee33c0300d49f +size 31051 diff --git a/doc/talks/2022-06-23-stack/assets/garage_tables.svg b/doc/talks/2022-06-23-stack/assets/garage_tables.svg new file mode 100644 index 00000000..fc3d8fc5 --- /dev/null +++ b/doc/talks/2022-06-23-stack/assets/garage_tables.svg @@ -0,0 +1,502 @@ + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + Object + + + + + + bucket + + + + + + file path + + + + + + Version 1 + deleted + + + + + + + Version 2 + id + + size + MIME type + ... + + + + + + Version + + id + h(block 1) + h(block 2) + ... + + + + + Data block + + hash + data + + + + Objects table + Versions table + Blocks table + + diff --git a/doc/talks/2022-06-23-stack/assets/garageuses.png b/doc/talks/2022-06-23-stack/assets/garageuses.png new file mode 100644 index 00000000..b66d7f30 Binary files /dev/null and b/doc/talks/2022-06-23-stack/assets/garageuses.png differ diff --git a/doc/talks/2022-06-23-stack/assets/inframap.jpg b/doc/talks/2022-06-23-stack/assets/inframap.jpg new file mode 100644 index 00000000..19905a99 Binary files /dev/null and b/doc/talks/2022-06-23-stack/assets/inframap.jpg differ diff --git a/doc/talks/2022-06-23-stack/assets/location-aware.png b/doc/talks/2022-06-23-stack/assets/location-aware.png new file mode 100644 index 00000000..8b55f253 Binary files /dev/null and b/doc/talks/2022-06-23-stack/assets/location-aware.png differ diff --git a/doc/talks/2022-06-23-stack/assets/map.png b/doc/talks/2022-06-23-stack/assets/map.png new file mode 100644 index 00000000..1dff3ab6 Binary files /dev/null and b/doc/talks/2022-06-23-stack/assets/map.png differ diff --git a/doc/talks/2022-06-23-stack/assets/minio.png b/doc/talks/2022-06-23-stack/assets/minio.png new file mode 100644 index 00000000..a71e9ccc Binary files /dev/null and b/doc/talks/2022-06-23-stack/assets/minio.png differ diff --git a/doc/talks/2022-06-23-stack/assets/neptune.jpg b/doc/talks/2022-06-23-stack/assets/neptune.jpg new file mode 100644 index 00000000..e59f0bfa Binary files /dev/null and b/doc/talks/2022-06-23-stack/assets/neptune.jpg differ diff --git a/doc/talks/2022-06-23-stack/assets/rust_logo.png b/doc/talks/2022-06-23-stack/assets/rust_logo.png new file mode 100644 index 00000000..0e4809ec Binary files /dev/null and b/doc/talks/2022-06-23-stack/assets/rust_logo.png differ diff --git a/doc/talks/2022-06-23-stack/assets/slide1.png b/doc/talks/2022-06-23-stack/assets/slide1.png new file mode 100644 index 00000000..eb2e67a0 Binary files /dev/null and b/doc/talks/2022-06-23-stack/assets/slide1.png differ diff --git a/doc/talks/2022-06-23-stack/assets/slide2.png b/doc/talks/2022-06-23-stack/assets/slide2.png new file mode 100644 index 00000000..126a39b8 Binary files /dev/null and b/doc/talks/2022-06-23-stack/assets/slide2.png differ diff --git a/doc/talks/2022-06-23-stack/assets/slide3.png b/doc/talks/2022-06-23-stack/assets/slide3.png new file mode 100644 index 00000000..a39f96bf Binary files /dev/null and b/doc/talks/2022-06-23-stack/assets/slide3.png differ diff --git a/doc/talks/2022-06-23-stack/assets/slides.svg b/doc/talks/2022-06-23-stack/assets/slides.svg new file mode 100644 index 00000000..9946c6fb --- /dev/null +++ b/doc/talks/2022-06-23-stack/assets/slides.svg @@ -0,0 +1,4326 @@ + + + + + + + + + + + + + + + + + + + + + + User-facing application + Database + Filesystem + + + + + + + + + + diff --git a/doc/talks/2022-06-23-stack/talk.pdf b/doc/talks/2022-06-23-stack/talk.pdf new file mode 100644 index 00000000..048de5dd --- /dev/null +++ b/doc/talks/2022-06-23-stack/talk.pdf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:165fed1968c12fcac02f0735d5ea79e7953feddd0ed71ba77b4e7cbc121f6640 +size 1867699 diff --git a/doc/talks/2022-06-23-stack/talk.tex b/doc/talks/2022-06-23-stack/talk.tex new file mode 100644 index 00000000..a0ffed14 --- /dev/null +++ b/doc/talks/2022-06-23-stack/talk.tex @@ -0,0 +1,305 @@ +%\nonstopmode +\documentclass[aspectratio=169]{beamer} +\usepackage[utf8]{inputenc} +% \usepackage[frenchb]{babel} +\usepackage{amsmath} +\usepackage{mathtools} +\usepackage{breqn} +\usepackage{multirow} +\usetheme{boxes} +\usepackage{graphicx} +%\useoutertheme[footline=authortitle,subsection=false]{miniframes} + +\beamertemplatenavigationsymbolsempty + +\definecolor{TitleOrange}{RGB}{255,137,0} +\setbeamercolor{title}{fg=TitleOrange} +\setbeamercolor{frametitle}{fg=TitleOrange} + +\definecolor{ListOrange}{RGB}{255,145,5} +\setbeamertemplate{itemize item}{\color{ListOrange}$\blacktriangleright$} + +\definecolor{verygrey}{RGB}{70,70,70} +\setbeamercolor{normal text}{fg=verygrey} + + +\usepackage{tabu} +\usepackage{multicol} +\usepackage{vwcol} +\usepackage{stmaryrd} +\usepackage{graphicx} + +\usepackage[normalem]{ulem} + +\title{Introducing Garage} +\subtitle{a new storage platform for self-hosted geo-distributed clusters} +\author{Deuxfleurs Association} +\date{IMT Atlantique, 2022-06-23} + +\begin{document} + +\begin{frame} + \centering + \includegraphics[width=.3\linewidth]{../../sticker/Garage.pdf} + \vspace{1em} + + {\large\bf Deuxfleurs Association} + \vspace{1em} + + \url{https://garagehq.deuxfleurs.fr/} + + Matrix channel: \texttt{\#garage:deuxfleurs.fr} +\end{frame} + +\begin{frame} + \frametitle{Our objective at Deuxfleurs} + + \begin{center} + \textbf{Promote self-hosting and small-scale hosting\\ + as an alternative to large cloud providers} + \end{center} + \vspace{2em} + \visible<2->{ + Why is it hard? + } + \visible<3->{ + \vspace{2em} + \begin{center} + \textbf{\underline{Resilience}}\\ + {\footnotesize (we want good uptime/availability with low supervision)} + \end{center} + } +\end{frame} + +\begin{frame} + \frametitle{How to be resilient (the hard way)} + + Entreprise-grade systems typically employ: + \vspace{1em} + \begin{itemize} + \item RAID + \item Redundant power grid + UPS + \item Redundant Internet connections + \item Low-latency links + \item ... + \end{itemize} + \vspace{1em} + $\to$ it's costly and only worth it at DC scale +\end{frame} + +\begin{frame} + \frametitle{How to be resilient (the \underline{\textbf{cheap}} way)} + + \only<1,4-5>{ + Instead, we use: + \vspace{1em} + \begin{itemize} + \item \textcolor<2->{gray}{Commodity hardware (e.g. old desktop PCs)} + \vspace{.5em} + \item<4-> \textcolor<5->{gray}{Commodity Internet (e.g. FTTB, FTTH) and power grid} + \vspace{.5em} + \item<5-> \textcolor<6->{gray}{\textbf{Geographical redundancy} (multi-site replication)} + \end{itemize} + } + \only<2>{ + \begin{center} + \includegraphics[width=.8\linewidth]{assets/atuin.jpg} + \end{center} + } + \only<3>{ + \begin{center} + \includegraphics[width=.8\linewidth]{assets/neptune.jpg} + \end{center} + } + \only<6>{ + \begin{center} + \includegraphics[width=.5\linewidth]{assets/inframap.jpg} + \end{center} + } +\end{frame} + +\begin{frame} + \frametitle{How to make this happen} + \begin{center} + \only<1>{\includegraphics[width=.8\linewidth]{assets/slide1.png}}% + \only<2>{\includegraphics[width=.8\linewidth]{assets/slide2.png}}% + \only<3>{\includegraphics[width=.8\linewidth]{assets/slide3.png}}% + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Distributed file systems are slow} + File systems are complex, for example: + \vspace{1em} + \begin{itemize} + \item Concurrent modification by several processes + \vspace{1em} + \item Folder hierarchies + \vspace{1em} + \item Other requirements of the POSIX spec + \end{itemize} + \vspace{1em} + Coordination in a distributed system is costly + + \vspace{1em} + Costs explode with commodity hardware / Internet connections\\ + {\small (we experienced this!)} +\end{frame} + +\begin{frame} + \frametitle{A simpler solution: object storage} + Only two operations: + \vspace{1em} + \begin{itemize} + \item Put an object at a key + \vspace{1em} + \item Retrieve an object from its key + \end{itemize} + \vspace{1em} + {\footnotesize (and a few others)} + + \vspace{1em} + Sufficient for many applications! +\end{frame} + +\begin{frame} + \frametitle{A simpler solution: object storage} + \begin{center} + \includegraphics[width=.2\linewidth]{../2020-12-02_wide-team/img/Amazon-S3.jpg} + \hspace{5em} + \includegraphics[width=.2\linewidth]{assets/minio.png} + \end{center} + \vspace{1em} + S3: a de-facto standard, many compatible applications + + \vspace{1em} + + MinIO is self-hostable but not suited for geo-distributed deployments +\end{frame} + + +\begin{frame} + \frametitle{But what is Garage, exactly?} + \textbf{Garage is a self-hosted drop-in replacement for the Amazon S3 object store}\\ + \vspace{.5em} + that implements resilience through geographical redundancy on commodity hardware + \begin{center} + \includegraphics[width=.8\linewidth]{assets/garageuses.png} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{What makes Garage different?} + \textbf{Coordination-free:} + \vspace{2em} + \begin{itemize} + \item No Raft or Paxos + \vspace{1em} + \item Internal data types are CRDTs + \vspace{1em} + \item All nodes are equivalent (no master/leader/index node) + \end{itemize} + \vspace{2em} + $\to$ less sensitive to higher latencies between nodes +\end{frame} + +\begin{frame} + \frametitle{Storing and retrieving files} + \begin{center} + \only<1>{\includegraphics[width=.5\linewidth]{assets/garage2a.drawio.pdf}}% + \only<2>{\includegraphics[width=.5\linewidth]{assets/garage2b.drawio.pdf}}% + \end{center} +\end{frame} + +\begin{frame} + \frametitle{What makes Garage different?} + \begin{center} + \includegraphics[width=.9\linewidth]{assets/endpoint-latency-dc.png} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{What makes Garage different?} + \textbf{Consistency model:} + \vspace{2em} + \begin{itemize} + \item Not ACID (not required by S3 spec) / not linearizable + \vspace{1em} + \item \textbf{Read-after-write consistency}\\ + {\footnotesize (stronger than eventual consistency)} + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{What makes Garage different?} + \textbf{Location-aware:} + \vspace{2em} + \begin{center} + \includegraphics[width=\linewidth]{assets/location-aware.png} + \end{center} + \vspace{2em} + Garage replicates data on different zones when possible +\end{frame} + +\begin{frame} + \frametitle{What makes Garage different?} + \begin{center} + \includegraphics[width=.8\linewidth]{assets/map.png} + \end{center} +\end{frame} + + +\begin{frame} + \frametitle{Garage's architecture} + \begin{center} + \includegraphics[width=.5\linewidth]{assets/garage.drawio.pdf} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Consistent Hashing (DynamoDB)} + \textbf{How to spred files over different cluster nodes?} + \vspace{1em} + + \centering + + \only<1>{\includegraphics[width=.55\columnwidth]{assets/consistent_hashing_1.pdf}}% + \only<2>{\includegraphics[width=.55\columnwidth]{assets/consistent_hashing_2.pdf}}% + \only<3>{\includegraphics[width=.55\columnwidth]{assets/consistent_hashing_3.pdf}}% + \only<4>{\includegraphics[width=.55\columnwidth]{assets/consistent_hashing_4.pdf}}% +\end{frame} + +\begin{frame} + \frametitle{Garage data structures: 3 levels of consistent hashing} + \centering + \includegraphics[width=.85\columnwidth]{assets/garage_tables.pdf} +\end{frame} + +\begin{frame} + \frametitle{An ever-increasing compatibility list} + \begin{center} + \includegraphics[width=.7\linewidth]{assets/compatibility.png} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Get Garage now!} + \begin{center} + \includegraphics[width=.3\linewidth]{../../logo/garage_hires.png}\\ + \vspace{-1em} + \url{https://garagehq.deuxfleurs.fr/}\\ + Matrix channel: \texttt{\#garage:deuxfleurs.fr} + + \vspace{2em} + \includegraphics[width=.09\linewidth]{assets/rust_logo.png} + \includegraphics[width=.2\linewidth]{assets/AGPLv3_Logo.png} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Demo time!} +\end{frame} + +\end{document} + +%% vim: set ts=4 sw=4 tw=0 noet spelllang=fr :