diff --git a/2025-04-03-stack/.gitignore b/2025-04-03-stack/.gitignore new file mode 100644 index 0000000..4bfdf16 --- /dev/null +++ b/2025-04-03-stack/.gitignore @@ -0,0 +1,10 @@ +*.nav +*.out +*.snm +*.toc +*.aux +*.log +*.fls +*.fdb_latexmk +.auctex-auto +*~ \ No newline at end of file diff --git a/2025-04-03-stack/Makefile b/2025-04-03-stack/Makefile new file mode 100644 index 0000000..48fef8c --- /dev/null +++ b/2025-04-03-stack/Makefile @@ -0,0 +1,23 @@ +ASSETS=assets/lattice/lattice1.pdf_tex \ + assets/lattice/lattice2.pdf_tex \ + assets/lattice/lattice3.pdf_tex \ + assets/lattice/lattice4.pdf_tex \ + assets/lattice/lattice5.pdf_tex \ + assets/lattice/lattice6.pdf_tex \ + assets/lattice/lattice7.pdf_tex \ + assets/lattice/lattice8.pdf_tex \ + assets/logos/deuxfleurs.pdf \ + assets/timeline-22-24.pdf + +talk.pdf: talk.tex $(ASSETS) + latexmk -pdf talk.tex + +.PHONY: loop +loop: + latexmk -pdf -pvc talk.tex + +%.pdf: %.svg + inkscape -D -z --file=$^ --export-pdf=$@ + +%.pdf_tex: %.svg + inkscape -D -z --file=$^ --export-pdf=$@ --export-latex diff --git a/2025-04-03-stack/assets/.gitignore b/2025-04-03-stack/assets/.gitignore new file mode 100644 index 0000000..9ce0997 --- /dev/null +++ b/2025-04-03-stack/assets/.gitignore @@ -0,0 +1,8 @@ +# Files that are auto-generated when building pdfs +deuxfleurs.pdf +timeline-22-24.pdf +lattice*.pdf_tex +lattice*.pdf + +# tmp files generated by krita +*~ diff --git a/2025-04-03-stack/assets/alex.jpg b/2025-04-03-stack/assets/alex.jpg new file mode 100644 index 0000000..eac0f0a Binary files /dev/null and b/2025-04-03-stack/assets/alex.jpg differ diff --git a/2025-04-03-stack/assets/atuin.jpg b/2025-04-03-stack/assets/atuin.jpg new file mode 100644 index 0000000..f2fbd61 Binary files /dev/null and b/2025-04-03-stack/assets/atuin.jpg differ diff --git a/2025-04-03-stack/assets/garage.drawio.pdf b/2025-04-03-stack/assets/garage.drawio.pdf new file mode 100644 index 0000000..a54a163 Binary files /dev/null and b/2025-04-03-stack/assets/garage.drawio.pdf differ diff --git a/2025-04-03-stack/assets/garage.drawio.png b/2025-04-03-stack/assets/garage.drawio.png new file mode 100644 index 0000000..386dd86 Binary files /dev/null and b/2025-04-03-stack/assets/garage.drawio.png differ diff --git a/2025-04-03-stack/assets/garage2.drawio.png b/2025-04-03-stack/assets/garage2.drawio.png new file mode 100644 index 0000000..8562fbc Binary files /dev/null and b/2025-04-03-stack/assets/garage2.drawio.png differ diff --git a/2025-04-03-stack/assets/garage2a.drawio.pdf b/2025-04-03-stack/assets/garage2a.drawio.pdf new file mode 100644 index 0000000..422c934 Binary files /dev/null and b/2025-04-03-stack/assets/garage2a.drawio.pdf differ diff --git a/2025-04-03-stack/assets/garage2b.drawio.pdf b/2025-04-03-stack/assets/garage2b.drawio.pdf new file mode 100644 index 0000000..05a9710 Binary files /dev/null and b/2025-04-03-stack/assets/garage2b.drawio.pdf differ diff --git a/2025-04-03-stack/assets/garage_sync.drawio.pdf b/2025-04-03-stack/assets/garage_sync.drawio.pdf new file mode 100644 index 0000000..a94b357 Binary files /dev/null and b/2025-04-03-stack/assets/garage_sync.drawio.pdf differ diff --git a/2025-04-03-stack/assets/garage_sync.drawio.png b/2025-04-03-stack/assets/garage_sync.drawio.png new file mode 100644 index 0000000..2e7b5af Binary files /dev/null and b/2025-04-03-stack/assets/garage_sync.drawio.png differ diff --git a/2025-04-03-stack/assets/garage_tables.pdf b/2025-04-03-stack/assets/garage_tables.pdf new file mode 100644 index 0000000..3c54cdc Binary files /dev/null and b/2025-04-03-stack/assets/garage_tables.pdf differ diff --git a/2025-04-03-stack/assets/geodistrib_paper.png b/2025-04-03-stack/assets/geodistrib_paper.png new file mode 100644 index 0000000..318d8e5 Binary files /dev/null and b/2025-04-03-stack/assets/geodistrib_paper.png differ diff --git a/2025-04-03-stack/assets/inframap_jdll2023.pdf b/2025-04-03-stack/assets/inframap_jdll2023.pdf new file mode 100644 index 0000000..77f97ba Binary files /dev/null and b/2025-04-03-stack/assets/inframap_jdll2023.pdf differ diff --git a/2025-04-03-stack/assets/intro/slide1.png b/2025-04-03-stack/assets/intro/slide1.png new file mode 100644 index 0000000..eb2e67a Binary files /dev/null and b/2025-04-03-stack/assets/intro/slide1.png differ diff --git a/2025-04-03-stack/assets/intro/slide2.png b/2025-04-03-stack/assets/intro/slide2.png new file mode 100644 index 0000000..126a39b Binary files /dev/null and b/2025-04-03-stack/assets/intro/slide2.png differ diff --git a/2025-04-03-stack/assets/intro/slide3.png b/2025-04-03-stack/assets/intro/slide3.png new file mode 100644 index 0000000..a39f96b Binary files /dev/null and b/2025-04-03-stack/assets/intro/slide3.png differ diff --git a/2025-04-03-stack/assets/intro/slideB1.png b/2025-04-03-stack/assets/intro/slideB1.png new file mode 100644 index 0000000..b14b607 Binary files /dev/null and b/2025-04-03-stack/assets/intro/slideB1.png differ diff --git a/2025-04-03-stack/assets/intro/slideB2.png b/2025-04-03-stack/assets/intro/slideB2.png new file mode 100644 index 0000000..a881a79 Binary files /dev/null and b/2025-04-03-stack/assets/intro/slideB2.png differ diff --git a/2025-04-03-stack/assets/intro/slideB3.png b/2025-04-03-stack/assets/intro/slideB3.png new file mode 100644 index 0000000..830709d Binary files /dev/null and b/2025-04-03-stack/assets/intro/slideB3.png differ diff --git a/2025-04-03-stack/assets/intro/slides.svg b/2025-04-03-stack/assets/intro/slides.svg new file mode 100644 index 0000000..9946c6f --- /dev/null +++ b/2025-04-03-stack/assets/intro/slides.svg @@ -0,0 +1,4326 @@ + + + + + + + + + + + + + + + + + + + + + + User-facing application + Database + Filesystem + + + + + + + + + + diff --git a/2025-04-03-stack/assets/intro/slidesB.svg b/2025-04-03-stack/assets/intro/slidesB.svg new file mode 100644 index 0000000..c0a6e97 --- /dev/null +++ b/2025-04-03-stack/assets/intro/slidesB.svg @@ -0,0 +1,444 @@ + + + +User-facing applicationDatabase*K2VObject storage*(not really a database)Database diff --git a/2025-04-03-stack/assets/jepsen-0.10.png b/2025-04-03-stack/assets/jepsen-0.10.png new file mode 100644 index 0000000..d7895e3 Binary files /dev/null and b/2025-04-03-stack/assets/jepsen-0.10.png differ diff --git a/2025-04-03-stack/assets/jepsen-0.9.png b/2025-04-03-stack/assets/jepsen-0.9.png new file mode 100644 index 0000000..3f65403 Binary files /dev/null and b/2025-04-03-stack/assets/jepsen-0.9.png differ diff --git a/2025-04-03-stack/assets/lattice/lattice1.svg b/2025-04-03-stack/assets/lattice/lattice1.svg new file mode 100644 index 0000000..8bfa5aa --- /dev/null +++ b/2025-04-03-stack/assets/lattice/lattice1.svg @@ -0,0 +1,433 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + diff --git a/2025-04-03-stack/assets/lattice/lattice2.svg b/2025-04-03-stack/assets/lattice/lattice2.svg new file mode 100644 index 0000000..adcd92c --- /dev/null +++ b/2025-04-03-stack/assets/lattice/lattice2.svg @@ -0,0 +1,514 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + + diff --git a/2025-04-03-stack/assets/lattice/lattice3.svg b/2025-04-03-stack/assets/lattice/lattice3.svg new file mode 100644 index 0000000..640dc46 --- /dev/null +++ b/2025-04-03-stack/assets/lattice/lattice3.svg @@ -0,0 +1,515 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + $\not\sqsupseteq \{a\}$ + + diff --git a/2025-04-03-stack/assets/lattice/lattice4.svg b/2025-04-03-stack/assets/lattice/lattice4.svg new file mode 100644 index 0000000..b2a99e2 --- /dev/null +++ b/2025-04-03-stack/assets/lattice/lattice4.svg @@ -0,0 +1,525 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $\sqsupseteq \{a\} \to$ OK + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + return OK + + diff --git a/2025-04-03-stack/assets/lattice/lattice5.svg b/2025-04-03-stack/assets/lattice/lattice5.svg new file mode 100644 index 0000000..bc6b719 --- /dev/null +++ b/2025-04-03-stack/assets/lattice/lattice5.svg @@ -0,0 +1,536 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $read()$: + $\sqsupseteq \{a\} \to$ OK + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + return OK + + diff --git a/2025-04-03-stack/assets/lattice/lattice6.svg b/2025-04-03-stack/assets/lattice/lattice6.svg new file mode 100644 index 0000000..176b171 --- /dev/null +++ b/2025-04-03-stack/assets/lattice/lattice6.svg @@ -0,0 +1,553 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $read()$: + $\sqsupseteq \{a\} \to$ OK + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + + $\to \{\}$ + return OK + + diff --git a/2025-04-03-stack/assets/lattice/lattice7.svg b/2025-04-03-stack/assets/lattice/lattice7.svg new file mode 100644 index 0000000..7ce8bda --- /dev/null +++ b/2025-04-03-stack/assets/lattice/lattice7.svg @@ -0,0 +1,581 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $read()$: + $\sqsupseteq \{a\} \to$ OK + $\sqsupseteq \{a\} \to$ OK + $\not\sqsupseteq \{a\}$ + + $\to \{\}$ + return OK + return $\{\}\sqcup\{a\}=\{a\}$ + + $\to \{a\}$ + + diff --git a/2025-04-03-stack/assets/lattice/lattice8.svg b/2025-04-03-stack/assets/lattice/lattice8.svg new file mode 100644 index 0000000..c94a69b --- /dev/null +++ b/2025-04-03-stack/assets/lattice/lattice8.svg @@ -0,0 +1,581 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $\{\}$ + $\{a,b,c\}$ + + $\{a\}$ + $\{b\}$ + $\{c\}$ + + + $\{a,c\}$ + $\{a,b\}$ + $\{b,c\}$ + + + + + + + + + + + + + + + + + + + + $write(\{a\})$: + $read()$: + $\sqsupseteq \{a\} \to$ OK + $\sqsupseteq \{a\} \to$ OK + $\sqsupseteq \{a\}$ + + $\to \{\}$ + return OK + return $\{\}\sqcup\{a\}=\{a\}$ + + $\to \{a\}$ + + diff --git a/2025-04-03-stack/assets/logos/AGPLv3_Logo.png b/2025-04-03-stack/assets/logos/AGPLv3_Logo.png new file mode 100644 index 0000000..445284a Binary files /dev/null and b/2025-04-03-stack/assets/logos/AGPLv3_Logo.png differ diff --git a/2025-04-03-stack/assets/logos/Amazon-S3.jpg b/2025-04-03-stack/assets/logos/Amazon-S3.jpg new file mode 100644 index 0000000..a950197 Binary files /dev/null and b/2025-04-03-stack/assets/logos/Amazon-S3.jpg differ diff --git a/2025-04-03-stack/assets/logos/NGI0Entrust_tag.png b/2025-04-03-stack/assets/logos/NGI0Entrust_tag.png new file mode 100644 index 0000000..064c4ae Binary files /dev/null and b/2025-04-03-stack/assets/logos/NGI0Entrust_tag.png differ diff --git a/2025-04-03-stack/assets/logos/NGI0Entrust_tag.svg b/2025-04-03-stack/assets/logos/NGI0Entrust_tag.svg new file mode 100644 index 0000000..ca0d281 --- /dev/null +++ b/2025-04-03-stack/assets/logos/NGI0Entrust_tag.svg @@ -0,0 +1,149 @@ + + + +image/svg+xml + + + + + + +NGI Zero Entrust + + + + diff --git a/2025-04-03-stack/assets/logos/capitole_du_libre.png b/2025-04-03-stack/assets/logos/capitole_du_libre.png new file mode 100644 index 0000000..eaa521a Binary files /dev/null and b/2025-04-03-stack/assets/logos/capitole_du_libre.png differ diff --git a/2025-04-03-stack/assets/logos/deuxfleurs.svg b/2025-04-03-stack/assets/logos/deuxfleurs.svg new file mode 100644 index 0000000..c298c22 --- /dev/null +++ b/2025-04-03-stack/assets/logos/deuxfleurs.svg @@ -0,0 +1,91 @@ + + + + + + + + + + + + + + + D + F + diff --git a/2025-04-03-stack/assets/logos/fosdem22.png b/2025-04-03-stack/assets/logos/fosdem22.png new file mode 100644 index 0000000..17332e4 Binary files /dev/null and b/2025-04-03-stack/assets/logos/fosdem22.png differ diff --git a/2025-04-03-stack/assets/logos/fosdem24.png b/2025-04-03-stack/assets/logos/fosdem24.png new file mode 100644 index 0000000..1163169 Binary files /dev/null and b/2025-04-03-stack/assets/logos/fosdem24.png differ diff --git a/2025-04-03-stack/assets/logos/jdll.png b/2025-04-03-stack/assets/logos/jdll.png new file mode 100644 index 0000000..a7cbe09 Binary files /dev/null and b/2025-04-03-stack/assets/logos/jdll.png differ diff --git a/2025-04-03-stack/assets/logos/logo_chatons.png b/2025-04-03-stack/assets/logos/logo_chatons.png new file mode 100644 index 0000000..890cf17 Binary files /dev/null and b/2025-04-03-stack/assets/logos/logo_chatons.png differ diff --git a/2025-04-03-stack/assets/logos/minio.png b/2025-04-03-stack/assets/logos/minio.png new file mode 100644 index 0000000..a71e9cc Binary files /dev/null and b/2025-04-03-stack/assets/logos/minio.png differ diff --git a/2025-04-03-stack/assets/logos/ngi-pointer.png b/2025-04-03-stack/assets/logos/ngi-pointer.png new file mode 100644 index 0000000..730ecec Binary files /dev/null and b/2025-04-03-stack/assets/logos/ngi-pointer.png differ diff --git a/2025-04-03-stack/assets/logos/nlnet.svg b/2025-04-03-stack/assets/logos/nlnet.svg new file mode 100644 index 0000000..373c8d8 --- /dev/null +++ b/2025-04-03-stack/assets/logos/nlnet.svg @@ -0,0 +1,34 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/2025-04-03-stack/assets/logos/pses.png b/2025-04-03-stack/assets/logos/pses.png new file mode 100644 index 0000000..da871f8 Binary files /dev/null and b/2025-04-03-stack/assets/logos/pses.png differ diff --git a/2025-04-03-stack/assets/logos/rust_logo.png b/2025-04-03-stack/assets/logos/rust_logo.png new file mode 100644 index 0000000..0e4809e Binary files /dev/null and b/2025-04-03-stack/assets/logos/rust_logo.png differ diff --git a/2025-04-03-stack/assets/map.png b/2025-04-03-stack/assets/map.png new file mode 100644 index 0000000..1dff3ab Binary files /dev/null and b/2025-04-03-stack/assets/map.png differ diff --git a/2025-04-03-stack/assets/neptune.jpg b/2025-04-03-stack/assets/neptune.jpg new file mode 100644 index 0000000..61fcbff Binary files /dev/null and b/2025-04-03-stack/assets/neptune.jpg differ diff --git a/2025-04-03-stack/assets/perf/db_engine.png b/2025-04-03-stack/assets/perf/db_engine.png new file mode 100644 index 0000000..b1124b0 Binary files /dev/null and b/2025-04-03-stack/assets/perf/db_engine.png differ diff --git a/2025-04-03-stack/assets/perf/endpoint_latency_0.7_0.8_minio.png b/2025-04-03-stack/assets/perf/endpoint_latency_0.7_0.8_minio.png new file mode 100644 index 0000000..92eac3f Binary files /dev/null and b/2025-04-03-stack/assets/perf/endpoint_latency_0.7_0.8_minio.png differ diff --git a/2025-04-03-stack/assets/perf/io-0.7-0.8-minio.png b/2025-04-03-stack/assets/perf/io-0.7-0.8-minio.png new file mode 100644 index 0000000..f581a22 Binary files /dev/null and b/2025-04-03-stack/assets/perf/io-0.7-0.8-minio.png differ diff --git a/2025-04-03-stack/assets/perf/ttfb.png b/2025-04-03-stack/assets/perf/ttfb.png new file mode 100644 index 0000000..c0335bd Binary files /dev/null and b/2025-04-03-stack/assets/perf/ttfb.png differ diff --git a/2025-04-03-stack/assets/schema-streaming-1.png b/2025-04-03-stack/assets/schema-streaming-1.png new file mode 100644 index 0000000..4113f03 Binary files /dev/null and b/2025-04-03-stack/assets/schema-streaming-1.png differ diff --git a/2025-04-03-stack/assets/schema-streaming-2.png b/2025-04-03-stack/assets/schema-streaming-2.png new file mode 100644 index 0000000..2f97bb3 Binary files /dev/null and b/2025-04-03-stack/assets/schema-streaming-2.png differ diff --git a/2025-04-03-stack/assets/schema-streaming.png b/2025-04-03-stack/assets/schema-streaming.png new file mode 100644 index 0000000..3b24d91 Binary files /dev/null and b/2025-04-03-stack/assets/schema-streaming.png differ diff --git a/2025-04-03-stack/assets/screenshots/garage_stats_0.10.png b/2025-04-03-stack/assets/screenshots/garage_stats_0.10.png new file mode 100644 index 0000000..78a36ea Binary files /dev/null and b/2025-04-03-stack/assets/screenshots/garage_stats_0.10.png differ diff --git a/2025-04-03-stack/assets/screenshots/garage_status_0.10.png b/2025-04-03-stack/assets/screenshots/garage_status_0.10.png new file mode 100644 index 0000000..510717c Binary files /dev/null and b/2025-04-03-stack/assets/screenshots/garage_status_0.10.png differ diff --git a/2025-04-03-stack/assets/screenshots/garage_status_0.9_prod.png b/2025-04-03-stack/assets/screenshots/garage_status_0.9_prod.png new file mode 100644 index 0000000..e1a0589 Binary files /dev/null and b/2025-04-03-stack/assets/screenshots/garage_status_0.9_prod.png differ diff --git a/2025-04-03-stack/assets/screenshots/garage_status_0.9_prod_zonehl.png b/2025-04-03-stack/assets/screenshots/garage_status_0.9_prod_zonehl.png new file mode 100644 index 0000000..99d46f9 Binary files /dev/null and b/2025-04-03-stack/assets/screenshots/garage_status_0.9_prod_zonehl.png differ diff --git a/2025-04-03-stack/assets/screenshots/garage_status_unhealthy_0.10.png b/2025-04-03-stack/assets/screenshots/garage_status_unhealthy_0.10.png new file mode 100644 index 0000000..b167687 Binary files /dev/null and b/2025-04-03-stack/assets/screenshots/garage_status_unhealthy_0.10.png differ diff --git a/2025-04-03-stack/assets/screenshots/garage_worker_list_0.10.png b/2025-04-03-stack/assets/screenshots/garage_worker_list_0.10.png new file mode 100644 index 0000000..078f060 Binary files /dev/null and b/2025-04-03-stack/assets/screenshots/garage_worker_list_0.10.png differ diff --git a/2025-04-03-stack/assets/screenshots/garage_worker_param_0.10.png b/2025-04-03-stack/assets/screenshots/garage_worker_param_0.10.png new file mode 100644 index 0000000..39ee11a Binary files /dev/null and b/2025-04-03-stack/assets/screenshots/garage_worker_param_0.10.png differ diff --git a/2025-04-03-stack/assets/screenshots/grafana_dashboard.png b/2025-04-03-stack/assets/screenshots/grafana_dashboard.png new file mode 100644 index 0000000..bda2d73 Binary files /dev/null and b/2025-04-03-stack/assets/screenshots/grafana_dashboard.png differ diff --git a/2025-04-03-stack/assets/screenshots/jaeger_listobjects.png b/2025-04-03-stack/assets/screenshots/jaeger_listobjects.png new file mode 100644 index 0000000..ab7d68f Binary files /dev/null and b/2025-04-03-stack/assets/screenshots/jaeger_listobjects.png differ diff --git a/2025-04-03-stack/assets/screenshots/jaeger_putobject.png b/2025-04-03-stack/assets/screenshots/jaeger_putobject.png new file mode 100644 index 0000000..7966275 Binary files /dev/null and b/2025-04-03-stack/assets/screenshots/jaeger_putobject.png differ diff --git a/2025-04-03-stack/assets/survey_requested_features.png b/2025-04-03-stack/assets/survey_requested_features.png new file mode 100644 index 0000000..07247c4 Binary files /dev/null and b/2025-04-03-stack/assets/survey_requested_features.png differ diff --git a/2025-04-03-stack/assets/timeline-22-24.svg b/2025-04-03-stack/assets/timeline-22-24.svg new file mode 100644 index 0000000..90fe5de --- /dev/null +++ b/2025-04-03-stack/assets/timeline-22-24.svg @@ -0,0 +1,327 @@ + + + +Feb 6, 2022Feb 3, 2024v0.6.0Feb 2, 2022v0.7.0Apr 8, 2022v0.8.0Nov 21, 2022v0.9.0Oct 10, 2023v0.10.0 betaTowards v1.0 ?Apr/May, 2024Nov 19, 2022Apr 1, 2023PSESJun 6, 2023 diff --git a/2025-04-03-stack/bespin.jpg b/2025-04-03-stack/bespin.jpg new file mode 100644 index 0000000..05bbc66 Binary files /dev/null and b/2025-04-03-stack/bespin.jpg differ diff --git a/2025-04-03-stack/carte.png b/2025-04-03-stack/carte.png new file mode 100644 index 0000000..0582dce Binary files /dev/null and b/2025-04-03-stack/carte.png differ diff --git a/2025-04-03-stack/corrin.jpg b/2025-04-03-stack/corrin.jpg new file mode 100644 index 0000000..0bfe2cc Binary files /dev/null and b/2025-04-03-stack/corrin.jpg differ diff --git a/2025-04-03-stack/infra.jpg b/2025-04-03-stack/infra.jpg new file mode 100644 index 0000000..91dbf0f Binary files /dev/null and b/2025-04-03-stack/infra.jpg differ diff --git a/2025-04-03-stack/infra.xcf b/2025-04-03-stack/infra.xcf new file mode 100644 index 0000000..3ac3796 Binary files /dev/null and b/2025-04-03-stack/infra.xcf differ diff --git a/2025-04-03-stack/logo/garage-dark-notext.png b/2025-04-03-stack/logo/garage-dark-notext.png new file mode 100644 index 0000000..a2c7f5e Binary files /dev/null and b/2025-04-03-stack/logo/garage-dark-notext.png differ diff --git a/2025-04-03-stack/logo/garage-dark-notext.svg b/2025-04-03-stack/logo/garage-dark-notext.svg new file mode 100644 index 0000000..2d1c187 --- /dev/null +++ b/2025-04-03-stack/logo/garage-dark-notext.svg @@ -0,0 +1,113 @@ + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + diff --git a/2025-04-03-stack/logo/garage-dark.svg b/2025-04-03-stack/logo/garage-dark.svg new file mode 100644 index 0000000..3c0699a --- /dev/null +++ b/2025-04-03-stack/logo/garage-dark.svg @@ -0,0 +1,174 @@ + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/2025-04-03-stack/logo/garage-notext.png b/2025-04-03-stack/logo/garage-notext.png new file mode 100644 index 0000000..3ac607f Binary files /dev/null and b/2025-04-03-stack/logo/garage-notext.png differ diff --git a/2025-04-03-stack/logo/garage-notext.svg b/2025-04-03-stack/logo/garage-notext.svg new file mode 100644 index 0000000..e48e14e --- /dev/null +++ b/2025-04-03-stack/logo/garage-notext.svg @@ -0,0 +1,146 @@ + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/2025-04-03-stack/logo/garage.png b/2025-04-03-stack/logo/garage.png new file mode 100644 index 0000000..2519e21 Binary files /dev/null and b/2025-04-03-stack/logo/garage.png differ diff --git a/2025-04-03-stack/logo/garage.svg b/2025-04-03-stack/logo/garage.svg new file mode 100644 index 0000000..130c607 --- /dev/null +++ b/2025-04-03-stack/logo/garage.svg @@ -0,0 +1,206 @@ + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/2025-04-03-stack/logo/garage_hires.png b/2025-04-03-stack/logo/garage_hires.png new file mode 100644 index 0000000..488836f Binary files /dev/null and b/2025-04-03-stack/logo/garage_hires.png differ diff --git a/2025-04-03-stack/logo/garage_hires_crop.png b/2025-04-03-stack/logo/garage_hires_crop.png new file mode 100644 index 0000000..2fd0bab Binary files /dev/null and b/2025-04-03-stack/logo/garage_hires_crop.png differ diff --git a/2025-04-03-stack/scorpio.jpg b/2025-04-03-stack/scorpio.jpg new file mode 100644 index 0000000..9290a60 Binary files /dev/null and b/2025-04-03-stack/scorpio.jpg differ diff --git a/2025-04-03-stack/sticker/Garage.pdf b/2025-04-03-stack/sticker/Garage.pdf new file mode 100644 index 0000000..1bfb4a1 Binary files /dev/null and b/2025-04-03-stack/sticker/Garage.pdf differ diff --git a/2025-04-03-stack/sticker/Garage.png b/2025-04-03-stack/sticker/Garage.png new file mode 100644 index 0000000..1bfd61b Binary files /dev/null and b/2025-04-03-stack/sticker/Garage.png differ diff --git a/2025-04-03-stack/sticker/Garage.svg b/2025-04-03-stack/sticker/Garage.svg new file mode 100644 index 0000000..9464986 --- /dev/null +++ b/2025-04-03-stack/sticker/Garage.svg @@ -0,0 +1,181 @@ + + + + + + + + + + texte garage + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/2025-04-03-stack/sticker/Garage_NGI.pdf b/2025-04-03-stack/sticker/Garage_NGI.pdf new file mode 100644 index 0000000..43f511e Binary files /dev/null and b/2025-04-03-stack/sticker/Garage_NGI.pdf differ diff --git a/2025-04-03-stack/sticker/Garage_NGI.png b/2025-04-03-stack/sticker/Garage_NGI.png new file mode 100644 index 0000000..ab95b73 Binary files /dev/null and b/2025-04-03-stack/sticker/Garage_NGI.png differ diff --git a/2025-04-03-stack/sticker/Garage_NGI.svg b/2025-04-03-stack/sticker/Garage_NGI.svg new file mode 100644 index 0000000..da04692 --- /dev/null +++ b/2025-04-03-stack/sticker/Garage_NGI.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/2025-04-03-stack/talk.tex b/2025-04-03-stack/talk.tex new file mode 100644 index 0000000..9e7648f --- /dev/null +++ b/2025-04-03-stack/talk.tex @@ -0,0 +1,841 @@ +\nonstopmode +\documentclass[aspectratio=169,xcolor={svgnames}]{beamer} +\usepackage[utf8]{inputenc} +% \usepackage[frenchb]{babel} +\usepackage{amsmath} +\usepackage{mathtools} +\usepackage{breqn} +\usepackage{multirow} +\usetheme{boxes} +\usepackage{graphicx} +\usepackage{import} +\usepackage{adjustbox} +\usepackage[absolute,overlay]{textpos} +%\useoutertheme[footline=authortitle,subsection=false]{miniframes} +%\useoutertheme[footline=authorinstitute,subsection=false]{miniframes} +\useoutertheme{infolines} +\setbeamertemplate{headline}{} + +\beamertemplatenavigationsymbolsempty + +\definecolor{TitleOrange}{RGB}{255,137,0} +\setbeamercolor{title}{fg=TitleOrange} +\setbeamercolor{frametitle}{fg=TitleOrange} + +\definecolor{ListOrange}{RGB}{255,145,5} +\setbeamertemplate{itemize item}{\color{ListOrange}$\blacktriangleright$} + +\definecolor{verygrey}{RGB}{70,70,70} +\setbeamercolor{normal text}{fg=verygrey} + + +\usepackage{tabu} +\usepackage{multicol} +\usepackage{vwcol} +\usepackage{stmaryrd} +\usepackage{graphicx} + +\usepackage[normalem]{ulem} + +\AtBeginSection[]{ + \begin{frame} + \vfill + \centering + \begin{beamercolorbox}[sep=8pt,center,shadow=true,rounded=true]{title} + \usebeamerfont{title}\insertsectionhead\par% + \end{beamercolorbox} + \vfill + \end{frame} +} + +\title{Deuxfleurs : une infrastructure numérique distribuée low-tech} +\author{Armaël Guéneau, Baptiste Jonglez} +\date{2024-04-03} + +\begin{document} + +\begin{frame} + \centering + \includegraphics[width=.2\linewidth]{assets/logos/deuxfleurs.pdf} + \vfill + + {\large\bf Deuxfleurs : une infrastructure numérique distribuée low-tech} + \vspace{1em} + + {Armaël Guéneau, Inria Toccata} \\ + {Baptiste Jonglez, Inria Stack} \\ +\end{frame} + +\begin{frame}{Qui sommes nous} + +\end{frame} + +\begin{frame}{L'association Deuxfleurs} + + + + Deuxfleurs est une association loi 1901 qui héberge des services numériques + + \vfill + + Objectif : \\ \textbf{Construire une infrastructure numérique robuste, + utile, et gérée en commun} + + \vfill + + membre du collectif CHATONS (hébergeurs associatifs) +\end{frame} + +\begin{frame}{Choix politiques} + +\begin{enumerate} +\setlength\itemsep{1.5em} + \item Réduire les dépendances aux acteurs dominants, maîtriser les dépendances + inévitables (électricité, réseau). Jusqu'à maîtriser toute la pile + logicielle, matérielle et hébergement. + + \item Sobriété radicale : empreinte matérielle minimale et fixe. Adapter les + usages aux ressources et non l'inverse. + + \item Bonne qualité de service : alternative crédible aux GAFAM (CHATONS + souvent critiqués pour leur qualité de service) +\end{enumerate} + +\end{frame} + +\begin{frame}{Conséquences de ces choix} + +\begin{itemize} +\setlength\itemsep{1.5em} +\item \textbf{hébergement hors datacenters}, à la maison \\ + $\Longrightarrow$ électricité et réseau domestique pas fiables +\item \textbf{matériel reconditionné peu puissant}, de capacité fixe \\ + $\Longrightarrow$ + adapter les services et usages en fonction des limites (RAM, CPU, stockage) +\item \textbf{infrastructure géo-distribuée et tolérante aux fautes} pour assurer une bonne + disponibilité \\ + $\Longrightarrow$ besoin de logiciels adaptés +\end{itemize} + +\end{frame} + +\begin{frame}[plain] +\includegraphics[width=\textwidth]{infra.jpg} +\end{frame} + +\begin{frame} +\begin{center} +\textbf{ +Qu'est-ce qui rend l'infra Deuxfleurs intéressante dans un contexte recherche ? +} +\end{center} + +\begin{itemize} +\item +\item +\item +\end{itemize} + +\end{frame} + + + +\begin{frame} + +\end{frame} + +\begin{frame} + \frametitle{Who I am} + \begin{columns}[t] + \begin{column}{.2\textwidth} + \centering + \adjincludegraphics[width=.4\linewidth, valign=t]{assets/alex.jpg} + \end{column} + \begin{column}{.6\textwidth} + \textbf{Alex Auvolat}\\ + PhD; co-founder of Deuxfleurs + \end{column} + \begin{column}{.2\textwidth} + ~ + \end{column} + \end{columns} + \vspace{2em} + + \begin{columns}[t] + \begin{column}{.2\textwidth} + \centering + \adjincludegraphics[width=.5\linewidth, valign=t]{assets/logos/deuxfleurs.pdf} + \end{column} + \begin{column}{.6\textwidth} + \textbf{Deuxfleurs}\\ + A non-profit self-hosting collective,\\ + member of the CHATONS network + \end{column} + \begin{column}{.2\textwidth} + \centering + \adjincludegraphics[width=.7\linewidth, valign=t]{assets/logos/logo_chatons.png} + \end{column} + \end{columns} + +\end{frame} + +\begin{frame} + \frametitle{Our objective at Deuxfleurs} + + \begin{center} + \textbf{Promote self-hosting and small-scale hosting\\ + as an alternative to large cloud providers} + \end{center} + \vspace{2em} + \visible<2->{ + Why is it hard? + } + \visible<3->{ + \vspace{2em} + \begin{center} + \textbf{\underline{Resilience}}\\ + {\footnotesize we want good uptime/availability with low supervision} + \end{center} + } +\end{frame} + +\begin{frame} + \frametitle{Building a resilient system with cheap stuff} + + \only<1,4-7>{ + \begin{itemize} + \item \textcolor<5->{gray}{Commodity hardware (e.g. old desktop PCs)\\ + \vspace{.5em} + \visible<4->{{\footnotesize (can die at any time)}}} + \vspace{1.5em} + \item<5-> \textcolor<7->{gray}{Regular Internet (e.g. FTTB, FTTH) and power grid connections\\ + \vspace{.5em} + \visible<6->{{\footnotesize (can be unavailable randomly)}}} + \vspace{1.5em} + \item<7-> \textbf{Geographical redundancy} (multi-site replication) + \end{itemize} + } + \only<2>{ + \begin{center} + \includegraphics[width=.8\linewidth]{assets/neptune.jpg} + \end{center} + } + \only<3>{ + \begin{center} + \includegraphics[width=.8\linewidth]{assets/atuin.jpg} + \end{center} + } + \only<8>{ + \begin{center} + \includegraphics[width=.8\linewidth]{assets/inframap_jdll2023.pdf} + \end{center} + } +\end{frame} + +\begin{frame} + \frametitle{Object storage: a crucial component} + \begin{center} + \includegraphics[height=6em]{assets/logos/Amazon-S3.jpg} + \hspace{3em} + \visible<2->{\includegraphics[height=5em]{assets/logos/minio.png}} + \hspace{3em} + \visible<3>{\includegraphics[height=6em]{logo/garage_hires_crop.png}} + \end{center} + \vspace{1em} + S3: a de-facto standard, many compatible applications + + \vspace{1em} + \visible<2->{MinIO is self-hostable but not suited for geo-distributed deployments} + + \vspace{1em} + \visible<3->{\textbf{Garage is a self-hosted drop-in replacement for the Amazon S3 object store}} +\end{frame} + +\begin{frame} + \frametitle{CRDTs / weak consistency instead of consensus} + + \underline{Internally, Garage uses only CRDTs} (conflict-free replicated data types) + + \vspace{2em} + Why not Raft, Paxos, ...? Issues of consensus algorithms: + + \vspace{1em} + \begin{itemize} + \item<2-> \textbf{Software complexity} + \vspace{1em} + \item<3-> \textbf{Performance issues:} + \vspace{.5em} + \begin{itemize} + \item<4-> The leader is a \textbf{bottleneck} for all requests\\ + \vspace{.5em} + \item<5-> \textbf{Sensitive to higher latency} between nodes + \vspace{.5em} + \item<6-> \textbf{Takes time to reconverge} when disrupted (e.g. node going down) + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{The data model of object storage} + Object storage is basically a \textbf{key-value store}: + \vspace{.5em} + + {\scriptsize + \begin{center} + \begin{tabular}{|l|p{7cm}|} + \hline + \textbf{Key: file path + name} & \textbf{Value: file data + metadata} \\ + \hline + \hline + \texttt{index.html} & + \texttt{Content-Type: text/html; charset=utf-8} \newline + \texttt{Content-Length: 24929} \newline + \texttt{} \\ + \hline + \texttt{img/logo.svg} & + \texttt{Content-Type: text/svg+xml} \newline + \texttt{Content-Length: 13429} \newline + \texttt{} \\ + \hline + \texttt{download/index.html} & + \texttt{Content-Type: text/html; charset=utf-8} \newline + \texttt{Content-Length: 26563} \newline + \texttt{} \\ + \hline + \end{tabular} + \end{center} + } + + \vspace{1em} + \begin{itemize} + \item<2> Maps well to CRDT data types + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Performance gains in practice} + \begin{center} + \includegraphics[width=.8\linewidth]{assets/perf/endpoint_latency_0.7_0.8_minio.png} + \end{center} +\end{frame} + + +% ======================================== TIMELINE +% ======================================== TIMELINE +% ======================================== TIMELINE + +\section{Recent developments} + +% ====================== v0.7.0 =============================== + +\begin{frame} + \begin{center} + \includegraphics[width=.8\linewidth]{assets/timeline-22-24.pdf} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{April 2022 - Garage v0.7.0} + Focus on \underline{observability and ecosystem integration} + \vspace{2em} + \begin{itemize} + \item \textbf{Monitoring:} metrics and traces, using OpenTelemetry + \vspace{1em} + \item Replication modes with 1 or 2 copies / weaker consistency + \vspace{1em} + \item Kubernetes integration for node discovery + \vspace{1em} + \item Admin API (v0.7.2) + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Metrics (Prometheus + Grafana)} + \begin{center} + \includegraphics[width=.9\linewidth]{assets/screenshots/grafana_dashboard.png} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Traces (Jaeger)} + \begin{center} + \includegraphics[width=.8\linewidth]{assets/screenshots/jaeger_listobjects.png} + \end{center} +\end{frame} + +% ====================== v0.8.0 =============================== + +\begin{frame} + \begin{center} + \includegraphics[width=.8\linewidth]{assets/timeline-22-24.pdf} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{November 2022 - Garage v0.8.0} + Focus on \underline{performance} + \vspace{2em} + \begin{itemize} + \item \textbf{Alternative metadata DB engines} (LMDB, Sqlite) + \vspace{1em} + \item \textbf{Performance improvements:} block streaming, various optimizations... + \vspace{1em} + \item Bucket quotas (max size, max \#objects) + \vspace{1em} + \item Quality of life improvements, observability, etc. + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{About metadata DB engines} + \textbf{Issues with Sled:} + \vspace{1em} + \begin{itemize} + \item Huge files on disk + \vspace{.5em} + \item Unpredictable performance, especially on HDD + \vspace{.5em} + \item API limitations + \vspace{.5em} + \item Not actively maintained + \end{itemize} + + \vspace{2em} + \textbf{LMDB:} very stable, good performance, file size is reasonable\\ + \textbf{Sqlite} also available as a second choice + + \vspace{1em} + Sled will be removed in Garage v1.0 +\end{frame} + +\begin{frame} + \frametitle{DB engine performance comparison} + \begin{center} + \includegraphics[width=.6\linewidth]{assets/perf/db_engine.png} + \end{center} + NB: Sqlite was slow due to synchronous mode, now configurable +\end{frame} + +\begin{frame} + \frametitle{Block streaming} + \begin{center} + \only<1>{\includegraphics[width=.8\linewidth]{assets/schema-streaming-1.png}} + \only<2>{\includegraphics[width=.8\linewidth]{assets/schema-streaming-2.png}} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{TTFB benchmark} + \begin{center} + \includegraphics[width=.8\linewidth]{assets/perf/ttfb.png} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Throughput benchmark} + \begin{center} + \includegraphics[width=.7\linewidth]{assets/perf/io-0.7-0.8-minio.png} + \end{center} +\end{frame} + +% ====================== v0.9.0 =============================== + +\begin{frame} + \begin{center} + \includegraphics[width=.8\linewidth]{assets/timeline-22-24.pdf} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{October 2023 - Garage v0.9.0} + Focus on \underline{streamlining \& usability} + \vspace{2em} + \begin{itemize} + \item Support multiple HDDs per node + \vspace{1em} + \item S3 compatibility: + \vspace{1em} + \begin{itemize} + \item support basic lifecycle configurations + \vspace{.5em} + \item allow for multipart upload part retries + \end{itemize} + \vspace{1em} + \item LMDB by default, deprecation of Sled + \vspace{1em} + \item New layout computation algorithm + \end{itemize} +\end{frame} + + +\begin{frame} + \frametitle{Layout computation} + \begin{overprint} + \onslide<1> + \begin{center} + \includegraphics[width=\linewidth, trim=0 0 0 -4cm]{assets/screenshots/garage_status_0.9_prod_zonehl.png} + \end{center} + \onslide<2> + \begin{center} + \includegraphics[width=.7\linewidth]{assets/map.png} + \end{center} + \end{overprint} + \vspace{1em} + Garage stores replicas on different zones when possible +\end{frame} + +\begin{frame} + \frametitle{What a "layout" is} + \textbf{A layout is a precomputed index table:} + \vspace{1em} + + {\footnotesize + \begin{center} + \begin{tabular}{|l|l|l|l|} + \hline + \textbf{Partition} & \textbf{Node 1} & \textbf{Node 2} & \textbf{Node 3} \\ + \hline + \hline + Partition 0 & df-ymk (bespin) & Abricot (scorpio) & Courgette (neptune) \\ + \hline + Partition 1 & Ananas (scorpio) & Courgette (neptune) & df-ykl (bespin) \\ + \hline + Partition 2 & df-ymf (bespin) & Celeri (neptune) & Abricot (scorpio) \\ + \hline + \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ \\ + \hline + Partition 255 & Concombre (neptune) & df-ykl (bespin) & Abricot (scorpio) \\ + \hline + \end{tabular} + \end{center} + } + + \vspace{2em} + \visible<2->{ + The index table is built centrally using an optimal algorithm,\\ + then propagated to all nodes + } + + \vspace{1em} + \visible<3->{ + \footnotesize + Oulamara, M., \& Auvolat, A. (2023). \emph{An algorithm for geo-distributed and redundant storage in Garage}.\\ arXiv preprint arXiv:2302.13798. + } +\end{frame} + + + +% ====================== v0.10.0 =============================== + +\begin{frame} + \begin{center} + \includegraphics[width=.8\linewidth]{assets/timeline-22-24.pdf} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{October 2023 - Garage v0.10.0 beta} + Focus on \underline{consistency} + \vspace{2em} + \begin{itemize} + \item Fix consistency issues when reshuffling data + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Working with weak consistency} + Not using consensus limits us to the following: + \vspace{2em} + \begin{itemize} + \item<2-> \textbf{Conflict-free replicated data types} (CRDT)\\ + \vspace{1em} + {\footnotesize Non-transactional key-value stores such as S3 are equivalent to a simple CRDT:\\ + a map of \textbf{last-writer-wins registers} (each key is its own CRDT)} + \vspace{1.5em} + \item<3-> \textbf{Read-after-write consistency}\\ + \vspace{1em} + {\footnotesize Can be implemented using quorums on read and write operations} + \end{itemize} +\end{frame} + +\begin{frame}[t] + \frametitle{CRDT read-after-write consistency using quorums} + + \vspace{1em} + {\small + \textbf{Property:} If client 1 did an operation $write(x)$ and received an OK response,\\ + \hspace{2cm} and client 2 starts an operation $read()$ after client 1 received OK,\\ + \hspace{2cm} then client 2 will read a value $x' \sqsupseteq x$. + } + + \vspace{1.5em} + \begin{overprint} + \onslide<2-9> + \begin{figure} + \centering + \footnotesize + \def\svgwidth{.7\textwidth} + \only<2>{\import{assets/lattice/}{lattice1.pdf_tex}}% + \only<3>{\import{assets/lattice/}{lattice2.pdf_tex}}% + \only<4>{\import{assets/lattice/}{lattice3.pdf_tex}}% + \only<5>{\import{assets/lattice/}{lattice4.pdf_tex}}% + \only<6>{\import{assets/lattice/}{lattice5.pdf_tex}}% + \only<7>{\import{assets/lattice/}{lattice6.pdf_tex}}% + \only<8>{\import{assets/lattice/}{lattice7.pdf_tex}}% + \only<9>{\import{assets/lattice/}{lattice8.pdf_tex}}% + \end{figure} + + \onslide<10> + \begin{minipage}{.10\textwidth} + ~ + \end{minipage} + \begin{minipage}{.40\textwidth} + \footnotesize + \textbf{Algorithm $write(x)$:} + \begin{enumerate} + \item Broadcast $write(x)$ to all nodes + \item Wait for $k > n/2$ nodes to reply OK + \item Return OK + \end{enumerate} + \end{minipage} + \begin{minipage}{.40\textwidth} + \footnotesize + \vspace{1em} + \textbf{Algorithm $read()$:} + \begin{enumerate} + \item Broadcast $read()$ to all nodes + \item Wait for $k > n/2$ nodes to reply\\ + with values $x_1, \dots, x_k$ + \item Return $x_1 \sqcup \dots \sqcup x_k$ + \end{enumerate} + \end{minipage} + \end{overprint} +\end{frame} + +\begin{frame} + \frametitle{A hard problem: layout changes} + \begin{itemize} + \item We rely on quorums $k > n/2$ within each partition:\\ + $$n=3,~~~~~~~k\ge 2$$ + \item<2-> When rebalancing, the set of nodes responsible for a partition can change:\\ + + \vspace{1em} + \begin{minipage}{.04\linewidth}~ + \end{minipage} + \begin{minipage}{.40\linewidth} + {\tiny + \begin{tabular}{|l|l|l|l|} + \hline + \textbf{Partition} & \textbf{Node 1} & \textbf{Node 2} & \textbf{Node 3} \\ + \hline + \hline + Partition 0 & \textcolor{Crimson}{df-ymk} & Abricot & \textcolor{Crimson}{Courgette} \\ + \hline + Partition 1 & Ananas & \textcolor{Crimson}{Courgette} & \textcolor{Crimson}{df-ykl} \\ + \hline + Partition 2 & \textcolor{Crimson}{df-ymf} & \textcolor{Crimson}{Celeri} & Abricot \\ + \hline + \hspace{1em}$\dots$ & \hspace{1em}$\dots$ & \hspace{1em}$\dots$ & \hspace{1em}$\dots$ \\ + \hline + \end{tabular} + } + \end{minipage} + \begin{minipage}{.04\linewidth} + $\to$ + \end{minipage} + \begin{minipage}{.40\linewidth} + {\tiny + \begin{tabular}{|l|l|l|l|} + \hline + \textbf{Partition} & \textbf{Node 1} & \textbf{Node 2} & \textbf{Node 3} \\ + \hline + \hline + Partition 0 & \textcolor{ForestGreen}{Dahlia} & Abricot & \textcolor{ForestGreen}{Eucalyptus} \\ + \hline + Partition 1 & Ananas & \textcolor{ForestGreen}{Euphorbe} & \textcolor{ForestGreen}{Doradille} \\ + \hline + Partition 2 & \textcolor{ForestGreen}{Dahlia} & \textcolor{ForestGreen}{Echinops} & Abricot \\ + \hline + \hspace{1em}$\dots$ & \hspace{1em}$\dots$ & \hspace{1em}$\dots$ & \hspace{1em}$\dots$ \\ + \hline + \end{tabular} + } + \end{minipage} + + \vspace{2em} + \item<3-> During the rebalancing, new nodes don't yet have the data,\\ + ~~~~~~~~~~~~~~~~~~~and old nodes want to get rid of the data to free up space\\ + \vspace{1.2em} + $\to$ risk of inconsistency, \textbf{how to coordinate?} + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Handling layout changes without losing consistency} + \begin{minipage}{.55\textwidth} + \begin{itemize} + \item \textbf{Solution:}\\ + \vspace{.5em} + \begin{itemize} + \item keep track of data transfer to new nodes + \vspace{.5em} + \item use multiple write quorums\\ + (new nodes + old nodes\\ + while data transfer is in progress) + \vspace{.5em} + \item switching reads to new nodes\\ + only once copy is finished + \end{itemize} + \vspace{1em} + \item \textbf{Implemented} in v0.10 + \vspace{1em} + \item \textbf{Validated} with Jepsen testing + \end{itemize} + \end{minipage} + \begin{minipage}{.23\textwidth} + \includegraphics[width=3cm]{assets/jepsen-0.9.png}\\ + {\footnotesize Garage v0.9.0} + \end{minipage} + \begin{minipage}{.2\textwidth} + \includegraphics[width=3cm]{assets/jepsen-0.10.png}\\ + {\footnotesize Garage v0.10 beta} + \end{minipage} +\end{frame} + +% ====================== v0.10.0 =============================== + +\begin{frame} + \begin{center} + \includegraphics[width=.8\linewidth]{assets/timeline-22-24.pdf} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Towards v1.0...} + Focus on \underline{security \& stability} + \vspace{2em} + \begin{itemize} + \item \textbf{Security audit} in progress by Radically Open Security + \vspace{1em} + \item Misc. S3 features (SSE-C, ...) and compatibility fixes + \vspace{1em} + \item Improve UX + \vspace{1em} + \item Fix bugs + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{...and beyond!} + \begin{center} + \includegraphics[width=.6\linewidth]{assets/survey_requested_features.png} + \end{center} +\end{frame} + +% ======================================== OPERATING +% ======================================== OPERATING +% ======================================== OPERATING + + +\section{Operating big Garage clusters} + +\begin{frame} + \frametitle{Operating Garage} + \begin{center} + \only<1-2>{ + \includegraphics[width=.9\linewidth]{assets/screenshots/garage_status_0.10.png} + \\\vspace{1em} + \visible<2>{\includegraphics[width=.9\linewidth]{assets/screenshots/garage_status_unhealthy_0.10.png}} + } + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Garage's architecture} + \begin{center} + \only<1>{\includegraphics[width=.45\linewidth]{assets/garage.drawio.pdf}}% + \only<2>{\includegraphics[width=.6\linewidth]{assets/garage_sync.drawio.pdf}}% + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Digging deeper} + \begin{center} + \only<1>{\includegraphics[width=.9\linewidth]{assets/screenshots/garage_stats_0.10.png}} + \only<2>{\includegraphics[width=.5\linewidth]{assets/screenshots/garage_worker_list_0.10.png}} + \only<3>{\includegraphics[width=.6\linewidth]{assets/screenshots/garage_worker_param_0.10.png}} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Potential limitations and bottlenecks} + \begin{itemize} + \item Global: + \begin{itemize} + \item Max. $\sim$100 nodes per cluster (excluding gateways) + \end{itemize} + \vspace{1em} + \item Metadata: + \begin{itemize} + \item One big bucket = bottleneck, object list on 3 nodes only + \end{itemize} + \vspace{1em} + \item Block manager: + \begin{itemize} + \item Lots of small files on disk + \item Processing the resync queue can be slow + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Deployment advice for very large clusters} + \begin{itemize} + \item Metadata storage: + \begin{itemize} + \item ZFS mirror (x2) on fast NVMe + \item Use LMDB storage engine + \end{itemize} + \vspace{.5em} + \item Data block storage: + \begin{itemize} + \item Use Garage's native multi-HDD support + \item XFS on individual drives + \item Increase block size (1MB $\to$ 10MB, requires more RAM and good networking) + \item Tune \texttt{resync-tranquility} and \texttt{resync-worker-count} dynamically + \end{itemize} + \vspace{.5em} + \item Other : + \begin{itemize} + \item Split data over several buckets + \item Use less than 100 storage nodes + \item Use gateway nodes + \end{itemize} + \vspace{.5em} + \end{itemize} + Our deployments: $< 10$ TB. Some people have done more! +\end{frame} + + +% ======================================== END +% ======================================== END +% ======================================== END + +\begin{frame} + \frametitle{Where to find us} + \begin{center} + \includegraphics[width=.25\linewidth]{logo/garage_hires.png}\\ + \vspace{-1em} + \url{https://garagehq.deuxfleurs.fr/}\\ + \url{mailto:garagehq@deuxfleurs.fr}\\ + \texttt{\#garage:deuxfleurs.fr} on Matrix + + \vspace{1.5em} + \includegraphics[width=.06\linewidth]{assets/logos/rust_logo.png} + \includegraphics[width=.13\linewidth]{assets/logos/AGPLv3_Logo.png} + \end{center} +\end{frame} + +\end{document} + +%% vim: set ts=4 sw=4 tw=0 noet spelllang=en :