LMDB s3 million objects

This commit is contained in:
Quentin 2022-09-24 17:01:34 +02:00
parent a744f72f15
commit a869e3614b
Signed by: quentin
GPG key ID: E9602264D639FF68
5 changed files with 191 additions and 26 deletions

View file

@ -0,0 +1,129 @@
total_objects,batch_dur_nanoseconds
0,2018191689
8192,1819596313
16384,2102802010
24576,2473935158
32768,4510296173
40960,6613958698
49152,7977651423
57344,13671435594
65536,8060203586
73728,3795495684
81920,5794535775
90112,5543820710
98304,4519985809
106496,5110068206
114688,6708672963
122880,7857479433
131072,20642163540
139264,5635269044
147456,9351347336
155648,5666222073
163840,6501995906
172032,5293598878
180224,9350079270
188416,13303962641
196608,17428285824
204800,11733919058
212992,8372494836
221184,26574754854
229376,6314078661
237568,9683594309
245760,21850576414
253952,8991130243
262144,6567887673
270336,9454781018
278528,25552173846
286720,9580711949
294912,10257632597
303104,7478191638
311296,10501501215
319488,3816776322
327680,26795462810
335872,9396360089
344064,11208895014
352256,21126662513
360448,14198029001
368640,5093580346
376832,21004203835
385024,14433396301
393216,8933953570
401408,5461923067
409600,30844223516
417792,8315342751
425984,6166609079
434176,29359253375
442368,7642009005
450560,7036982475
458752,27168386971
466944,10015226557
475136,9245612551
483328,24224252348
491520,13827842768
499712,9702435577
507904,26083366912
516096,7829114232
524288,10792095771
532480,28058503484
540672,9984663485
548864,10714515140
557056,28472749292
565248,10832278691
573440,7594452219
581632,23858923965
589824,38909249537
598016,15112735865
606208,9644628857
614400,28367134578
622592,35180699079
630784,14207640136
638976,13236547125
647168,37982462879
655360,8448278050
663552,11632520113
671744,22661429490
679936,29261192164
688128,11982484321
696320,30381827880
704512,23530964587
712704,8219614193
720896,34002514929
729088,21279203762
737280,6880671212
745472,10884265763
753664,41999879903
761856,13465319479
770048,10792229369
778240,29004881357
786432,24372150617
794624,13327280319
802816,18478244110
811008,48319469581
819200,14174987827
827392,14368184404
835584,56647745264
843776,9880099642
851968,12808097118
860160,28380242726
868352,44970657202
876544,18733551704
884736,9340561806
892928,25246663514
901120,46462124425
909312,12856639771
917504,9028224897
925696,38580954489
933888,48184562190
942080,11204051215
950272,19754687230
958464,36364975788
966656,40759996520
974848,10562565477
983040,13905331718
991232,54140254529
999424,46643736817
1007616,8362351299
1015808,13514305332
1024000,39389278472
1032192,40328458712
1040384,15958365846
1 total_objects batch_dur_nanoseconds
2 0 2018191689
3 8192 1819596313
4 16384 2102802010
5 24576 2473935158
6 32768 4510296173
7 40960 6613958698
8 49152 7977651423
9 57344 13671435594
10 65536 8060203586
11 73728 3795495684
12 81920 5794535775
13 90112 5543820710
14 98304 4519985809
15 106496 5110068206
16 114688 6708672963
17 122880 7857479433
18 131072 20642163540
19 139264 5635269044
20 147456 9351347336
21 155648 5666222073
22 163840 6501995906
23 172032 5293598878
24 180224 9350079270
25 188416 13303962641
26 196608 17428285824
27 204800 11733919058
28 212992 8372494836
29 221184 26574754854
30 229376 6314078661
31 237568 9683594309
32 245760 21850576414
33 253952 8991130243
34 262144 6567887673
35 270336 9454781018
36 278528 25552173846
37 286720 9580711949
38 294912 10257632597
39 303104 7478191638
40 311296 10501501215
41 319488 3816776322
42 327680 26795462810
43 335872 9396360089
44 344064 11208895014
45 352256 21126662513
46 360448 14198029001
47 368640 5093580346
48 376832 21004203835
49 385024 14433396301
50 393216 8933953570
51 401408 5461923067
52 409600 30844223516
53 417792 8315342751
54 425984 6166609079
55 434176 29359253375
56 442368 7642009005
57 450560 7036982475
58 458752 27168386971
59 466944 10015226557
60 475136 9245612551
61 483328 24224252348
62 491520 13827842768
63 499712 9702435577
64 507904 26083366912
65 516096 7829114232
66 524288 10792095771
67 532480 28058503484
68 540672 9984663485
69 548864 10714515140
70 557056 28472749292
71 565248 10832278691
72 573440 7594452219
73 581632 23858923965
74 589824 38909249537
75 598016 15112735865
76 606208 9644628857
77 614400 28367134578
78 622592 35180699079
79 630784 14207640136
80 638976 13236547125
81 647168 37982462879
82 655360 8448278050
83 663552 11632520113
84 671744 22661429490
85 679936 29261192164
86 688128 11982484321
87 696320 30381827880
88 704512 23530964587
89 712704 8219614193
90 720896 34002514929
91 729088 21279203762
92 737280 6880671212
93 745472 10884265763
94 753664 41999879903
95 761856 13465319479
96 770048 10792229369
97 778240 29004881357
98 786432 24372150617
99 794624 13327280319
100 802816 18478244110
101 811008 48319469581
102 819200 14174987827
103 827392 14368184404
104 835584 56647745264
105 843776 9880099642
106 851968 12808097118
107 860160 28380242726
108 868352 44970657202
109 876544 18733551704
110 884736 9340561806
111 892928 25246663514
112 901120 46462124425
113 909312 12856639771
114 917504 9028224897
115 925696 38580954489
116 933888 48184562190
117 942080 11204051215
118 950272 19754687230
119 958464 36364975788
120 966656 40759996520
121 974848 10562565477
122 983040 13905331718
123 991232 54140254529
124 999424 46643736817
125 1007616 8362351299
126 1015808 13514305332
127 1024000 39389278472
128 1032192 40328458712
129 1040384 15958365846

View file

@ -0,0 +1,14 @@
library(tidyverse)
read_csv("garage-v0.8-beta2-lmdb.csv") %>% mutate(batch_dur_sec = batch_dur_nanoseconds / 1000 / 1000 / 1000 ) -> s
ggplot(s, aes(x=total_objects, y=batch_dur_sec)) +
geom_point() +
geom_smooth(method = "gam", se = FALSE) +
labs(
y="Time spent sending a batch (8192 objects)",
x="Total number of objects stored in the cluster",
caption="Get the code to reproduce this graph at https://git.deuxfleurs.fr/Deuxfleurs/mknet",
title="Storing 1M+ files on Garage! Impact of existing data on cluster interactiveness",
subtitle="Daemon: Garage v0.8 beta 2 with LMDB as db_engine\nBenchmark: 128 batch. 192 objects/batch. 32 threads/batch. 256 objects/thread. 16-byte/objects.\nEnvironment: mknet (Ryzen 5 1400, 16GB RAM, SSD). DC topo (3 nodes, 1Gb/s, 1ms latency).") +
theme_classic()
ggsave("./plot.png", width=200, height=120, units="mm")

Binary file not shown.

After

Width:  |  Height:  |  Size: 221 KiB

View file

@ -38,11 +38,11 @@ if 'HOST' in env:
config = storage_path / "garage.toml"
env['GARAGE_CONFIG_FILE'] = str(config)
def deploy_coord(gw=None):
def deploy_coord(gw=None, uroot={}, us3_api={}, us3_web={}, uadmin={}):
destroy()
shared.log("start daemon")
shared.exec(f"{version['path']} --version")
daemon()
daemon(uroot,us3_api,us3_web,uadmin)
shared.log("discover nodes")
connect()
shared.log("build layout")
@ -51,43 +51,62 @@ def deploy_coord(gw=None):
create_key()
shared.log("ready")
def deploy_follow():
def deploy_follow(uroot={}, us3_api={}, us3_web={}, uadmin={}):
destroy()
shared.log("start daemon")
daemon()
daemon(uroot,us3_api,us3_web,uadmin)
shared.log("wait for coord")
sync_on_key_up()
shared.log("ready")
def daemon():
def to_toml(d): return "\n".join([ f"{k} = \"{v}\"" for k, v in d.items() ])
def daemon(uroot={}, us3_api={}, us3_web={}, uadmin={}):
root = {
"metadata_dir": f"{storage_path}/meta",
"data_dir": f"{storage_path}/data",
"replication_mode": "3",
"rpc_bind_addr": "[::]:3901",
"rpc_public_addr": f"[{env['IP']}]:3901",
"rpc_secret": f"{rpc_secret}",
}
root.update(uroot)
s3_api = {
"s3_region": "garage",
"api_bind_addr": "[::]:3900",
"root_domain": ".s3.garage",
}
s3_api.update(us3_api)
s3_web = {
"bind_addr": "[::]:3902",
"root_domain": ".web.garage",
"index": "index.html",
}
s3_web.update(us3_web)
sect_admin = {
"api_bind_addr": "0.0.0.0:3903",
"metrics_token": f"{metrics}",
"admin_token": f"{admin}",
}
sect_admin.update(uadmin)
shared.exec(f"mkdir -p {storage_path}")
with open(config, 'w+') as f:
f.write(f"""
metadata_dir = "{storage_path}/meta"
data_dir = "{storage_path}/data"
replication_mode = "3"
rpc_bind_addr = "[::]:3901"
rpc_public_addr = "[{env['IP']}]:3901"
rpc_secret = "{rpc_secret}"
bootstrap_peers=[]
{to_toml(root)}
bootstrap_peers = []
[s3_api]
s3_region = "garage"
api_bind_addr = "[::]:3900"
root_domain = ".s3.garage"
{to_toml(s3_api)}
[s3_web]
bind_addr = "[::]:3902"
root_domain = ".web.garage"
index = "index.html"
{to_toml(s3_web)}
[admin]
api_bind_addr = "0.0.0.0:3903"
metrics_token = "{metrics}"
admin_token = "{admin}"
{to_toml(sect_admin)}
""")
shared.exec(f"{version['path']} server 2>> {storage_path}/logs.stderr 1>> {storage_path}/logs.stdout & echo $! > {storage_path}/daemon.pid")

View file

@ -8,14 +8,17 @@ for fl in sys.argv[1:]:
#os.environ['BATCH_COUNT'] = "2"
shared.exec("ulimit -n 65535")
conf_root = {
"db_engine": "lmdb",
}
if shared.id() == 1:
garage.deploy_coord()
garage.deploy_coord(uroot=conf_root)
s3billion.on_garage()
garage.delete_key()
garage.destroy()
else:
garage.deploy_follow()
garage.deploy_follow(uroot=conf_root)
garage.sync_on_key_down()
garage.destroy()
shared.log("bye")