diff --git a/artifacts/2022-09-14-s3billion/garage-v0.8-beta2-lmdb.csv b/artifacts/2022-09-14-s3billion/garage-v0.8-beta2-lmdb.csv new file mode 100644 index 0000000..66552f2 --- /dev/null +++ b/artifacts/2022-09-14-s3billion/garage-v0.8-beta2-lmdb.csv @@ -0,0 +1,129 @@ +total_objects,batch_dur_nanoseconds +0,2018191689 +8192,1819596313 +16384,2102802010 +24576,2473935158 +32768,4510296173 +40960,6613958698 +49152,7977651423 +57344,13671435594 +65536,8060203586 +73728,3795495684 +81920,5794535775 +90112,5543820710 +98304,4519985809 +106496,5110068206 +114688,6708672963 +122880,7857479433 +131072,20642163540 +139264,5635269044 +147456,9351347336 +155648,5666222073 +163840,6501995906 +172032,5293598878 +180224,9350079270 +188416,13303962641 +196608,17428285824 +204800,11733919058 +212992,8372494836 +221184,26574754854 +229376,6314078661 +237568,9683594309 +245760,21850576414 +253952,8991130243 +262144,6567887673 +270336,9454781018 +278528,25552173846 +286720,9580711949 +294912,10257632597 +303104,7478191638 +311296,10501501215 +319488,3816776322 +327680,26795462810 +335872,9396360089 +344064,11208895014 +352256,21126662513 +360448,14198029001 +368640,5093580346 +376832,21004203835 +385024,14433396301 +393216,8933953570 +401408,5461923067 +409600,30844223516 +417792,8315342751 +425984,6166609079 +434176,29359253375 +442368,7642009005 +450560,7036982475 +458752,27168386971 +466944,10015226557 +475136,9245612551 +483328,24224252348 +491520,13827842768 +499712,9702435577 +507904,26083366912 +516096,7829114232 +524288,10792095771 +532480,28058503484 +540672,9984663485 +548864,10714515140 +557056,28472749292 +565248,10832278691 +573440,7594452219 +581632,23858923965 +589824,38909249537 +598016,15112735865 +606208,9644628857 +614400,28367134578 +622592,35180699079 +630784,14207640136 +638976,13236547125 +647168,37982462879 +655360,8448278050 +663552,11632520113 +671744,22661429490 +679936,29261192164 +688128,11982484321 +696320,30381827880 +704512,23530964587 +712704,8219614193 +720896,34002514929 +729088,21279203762 +737280,6880671212 +745472,10884265763 +753664,41999879903 +761856,13465319479 +770048,10792229369 +778240,29004881357 +786432,24372150617 +794624,13327280319 +802816,18478244110 +811008,48319469581 +819200,14174987827 +827392,14368184404 +835584,56647745264 +843776,9880099642 +851968,12808097118 +860160,28380242726 +868352,44970657202 +876544,18733551704 +884736,9340561806 +892928,25246663514 +901120,46462124425 +909312,12856639771 +917504,9028224897 +925696,38580954489 +933888,48184562190 +942080,11204051215 +950272,19754687230 +958464,36364975788 +966656,40759996520 +974848,10562565477 +983040,13905331718 +991232,54140254529 +999424,46643736817 +1007616,8362351299 +1015808,13514305332 +1024000,39389278472 +1032192,40328458712 +1040384,15958365846 diff --git a/artifacts/2022-09-14-s3billion/plot.R b/artifacts/2022-09-14-s3billion/plot.R new file mode 100644 index 0000000..6929a6e --- /dev/null +++ b/artifacts/2022-09-14-s3billion/plot.R @@ -0,0 +1,14 @@ +library(tidyverse) +read_csv("garage-v0.8-beta2-lmdb.csv") %>% mutate(batch_dur_sec = batch_dur_nanoseconds / 1000 / 1000 / 1000 ) -> s + +ggplot(s, aes(x=total_objects, y=batch_dur_sec)) + + geom_point() + + geom_smooth(method = "gam", se = FALSE) + + labs( + y="Time spent sending a batch (8192 objects)", + x="Total number of objects stored in the cluster", + caption="Get the code to reproduce this graph at https://git.deuxfleurs.fr/Deuxfleurs/mknet", + title="Storing 1M+ files on Garage! Impact of existing data on cluster interactiveness", + subtitle="Daemon: Garage v0.8 beta 2 with LMDB as db_engine\nBenchmark: 128 batch. 192 objects/batch. 32 threads/batch. 256 objects/thread. 16-byte/objects.\nEnvironment: mknet (Ryzen 5 1400, 16GB RAM, SSD). DC topo (3 nodes, 1Gb/s, 1ms latency).") + + theme_classic() +ggsave("./plot.png", width=200, height=120, units="mm") diff --git a/artifacts/2022-09-14-s3billion/plot.png b/artifacts/2022-09-14-s3billion/plot.png new file mode 100644 index 0000000..5c55897 Binary files /dev/null and b/artifacts/2022-09-14-s3billion/plot.png differ diff --git a/scenarios/fragments/garage.py b/scenarios/fragments/garage.py index b291321..e19a09e 100644 --- a/scenarios/fragments/garage.py +++ b/scenarios/fragments/garage.py @@ -38,11 +38,11 @@ if 'HOST' in env: config = storage_path / "garage.toml" env['GARAGE_CONFIG_FILE'] = str(config) -def deploy_coord(gw=None): +def deploy_coord(gw=None, uroot={}, us3_api={}, us3_web={}, uadmin={}): destroy() shared.log("start daemon") shared.exec(f"{version['path']} --version") - daemon() + daemon(uroot,us3_api,us3_web,uadmin) shared.log("discover nodes") connect() shared.log("build layout") @@ -51,43 +51,62 @@ def deploy_coord(gw=None): create_key() shared.log("ready") -def deploy_follow(): +def deploy_follow(uroot={}, us3_api={}, us3_web={}, uadmin={}): destroy() shared.log("start daemon") - daemon() + daemon(uroot,us3_api,us3_web,uadmin) shared.log("wait for coord") sync_on_key_up() shared.log("ready") -def daemon(): +def to_toml(d): return "\n".join([ f"{k} = \"{v}\"" for k, v in d.items() ]) +def daemon(uroot={}, us3_api={}, us3_web={}, uadmin={}): + root = { + "metadata_dir": f"{storage_path}/meta", + "data_dir": f"{storage_path}/data", + "replication_mode": "3", + "rpc_bind_addr": "[::]:3901", + "rpc_public_addr": f"[{env['IP']}]:3901", + "rpc_secret": f"{rpc_secret}", + } + root.update(uroot) + + s3_api = { + "s3_region": "garage", + "api_bind_addr": "[::]:3900", + "root_domain": ".s3.garage", + } + s3_api.update(us3_api) + + s3_web = { + "bind_addr": "[::]:3902", + "root_domain": ".web.garage", + "index": "index.html", + } + s3_web.update(us3_web) + + sect_admin = { + "api_bind_addr": "0.0.0.0:3903", + "metrics_token": f"{metrics}", + "admin_token": f"{admin}", + } + sect_admin.update(uadmin) + + shared.exec(f"mkdir -p {storage_path}") with open(config, 'w+') as f: f.write(f""" -metadata_dir = "{storage_path}/meta" -data_dir = "{storage_path}/data" - -replication_mode = "3" - -rpc_bind_addr = "[::]:3901" -rpc_public_addr = "[{env['IP']}]:3901" -rpc_secret = "{rpc_secret}" - -bootstrap_peers=[] +{to_toml(root)} +bootstrap_peers = [] [s3_api] -s3_region = "garage" -api_bind_addr = "[::]:3900" -root_domain = ".s3.garage" +{to_toml(s3_api)} [s3_web] -bind_addr = "[::]:3902" -root_domain = ".web.garage" -index = "index.html" +{to_toml(s3_web)} [admin] -api_bind_addr = "0.0.0.0:3903" -metrics_token = "{metrics}" -admin_token = "{admin}" +{to_toml(sect_admin)} """) shared.exec(f"{version['path']} server 2>> {storage_path}/logs.stderr 1>> {storage_path}/logs.stdout & echo $! > {storage_path}/daemon.pid") diff --git a/scenarios/garage-s3billion b/scenarios/garage-s3billion index 55653f6..6a191e9 100755 --- a/scenarios/garage-s3billion +++ b/scenarios/garage-s3billion @@ -8,14 +8,17 @@ for fl in sys.argv[1:]: #os.environ['BATCH_COUNT'] = "2" shared.exec("ulimit -n 65535") +conf_root = { + "db_engine": "lmdb", +} if shared.id() == 1: - garage.deploy_coord() + garage.deploy_coord(uroot=conf_root) s3billion.on_garage() garage.delete_key() garage.destroy() else: - garage.deploy_follow() + garage.deploy_follow(uroot=conf_root) garage.sync_on_key_down() garage.destroy() shared.log("bye")