LMDB s3 million objects

5 files changed, 191 insertions, 26 deletions

diff --git a/artifacts/2022-09-14-s3billion/garage-v0.8-beta2-lmdb.csv b/artifacts/2022-09-14-s3billion/garage-v0.8-beta2-lmdb.csv
new file mode 100644
index 0000000..66552f2
--- /dev/null
+++ b/artifacts/2022-09-14-s3billion/garage-v0.8-beta2-lmdb.csv
@@ -0,0 +1,129 @@
+total_objects,batch_dur_nanoseconds
+0,2018191689
+8192,1819596313
+16384,2102802010
+24576,2473935158
+32768,4510296173
+40960,6613958698
+49152,7977651423
+57344,13671435594
+65536,8060203586
+73728,3795495684
+81920,5794535775
+90112,5543820710
+98304,4519985809
+106496,5110068206
+114688,6708672963
+122880,7857479433
+131072,20642163540
+139264,5635269044
+147456,9351347336
+155648,5666222073
+163840,6501995906
+172032,5293598878
+180224,9350079270
+188416,13303962641
+196608,17428285824
+204800,11733919058
+212992,8372494836
+221184,26574754854
+229376,6314078661
+237568,9683594309
+245760,21850576414
+253952,8991130243
+262144,6567887673
+270336,9454781018
+278528,25552173846
+286720,9580711949
+294912,10257632597
+303104,7478191638
+311296,10501501215
+319488,3816776322
+327680,26795462810
+335872,9396360089
+344064,11208895014
+352256,21126662513
+360448,14198029001
+368640,5093580346
+376832,21004203835
+385024,14433396301
+393216,8933953570
+401408,5461923067
+409600,30844223516
+417792,8315342751
+425984,6166609079
+434176,29359253375
+442368,7642009005
+450560,7036982475
+458752,27168386971
+466944,10015226557
+475136,9245612551
+483328,24224252348
+491520,13827842768
+499712,9702435577
+507904,26083366912
+516096,7829114232
+524288,10792095771
+532480,28058503484
+540672,9984663485
+548864,10714515140
+557056,28472749292
+565248,10832278691
+573440,7594452219
+581632,23858923965
+589824,38909249537
+598016,15112735865
+606208,9644628857
+614400,28367134578
+622592,35180699079
+630784,14207640136
+638976,13236547125
+647168,37982462879
+655360,8448278050
+663552,11632520113
+671744,22661429490
+679936,29261192164
+688128,11982484321
+696320,30381827880
+704512,23530964587
+712704,8219614193
+720896,34002514929
+729088,21279203762
+737280,6880671212
+745472,10884265763
+753664,41999879903
+761856,13465319479
+770048,10792229369
+778240,29004881357
+786432,24372150617
+794624,13327280319
+802816,18478244110
+811008,48319469581
+819200,14174987827
+827392,14368184404
+835584,56647745264
+843776,9880099642
+851968,12808097118
+860160,28380242726
+868352,44970657202
+876544,18733551704
+884736,9340561806
+892928,25246663514
+901120,46462124425
+909312,12856639771
+917504,9028224897
+925696,38580954489
+933888,48184562190
+942080,11204051215
+950272,19754687230
+958464,36364975788
+966656,40759996520
+974848,10562565477
+983040,13905331718
+991232,54140254529
+999424,46643736817
+1007616,8362351299
+1015808,13514305332
+1024000,39389278472
+1032192,40328458712
+1040384,15958365846
diff --git a/artifacts/2022-09-14-s3billion/plot.R b/artifacts/2022-09-14-s3billion/plot.R
new file mode 100644
index 0000000..6929a6e
--- /dev/null
+++ b/artifacts/2022-09-14-s3billion/plot.R
@@ -0,0 +1,14 @@
+library(tidyverse)
+read_csv("garage-v0.8-beta2-lmdb.csv") %>% mutate(batch_dur_sec = batch_dur_nanoseconds / 1000 / 1000 / 1000 ) -> s
+
+ggplot(s, aes(x=total_objects, y=batch_dur_sec)) +
+  geom_point() +
+  geom_smooth(method = "gam", se = FALSE) + 
+  labs(
+    y="Time spent sending a batch (8192 objects)",
+    x="Total number of objects stored in the cluster",
+    caption="Get the code to reproduce this graph at https://git.deuxfleurs.fr/Deuxfleurs/mknet",
+    title="Storing 1M+ files on Garage! Impact of existing data on cluster interactiveness",
+    subtitle="Daemon: Garage v0.8 beta 2 with LMDB as db_engine\nBenchmark: 128 batch. 192 objects/batch. 32 threads/batch. 256 objects/thread. 16-byte/objects.\nEnvironment: mknet (Ryzen 5 1400, 16GB RAM, SSD). DC topo (3 nodes, 1Gb/s, 1ms latency).") +
+  theme_classic()
+ggsave("./plot.png", width=200, height=120, units="mm")
diff --git a/artifacts/2022-09-14-s3billion/plot.png b/artifacts/2022-09-14-s3billion/plot.png
new file mode 100644
index 0000000..5c55897
--- /dev/null
+++ b/artifacts/2022-09-14-s3billion/plot.png
diff --git a/scenarios/fragments/garage.py b/scenarios/fragments/garage.py
index b291321..e19a09e 100644
--- a/scenarios/fragments/garage.py
+++ b/scenarios/fragments/garage.py
@@ -38,11 +38,11 @@ if 'HOST' in env:
     config = storage_path / "garage.toml"
     env['GARAGE_CONFIG_FILE'] = str(config)
 
-def deploy_coord(gw=None):
+def deploy_coord(gw=None, uroot={}, us3_api={}, us3_web={}, uadmin={}):
     destroy()
     shared.log("start daemon")
     shared.exec(f"{version['path']} --version")
-    daemon()
+    daemon(uroot,us3_api,us3_web,uadmin)
     shared.log("discover nodes")
     connect()
     shared.log("build layout")
@@ -51,43 +51,62 @@ def deploy_coord(gw=None):
     create_key()
     shared.log("ready")
 
-def deploy_follow():
+def deploy_follow(uroot={}, us3_api={}, us3_web={}, uadmin={}):
     destroy()
     shared.log("start daemon")
-    daemon()
+    daemon(uroot,us3_api,us3_web,uadmin)
     shared.log("wait for coord")
     sync_on_key_up()
     shared.log("ready")
 
-def daemon():
+def to_toml(d): return "\n".join([ f"{k} = \"{v}\"" for k, v in d.items() ])
+def daemon(uroot={}, us3_api={}, us3_web={}, uadmin={}):
+    root = {
+        "metadata_dir": f"{storage_path}/meta",
+        "data_dir": f"{storage_path}/data",
+        "replication_mode": "3",
+        "rpc_bind_addr": "[::]:3901",
+        "rpc_public_addr": f"[{env['IP']}]:3901",
+        "rpc_secret": f"{rpc_secret}",
+    }
+    root.update(uroot)
+
+    s3_api = {
+        "s3_region": "garage",
+        "api_bind_addr": "[::]:3900",
+        "root_domain": ".s3.garage",
+    }
+    s3_api.update(us3_api)
+    
+    s3_web = {
+        "bind_addr": "[::]:3902",
+        "root_domain": ".web.garage",
+        "index": "index.html",
+    }
+    s3_web.update(us3_web)
+
+    sect_admin = {
+        "api_bind_addr": "0.0.0.0:3903",
+        "metrics_token": f"{metrics}",
+        "admin_token": f"{admin}",
+    }
+    sect_admin.update(uadmin)
+
+
     shared.exec(f"mkdir -p {storage_path}")
     with open(config, 'w+') as f:
         f.write(f"""
-metadata_dir = "{storage_path}/meta"
-data_dir = "{storage_path}/data"
-
-replication_mode = "3"
-
-rpc_bind_addr = "[::]:3901"
-rpc_public_addr = "[{env['IP']}]:3901"
-rpc_secret = "{rpc_secret}"
-
-bootstrap_peers=[]
+{to_toml(root)}
+bootstrap_peers = []
 
 [s3_api]
-s3_region = "garage"
-api_bind_addr = "[::]:3900"
-root_domain = ".s3.garage"
+{to_toml(s3_api)}
 
 [s3_web]
-bind_addr = "[::]:3902"
-root_domain = ".web.garage"
-index = "index.html"
+{to_toml(s3_web)}
 
 [admin]
-api_bind_addr = "0.0.0.0:3903"
-metrics_token = "{metrics}"
-admin_token = "{admin}"
+{to_toml(sect_admin)}
         """)
 
     shared.exec(f"{version['path']} server 2>> {storage_path}/logs.stderr 1>> {storage_path}/logs.stdout & echo $! > {storage_path}/daemon.pid")
diff --git a/scenarios/garage-s3billion b/scenarios/garage-s3billion
index 55653f6..6a191e9 100755
--- a/scenarios/garage-s3billion
+++ b/scenarios/garage-s3billion
@@ -8,14 +8,17 @@ for fl in sys.argv[1:]:
 
 #os.environ['BATCH_COUNT'] = "2"
 shared.exec("ulimit -n 65535")
+conf_root = {
+    "db_engine": "lmdb",
+}
 
 if shared.id() == 1:
-    garage.deploy_coord()
+    garage.deploy_coord(uroot=conf_root)
     s3billion.on_garage()
     garage.delete_key()
     garage.destroy()
 else:
-    garage.deploy_follow()
+    garage.deploy_follow(uroot=conf_root)
     garage.sync_on_key_down()
     garage.destroy()
 shared.log("bye")