LMDB s3 million objects

2 files changed, 48 insertions, 26 deletions

diff --git a/scenarios/fragments/garage.py b/scenarios/fragments/garage.py
index b291321..e19a09e 100644
--- a/scenarios/fragments/garage.py
+++ b/scenarios/fragments/garage.py
@@ -38,11 +38,11 @@ if 'HOST' in env:
     config = storage_path / "garage.toml"
     env['GARAGE_CONFIG_FILE'] = str(config)
 
-def deploy_coord(gw=None):
+def deploy_coord(gw=None, uroot={}, us3_api={}, us3_web={}, uadmin={}):
     destroy()
     shared.log("start daemon")
     shared.exec(f"{version['path']} --version")
-    daemon()
+    daemon(uroot,us3_api,us3_web,uadmin)
     shared.log("discover nodes")
     connect()
     shared.log("build layout")
@@ -51,43 +51,62 @@ def deploy_coord(gw=None):
     create_key()
     shared.log("ready")
 
-def deploy_follow():
+def deploy_follow(uroot={}, us3_api={}, us3_web={}, uadmin={}):
     destroy()
     shared.log("start daemon")
-    daemon()
+    daemon(uroot,us3_api,us3_web,uadmin)
     shared.log("wait for coord")
     sync_on_key_up()
     shared.log("ready")
 
-def daemon():
+def to_toml(d): return "\n".join([ f"{k} = \"{v}\"" for k, v in d.items() ])
+def daemon(uroot={}, us3_api={}, us3_web={}, uadmin={}):
+    root = {
+        "metadata_dir": f"{storage_path}/meta",
+        "data_dir": f"{storage_path}/data",
+        "replication_mode": "3",
+        "rpc_bind_addr": "[::]:3901",
+        "rpc_public_addr": f"[{env['IP']}]:3901",
+        "rpc_secret": f"{rpc_secret}",
+    }
+    root.update(uroot)
+
+    s3_api = {
+        "s3_region": "garage",
+        "api_bind_addr": "[::]:3900",
+        "root_domain": ".s3.garage",
+    }
+    s3_api.update(us3_api)
+    
+    s3_web = {
+        "bind_addr": "[::]:3902",
+        "root_domain": ".web.garage",
+        "index": "index.html",
+    }
+    s3_web.update(us3_web)
+
+    sect_admin = {
+        "api_bind_addr": "0.0.0.0:3903",
+        "metrics_token": f"{metrics}",
+        "admin_token": f"{admin}",
+    }
+    sect_admin.update(uadmin)
+
+
     shared.exec(f"mkdir -p {storage_path}")
     with open(config, 'w+') as f:
         f.write(f"""
-metadata_dir = "{storage_path}/meta"
-data_dir = "{storage_path}/data"
-
-replication_mode = "3"
-
-rpc_bind_addr = "[::]:3901"
-rpc_public_addr = "[{env['IP']}]:3901"
-rpc_secret = "{rpc_secret}"
-
-bootstrap_peers=[]
+{to_toml(root)}
+bootstrap_peers = []
 
 [s3_api]
-s3_region = "garage"
-api_bind_addr = "[::]:3900"
-root_domain = ".s3.garage"
+{to_toml(s3_api)}
 
 [s3_web]
-bind_addr = "[::]:3902"
-root_domain = ".web.garage"
-index = "index.html"
+{to_toml(s3_web)}
 
 [admin]
-api_bind_addr = "0.0.0.0:3903"
-metrics_token = "{metrics}"
-admin_token = "{admin}"
+{to_toml(sect_admin)}
         """)
 
     shared.exec(f"{version['path']} server 2>> {storage_path}/logs.stderr 1>> {storage_path}/logs.stdout & echo $! > {storage_path}/daemon.pid")
diff --git a/scenarios/garage-s3billion b/scenarios/garage-s3billion
index 55653f6..6a191e9 100755
--- a/scenarios/garage-s3billion
+++ b/scenarios/garage-s3billion
@@ -8,14 +8,17 @@ for fl in sys.argv[1:]:
 
 #os.environ['BATCH_COUNT'] = "2"
 shared.exec("ulimit -n 65535")
+conf_root = {
+    "db_engine": "lmdb",
+}
 
 if shared.id() == 1:
-    garage.deploy_coord()
+    garage.deploy_coord(uroot=conf_root)
     s3billion.on_garage()
     garage.delete_key()
     garage.destroy()
 else:
-    garage.deploy_follow()
+    garage.deploy_follow(uroot=conf_root)
     garage.sync_on_key_down()
     garage.destroy()
 shared.log("bye")