From a40c12354257418e686892a7612d7e11a1a58235 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Fri, 23 Sep 2022 22:50:50 +0200
Subject: Add S3TTFB

---
 scenarios/fragments/garage.py | 35 +++++++++++++++++++++++++++--------
 scenarios/fragments/s3ttfb.py | 26 ++++++++++++++++++++++++++
 scenarios/garage-s3ttfb       | 18 ++++++++++++++++++
 scenarios/minio-s3ttfb        | 19 +++++++++++++++++++
 scenarios/requirements.txt    |  2 +-
 5 files changed, 91 insertions(+), 9 deletions(-)
 create mode 100644 scenarios/fragments/s3ttfb.py
 create mode 100755 scenarios/garage-s3ttfb
 create mode 100755 scenarios/minio-s3ttfb

(limited to 'scenarios')

diff --git a/scenarios/fragments/garage.py b/scenarios/fragments/garage.py
index 84ca90a..b291321 100644
--- a/scenarios/fragments/garage.py
+++ b/scenarios/fragments/garage.py
@@ -38,7 +38,7 @@ if 'HOST' in env:
     config = storage_path / "garage.toml"
     env['GARAGE_CONFIG_FILE'] = str(config)
 
-def deploy_coord():
+def deploy_coord(gw=None):
     destroy()
     shared.log("start daemon")
     shared.exec(f"{version['path']} --version")
@@ -46,7 +46,7 @@ def deploy_coord():
     shared.log("discover nodes")
     connect()
     shared.log("build layout")
-    create_layout()
+    create_layout(gw=gw)
     shared.log("create key")
     create_key()
     shared.log("ready")
@@ -136,26 +136,45 @@ def cluster_info():
 def connect():
     cinf = cluster_info()
     shared.log("start connections...")
-    ret = nodes.add_node([n['node_addr'] for n in cinf])
-    for st in ret:
-        if not st.success:
-            raise Exception("Node connect failed", ret)
+    while True:
+        try:
+            ret = nodes.add_node([n['node_addr'] for n in cinf], _request_timeout=3)
+        except:
+            shared.log("not ready, retry in 1sec")
+            time.sleep(1)
+            continue
+
+        for st in ret:
+            if not st.success:
+                continue
+                #raise Exception("Node connect failed", ret)
+        break
+
     shared.log("all nodes connected")
 
-def create_layout():
+def create_layout(gw=None):
+    if gw is None:
+        gw = []
+
     v = layout.get_layout().version
 
     cinf = cluster_info()
     nlay = dict()
     for n in cinf:
+        capa = 1
+        if n['host'] in gw:
+            capa = None
+
         nlay[n['node_id']] = NodeClusterInfo(
             zone = n['zone'],
-            capacity = 1,
+            capacity = capa,
             tags = [ n['host'] ],
         )
     layout.add_layout(nlay)
     layout.apply_layout(LayoutVersion(version=v+1))
 
+    shared.log(layout.get_layout())
+
 def create_key():
     global key
     kinfo = shared.fn_retry(lambda: keys.add_key(AddKeyRequest(name="mknet")))
diff --git a/scenarios/fragments/s3ttfb.py b/scenarios/fragments/s3ttfb.py
new file mode 100644
index 0000000..66fdcc6
--- /dev/null
+++ b/scenarios/fragments/s3ttfb.py
@@ -0,0 +1,26 @@
+import os
+from os.path import exists
+from pathlib import Path
+from fragments import shared, minio, garage
+
+s3bin = Path(os.path.dirname(__file__)) / "../../benchmarks/s3ttfb/s3ttfb"
+
+def common():
+    out = Path(shared.storage_path) / "s3ttfb.csv"
+    shared.log(f"launching s3ttfb ({s3bin})")
+    shared.exec(f"{s3bin} > {out}")
+    shared.log(f"execution done, output written to {out}")
+
+def on_garage():
+    os.environ['AWS_ACCESS_KEY_ID'] = garage.key.access_key_id
+    os.environ['AWS_SECRET_ACCESS_KEY'] = garage.key.secret_access_key
+    os.environ['ENDPOINT'] = "localhost:3900"
+    os.environ['AWS_REGION'] = "garage"
+    common()
+
+def on_minio():
+    os.environ['AWS_ACCESS_KEY_ID'] = minio.access_key
+    os.environ['AWS_SECRET_ACCESS_KEY'] = minio.secret_key
+    os.environ['ENDPOINT'] = "localhost:9000"
+    os.environ['AWS_REGION'] = "us-east-1"
+    common()
diff --git a/scenarios/garage-s3ttfb b/scenarios/garage-s3ttfb
new file mode 100755
index 0000000..20bb6d0
--- /dev/null
+++ b/scenarios/garage-s3ttfb
@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+from fragments import garage, s3ttfb, shared, flavor
+import sys
+
+for fl in sys.argv[1:]:
+    if fl in flavor.garage:
+        garage.version = flavor.garage[fl]
+
+if shared.id() == 1:
+    garage.deploy_coord(gw = [ 'node1' ])
+    s3ttfb.on_garage()
+    garage.delete_key()
+    garage.destroy()
+else:
+    garage.deploy_follow()
+    garage.sync_on_key_down()
+    garage.destroy()
+shared.log("bye")
diff --git a/scenarios/minio-s3ttfb b/scenarios/minio-s3ttfb
new file mode 100755
index 0000000..ed78aea
--- /dev/null
+++ b/scenarios/minio-s3ttfb
@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+from fragments import minio, s3ttfb, shared, flavor
+import sys
+
+
+for fl in sys.argv[1:]:
+    if fl in flavor.minio:
+        minio.version = flavor.minio[fl]
+
+if shared.id() == 1:
+    minio.deploy_coord()
+    s3ttfb.on_minio()
+    minio.delete_sync_bucket()
+    minio.destroy()
+else:
+    minio.deploy_follow()
+    minio.sync_on_bucket_down()
+    minio.destroy()
+shared.log("bye")
diff --git a/scenarios/requirements.txt b/scenarios/requirements.txt
index b85592f..1d9ce28 100644
--- a/scenarios/requirements.txt
+++ b/scenarios/requirements.txt
@@ -1,2 +1,2 @@
-git+https://git.deuxfleurs.fr/quentin/garage-admin-sdk@7b1c1faf7a#egg=garage-admin-sdk&subdirectory=python
+git+https://git.deuxfleurs.fr/quentin/garage-admin-sdk@2b685b90be9aa848a9450dd582d1d322b25b5568#egg=garage-admin-sdk&subdirectory=python
 minio
-- 
cgit v1.2.3