From c77d8dcfa7333b32e07df7326b8943838f34a59b Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Wed, 14 Sep 2022 18:01:44 +0200 Subject: Working s3lat with Garage --- .gitignore | 2 + benchmarks/clean | 8 -- benchmarks/fragments/.shared.py.swo | Bin 12288 -> 0 bytes benchmarks/fragments/garage.py | 202 ------------------------------------ benchmarks/fragments/minio.py | 62 ----------- benchmarks/fragments/s3lat.py | 18 ---- benchmarks/fragments/shared.py | 28 ----- benchmarks/garage-s3lat | 12 --- benchmarks/requirements.txt | 1 - benchmarks/s3concurrent/go.mod | 24 +++++ benchmarks/s3concurrent/go.sum | 52 ++++++++++ benchmarks/s3concurrent/main.go | 121 +++++++++++++++++++++ benchmarks/s3lat/README.md | 22 ++++ benchmarks/s3lat/go.mod | 8 ++ benchmarks/s3lat/go.sum | 78 ++++++++++++++ benchmarks/s3lat/main.go | 135 ++++++++++++++++++++++++ benchmarks/s3lat/s3lat | Bin 0 -> 8600112 bytes benchmarks/warp | 1 + scenarios/.garage-s3lat.swo | Bin 0 -> 12288 bytes scenarios/clean | 8 ++ scenarios/fragments/.shared.py.swo | Bin 0 -> 12288 bytes scenarios/fragments/garage.py | 201 +++++++++++++++++++++++++++++++++++ scenarios/fragments/minio.py | 62 +++++++++++ scenarios/fragments/s3lat.py | 16 +++ scenarios/fragments/shared.py | 28 +++++ scenarios/garage-s3lat | 13 +++ scenarios/requirements.txt | 1 + 27 files changed, 772 insertions(+), 331 deletions(-) delete mode 100755 benchmarks/clean delete mode 100644 benchmarks/fragments/.shared.py.swo delete mode 100644 benchmarks/fragments/garage.py delete mode 100644 benchmarks/fragments/minio.py delete mode 100644 benchmarks/fragments/s3lat.py delete mode 100644 benchmarks/fragments/shared.py delete mode 100755 benchmarks/garage-s3lat delete mode 100644 benchmarks/requirements.txt create mode 100644 benchmarks/s3concurrent/go.mod create mode 100644 benchmarks/s3concurrent/go.sum create mode 100644 benchmarks/s3concurrent/main.go create mode 100644 benchmarks/s3lat/README.md create mode 100644 benchmarks/s3lat/go.mod create mode 100644 benchmarks/s3lat/go.sum create mode 100644 benchmarks/s3lat/main.go create mode 100755 benchmarks/s3lat/s3lat create mode 160000 benchmarks/warp create mode 100644 scenarios/.garage-s3lat.swo create mode 100755 scenarios/clean create mode 100644 scenarios/fragments/.shared.py.swo create mode 100644 scenarios/fragments/garage.py create mode 100644 scenarios/fragments/minio.py create mode 100644 scenarios/fragments/s3lat.py create mode 100644 scenarios/fragments/shared.py create mode 100755 scenarios/garage-s3lat create mode 100644 scenarios/requirements.txt diff --git a/.gitignore b/.gitignore index fdc002f..730ff55 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ .current_state.yml __pycache__ *.swp +build +mknet.egg-info diff --git a/benchmarks/clean b/benchmarks/clean deleted file mode 100755 index 325edec..0000000 --- a/benchmarks/clean +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env python3 - -import os -from fragments import garage, shared - -garage.destroy() -shared.log("clean done") - diff --git a/benchmarks/fragments/.shared.py.swo b/benchmarks/fragments/.shared.py.swo deleted file mode 100644 index 02b9acb..0000000 Binary files a/benchmarks/fragments/.shared.py.swo and /dev/null differ diff --git a/benchmarks/fragments/garage.py b/benchmarks/fragments/garage.py deleted file mode 100644 index ac77475..0000000 --- a/benchmarks/fragments/garage.py +++ /dev/null @@ -1,202 +0,0 @@ -import glob, json, requests, time, garage_admin_sdk -from os.path import exists -from os import environ as env -from pathlib import Path -from fragments import shared -from garage_admin_sdk.api import nodes_api, layout_api, key_api -from garage_admin_sdk.model.node_cluster_info import NodeClusterInfo -from garage_admin_sdk.model.layout_version import LayoutVersion -from garage_admin_sdk.model.add_key_request import AddKeyRequest -from garage_admin_sdk.model.update_key_request import UpdateKeyRequest -from garage_admin_sdk.model.update_key_request_allow import UpdateKeyRequestAllow - -storage_path = "./i/am/not/defined" -rpc_secret = "3e9abff5f9e480afbadb46a77b7a26fe0e404258f0dc3fd5386b0ba8e0ad2fba" -metrics = "cacce0b2de4bc2d9f5b5fdff551e01ac1496055aed248202d415398987e35f81" -admin = "ae8cb40ea7368bbdbb6430af11cca7da833d3458a5f52086f4e805a570fb5c2a" -path = None -access_key = None -secret_key = None - - -configuration = garage_admin_sdk.Configuration( - host = "http://localhost:3903/v0", - access_token = admin -) -api = garage_admin_sdk.ApiClient(configuration) -nodes = nodes_api.NodesApi(api) -layout = layout_api.LayoutApi(api) -keys = key_api.KeyApi(api) - - -# Setup, launch on import -storage_path = Path(shared.storage_path) / "garage" / env['HOST'] -if 'ZONE' in env: - storage_path = Path(shared.storage_path) / "garage" / env['ZONE'] / env['HOST'] -config = storage_path / "garage.toml" -env['GARAGE_CONFIG_FILE'] = str(config) - -def deploy_coord(version=None, target=None): - destroy() - from_ci(version, target) - shared.log("start daemon") - daemon() - shared.log("discover nodes") - connect() - shared.log("build layout") - create_layout() - shared.log("create key") - create_key() - shared.log("ready") - -def deploy_follow(version=None, target=None): - destroy() - from_ci(version, target) - shared.log("start daemon") - daemon() - shared.log("wait for coord") - sync_on_key_up() - shared.log("ready") - -def from_local(p): - global path - path = p - shared.exec(f"{p} --version") - -def from_ci(version=None, target=None): - global path - version = version or "v0.7.3" - target = target or "x86_64-unknown-linux-musl" - - binary = f"garage-{target}-{version}" - path = Path(shared.binary_path) / binary - if shared.id() != 1: return - - if not exists(path): - shared.exec(f"mkdir -p {shared.binary_path}") - shared.exec(f"wget https://garagehq.deuxfleurs.fr/_releases/{version}/{target}/garage -O {path}") - shared.exec(f"chmod +x {path}") - shared.exec(f"{path} --version") - -def daemon(): - shared.exec(f"mkdir -p {storage_path}") - with open(config, 'w+') as f: - f.write(f""" -metadata_dir = "{storage_path}/meta" -data_dir = "{storage_path}/data" - -replication_mode = "3" - -rpc_bind_addr = "[::]:3901" -rpc_public_addr = "[{env['IP']}]:3901" -rpc_secret = "{rpc_secret}" - -bootstrap_peers=[] - -[s3_api] -s3_region = "garage" -api_bind_addr = "[::]:3900" -root_domain = ".s3.garage" - -[s3_web] -bind_addr = "[::]:3902" -root_domain = ".web.garage" -index = "index.html" - -[admin] -api_bind_addr = "0.0.0.0:3903" -metrics_token = "{metrics}" -admin_token = "{admin}" - """) - - shared.exec(f"{path} server 2>> {storage_path}/logs.stderr 1>> {storage_path}/logs.stdout & echo $! > {storage_path}/daemon.pid") - time.sleep(1) - - node_info = storage_path / "node_info" - node_id = nodes.get_nodes().node - with open(node_info, 'w+') as f: - f.write(json.dumps({ - "node_addr": f"{node_id}@{env['IP']}:3901", - "node_id": node_id, - "zone": env['ZONE'], - "host": env['HOST'], - })) - -def destroy(): - dpid = Path(storage_path) / "daemon.pid" - if exists(dpid): - shared.exec(f"kill -9 $(cat {dpid})") - shared.exec(f"rm -f {dpid}") - if len(str(storage_path)) < 8: # arbitrary, stupid safe guard - print(storage_path) - raise Exception("You tried to clean a storage path that might be the root of your FS, panicking...") - shared.exec(f"rm -fr {storage_path}") - -# this function is ugly, sorry :s -_cluster_info = None -def cluster_info(): - global _cluster_info - if _cluster_info is not None: return _cluster_info - - while True: - time.sleep(1) - node_files = glob.glob(f"{shared.storage_path}/**/node_info", recursive=True) - if len(node_files) == shared.count(): break - - _cluster_info = [ json.loads(Path(f).read_text()) for f in node_files ] - return _cluster_info - - -def connect(): - cinf = cluster_info() - ret = nodes.add_node([n['node_addr'] for n in cinf]) - for st in ret: - if not st.success: - raise Exception("Node connect failed", ret) - shared.log("all nodes connected") - -def create_layout(): - v = layout.get_layout().version - - cinf = cluster_info() - nlay = dict() - for n in cinf: - nlay[n['node_id']] = NodeClusterInfo( - zone = n['zone'], - capacity = 1, - tags = [ n['host'] ], - ) - layout.add_layout(nlay) - layout.apply_layout(LayoutVersion(version=v+1)) - -def create_key(): - global key - kinfo = shared.fn_retry(lambda: keys.add_key(AddKeyRequest(name="mknet"))) - allow_create = UpdateKeyRequestAllow(create_bucket=True) - keys.update_key(kinfo.access_key_id, UpdateKeyRequest(allow=allow_create)) - key = kinfo - - -def delete_key(): - global key - delete_key(key.access_key_id) - key = None - -def sync_on_key_up(): - global key - while True: - try: - key = keys.search_key("mknet") - return key - except: - pass - time.sleep(1) - -def sync_on_key_down(): - while True: - try: - keys.search_key("mknet") - except: - return - time.sleep(1) - diff --git a/benchmarks/fragments/minio.py b/benchmarks/fragments/minio.py deleted file mode 100644 index 431b983..0000000 --- a/benchmarks/fragments/minio.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python3 -import json, os, sys, time, pathlib, socket, shutil - -STORAGE_PATH = os.path.join(os.getcwd(), '.minio-testnet') -HOSTS_PATH = os.path.join(STORAGE_PATH, 'hosts.txt') -UNIX_SOCK = os.path.join(STORAGE_PATH, 'deploy.sock') -DATA_PATH = lambda nid: os.path.join(STORAGE_PATH, 'data'+str(nid)) - -def main(): - if int(os.environ['ID']) == 1: leader() - else: follower() - -def leader(): - shutil.rmtree(STORAGE_PATH, ignore_errors=True) - os.makedirs(STORAGE_PATH) - print(STORAGE_PATH) - - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - sock.bind(UNIX_SOCK) - sock.listen() - - n_serv = int(os.environ['SERVER_COUNT']) - fl = [ co for co, addr in [ sock.accept() for i in range(n_serv - 1) ]] - - identities = [ json.loads(co.makefile().readline()) for co in fl ] + [ { "ip": os.environ['IP'], "path": make_data() } ] - print(f"ident: {identities}") - msg = f"{json.dumps(identities)}\n".encode() - [ co.send(msg) for co in fl ] - - run_minio(identities) - -def follower(): - co = None - while True: - time.sleep(1) - try: - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - sock.connect(UNIX_SOCK) - co = sock.makefile() - break - except Exception as err: - print('conn failed, wait,', err) - my_identity = json.dumps({ "ip": os.environ['IP'], "path": make_data() }) - sock.send(f"{my_identity}\n".encode()) - identities = json.loads(co.readline()) - - run_minio(identities) - -def make_data(): - data_path = DATA_PATH(os.environ['ID']) - os.makedirs(data_path) - return data_path - -def run_minio(identities): - cmd = f"minio server --console-address ':9001' --address ':9000'" - for ident in identities: - cmd += f" http://[{ident['ip']}]:9000{ident['path']}" - cmd += f" > {os.path.join(STORAGE_PATH, 'minio'+os.environ['ID']+'.log')} 2>&1" - print("launch: ", cmd) - os.system(cmd) - -__name__ == '__main__' and main() diff --git a/benchmarks/fragments/s3lat.py b/benchmarks/fragments/s3lat.py deleted file mode 100644 index c25594e..0000000 --- a/benchmarks/fragments/s3lat.py +++ /dev/null @@ -1,18 +0,0 @@ -a = """ -echo "sleep 3 min to wait for minio bootstrap" -sleep 180 - -export ENDPOINT=localhost:9000 -export AWS_ACCESS_KEY_ID=minioadmin -export AWS_SECRET_ACCESS_KEY=minioadmin - -mc alias set minio-bench http://$ENDPOINT $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY -for i in $(seq 1 10); do - mc mb minio-bench/bench$i -done - -s3lat | tee 50ms.minio.csv -""" - -def on_garage(): - raise Exception("Not yet implemented") diff --git a/benchmarks/fragments/shared.py b/benchmarks/fragments/shared.py deleted file mode 100644 index e0cd449..0000000 --- a/benchmarks/fragments/shared.py +++ /dev/null @@ -1,28 +0,0 @@ -import os, time - -binary_path = "/tmp/mknet-bin" -storage_path = "/tmp/mknet-store" - -def exec(s): - if os.system(s) != 0: - raise Exception("Command terminated with an error") -def exec_retry(s, cnt=16): - print(s) - for i in range(cnt): - time.sleep(i) # this is expected to sleep before running the command to reduce the noise - if os.system(s) == 0: return - raise Exception("Command terminated with an error too many times") -def fn_retry(f, cnt=5): - for i in range(cnt): - try: - r = f() - return r - except Exception as e: - if i+1 == cnt: raise e - log(f"failed call, retry in {i} sec") - time.sleep(i) - -def id(): return int(os.environ['ID']) -def count(): return int(os.environ['SERVER_COUNT']) -def log(*args): print(f"[{id()}/{count()} - {os.environ['HOST']}]", *args) - diff --git a/benchmarks/garage-s3lat b/benchmarks/garage-s3lat deleted file mode 100755 index 361ed26..0000000 --- a/benchmarks/garage-s3lat +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env python3 -from fragments import garage, s3lat, shared - -if shared.id() == 1: - garage.deploy_coord() - s3lat.on_garage() - garage.delete_key() - garage.destroy() -else: - garage.deploy_follow() - garage.sync_on_key_down() - garage.destroy() diff --git a/benchmarks/requirements.txt b/benchmarks/requirements.txt deleted file mode 100644 index 41a5912..0000000 --- a/benchmarks/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -git+https://git.deuxfleurs.fr/quentin/garage-admin-sdk@7b1c1faf7a#egg=garage-admin-sdk&subdirectory=python diff --git a/benchmarks/s3concurrent/go.mod b/benchmarks/s3concurrent/go.mod new file mode 100644 index 0000000..d3b6113 --- /dev/null +++ b/benchmarks/s3concurrent/go.mod @@ -0,0 +1,24 @@ +module git.deuxfleurs.fr/quentin/s3concurrent + +go 1.18 + +require github.com/minio/minio-go/v7 v7.0.34 + +require ( + github.com/dustin/go-humanize v1.0.0 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.15.9 // indirect + github.com/klauspost/cpuid/v2 v2.1.0 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/sha256-simd v1.0.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/rs/xid v1.4.0 // indirect + github.com/sirupsen/logrus v1.9.0 // indirect + golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect + golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect + golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect + golang.org/x/text v0.3.7 // indirect + gopkg.in/ini.v1 v1.66.6 // indirect +) diff --git a/benchmarks/s3concurrent/go.sum b/benchmarks/s3concurrent/go.sum new file mode 100644 index 0000000..598b838 --- /dev/null +++ b/benchmarks/s3concurrent/go.sum @@ -0,0 +1,52 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.1.0 h1:eyi1Ad2aNJMW95zcSbmGg7Cg6cq3ADwLpMAP96d8rF0= +github.com/klauspost/cpuid/v2 v2.1.0/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.34 h1:JMfS5fudx1mN6V2MMNyCJ7UMrjEzZzIvMgfkWc1Vnjk= +github.com/minio/minio-go/v7 v7.0.34/go.mod h1:nCrRzjoSUQh8hgKKtu3Y708OLvRLtuASMg2/nvmbarw= +github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= +github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY= +github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.66.6 h1:LATuAqN/shcYAOkv3wl2L4rkaKqkcgTBQjOyYDvcPKI= +gopkg.in/ini.v1 v1.66.6/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/benchmarks/s3concurrent/main.go b/benchmarks/s3concurrent/main.go new file mode 100644 index 0000000..a67fc5b --- /dev/null +++ b/benchmarks/s3concurrent/main.go @@ -0,0 +1,121 @@ +package main + +import ( + "context" + "crypto/tls" + "io" + "log" + "math/rand" + "net/http" + "os" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/google/uuid" +) + +func buildMc() (*minio.Client, error) { + _, isSSL := os.LookupEnv("SSL") + opts := minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv("AWS_ACCESS_KEY_ID"), os.Getenv("AWS_SECRET_ACCESS_KEY"), ""), + Secure: isSSL, + } + + if region, ok := os.LookupEnv("REGION"); ok { + opts.Region = region + } + + if _, ok := os.LookupEnv("SSL_INSECURE"); ok { + opts.Transport = &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} + } + + mc, err := minio.New(os.Getenv("ENDPOINT"), &opts) + return mc, err +} + +type PRNG struct { + rem int64 +} +func (r *PRNG) Read(p []byte) (n int, err error) { + //log.Printf("rem=%d, buf=%d\n", r.rem, len(p)) + if int64(len(p)) > r.rem { + p = p[:r.rem] + } + + if int64(len(p)) > r.rem { + log.Fatal("LOGIC ERROR") + } + + n, err = rand.Read(p) + if err != nil { + return + } + r.rem -= int64(n) + if r.rem <= 0 { + err = io.EOF + //log.Printf("PRNG file has been fully read. rem=%d,n=%d,err=%s\n", r.rem, n, err) + } + return +} + +func putObj(buck string, size int64) error { + mc, err := buildMc() + if err != nil { + return err + } + + prng := new(PRNG) + prng.rem = size + + key := uuid.New().String() + + _, err = mc.PutObject( + context.Background(), + buck, + key, + prng, + size, + minio.PutObjectOptions{ContentType:"application/octet-stream"}, + ) + + return err +} + +func main() { + minio.MaxRetry = 1 + mc, err := buildMc() + if err != nil { + log.Fatal("failed connect", err) + return + } + + // Create Bucket + buck := uuid.New().String() + err = mc.MakeBucket(context.Background(), buck, minio.MakeBucketOptions{ }) + if err != nil { + log.Fatal(err) + return + } + log.Printf("created bucket %s\n", buck) + + // Send to bucket + for i := 1; i <= 16; i++ { + log.Printf("start concurrent loop with %d coroutines\n", i) + syn := make(chan error) + for j := 1; j <= i; j++ { + go func() { + syn <- putObj(buck, 1024 * 1024) + }() + } + + for j := 1; j <= i; j++ { + cerr := <-syn + if cerr != nil { + log.Printf("%d/%d failed with %s\n", j, i, cerr) + } + } + log.Printf("done, %d coroutines returned\n", i) + } + + log.Println("done") +} diff --git a/benchmarks/s3lat/README.md b/benchmarks/s3lat/README.md new file mode 100644 index 0000000..95936bc --- /dev/null +++ b/benchmarks/s3lat/README.md @@ -0,0 +1,22 @@ +# s3lat + +## installation + +```bash +go get git.deuxfleurs.fr/quentin/s3lat@latest +``` + +## usage + +```bash +export ENDPOINT=[fc00:9a7a:9e::1]:9000 +export AWS_ACCESS_KEY_ID=minioadmin +export AWS_SECRET_ACCESS_KEY=minioadmin + +s3lat +``` + +## see also + + - https://git.deuxfleurs.fr/quentin/benchmarks + - https://git.deuxfleurs.fr/trinity-1686a/mknet diff --git a/benchmarks/s3lat/go.mod b/benchmarks/s3lat/go.mod new file mode 100644 index 0000000..627a83a --- /dev/null +++ b/benchmarks/s3lat/go.mod @@ -0,0 +1,8 @@ +module git.deuxfleurs.fr/quentin/s3lat + +go 1.16 + +require ( + github.com/google/uuid v1.1.1 + github.com/minio/minio-go/v7 v7.0.16 +) diff --git a/benchmarks/s3lat/go.sum b/benchmarks/s3lat/go.sum new file mode 100644 index 0000000..207d6f9 --- /dev/null +++ b/benchmarks/s3lat/go.sum @@ -0,0 +1,78 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/klauspost/compress v1.13.5 h1:9O69jUPDcsT9fEm74W92rZL9FQY7rCdaXVneq+yyzl4= +github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= +github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= +github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= +github.com/minio/minio-go/v7 v7.0.16 h1:GspaSBS8lOuEUCAqMe0W3UxSoyOA4b4F8PTspRVI+k4= +github.com/minio/minio-go/v7 v7.0.16/go.mod h1:pUV0Pc+hPd1nccgmzQF/EXh48l/Z/yps6QPF1aaie4g= +github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= +github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f h1:aZp0e2vLN4MToVqnjNEYEtrEA8RH8U8FN1CU7JgqsPU= +golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= +gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/benchmarks/s3lat/main.go b/benchmarks/s3lat/main.go new file mode 100644 index 0000000..32b03cf --- /dev/null +++ b/benchmarks/s3lat/main.go @@ -0,0 +1,135 @@ +package main + +import ( + "context" + "log" + "os" + "fmt" + "time" + "io" + "io/ioutil" + "strings" + "net/http" + "crypto/tls" + "strconv" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/google/uuid" +) + +func main() { + fmt.Printf("endpoint,nanoseconds\n") + + // Initial setup + _, isSSL := os.LookupEnv("SSL"); + opts := minio.Options { + Creds: credentials.NewStaticV4(os.Getenv("AWS_ACCESS_KEY_ID"), os.Getenv("AWS_SECRET_ACCESS_KEY"), ""), + Secure: isSSL, + } + + if region, ok := os.LookupEnv("REGION"); ok { + opts.Region = region + } + + if _, ok := os.LookupEnv("SSL_INSECURE"); ok { + opts.Transport = &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} + } + + mc, err := minio.New(os.Getenv("ENDPOINT"), &opts) + + if err != nil { + log.Fatal("failed connect", err) + return + } + + // Create Bucket + buck := uuid.New().String() + err = mc.MakeBucket(context.Background(), buck, minio.MakeBucketOptions{ }) + if err != nil { + log.Fatal(err) + return + } + + // List Buckets + for i := 0; i < 100; i++ { + start := time.Now() + _, err := mc.ListBuckets(context.Background()) + elapsed := time.Since(start) + if err != nil { + log.Fatal("failed listbucket: ", err) + return + } + fmt.Printf("listbuckets,%v\n", elapsed.Nanoseconds()) + } + + // PutObject + for i := 0; i < 100; i++ { + istr := strconv.Itoa(i) + content := istr + " hello world " + istr + start := time.Now() + _, err := mc.PutObject(context.Background(), buck, "element"+istr, strings.NewReader(content), int64(len(content)), minio.PutObjectOptions{ContentType:"application/octet-stream"}) + elapsed := time.Since(start) + if err != nil { + log.Fatal("failed putObject: ",err) + return + } + fmt.Printf("putobject,%v\n", elapsed.Nanoseconds()) + } + + // ListObject + for i := 0; i < 100; i++ { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + start := time.Now() + objectCh := mc.ListObjects(ctx, buck, minio.ListObjectsOptions{ + Recursive: true, + }) + for object := range objectCh { + if object.Err != nil { + log.Fatal(object.Err) + return + } + } + elapsed := time.Since(start) + fmt.Printf("listobjects,%v\n", elapsed.Nanoseconds()) + } + + // GetObject + for i := 0; i < 100; i++ { + istr := strconv.Itoa(i) + start := time.Now() + object, err := mc.GetObject(context.Background(), buck, "element"+istr, minio.GetObjectOptions{}) + if err != nil { + log.Fatal(err) + return + } + if _, err = io.Copy(ioutil.Discard, object) ; err != nil { + log.Fatal("failed getobject: ", err) + return + } + elapsed := time.Since(start) + fmt.Printf("getobject,%v\n", elapsed.Nanoseconds()) + } + + // RemoveObject + for i := 0; i < 100; i++ { + istr := strconv.Itoa(i) + start := time.Now() + err = mc.RemoveObject(context.Background(), buck, "element"+istr, minio.RemoveObjectOptions{}) + elapsed := time.Since(start) + if err != nil { + log.Fatal(err) + return + } + fmt.Printf("removeobject,%v\n", elapsed.Nanoseconds()) + } + + // RemoveBucket + err = mc.RemoveBucket(context.Background(), buck) + if err != nil { + log.Fatal(err) + return + } +} diff --git a/benchmarks/s3lat/s3lat b/benchmarks/s3lat/s3lat new file mode 100755 index 0000000..8b1f095 Binary files /dev/null and b/benchmarks/s3lat/s3lat differ diff --git a/benchmarks/warp b/benchmarks/warp new file mode 160000 index 0000000..df8f2cd --- /dev/null +++ b/benchmarks/warp @@ -0,0 +1 @@ +Subproject commit df8f2cd7b7a8523755b4d3c04271a69571a455ca diff --git a/scenarios/.garage-s3lat.swo b/scenarios/.garage-s3lat.swo new file mode 100644 index 0000000..ec47fd0 Binary files /dev/null and b/scenarios/.garage-s3lat.swo differ diff --git a/scenarios/clean b/scenarios/clean new file mode 100755 index 0000000..325edec --- /dev/null +++ b/scenarios/clean @@ -0,0 +1,8 @@ +#!/usr/bin/env python3 + +import os +from fragments import garage, shared + +garage.destroy() +shared.log("clean done") + diff --git a/scenarios/fragments/.shared.py.swo b/scenarios/fragments/.shared.py.swo new file mode 100644 index 0000000..02b9acb Binary files /dev/null and b/scenarios/fragments/.shared.py.swo differ diff --git a/scenarios/fragments/garage.py b/scenarios/fragments/garage.py new file mode 100644 index 0000000..cd8b888 --- /dev/null +++ b/scenarios/fragments/garage.py @@ -0,0 +1,201 @@ +import glob, json, requests, time, garage_admin_sdk +from os.path import exists +from os import environ as env +from pathlib import Path +from fragments import shared +from garage_admin_sdk.api import nodes_api, layout_api, key_api +from garage_admin_sdk.model.node_cluster_info import NodeClusterInfo +from garage_admin_sdk.model.layout_version import LayoutVersion +from garage_admin_sdk.model.add_key_request import AddKeyRequest +from garage_admin_sdk.model.update_key_request import UpdateKeyRequest +from garage_admin_sdk.model.update_key_request_allow import UpdateKeyRequestAllow + +storage_path = "./i/am/not/defined" +rpc_secret = "3e9abff5f9e480afbadb46a77b7a26fe0e404258f0dc3fd5386b0ba8e0ad2fba" +metrics = "cacce0b2de4bc2d9f5b5fdff551e01ac1496055aed248202d415398987e35f81" +admin = "ae8cb40ea7368bbdbb6430af11cca7da833d3458a5f52086f4e805a570fb5c2a" +path = None +key = None + + +configuration = garage_admin_sdk.Configuration( + host = "http://localhost:3903/v0", + access_token = admin +) +api = garage_admin_sdk.ApiClient(configuration) +nodes = nodes_api.NodesApi(api) +layout = layout_api.LayoutApi(api) +keys = key_api.KeyApi(api) + + +# Setup, launch on import +storage_path = Path(shared.storage_path) / "garage" / env['HOST'] +if 'ZONE' in env: + storage_path = Path(shared.storage_path) / "garage" / env['ZONE'] / env['HOST'] +config = storage_path / "garage.toml" +env['GARAGE_CONFIG_FILE'] = str(config) + +def deploy_coord(version=None, target=None): + destroy() + from_ci(version, target) + shared.log("start daemon") + daemon() + shared.log("discover nodes") + connect() + shared.log("build layout") + create_layout() + shared.log("create key") + create_key() + shared.log("ready") + +def deploy_follow(version=None, target=None): + destroy() + from_ci(version, target) + shared.log("start daemon") + daemon() + shared.log("wait for coord") + sync_on_key_up() + shared.log("ready") + +def from_local(p): + global path + path = p + shared.exec(f"{p} --version") + +def from_ci(version=None, target=None): + global path + version = version or "v0.7.3" + target = target or "x86_64-unknown-linux-musl" + + binary = f"garage-{target}-{version}" + path = Path(shared.binary_path) / binary + if shared.id() != 1: return + + if not exists(path): + shared.exec(f"mkdir -p {shared.binary_path}") + shared.exec(f"wget https://garagehq.deuxfleurs.fr/_releases/{version}/{target}/garage -O {path}") + shared.exec(f"chmod +x {path}") + shared.exec(f"{path} --version") + +def daemon(): + shared.exec(f"mkdir -p {storage_path}") + with open(config, 'w+') as f: + f.write(f""" +metadata_dir = "{storage_path}/meta" +data_dir = "{storage_path}/data" + +replication_mode = "3" + +rpc_bind_addr = "[::]:3901" +rpc_public_addr = "[{env['IP']}]:3901" +rpc_secret = "{rpc_secret}" + +bootstrap_peers=[] + +[s3_api] +s3_region = "garage" +api_bind_addr = "[::]:3900" +root_domain = ".s3.garage" + +[s3_web] +bind_addr = "[::]:3902" +root_domain = ".web.garage" +index = "index.html" + +[admin] +api_bind_addr = "0.0.0.0:3903" +metrics_token = "{metrics}" +admin_token = "{admin}" + """) + + shared.exec(f"{path} server 2>> {storage_path}/logs.stderr 1>> {storage_path}/logs.stdout & echo $! > {storage_path}/daemon.pid") + time.sleep(1) + + node_info = storage_path / "node_info" + node_id = shared.fn_retry(lambda: nodes.get_nodes().node) + with open(node_info, 'w+') as f: + f.write(json.dumps({ + "node_addr": f"{node_id}@{env['IP']}:3901", + "node_id": node_id, + "zone": env['ZONE'], + "host": env['HOST'], + })) + +def destroy(): + dpid = Path(storage_path) / "daemon.pid" + if exists(dpid): + shared.exec(f"kill -9 $(cat {dpid})") + shared.exec(f"rm -f {dpid}") + if len(str(storage_path)) < 8: # arbitrary, stupid safe guard + print(storage_path) + raise Exception("You tried to clean a storage path that might be the root of your FS, panicking...") + shared.exec(f"rm -fr {storage_path}") + +# this function is ugly, sorry :s +_cluster_info = None +def cluster_info(): + global _cluster_info + if _cluster_info is not None: return _cluster_info + + while True: + time.sleep(1) + node_files = glob.glob(f"{shared.storage_path}/**/node_info", recursive=True) + if len(node_files) == shared.count(): break + + _cluster_info = [ json.loads(Path(f).read_text()) for f in node_files ] + return _cluster_info + + +def connect(): + cinf = cluster_info() + ret = nodes.add_node([n['node_addr'] for n in cinf]) + for st in ret: + if not st.success: + raise Exception("Node connect failed", ret) + shared.log("all nodes connected") + +def create_layout(): + v = layout.get_layout().version + + cinf = cluster_info() + nlay = dict() + for n in cinf: + nlay[n['node_id']] = NodeClusterInfo( + zone = n['zone'], + capacity = 1, + tags = [ n['host'] ], + ) + layout.add_layout(nlay) + layout.apply_layout(LayoutVersion(version=v+1)) + +def create_key(): + global key + kinfo = shared.fn_retry(lambda: keys.add_key(AddKeyRequest(name="mknet"))) + allow_create = UpdateKeyRequestAllow(create_bucket=True) + keys.update_key(kinfo.access_key_id, UpdateKeyRequest(allow=allow_create)) + key = kinfo + + +def delete_key(): + global key + keys.delete_key(key.access_key_id) + key = None + +def sync_on_key_up(): + global key + while True: + try: + key = keys.search_key("mknet") + return key + except: + pass + time.sleep(1) + +def sync_on_key_down(): + while True: + try: + keys.search_key("mknet") + except: + return + time.sleep(1) + diff --git a/scenarios/fragments/minio.py b/scenarios/fragments/minio.py new file mode 100644 index 0000000..431b983 --- /dev/null +++ b/scenarios/fragments/minio.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +import json, os, sys, time, pathlib, socket, shutil + +STORAGE_PATH = os.path.join(os.getcwd(), '.minio-testnet') +HOSTS_PATH = os.path.join(STORAGE_PATH, 'hosts.txt') +UNIX_SOCK = os.path.join(STORAGE_PATH, 'deploy.sock') +DATA_PATH = lambda nid: os.path.join(STORAGE_PATH, 'data'+str(nid)) + +def main(): + if int(os.environ['ID']) == 1: leader() + else: follower() + +def leader(): + shutil.rmtree(STORAGE_PATH, ignore_errors=True) + os.makedirs(STORAGE_PATH) + print(STORAGE_PATH) + + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.bind(UNIX_SOCK) + sock.listen() + + n_serv = int(os.environ['SERVER_COUNT']) + fl = [ co for co, addr in [ sock.accept() for i in range(n_serv - 1) ]] + + identities = [ json.loads(co.makefile().readline()) for co in fl ] + [ { "ip": os.environ['IP'], "path": make_data() } ] + print(f"ident: {identities}") + msg = f"{json.dumps(identities)}\n".encode() + [ co.send(msg) for co in fl ] + + run_minio(identities) + +def follower(): + co = None + while True: + time.sleep(1) + try: + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.connect(UNIX_SOCK) + co = sock.makefile() + break + except Exception as err: + print('conn failed, wait,', err) + my_identity = json.dumps({ "ip": os.environ['IP'], "path": make_data() }) + sock.send(f"{my_identity}\n".encode()) + identities = json.loads(co.readline()) + + run_minio(identities) + +def make_data(): + data_path = DATA_PATH(os.environ['ID']) + os.makedirs(data_path) + return data_path + +def run_minio(identities): + cmd = f"minio server --console-address ':9001' --address ':9000'" + for ident in identities: + cmd += f" http://[{ident['ip']}]:9000{ident['path']}" + cmd += f" > {os.path.join(STORAGE_PATH, 'minio'+os.environ['ID']+'.log')} 2>&1" + print("launch: ", cmd) + os.system(cmd) + +__name__ == '__main__' and main() diff --git a/scenarios/fragments/s3lat.py b/scenarios/fragments/s3lat.py new file mode 100644 index 0000000..7582350 --- /dev/null +++ b/scenarios/fragments/s3lat.py @@ -0,0 +1,16 @@ +import os +from os.path import exists +from pathlib import Path +from fragments import shared, garage + +s3bin = Path(os.path.dirname(__file__)) / "../../benchmarks/s3lat/s3lat" + +def on_garage(): + os.environ['AWS_ACCESS_KEY_ID'] = garage.key.access_key_id + os.environ['AWS_SECRET_ACCESS_KEY'] = garage.key.secret_access_key + os.environ['ENDPOINT'] = "localhost:3900" + + out = Path(shared.storage_path) / "s3lat.csv" + shared.log(f"launching s3lat ({s3bin})") + shared.exec(f"{s3bin} > {out}") + shared.log(f"execution done, output written to {out}") diff --git a/scenarios/fragments/shared.py b/scenarios/fragments/shared.py new file mode 100644 index 0000000..e0cd449 --- /dev/null +++ b/scenarios/fragments/shared.py @@ -0,0 +1,28 @@ +import os, time + +binary_path = "/tmp/mknet-bin" +storage_path = "/tmp/mknet-store" + +def exec(s): + if os.system(s) != 0: + raise Exception("Command terminated with an error") +def exec_retry(s, cnt=16): + print(s) + for i in range(cnt): + time.sleep(i) # this is expected to sleep before running the command to reduce the noise + if os.system(s) == 0: return + raise Exception("Command terminated with an error too many times") +def fn_retry(f, cnt=5): + for i in range(cnt): + try: + r = f() + return r + except Exception as e: + if i+1 == cnt: raise e + log(f"failed call, retry in {i} sec") + time.sleep(i) + +def id(): return int(os.environ['ID']) +def count(): return int(os.environ['SERVER_COUNT']) +def log(*args): print(f"[{id()}/{count()} - {os.environ['HOST']}]", *args) + diff --git a/scenarios/garage-s3lat b/scenarios/garage-s3lat new file mode 100755 index 0000000..c9dd695 --- /dev/null +++ b/scenarios/garage-s3lat @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +from fragments import garage, s3lat, shared + +if shared.id() == 1: + garage.deploy_coord() + s3lat.on_garage() + garage.delete_key() + garage.destroy() +else: + garage.deploy_follow() + garage.sync_on_key_down() + garage.destroy() +shared.log("bye") diff --git a/scenarios/requirements.txt b/scenarios/requirements.txt new file mode 100644 index 0000000..41a5912 --- /dev/null +++ b/scenarios/requirements.txt @@ -0,0 +1 @@ +git+https://git.deuxfleurs.fr/quentin/garage-admin-sdk@7b1c1faf7a#egg=garage-admin-sdk&subdirectory=python -- cgit v1.2.3