aboutsummaryrefslogtreecommitdiff
path: root/benchmarks
diff options
context:
space:
mode:
authorQuentin Dufour <quentin@deuxfleurs.fr>2022-09-14 18:01:44 +0200
committerQuentin Dufour <quentin@deuxfleurs.fr>2022-09-14 18:01:44 +0200
commitc77d8dcfa7333b32e07df7326b8943838f34a59b (patch)
tree7ef87c7597e9f6d3da345d407cad2f1eb3908149 /benchmarks
parente3409ce6b751fa92a98000ca4a2225b2ed10fdee (diff)
downloadmknet-c77d8dcfa7333b32e07df7326b8943838f34a59b.tar.gz
mknet-c77d8dcfa7333b32e07df7326b8943838f34a59b.zip
Working s3lat with Garage
Diffstat (limited to 'benchmarks')
-rwxr-xr-xbenchmarks/clean8
-rw-r--r--benchmarks/fragments/.shared.py.swobin12288 -> 0 bytes
-rw-r--r--benchmarks/fragments/garage.py202
-rw-r--r--benchmarks/fragments/minio.py62
-rw-r--r--benchmarks/fragments/s3lat.py18
-rw-r--r--benchmarks/fragments/shared.py28
-rwxr-xr-xbenchmarks/garage-s3lat12
-rw-r--r--benchmarks/requirements.txt1
-rw-r--r--benchmarks/s3concurrent/go.mod24
-rw-r--r--benchmarks/s3concurrent/go.sum52
-rw-r--r--benchmarks/s3concurrent/main.go121
-rw-r--r--benchmarks/s3lat/README.md22
-rw-r--r--benchmarks/s3lat/go.mod8
-rw-r--r--benchmarks/s3lat/go.sum78
-rw-r--r--benchmarks/s3lat/main.go135
-rwxr-xr-xbenchmarks/s3lat/s3latbin0 -> 8600112 bytes
m---------benchmarks/warp0
17 files changed, 440 insertions, 331 deletions
diff --git a/benchmarks/clean b/benchmarks/clean
deleted file mode 100755
index 325edec..0000000
--- a/benchmarks/clean
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-from fragments import garage, shared
-
-garage.destroy()
-shared.log("clean done")
-
diff --git a/benchmarks/fragments/.shared.py.swo b/benchmarks/fragments/.shared.py.swo
deleted file mode 100644
index 02b9acb..0000000
--- a/benchmarks/fragments/.shared.py.swo
+++ /dev/null
Binary files differ
diff --git a/benchmarks/fragments/garage.py b/benchmarks/fragments/garage.py
deleted file mode 100644
index ac77475..0000000
--- a/benchmarks/fragments/garage.py
+++ /dev/null
@@ -1,202 +0,0 @@
-import glob, json, requests, time, garage_admin_sdk
-from os.path import exists
-from os import environ as env
-from pathlib import Path
-from fragments import shared
-from garage_admin_sdk.api import nodes_api, layout_api, key_api
-from garage_admin_sdk.model.node_cluster_info import NodeClusterInfo
-from garage_admin_sdk.model.layout_version import LayoutVersion
-from garage_admin_sdk.model.add_key_request import AddKeyRequest
-from garage_admin_sdk.model.update_key_request import UpdateKeyRequest
-from garage_admin_sdk.model.update_key_request_allow import UpdateKeyRequestAllow
-
-storage_path = "./i/am/not/defined"
-rpc_secret = "3e9abff5f9e480afbadb46a77b7a26fe0e404258f0dc3fd5386b0ba8e0ad2fba"
-metrics = "cacce0b2de4bc2d9f5b5fdff551e01ac1496055aed248202d415398987e35f81"
-admin = "ae8cb40ea7368bbdbb6430af11cca7da833d3458a5f52086f4e805a570fb5c2a"
-path = None
-access_key = None
-secret_key = None
-
-
-configuration = garage_admin_sdk.Configuration(
- host = "http://localhost:3903/v0",
- access_token = admin
-)
-api = garage_admin_sdk.ApiClient(configuration)
-nodes = nodes_api.NodesApi(api)
-layout = layout_api.LayoutApi(api)
-keys = key_api.KeyApi(api)
-
-
-# Setup, launch on import
-storage_path = Path(shared.storage_path) / "garage" / env['HOST']
-if 'ZONE' in env:
- storage_path = Path(shared.storage_path) / "garage" / env['ZONE'] / env['HOST']
-config = storage_path / "garage.toml"
-env['GARAGE_CONFIG_FILE'] = str(config)
-
-def deploy_coord(version=None, target=None):
- destroy()
- from_ci(version, target)
- shared.log("start daemon")
- daemon()
- shared.log("discover nodes")
- connect()
- shared.log("build layout")
- create_layout()
- shared.log("create key")
- create_key()
- shared.log("ready")
-
-def deploy_follow(version=None, target=None):
- destroy()
- from_ci(version, target)
- shared.log("start daemon")
- daemon()
- shared.log("wait for coord")
- sync_on_key_up()
- shared.log("ready")
-
-def from_local(p):
- global path
- path = p
- shared.exec(f"{p} --version")
-
-def from_ci(version=None, target=None):
- global path
- version = version or "v0.7.3"
- target = target or "x86_64-unknown-linux-musl"
-
- binary = f"garage-{target}-{version}"
- path = Path(shared.binary_path) / binary
- if shared.id() != 1: return
-
- if not exists(path):
- shared.exec(f"mkdir -p {shared.binary_path}")
- shared.exec(f"wget https://garagehq.deuxfleurs.fr/_releases/{version}/{target}/garage -O {path}")
- shared.exec(f"chmod +x {path}")
- shared.exec(f"{path} --version")
-
-def daemon():
- shared.exec(f"mkdir -p {storage_path}")
- with open(config, 'w+') as f:
- f.write(f"""
-metadata_dir = "{storage_path}/meta"
-data_dir = "{storage_path}/data"
-
-replication_mode = "3"
-
-rpc_bind_addr = "[::]:3901"
-rpc_public_addr = "[{env['IP']}]:3901"
-rpc_secret = "{rpc_secret}"
-
-bootstrap_peers=[]
-
-[s3_api]
-s3_region = "garage"
-api_bind_addr = "[::]:3900"
-root_domain = ".s3.garage"
-
-[s3_web]
-bind_addr = "[::]:3902"
-root_domain = ".web.garage"
-index = "index.html"
-
-[admin]
-api_bind_addr = "0.0.0.0:3903"
-metrics_token = "{metrics}"
-admin_token = "{admin}"
- """)
-
- shared.exec(f"{path} server 2>> {storage_path}/logs.stderr 1>> {storage_path}/logs.stdout & echo $! > {storage_path}/daemon.pid")
- time.sleep(1)
-
- node_info = storage_path / "node_info"
- node_id = nodes.get_nodes().node
- with open(node_info, 'w+') as f:
- f.write(json.dumps({
- "node_addr": f"{node_id}@{env['IP']}:3901",
- "node_id": node_id,
- "zone": env['ZONE'],
- "host": env['HOST'],
- }))
-
-def destroy():
- dpid = Path(storage_path) / "daemon.pid"
- if exists(dpid):
- shared.exec(f"kill -9 $(cat {dpid})")
- shared.exec(f"rm -f {dpid}")
- if len(str(storage_path)) < 8: # arbitrary, stupid safe guard
- print(storage_path)
- raise Exception("You tried to clean a storage path that might be the root of your FS, panicking...")
- shared.exec(f"rm -fr {storage_path}")
-
-# this function is ugly, sorry :s
-_cluster_info = None
-def cluster_info():
- global _cluster_info
- if _cluster_info is not None: return _cluster_info
-
- while True:
- time.sleep(1)
- node_files = glob.glob(f"{shared.storage_path}/**/node_info", recursive=True)
- if len(node_files) == shared.count(): break
-
- _cluster_info = [ json.loads(Path(f).read_text()) for f in node_files ]
- return _cluster_info
-
-
-def connect():
- cinf = cluster_info()
- ret = nodes.add_node([n['node_addr'] for n in cinf])
- for st in ret:
- if not st.success:
- raise Exception("Node connect failed", ret)
- shared.log("all nodes connected")
-
-def create_layout():
- v = layout.get_layout().version
-
- cinf = cluster_info()
- nlay = dict()
- for n in cinf:
- nlay[n['node_id']] = NodeClusterInfo(
- zone = n['zone'],
- capacity = 1,
- tags = [ n['host'] ],
- )
- layout.add_layout(nlay)
- layout.apply_layout(LayoutVersion(version=v+1))
-
-def create_key():
- global key
- kinfo = shared.fn_retry(lambda: keys.add_key(AddKeyRequest(name="mknet")))
- allow_create = UpdateKeyRequestAllow(create_bucket=True)
- keys.update_key(kinfo.access_key_id, UpdateKeyRequest(allow=allow_create))
- key = kinfo
-
-
-def delete_key():
- global key
- delete_key(key.access_key_id)
- key = None
-
-def sync_on_key_up():
- global key
- while True:
- try:
- key = keys.search_key("mknet")
- return key
- except:
- pass
- time.sleep(1)
-
-def sync_on_key_down():
- while True:
- try:
- keys.search_key("mknet")
- except:
- return
- time.sleep(1)
-
diff --git a/benchmarks/fragments/minio.py b/benchmarks/fragments/minio.py
deleted file mode 100644
index 431b983..0000000
--- a/benchmarks/fragments/minio.py
+++ /dev/null
@@ -1,62 +0,0 @@
-#!/usr/bin/env python3
-import json, os, sys, time, pathlib, socket, shutil
-
-STORAGE_PATH = os.path.join(os.getcwd(), '.minio-testnet')
-HOSTS_PATH = os.path.join(STORAGE_PATH, 'hosts.txt')
-UNIX_SOCK = os.path.join(STORAGE_PATH, 'deploy.sock')
-DATA_PATH = lambda nid: os.path.join(STORAGE_PATH, 'data'+str(nid))
-
-def main():
- if int(os.environ['ID']) == 1: leader()
- else: follower()
-
-def leader():
- shutil.rmtree(STORAGE_PATH, ignore_errors=True)
- os.makedirs(STORAGE_PATH)
- print(STORAGE_PATH)
-
- sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
- sock.bind(UNIX_SOCK)
- sock.listen()
-
- n_serv = int(os.environ['SERVER_COUNT'])
- fl = [ co for co, addr in [ sock.accept() for i in range(n_serv - 1) ]]
-
- identities = [ json.loads(co.makefile().readline()) for co in fl ] + [ { "ip": os.environ['IP'], "path": make_data() } ]
- print(f"ident: {identities}")
- msg = f"{json.dumps(identities)}\n".encode()
- [ co.send(msg) for co in fl ]
-
- run_minio(identities)
-
-def follower():
- co = None
- while True:
- time.sleep(1)
- try:
- sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
- sock.connect(UNIX_SOCK)
- co = sock.makefile()
- break
- except Exception as err:
- print('conn failed, wait,', err)
- my_identity = json.dumps({ "ip": os.environ['IP'], "path": make_data() })
- sock.send(f"{my_identity}\n".encode())
- identities = json.loads(co.readline())
-
- run_minio(identities)
-
-def make_data():
- data_path = DATA_PATH(os.environ['ID'])
- os.makedirs(data_path)
- return data_path
-
-def run_minio(identities):
- cmd = f"minio server --console-address ':9001' --address ':9000'"
- for ident in identities:
- cmd += f" http://[{ident['ip']}]:9000{ident['path']}"
- cmd += f" > {os.path.join(STORAGE_PATH, 'minio'+os.environ['ID']+'.log')} 2>&1"
- print("launch: ", cmd)
- os.system(cmd)
-
-__name__ == '__main__' and main()
diff --git a/benchmarks/fragments/s3lat.py b/benchmarks/fragments/s3lat.py
deleted file mode 100644
index c25594e..0000000
--- a/benchmarks/fragments/s3lat.py
+++ /dev/null
@@ -1,18 +0,0 @@
-a = """
-echo "sleep 3 min to wait for minio bootstrap"
-sleep 180
-
-export ENDPOINT=localhost:9000
-export AWS_ACCESS_KEY_ID=minioadmin
-export AWS_SECRET_ACCESS_KEY=minioadmin
-
-mc alias set minio-bench http://$ENDPOINT $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY
-for i in $(seq 1 10); do
- mc mb minio-bench/bench$i
-done
-
-s3lat | tee 50ms.minio.csv
-"""
-
-def on_garage():
- raise Exception("Not yet implemented")
diff --git a/benchmarks/fragments/shared.py b/benchmarks/fragments/shared.py
deleted file mode 100644
index e0cd449..0000000
--- a/benchmarks/fragments/shared.py
+++ /dev/null
@@ -1,28 +0,0 @@
-import os, time
-
-binary_path = "/tmp/mknet-bin"
-storage_path = "/tmp/mknet-store"
-
-def exec(s):
- if os.system(s) != 0:
- raise Exception("Command terminated with an error")
-def exec_retry(s, cnt=16):
- print(s)
- for i in range(cnt):
- time.sleep(i) # this is expected to sleep before running the command to reduce the noise
- if os.system(s) == 0: return
- raise Exception("Command terminated with an error too many times")
-def fn_retry(f, cnt=5):
- for i in range(cnt):
- try:
- r = f()
- return r
- except Exception as e:
- if i+1 == cnt: raise e
- log(f"failed call, retry in {i} sec")
- time.sleep(i)
-
-def id(): return int(os.environ['ID'])
-def count(): return int(os.environ['SERVER_COUNT'])
-def log(*args): print(f"[{id()}/{count()} - {os.environ['HOST']}]", *args)
-
diff --git a/benchmarks/garage-s3lat b/benchmarks/garage-s3lat
deleted file mode 100755
index 361ed26..0000000
--- a/benchmarks/garage-s3lat
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/usr/bin/env python3
-from fragments import garage, s3lat, shared
-
-if shared.id() == 1:
- garage.deploy_coord()
- s3lat.on_garage()
- garage.delete_key()
- garage.destroy()
-else:
- garage.deploy_follow()
- garage.sync_on_key_down()
- garage.destroy()
diff --git a/benchmarks/requirements.txt b/benchmarks/requirements.txt
deleted file mode 100644
index 41a5912..0000000
--- a/benchmarks/requirements.txt
+++ /dev/null
@@ -1 +0,0 @@
-git+https://git.deuxfleurs.fr/quentin/garage-admin-sdk@7b1c1faf7a#egg=garage-admin-sdk&subdirectory=python
diff --git a/benchmarks/s3concurrent/go.mod b/benchmarks/s3concurrent/go.mod
new file mode 100644
index 0000000..d3b6113
--- /dev/null
+++ b/benchmarks/s3concurrent/go.mod
@@ -0,0 +1,24 @@
+module git.deuxfleurs.fr/quentin/s3concurrent
+
+go 1.18
+
+require github.com/minio/minio-go/v7 v7.0.34
+
+require (
+ github.com/dustin/go-humanize v1.0.0 // indirect
+ github.com/google/uuid v1.3.0 // indirect
+ github.com/json-iterator/go v1.1.12 // indirect
+ github.com/klauspost/compress v1.15.9 // indirect
+ github.com/klauspost/cpuid/v2 v2.1.0 // indirect
+ github.com/minio/md5-simd v1.1.2 // indirect
+ github.com/minio/sha256-simd v1.0.0 // indirect
+ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+ github.com/modern-go/reflect2 v1.0.2 // indirect
+ github.com/rs/xid v1.4.0 // indirect
+ github.com/sirupsen/logrus v1.9.0 // indirect
+ golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect
+ golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect
+ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect
+ golang.org/x/text v0.3.7 // indirect
+ gopkg.in/ini.v1 v1.66.6 // indirect
+)
diff --git a/benchmarks/s3concurrent/go.sum b/benchmarks/s3concurrent/go.sum
new file mode 100644
index 0000000..598b838
--- /dev/null
+++ b/benchmarks/s3concurrent/go.sum
@@ -0,0 +1,52 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY=
+github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
+github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.1.0 h1:eyi1Ad2aNJMW95zcSbmGg7Cg6cq3ADwLpMAP96d8rF0=
+github.com/klauspost/cpuid/v2 v2.1.0/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
+github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
+github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
+github.com/minio/minio-go/v7 v7.0.34 h1:JMfS5fudx1mN6V2MMNyCJ7UMrjEzZzIvMgfkWc1Vnjk=
+github.com/minio/minio-go/v7 v7.0.34/go.mod h1:nCrRzjoSUQh8hgKKtu3Y708OLvRLtuASMg2/nvmbarw=
+github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
+github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY=
+github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
+github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c=
+golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/ini.v1 v1.66.6 h1:LATuAqN/shcYAOkv3wl2L4rkaKqkcgTBQjOyYDvcPKI=
+gopkg.in/ini.v1 v1.66.6/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/benchmarks/s3concurrent/main.go b/benchmarks/s3concurrent/main.go
new file mode 100644
index 0000000..a67fc5b
--- /dev/null
+++ b/benchmarks/s3concurrent/main.go
@@ -0,0 +1,121 @@
+package main
+
+import (
+ "context"
+ "crypto/tls"
+ "io"
+ "log"
+ "math/rand"
+ "net/http"
+ "os"
+
+ "github.com/minio/minio-go/v7"
+ "github.com/minio/minio-go/v7/pkg/credentials"
+ "github.com/google/uuid"
+)
+
+func buildMc() (*minio.Client, error) {
+ _, isSSL := os.LookupEnv("SSL")
+ opts := minio.Options{
+ Creds: credentials.NewStaticV4(os.Getenv("AWS_ACCESS_KEY_ID"), os.Getenv("AWS_SECRET_ACCESS_KEY"), ""),
+ Secure: isSSL,
+ }
+
+ if region, ok := os.LookupEnv("REGION"); ok {
+ opts.Region = region
+ }
+
+ if _, ok := os.LookupEnv("SSL_INSECURE"); ok {
+ opts.Transport = &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
+ }
+
+ mc, err := minio.New(os.Getenv("ENDPOINT"), &opts)
+ return mc, err
+}
+
+type PRNG struct {
+ rem int64
+}
+func (r *PRNG) Read(p []byte) (n int, err error) {
+ //log.Printf("rem=%d, buf=%d\n", r.rem, len(p))
+ if int64(len(p)) > r.rem {
+ p = p[:r.rem]
+ }
+
+ if int64(len(p)) > r.rem {
+ log.Fatal("LOGIC ERROR")
+ }
+
+ n, err = rand.Read(p)
+ if err != nil {
+ return
+ }
+ r.rem -= int64(n)
+ if r.rem <= 0 {
+ err = io.EOF
+ //log.Printf("PRNG file has been fully read. rem=%d,n=%d,err=%s\n", r.rem, n, err)
+ }
+ return
+}
+
+func putObj(buck string, size int64) error {
+ mc, err := buildMc()
+ if err != nil {
+ return err
+ }
+
+ prng := new(PRNG)
+ prng.rem = size
+
+ key := uuid.New().String()
+
+ _, err = mc.PutObject(
+ context.Background(),
+ buck,
+ key,
+ prng,
+ size,
+ minio.PutObjectOptions{ContentType:"application/octet-stream"},
+ )
+
+ return err
+}
+
+func main() {
+ minio.MaxRetry = 1
+ mc, err := buildMc()
+ if err != nil {
+ log.Fatal("failed connect", err)
+ return
+ }
+
+ // Create Bucket
+ buck := uuid.New().String()
+ err = mc.MakeBucket(context.Background(), buck, minio.MakeBucketOptions{ })
+ if err != nil {
+ log.Fatal(err)
+ return
+ }
+ log.Printf("created bucket %s\n", buck)
+
+ // Send to bucket
+ for i := 1; i <= 16; i++ {
+ log.Printf("start concurrent loop with %d coroutines\n", i)
+ syn := make(chan error)
+ for j := 1; j <= i; j++ {
+ go func() {
+ syn <- putObj(buck, 1024 * 1024)
+ }()
+ }
+
+ for j := 1; j <= i; j++ {
+ cerr := <-syn
+ if cerr != nil {
+ log.Printf("%d/%d failed with %s\n", j, i, cerr)
+ }
+ }
+ log.Printf("done, %d coroutines returned\n", i)
+ }
+
+ log.Println("done")
+}
diff --git a/benchmarks/s3lat/README.md b/benchmarks/s3lat/README.md
new file mode 100644
index 0000000..95936bc
--- /dev/null
+++ b/benchmarks/s3lat/README.md
@@ -0,0 +1,22 @@
+# s3lat
+
+## installation
+
+```bash
+go get git.deuxfleurs.fr/quentin/s3lat@latest
+```
+
+## usage
+
+```bash
+export ENDPOINT=[fc00:9a7a:9e::1]:9000
+export AWS_ACCESS_KEY_ID=minioadmin
+export AWS_SECRET_ACCESS_KEY=minioadmin
+
+s3lat
+```
+
+## see also
+
+ - https://git.deuxfleurs.fr/quentin/benchmarks
+ - https://git.deuxfleurs.fr/trinity-1686a/mknet
diff --git a/benchmarks/s3lat/go.mod b/benchmarks/s3lat/go.mod
new file mode 100644
index 0000000..627a83a
--- /dev/null
+++ b/benchmarks/s3lat/go.mod
@@ -0,0 +1,8 @@
+module git.deuxfleurs.fr/quentin/s3lat
+
+go 1.16
+
+require (
+ github.com/google/uuid v1.1.1
+ github.com/minio/minio-go/v7 v7.0.16
+)
diff --git a/benchmarks/s3lat/go.sum b/benchmarks/s3lat/go.sum
new file mode 100644
index 0000000..207d6f9
--- /dev/null
+++ b/benchmarks/s3lat/go.sum
@@ -0,0 +1,78 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/klauspost/compress v1.13.5 h1:9O69jUPDcsT9fEm74W92rZL9FQY7rCdaXVneq+yyzl4=
+github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=
+github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
+github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
+github.com/minio/minio-go/v7 v7.0.16 h1:GspaSBS8lOuEUCAqMe0W3UxSoyOA4b4F8PTspRVI+k4=
+github.com/minio/minio-go/v7 v7.0.16/go.mod h1:pUV0Pc+hPd1nccgmzQF/EXh48l/Z/yps6QPF1aaie4g=
+github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
+github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
+github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f h1:aZp0e2vLN4MToVqnjNEYEtrEA8RH8U8FN1CU7JgqsPU=
+golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww=
+gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
diff --git a/benchmarks/s3lat/main.go b/benchmarks/s3lat/main.go
new file mode 100644
index 0000000..32b03cf
--- /dev/null
+++ b/benchmarks/s3lat/main.go
@@ -0,0 +1,135 @@
+package main
+
+import (
+ "context"
+ "log"
+ "os"
+ "fmt"
+ "time"
+ "io"
+ "io/ioutil"
+ "strings"
+ "net/http"
+ "crypto/tls"
+ "strconv"
+
+ "github.com/minio/minio-go/v7"
+ "github.com/minio/minio-go/v7/pkg/credentials"
+ "github.com/google/uuid"
+)
+
+func main() {
+ fmt.Printf("endpoint,nanoseconds\n")
+
+ // Initial setup
+ _, isSSL := os.LookupEnv("SSL");
+ opts := minio.Options {
+ Creds: credentials.NewStaticV4(os.Getenv("AWS_ACCESS_KEY_ID"), os.Getenv("AWS_SECRET_ACCESS_KEY"), ""),
+ Secure: isSSL,
+ }
+
+ if region, ok := os.LookupEnv("REGION"); ok {
+ opts.Region = region
+ }
+
+ if _, ok := os.LookupEnv("SSL_INSECURE"); ok {
+ opts.Transport = &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
+ }
+
+ mc, err := minio.New(os.Getenv("ENDPOINT"), &opts)
+
+ if err != nil {
+ log.Fatal("failed connect", err)
+ return
+ }
+
+ // Create Bucket
+ buck := uuid.New().String()
+ err = mc.MakeBucket(context.Background(), buck, minio.MakeBucketOptions{ })
+ if err != nil {
+ log.Fatal(err)
+ return
+ }
+
+ // List Buckets
+ for i := 0; i < 100; i++ {
+ start := time.Now()
+ _, err := mc.ListBuckets(context.Background())
+ elapsed := time.Since(start)
+ if err != nil {
+ log.Fatal("failed listbucket: ", err)
+ return
+ }
+ fmt.Printf("listbuckets,%v\n", elapsed.Nanoseconds())
+ }
+
+ // PutObject
+ for i := 0; i < 100; i++ {
+ istr := strconv.Itoa(i)
+ content := istr + " hello world " + istr
+ start := time.Now()
+ _, err := mc.PutObject(context.Background(), buck, "element"+istr, strings.NewReader(content), int64(len(content)), minio.PutObjectOptions{ContentType:"application/octet-stream"})
+ elapsed := time.Since(start)
+ if err != nil {
+ log.Fatal("failed putObject: ",err)
+ return
+ }
+ fmt.Printf("putobject,%v\n", elapsed.Nanoseconds())
+ }
+
+ // ListObject
+ for i := 0; i < 100; i++ {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ start := time.Now()
+ objectCh := mc.ListObjects(ctx, buck, minio.ListObjectsOptions{
+ Recursive: true,
+ })
+ for object := range objectCh {
+ if object.Err != nil {
+ log.Fatal(object.Err)
+ return
+ }
+ }
+ elapsed := time.Since(start)
+ fmt.Printf("listobjects,%v\n", elapsed.Nanoseconds())
+ }
+
+ // GetObject
+ for i := 0; i < 100; i++ {
+ istr := strconv.Itoa(i)
+ start := time.Now()
+ object, err := mc.GetObject(context.Background(), buck, "element"+istr, minio.GetObjectOptions{})
+ if err != nil {
+ log.Fatal(err)
+ return
+ }
+ if _, err = io.Copy(ioutil.Discard, object) ; err != nil {
+ log.Fatal("failed getobject: ", err)
+ return
+ }
+ elapsed := time.Since(start)
+ fmt.Printf("getobject,%v\n", elapsed.Nanoseconds())
+ }
+
+ // RemoveObject
+ for i := 0; i < 100; i++ {
+ istr := strconv.Itoa(i)
+ start := time.Now()
+ err = mc.RemoveObject(context.Background(), buck, "element"+istr, minio.RemoveObjectOptions{})
+ elapsed := time.Since(start)
+ if err != nil {
+ log.Fatal(err)
+ return
+ }
+ fmt.Printf("removeobject,%v\n", elapsed.Nanoseconds())
+ }
+
+ // RemoveBucket
+ err = mc.RemoveBucket(context.Background(), buck)
+ if err != nil {
+ log.Fatal(err)
+ return
+ }
+}
diff --git a/benchmarks/s3lat/s3lat b/benchmarks/s3lat/s3lat
new file mode 100755
index 0000000..8b1f095
--- /dev/null
+++ b/benchmarks/s3lat/s3lat
Binary files differ
diff --git a/benchmarks/warp b/benchmarks/warp
new file mode 160000
+Subproject df8f2cd7b7a8523755b4d3c04271a69571a455c