Add S3TTFB

16 files changed, 656 insertions, 10 deletions

diff --git a/artifacts/2022-09-23-s3ttfb/garage-v0.7.csv b/artifacts/2022-09-23-s3ttfb/garage-v0.7.csv
new file mode 100644
index 0000000..3b1b2d5
--- /dev/null
+++ b/artifacts/2022-09-23-s3ttfb/garage-v0.7.csv
@@ -0,0 +1,61 @@
+nanoseconds
+2043724295
+1819672397
+1856153241
+1814937942
+1818943177
+1814245475
+1828595614
+1817263829
+1857481054
+1816538404
+1816874169
+1856305403
+1857317361
+1814517488
+1817156893
+1815832311
+1857020274
+1822556545
+1816402439
+1815664395
+1818129711
+1855229810
+1817427508
+1817253968
+1855458740
+1818412087
+1816422102
+1814017661
+1821773596
+1814924392
+1819138108
+1814140079
+1814253338
+1814717178
+1823390172
+1856664915
+1817886566
+1815441938
+1815381667
+1864383216
+1815026715
+1816346397
+1814835227
+1855973303
+1814632968
+1823675273
+1817900846
+1816282804
+1817289580
+1817288129
+1860637032
+1855146585
+1816649086
+1822524427
+1814277922
+1901037764
+1818903030
+1815404774
+1815446844
+1815223628
diff --git a/artifacts/2022-09-23-s3ttfb/garage-v0.8.csv b/artifacts/2022-09-23-s3ttfb/garage-v0.8.csv
new file mode 100644
index 0000000..142184b
--- /dev/null
+++ b/artifacts/2022-09-23-s3ttfb/garage-v0.8.csv
@@ -0,0 +1,61 @@
+nanoseconds
+376764319
+268350943
+310303481
+266580460
+267172958
+46771899
+271623330
+270463024
+47401936
+45384570
+129505789
+46244307
+270594700
+267908944
+310163759
+271096844
+268930686
+309894956
+273312715
+269766963
+267947351
+270443270
+266292594
+215426567
+383550730
+84593093
+380970185
+419898767
+381574216
+422289246
+380535266
+268204508
+46449693
+270408730
+268572254
+308981862
+266303861
+46092770
+308073958
+383696451
+381725197
+379008890
+379902614
+382568588
+45370580
+83526768
+381230253
+267980416
+269187238
+84449786
+267524276
+46169934
+43509278
+308776528
+270626080
+271203190
+216167767
+239794865
+46230173
+267821837
diff --git a/artifacts/2022-09-23-s3ttfb/minio-20220917.csv b/artifacts/2022-09-23-s3ttfb/minio-20220917.csv
new file mode 100644
index 0000000..42e7e00
--- /dev/null
+++ b/artifacts/2022-09-23-s3ttfb/minio-20220917.csv
@@ -0,0 +1,61 @@
+nanoseconds
+941737601
+970059899
+956848578
+981034132
+968456906
+986209313
+947371347
+961890275
+951011001
+947766957
+946381130
+1172968280
+944672409
+946216350
+960072933
+947966145
+947998544
+957730811
+960898554
+950611253
+953195848
+974654976
+982578891
+957419979
+947448876
+945953214
+936687543
+950116533
+979971727
+941427574
+931898843
+928494073
+927924296
+940995728
+927965457
+928098595
+933759012
+929902652
+953753466
+931448442
+934503899
+933297864
+934775226
+936615695
+950945991
+935054323
+1162136321
+940502403
+1158490674
+945337099
+948016443
+939863143
+938112935
+933271354
+946325434
+954355353
+948997521
+1124820589
+989829976
+938639125
diff --git a/artifacts/2022-09-23-s3ttfb/plot.R b/artifacts/2022-09-23-s3ttfb/plot.R
new file mode 100644
index 0000000..84d4872
--- /dev/null
+++ b/artifacts/2022-09-23-s3ttfb/plot.R
@@ -0,0 +1,25 @@
+library(tidyverse)
+
+
+read_csv("garage-v0.7.csv") %>% add_column(daemon="garage 0.7.3") -> a
+read_csv("garage-v0.8.csv") %>% add_column(daemon="garage 0.8.0 beta") -> b
+read_csv("minio-20220917.csv") %>% add_column(daemon="minio RELEASE.2022-09-17") -> c
+bind_rows(a,b,c) %>% mutate(ms = nanoseconds / 1000 / 1000) -> d
+
+ggplot(d, aes(y = ms, group = daemon, fill=daemon)) +
+  #geom_violin(trim = TRUE, width=2) + 
+  geom_histogram(binwidth=50) +
+  scale_y_continuous(expand=c(0,0))+
+  scale_x_continuous(expand=c(0,0))+
+  coord_flip() + 
+  labs(
+    y="TTFB (ms)",
+    x="Count",
+    fill="Daemon",
+    caption="Get the code to reproduce this graph at https://git.deuxfleurs.fr/Deuxfleurs/mknet",
+    title="TTFB (Time To First Byte) on GetObject over a slow network (5 Mbps)",
+    subtitle="A 1MB file is uploaded and then fetched 60 times.\nExcept for Minio, the queried node does not store any data (gateway) to force net. communications.") +
+  theme_classic() +
+  theme(legend.position = c(.15, .8))
+
+ggsave("./plot.png", width=200, height=110, units="mm")
diff --git a/artifacts/2022-09-23-s3ttfb/plot.png b/artifacts/2022-09-23-s3ttfb/plot.png
new file mode 100644
index 0000000..774e0f8
--- /dev/null
+++ b/artifacts/2022-09-23-s3ttfb/plot.png
diff --git a/benchmarks/s3ttfb/.gitignore b/benchmarks/s3ttfb/.gitignore
new file mode 100644
index 0000000..a645be8
--- /dev/null
+++ b/benchmarks/s3ttfb/.gitignore
@@ -0,0 +1 @@
+s3ttfb
diff --git a/benchmarks/s3ttfb/go.mod b/benchmarks/s3ttfb/go.mod
new file mode 100644
index 0000000..71669c7
--- /dev/null
+++ b/benchmarks/s3ttfb/go.mod
@@ -0,0 +1,12 @@
+module git.deuxfleurs.fr/Deuxfleurs/mknet/benchmarks/s3ttfb
+
+go 1.16
+
+require (
+	github.com/aws/aws-sdk-go-v2 v1.16.16
+	github.com/aws/aws-sdk-go-v2/config v1.17.7
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.27.11
+	github.com/google/uuid v1.1.1
+	github.com/minio/minio-go/v7 v7.0.16
+	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
+)
diff --git a/benchmarks/s3ttfb/go.sum b/benchmarks/s3ttfb/go.sum
new file mode 100644
index 0000000..e52b7eb
--- /dev/null
+++ b/benchmarks/s3ttfb/go.sum
@@ -0,0 +1,123 @@
+github.com/aws/aws-sdk-go-v2 v1.16.16 h1:M1fj4FE2lB4NzRb9Y0xdWsn2P0+2UHVxwKyOa4YJNjk=
+github.com/aws/aws-sdk-go-v2 v1.16.16/go.mod h1:SwiyXi/1zTUZ6KIAmLK5V5ll8SiURNUYOqTerZPaF9k=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.8 h1:tcFliCWne+zOuUfKNRn8JdFBuWPDuISDH08wD2ULkhk=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.8/go.mod h1:JTnlBSot91steJeti4ryyu/tLd4Sk84O5W22L7O2EQU=
+github.com/aws/aws-sdk-go-v2/config v1.17.7 h1:odVM52tFHhpqZBKNjVW5h+Zt1tKHbhdTQRb+0WHrNtw=
+github.com/aws/aws-sdk-go-v2/config v1.17.7/go.mod h1:dN2gja/QXxFF15hQreyrqYhLBaQo1d9ZKe/v/uplQoI=
+github.com/aws/aws-sdk-go-v2/credentials v1.12.20 h1:9+ZhlDY7N9dPnUmf7CDfW9In4sW5Ff3bh7oy4DzS1IE=
+github.com/aws/aws-sdk-go-v2/credentials v1.12.20/go.mod h1:UKY5HyIux08bbNA7Blv4PcXQ8cTkGh7ghHMFklaviR4=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17 h1:r08j4sbZu/RVi+BNxkBJwPMUYY3P8mgSDuKkZ/ZN1lE=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17/go.mod h1:yIkQcCDYNsZfXpd5UX2Cy+sWA1jPgIhGTw9cOBzfVnQ=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23 h1:s4g/wnzMf+qepSNgTvaQQHNxyMLKSawNhKCPNy++2xY=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23/go.mod h1:2DFxAQ9pfIRy0imBCJv+vZ2X6RKxves6fbnEuSry6b4=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17 h1:/K482T5A3623WJgWT8w1yRAFK4RzGzEl7y39yhtn9eA=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17/go.mod h1:pRwaTYCJemADaqCbUAxltMoHKata7hmB5PjEXeu0kfg=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.3.24 h1:wj5Rwc05hvUSvKuOF29IYb9QrCLjU+rHAy/x/o0DK2c=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.3.24/go.mod h1:jULHjqqjDlbyTa7pfM7WICATnOv+iOhjletM3N0Xbu8=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.14 h1:ZSIPAkAsCCjYrhqfw2+lNzWDzxzHXEckFkTePL5RSWQ=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.14/go.mod h1:AyGgqiKv9ECM6IZeNQtdT8NnMvUb3/2wokeq2Fgryto=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.9 h1:Lh1AShsuIJTwMkoxVCAYPJgNG5H+eN6SmoUn8nOZ5wE=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.9/go.mod h1:a9j48l6yL5XINLHLcOKInjdvknN+vWqPBxqeIDw7ktw=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.18 h1:BBYoNQt2kUZUUK4bIPsKrCcjVPUMNsgQpNAwhznK/zo=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.18/go.mod h1:NS55eQ4YixUJPTC+INxi2/jCqe1y2Uw3rnh9wEOVJxY=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.17 h1:Jrd/oMh0PKQc6+BowB+pLEwLIgaQF29eYbe7E1Av9Ug=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.17/go.mod h1:4nYOrY41Lrbk2170/BGkcJKBhws9Pfn8MG3aGqjjeFI=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.17 h1:HfVVR1vItaG6le+Bpw6P4midjBDMKnjMyZnw9MXYUcE=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.17/go.mod h1:YqMdV+gEKCQ59NrB7rzrJdALeBIsYiVi8Inj3+KcqHI=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.27.11 h1:3/gm/JTX9bX8CpzTgIlrtYpB3EVBDxyg/GY/QdcIEZw=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.27.11/go.mod h1:fmgDANqTUCxciViKl9hb/zD5LFbvPINFRgWhDbR+vZo=
+github.com/aws/aws-sdk-go-v2/service/sso v1.11.23 h1:pwvCchFUEnlceKIgPUouBJwK81aCkQ8UDMORfeFtW10=
+github.com/aws/aws-sdk-go-v2/service/sso v1.11.23/go.mod h1:/w0eg9IhFGjGyyncHIQrXtU8wvNsTJOP0R6PPj0wf80=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.5 h1:GUnZ62TevLqIoDyHeiWj2P7EqaosgakBKVvWriIdLQY=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.5/go.mod h1:csZuQY65DAdFBt1oIjO5hhBR49kQqop4+lcuCjf2arA=
+github.com/aws/aws-sdk-go-v2/service/sts v1.16.19 h1:9pPi0PsFNAGILFfPCk8Y0iyEBGc6lu6OQ97U7hmdesg=
+github.com/aws/aws-sdk-go-v2/service/sts v1.16.19/go.mod h1:h4J3oPZQbxLhzGnk+j9dfYHi5qIOVJ5kczZd658/ydM=
+github.com/aws/smithy-go v1.13.3 h1:l7LYxGuzK6/K+NzJ2mC+VvLUbae0sL3bXU//04MkmnA=
+github.com/aws/smithy-go v1.13.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/klauspost/compress v1.13.5 h1:9O69jUPDcsT9fEm74W92rZL9FQY7rCdaXVneq+yyzl4=
+github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=
+github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
+github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
+github.com/minio/minio-go/v7 v7.0.16 h1:GspaSBS8lOuEUCAqMe0W3UxSoyOA4b4F8PTspRVI+k4=
+github.com/minio/minio-go/v7 v7.0.16/go.mod h1:pUV0Pc+hPd1nccgmzQF/EXh48l/Z/yps6QPF1aaie4g=
+github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
+github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
+github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f h1:aZp0e2vLN4MToVqnjNEYEtrEA8RH8U8FN1CU7JgqsPU=
+golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww=
+gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
diff --git a/benchmarks/s3ttfb/main.go b/benchmarks/s3ttfb/main.go
new file mode 100644
index 0000000..b5cf033
--- /dev/null
+++ b/benchmarks/s3ttfb/main.go
@@ -0,0 +1,214 @@
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"math/rand"
+	"net/http"
+	"net/http/httptrace"
+	"os"
+	"strconv"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+)
+
+type PRNG struct {
+	rem int64
+}
+
+func (r *PRNG) Read(p []byte) (n int, err error) {
+	//log.Printf("rem=%d, buf=%d\n", r.rem, len(p))
+	if int64(len(p)) > r.rem {
+		p = p[:r.rem]
+	}
+
+	if int64(len(p)) > r.rem {
+		log.Fatal("LOGIC ERROR")
+	}
+
+	n, err = rand.Read(p)
+	if err != nil {
+		return
+	}
+	r.rem -= int64(n)
+	if r.rem <= 0 {
+		err = io.EOF
+		//log.Printf("PRNG file has been fully read. rem=%d,n=%d,err=%s\n", r.rem, n, err)
+	}
+	return
+}
+
+func main() {
+	var err error
+	fmt.Printf("nanoseconds\n")
+
+	// Initial setup
+	_, isSSL := os.LookupEnv("SSL")
+	opts := minio.Options{
+		Creds:  credentials.NewStaticV4(os.Getenv("AWS_ACCESS_KEY_ID"), os.Getenv("AWS_SECRET_ACCESS_KEY"), ""),
+		Secure: isSSL,
+	}
+
+	if region, ok := os.LookupEnv("AWS_REGION"); ok {
+		opts.Region = region
+	}
+
+	if _, ok := os.LookupEnv("SSL_INSECURE"); ok {
+		opts.Transport = &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
+	}
+
+	size := 1 * 1024 * 1024 // 1 MB
+	if env_size, ok := os.LookupEnv("SIZE"); ok {
+		size, err = strconv.Atoi(env_size)
+		if err != nil {
+			log.Fatal(err)
+			return
+		}
+	}
+
+	repeat := 60
+	if env_repeat, ok := os.LookupEnv("REPEAT"); ok {
+		repeat, err = strconv.Atoi(env_repeat)
+		if err != nil {
+			log.Fatal(err)
+			return
+		}
+	}
+
+	mc, err := minio.New(os.Getenv("ENDPOINT"), &opts)
+	if err != nil {
+		log.Fatal("failed connect", err)
+		return
+	}
+
+	// AWS SDK stuff
+	customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
+        	return aws.Endpoint{
+            		PartitionID:   "aws",
+			URL:           "http://"+os.Getenv("ENDPOINT"),
+            		SigningRegion: region,
+        	}, nil
+	})
+
+
+	cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithEndpointResolverWithOptions(customResolver))
+	if err != nil {
+		log.Fatalf("failed to load configuration, %v", err)
+	}
+
+	// Create an Amazon S3 service client
+	client := s3.NewFromConfig(cfg, func(opts *s3.Options) {
+		opts.UsePathStyle = true
+	})
+
+
+	// Create Bucket
+	buck := uuid.New().String()
+	err = mc.MakeBucket(context.Background(), buck, minio.MakeBucketOptions{})
+	if err != nil {
+		log.Fatal(err)
+		return
+	}
+
+	// Send an object
+	prng := new(PRNG)
+	prng.rem = int64(size)
+
+
+	key := uuid.New().String()
+	_, err = mc.PutObject(
+		context.Background(),
+		buck,
+		key,
+		prng,
+		int64(size),
+		minio.PutObjectOptions{ContentType: "application/octet-stream"},
+	)
+	if err != nil {
+		log.Fatal("unable to upload our test object")
+		return
+	}
+
+	for i := 0; i < repeat; i++ {
+		// Now, try to fetch it *repeat* times (assume no caching)
+		var elapsed time.Duration
+
+		var start time.Time
+    		trace := &httptrace.ClientTrace{
+			GotFirstResponseByte: func() { elapsed = time.Since(start) },
+		}
+
+		start = time.Now()
+		/*object, err := mc.GetObject(httptrace.WithClientTrace(context.Background(), trace), buck, key, minio.GetObjectOptions{})
+		if err != nil {
+			log.Fatal(err)
+			return
+		}*/
+
+		out, err := client.GetObject(httptrace.WithClientTrace(context.Background(), trace), &s3.GetObjectInput {
+			Bucket: aws.String(buck),
+			Key: aws.String(key),
+		})
+		if err != nil {
+			log.Fatal(err)
+			return
+		}
+		object := out.Body
+
+
+		if _, err = io.Copy(ioutil.Discard, object) ; err != nil {
+			log.Fatal("failed getobject: ", err)
+			return
+		}
+
+		fmt.Printf("%v\n", elapsed.Nanoseconds())
+		//time.Sleep(10 * time.Second)
+	}
+}
+
+/*
+func timeGet(url string) {
+    req, _ := http.NewRequest("GET", url, nil)
+
+    var start, connect, dns, tlsHandshake time.Time
+
+    trace := &httptrace.ClientTrace{
+        DNSStart: func(dsi httptrace.DNSStartInfo) { dns = time.Now() },
+        DNSDone: func(ddi httptrace.DNSDoneInfo) {
+            fmt.Printf("DNS Done: %v\n", time.Since(dns))
+        },
+
+        TLSHandshakeStart: func() { tlsHandshake = time.Now() },
+        TLSHandshakeDone: func(cs tls.ConnectionState, err error) {
+            fmt.Printf("TLS Handshake: %v\n", time.Since(tlsHandshake))
+        },
+
+        ConnectStart: func(network, addr string) { connect = time.Now() },
+        ConnectDone: func(network, addr string, err error) {
+            fmt.Printf("Connect time: %v\n", time.Since(connect))
+        },
+
+        GotFirstResponseByte: func() {
+            fmt.Printf("Time from start to first byte: %v\n", time.Since(start))
+        },
+    }
+
+    req = req.WithContext(httptrace.WithClientTrace(req.Context(), trace))
+    start = time.Now()
+    if _, err := http.DefaultTransport.RoundTrip(req); err != nil {
+        log.Fatal(err)
+    }
+    fmt.Printf("Total time: %v\n", time.Since(start))
+}
+*/
diff --git a/prepare.py b/prepare.py
index 60d58c1..d6e2ab9 100755
--- a/prepare.py
+++ b/prepare.py
@@ -14,6 +14,8 @@ os.chdir(me / "benchmarks" / "s3concurrent")
 shared.exec("go build")
 os.chdir(me / "benchmarks" / "s3lat")
 shared.exec("go build")
+os.chdir(me / "benchmarks" / "s3ttfb")
+shared.exec("go build")
 os.chdir(me)
 
 print("--- install python dependencies ---")
@@ -29,8 +31,9 @@ print("--- download artifacts ---")
 from scenarios.fragments import flavor
 flavor.download()
 
-print("""done! now, run:
+print(f"""done! now, run:
 > sudo -i
+> cd {os.getcwd()}
 > source ./.venv/bin/activate
 > ./mknet scenario ./topo/with-vdsl.yml ./scenarios/garage-s3lat garage-v0.8
 """)
diff --git a/scenarios/fragments/garage.py b/scenarios/fragments/garage.py
index 84ca90a..b291321 100644
--- a/scenarios/fragments/garage.py
+++ b/scenarios/fragments/garage.py
@@ -38,7 +38,7 @@ if 'HOST' in env:
     config = storage_path / "garage.toml"
     env['GARAGE_CONFIG_FILE'] = str(config)
 
-def deploy_coord():
+def deploy_coord(gw=None):
     destroy()
     shared.log("start daemon")
     shared.exec(f"{version['path']} --version")
@@ -46,7 +46,7 @@ def deploy_coord():
     shared.log("discover nodes")
     connect()
     shared.log("build layout")
-    create_layout()
+    create_layout(gw=gw)
     shared.log("create key")
     create_key()
     shared.log("ready")
@@ -136,26 +136,45 @@ def cluster_info():
 def connect():
     cinf = cluster_info()
     shared.log("start connections...")
-    ret = nodes.add_node([n['node_addr'] for n in cinf])
-    for st in ret:
-        if not st.success:
-            raise Exception("Node connect failed", ret)
+    while True:
+        try:
+            ret = nodes.add_node([n['node_addr'] for n in cinf], _request_timeout=3)
+        except:
+            shared.log("not ready, retry in 1sec")
+            time.sleep(1)
+            continue
+
+        for st in ret:
+            if not st.success:
+                continue
+                #raise Exception("Node connect failed", ret)
+        break
+
     shared.log("all nodes connected")
 
-def create_layout():
+def create_layout(gw=None):
+    if gw is None:
+        gw = []
+
     v = layout.get_layout().version
 
     cinf = cluster_info()
     nlay = dict()
     for n in cinf:
+        capa = 1
+        if n['host'] in gw:
+            capa = None
+
         nlay[n['node_id']] = NodeClusterInfo(
             zone = n['zone'],
-            capacity = 1,
+            capacity = capa,
             tags = [ n['host'] ],
         )
     layout.add_layout(nlay)
     layout.apply_layout(LayoutVersion(version=v+1))
 
+    shared.log(layout.get_layout())
+
 def create_key():
     global key
     kinfo = shared.fn_retry(lambda: keys.add_key(AddKeyRequest(name="mknet")))
diff --git a/scenarios/fragments/s3ttfb.py b/scenarios/fragments/s3ttfb.py
new file mode 100644
index 0000000..66fdcc6
--- /dev/null
+++ b/scenarios/fragments/s3ttfb.py
@@ -0,0 +1,26 @@
+import os
+from os.path import exists
+from pathlib import Path
+from fragments import shared, minio, garage
+
+s3bin = Path(os.path.dirname(__file__)) / "../../benchmarks/s3ttfb/s3ttfb"
+
+def common():
+    out = Path(shared.storage_path) / "s3ttfb.csv"
+    shared.log(f"launching s3ttfb ({s3bin})")
+    shared.exec(f"{s3bin} > {out}")
+    shared.log(f"execution done, output written to {out}")
+
+def on_garage():
+    os.environ['AWS_ACCESS_KEY_ID'] = garage.key.access_key_id
+    os.environ['AWS_SECRET_ACCESS_KEY'] = garage.key.secret_access_key
+    os.environ['ENDPOINT'] = "localhost:3900"
+    os.environ['AWS_REGION'] = "garage"
+    common()
+
+def on_minio():
+    os.environ['AWS_ACCESS_KEY_ID'] = minio.access_key
+    os.environ['AWS_SECRET_ACCESS_KEY'] = minio.secret_key
+    os.environ['ENDPOINT'] = "localhost:9000"
+    os.environ['AWS_REGION'] = "us-east-1"
+    common()
diff --git a/scenarios/garage-s3ttfb b/scenarios/garage-s3ttfb
new file mode 100755
index 0000000..20bb6d0
--- /dev/null
+++ b/scenarios/garage-s3ttfb
@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+from fragments import garage, s3ttfb, shared, flavor
+import sys
+
+for fl in sys.argv[1:]:
+    if fl in flavor.garage:
+        garage.version = flavor.garage[fl]
+
+if shared.id() == 1:
+    garage.deploy_coord(gw = [ 'node1' ])
+    s3ttfb.on_garage()
+    garage.delete_key()
+    garage.destroy()
+else:
+    garage.deploy_follow()
+    garage.sync_on_key_down()
+    garage.destroy()
+shared.log("bye")
diff --git a/scenarios/minio-s3ttfb b/scenarios/minio-s3ttfb
new file mode 100755
index 0000000..ed78aea
--- /dev/null
+++ b/scenarios/minio-s3ttfb
@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+from fragments import minio, s3ttfb, shared, flavor
+import sys
+
+
+for fl in sys.argv[1:]:
+    if fl in flavor.minio:
+        minio.version = flavor.minio[fl]
+
+if shared.id() == 1:
+    minio.deploy_coord()
+    s3ttfb.on_minio()
+    minio.delete_sync_bucket()
+    minio.destroy()
+else:
+    minio.deploy_follow()
+    minio.sync_on_bucket_down()
+    minio.destroy()
+shared.log("bye")
diff --git a/scenarios/requirements.txt b/scenarios/requirements.txt
index b85592f..1d9ce28 100644
--- a/scenarios/requirements.txt
+++ b/scenarios/requirements.txt
@@ -1,2 +1,2 @@
-git+https://git.deuxfleurs.fr/quentin/garage-admin-sdk@7b1c1faf7a#egg=garage-admin-sdk&subdirectory=python
+git+https://git.deuxfleurs.fr/quentin/garage-admin-sdk@2b685b90be9aa848a9450dd582d1d322b25b5568#egg=garage-admin-sdk&subdirectory=python
 minio
diff --git a/topo/5mbps.yml b/topo/5mbps.yml
index 1e20d7c..7af1637 100644
--- a/topo/5mbps.yml
+++ b/topo/5mbps.yml
@@ -2,6 +2,7 @@ links:
   - &slow
     bandwidth: 5M
     latency: 500us
+    limit: 5
   - &1000
     bandwidth: 1000M
     latency: 100us
@@ -13,6 +14,8 @@ servers:
     <<: *slow
   - name: node3
     <<: *slow
+  - name: node4
+    <<: *slow
 
 global:
   subnet: