diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | s3_file.go | 11 |
2 files changed, 11 insertions, 1 deletions
@@ -1 +1,2 @@ bagage +.env @@ -98,7 +98,16 @@ func (f *S3File) Write(p []byte) (n int, err error) { contentType := mime.TypeByExtension(path.Ext(f.path.key)) go func() { - _, err := f.fs.mc.PutObject(context.Background(), f.path.bucket, f.path.key, r, -1, minio.PutObjectOptions{ContentType: contentType}) + /* @FIXME + PutObject has a strange behaviour when used with unknown size, it supposes the final size will be 5TiB. + Then it computes that, following the final size of the file, each part of the multipart upload should be 512MiB, which leads to big allocations. + The culprit is OptimalPartInfo: https://github.com/minio/minio-go/blob/62beca8cd87e9960d88793320220ad2c159bb5e5/api-put-object-common.go#L70 + We set this value to the minimum allowed one, 5MiB. + The minimum value is set here: https://github.com/minio/minio-go/blob/62beca8cd87e9960d88793320220ad2c159bb5e5/constants.go#L24 + Because Multipart uploads seems to be limited to 10 000 parts, it might be possible that we are limited to 50 GiB files, which is still good enough. + Ref: https://github.com/minio/minio-go/blob/62beca8cd87e9960d88793320220ad2c159bb5e5/api-put-object-common.go#L110-L112 + */ + _, err := f.fs.mc.PutObject(context.Background(), f.path.bucket, f.path.key, r, -1, minio.PutObjectOptions{ContentType: contentType, PartSize: 5*1024*1024}) f.donew <- err }() } |