From 93631b4e3d5195d446504db1c4a2bc7468b3ef28 Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Fri, 10 Sep 2021 17:28:08 +0200 Subject: [PATCH] Change part size to fix memory leak --- .gitignore | 1 + s3_file.go | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 9b6940b..76cf493 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ bagage +.env diff --git a/s3_file.go b/s3_file.go index a72397e..bbcfc8a 100644 --- a/s3_file.go +++ b/s3_file.go @@ -98,7 +98,16 @@ func (f *S3File) Write(p []byte) (n int, err error) { contentType := mime.TypeByExtension(path.Ext(f.path.key)) go func() { - _, err := f.fs.mc.PutObject(context.Background(), f.path.bucket, f.path.key, r, -1, minio.PutObjectOptions{ContentType: contentType}) + /* @FIXME + PutObject has a strange behaviour when used with unknown size, it supposes the final size will be 5TiB. + Then it computes that, following the final size of the file, each part of the multipart upload should be 512MiB, which leads to big allocations. + The culprit is OptimalPartInfo: https://github.com/minio/minio-go/blob/62beca8cd87e9960d88793320220ad2c159bb5e5/api-put-object-common.go#L70 + We set this value to the minimum allowed one, 5MiB. + The minimum value is set here: https://github.com/minio/minio-go/blob/62beca8cd87e9960d88793320220ad2c159bb5e5/constants.go#L24 + Because Multipart uploads seems to be limited to 10 000 parts, it might be possible that we are limited to 50 GiB files, which is still good enough. + Ref: https://github.com/minio/minio-go/blob/62beca8cd87e9960d88793320220ad2c159bb5e5/api-put-object-common.go#L110-L112 + */ + _, err := f.fs.mc.PutObject(context.Background(), f.path.bucket, f.path.key, r, -1, minio.PutObjectOptions{ContentType: contentType, PartSize: 5*1024*1024}) f.donew <- err }() }