Change part size to fix memory leak

This commit is contained in:
Quentin 2021-09-10 17:28:08 +02:00
parent 4982bdd839
commit 93631b4e3d
Signed by: quentin
GPG key ID: A98E9B769E4FF428
2 changed files with 11 additions and 1 deletions

1
.gitignore vendored
View file

@ -1 +1,2 @@
bagage bagage
.env

View file

@ -98,7 +98,16 @@ func (f *S3File) Write(p []byte) (n int, err error) {
contentType := mime.TypeByExtension(path.Ext(f.path.key)) contentType := mime.TypeByExtension(path.Ext(f.path.key))
go func() { go func() {
_, err := f.fs.mc.PutObject(context.Background(), f.path.bucket, f.path.key, r, -1, minio.PutObjectOptions{ContentType: contentType}) /* @FIXME
PutObject has a strange behaviour when used with unknown size, it supposes the final size will be 5TiB.
Then it computes that, following the final size of the file, each part of the multipart upload should be 512MiB, which leads to big allocations.
The culprit is OptimalPartInfo: https://github.com/minio/minio-go/blob/62beca8cd87e9960d88793320220ad2c159bb5e5/api-put-object-common.go#L70
We set this value to the minimum allowed one, 5MiB.
The minimum value is set here: https://github.com/minio/minio-go/blob/62beca8cd87e9960d88793320220ad2c159bb5e5/constants.go#L24
Because Multipart uploads seems to be limited to 10 000 parts, it might be possible that we are limited to 50 GiB files, which is still good enough.
Ref: https://github.com/minio/minio-go/blob/62beca8cd87e9960d88793320220ad2c159bb5e5/api-put-object-common.go#L110-L112
*/
_, err := f.fs.mc.PutObject(context.Background(), f.path.bucket, f.path.key, r, -1, minio.PutObjectOptions{ContentType: contentType, PartSize: 5*1024*1024})
f.donew <- err f.donew <- err
}() }()
} }