Change part size to fix memory leak
This commit is contained in:
parent
4982bdd839
commit
93631b4e3d
2 changed files with 11 additions and 1 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
@ -1 +1,2 @@
|
|||
bagage
|
||||
.env
|
||||
|
|
11
s3_file.go
11
s3_file.go
|
@ -98,7 +98,16 @@ func (f *S3File) Write(p []byte) (n int, err error) {
|
|||
|
||||
contentType := mime.TypeByExtension(path.Ext(f.path.key))
|
||||
go func() {
|
||||
_, err := f.fs.mc.PutObject(context.Background(), f.path.bucket, f.path.key, r, -1, minio.PutObjectOptions{ContentType: contentType})
|
||||
/* @FIXME
|
||||
PutObject has a strange behaviour when used with unknown size, it supposes the final size will be 5TiB.
|
||||
Then it computes that, following the final size of the file, each part of the multipart upload should be 512MiB, which leads to big allocations.
|
||||
The culprit is OptimalPartInfo: https://github.com/minio/minio-go/blob/62beca8cd87e9960d88793320220ad2c159bb5e5/api-put-object-common.go#L70
|
||||
We set this value to the minimum allowed one, 5MiB.
|
||||
The minimum value is set here: https://github.com/minio/minio-go/blob/62beca8cd87e9960d88793320220ad2c159bb5e5/constants.go#L24
|
||||
Because Multipart uploads seems to be limited to 10 000 parts, it might be possible that we are limited to 50 GiB files, which is still good enough.
|
||||
Ref: https://github.com/minio/minio-go/blob/62beca8cd87e9960d88793320220ad2c159bb5e5/api-put-object-common.go#L110-L112
|
||||
*/
|
||||
_, err := f.fs.mc.PutObject(context.Background(), f.path.bucket, f.path.key, r, -1, minio.PutObjectOptions{ContentType: contentType, PartSize: 5*1024*1024})
|
||||
f.donew <- err
|
||||
}()
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue