Add ListMultipartUploads + Refactor ListObjects
This commit is contained in:
parent
9cb2e9e57c
commit
ce357b9a12
17 changed files with 1228 additions and 283 deletions
20
default.nix
20
default.nix
|
@ -24,6 +24,11 @@ in let
|
||||||
rustChannel = pkgs.rustPlatform.rust;
|
rustChannel = pkgs.rustPlatform.rust;
|
||||||
|
|
||||||
overrides = pkgs.buildPackages.rustBuilder.overrides.all ++ [
|
overrides = pkgs.buildPackages.rustBuilder.overrides.all ++ [
|
||||||
|
/*
|
||||||
|
We want to inject the git version while keeping the build deterministic.
|
||||||
|
As we do not want to consider the .git folder as part of the input source,
|
||||||
|
we ask the user (the CI often) to pass the value to Nix.
|
||||||
|
*/
|
||||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||||
name = "garage";
|
name = "garage";
|
||||||
overrideAttrs = drv: if git_version != null then {
|
overrideAttrs = drv: if git_version != null then {
|
||||||
|
@ -33,6 +38,21 @@ in let
|
||||||
'';
|
'';
|
||||||
} else {};
|
} else {};
|
||||||
})
|
})
|
||||||
|
|
||||||
|
/*
|
||||||
|
On a sandbox pure NixOS environment, /usr/bin/file is not available.
|
||||||
|
This is a known problem: https://github.com/NixOS/nixpkgs/issues/98440
|
||||||
|
We simply patch the file as suggested
|
||||||
|
*/
|
||||||
|
/*(pkgs.rustBuilder.rustLib.makeOverride {
|
||||||
|
name = "libsodium-sys";
|
||||||
|
overrideAttrs = drv: {
|
||||||
|
preConfigure = ''
|
||||||
|
${drv.preConfigure or ""}
|
||||||
|
sed -i 's,/usr/bin/file,${file}/bin/file,g' ./configure
|
||||||
|
'';
|
||||||
|
}
|
||||||
|
})*/
|
||||||
];
|
];
|
||||||
|
|
||||||
packageFun = import ./Cargo.nix;
|
packageFun = import ./Cargo.nix;
|
||||||
|
|
|
@ -43,6 +43,8 @@ All APIs that are not mentionned are not implemented and will return a 400 bad r
|
||||||
| ListBuckets | Implemented |
|
| ListBuckets | Implemented |
|
||||||
| ListObjects | Implemented, bugs? (see below) |
|
| ListObjects | Implemented, bugs? (see below) |
|
||||||
| ListObjectsV2 | Implemented |
|
| ListObjectsV2 | Implemented |
|
||||||
|
| ListMultipartUpload | Implemented |
|
||||||
|
| ListParts | Missing |
|
||||||
| PutObject | Implemented |
|
| PutObject | Implemented |
|
||||||
| PutBucketWebsite | Partially implemented (see below)|
|
| PutBucketWebsite | Partially implemented (see below)|
|
||||||
| UploadPart | Implemented |
|
| UploadPart | Implemented |
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
#!/bin/bash
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
set -ex
|
set -ex
|
||||||
|
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
#!/bin/bash
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
set -ex
|
set -ex
|
||||||
|
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
#!/bin/bash
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
set -e
|
set -e
|
||||||
|
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
#!/bin/bash
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
set -ex
|
set -ex
|
||||||
|
|
||||||
|
|
|
@ -1,5 +1,3 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
export AWS_ACCESS_KEY_ID=`cat /tmp/garage.s3 |cut -d' ' -f1`
|
export AWS_ACCESS_KEY_ID=`cat /tmp/garage.s3 |cut -d' ' -f1`
|
||||||
export AWS_SECRET_ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2`
|
export AWS_SECRET_ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2`
|
||||||
export AWS_DEFAULT_REGION='garage'
|
export AWS_DEFAULT_REGION='garage'
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
#!/bin/bash
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
set -ex
|
set -ex
|
||||||
|
|
||||||
|
@ -10,6 +10,7 @@ GARAGE_DEBUG="${REPO_FOLDER}/target/debug/"
|
||||||
GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
|
GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
|
||||||
NIX_RELEASE="${REPO_FOLDER}/result/bin/"
|
NIX_RELEASE="${REPO_FOLDER}/result/bin/"
|
||||||
PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH"
|
PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH"
|
||||||
|
CMDOUT=/tmp/garage.cmd.tmp
|
||||||
|
|
||||||
# @FIXME Duck is not ready for testing, we have a bug
|
# @FIXME Duck is not ready for testing, we have a bug
|
||||||
SKIP_DUCK=1
|
SKIP_DUCK=1
|
||||||
|
@ -130,6 +131,142 @@ if [ -z "$SKIP_AWS" ]; then
|
||||||
rm /tmp/garage-index.html
|
rm /tmp/garage-index.html
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
if [ -z "$SKIP_AWS" ]; then
|
||||||
|
echo "🔌 Test S3API"
|
||||||
|
|
||||||
|
echo "Test Objects"
|
||||||
|
aws s3api put-object --bucket eprouvette --key a
|
||||||
|
aws s3api put-object --bucket eprouvette --key a/a
|
||||||
|
aws s3api put-object --bucket eprouvette --key a/b
|
||||||
|
aws s3api put-object --bucket eprouvette --key a/c
|
||||||
|
aws s3api put-object --bucket eprouvette --key a/d/a
|
||||||
|
aws s3api put-object --bucket eprouvette --key a/é
|
||||||
|
aws s3api put-object --bucket eprouvette --key b
|
||||||
|
aws s3api put-object --bucket eprouvette --key c
|
||||||
|
|
||||||
|
|
||||||
|
aws s3api list-objects-v2 --bucket eprouvette >$CMDOUT
|
||||||
|
[ $(jq '.Contents | length' $CMDOUT) == 8 ]
|
||||||
|
[ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ]
|
||||||
|
aws s3api list-objects-v2 --bucket eprouvette --page-size 0 >$CMDOUT
|
||||||
|
[ $(jq '.Contents | length' $CMDOUT) == 8 ]
|
||||||
|
[ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ]
|
||||||
|
aws s3api list-objects-v2 --bucket eprouvette --page-size 999999999999999 >$CMDOUT
|
||||||
|
[ $(jq '.Contents | length' $CMDOUT) == 8 ]
|
||||||
|
[ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ]
|
||||||
|
aws s3api list-objects-v2 --bucket eprouvette --page-size 1 >$CMDOUT
|
||||||
|
[ $(jq '.Contents | length' $CMDOUT) == 8 ]
|
||||||
|
[ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ]
|
||||||
|
aws s3api list-objects-v2 --bucket eprouvette --delimiter '/' >$CMDOUT
|
||||||
|
[ $(jq '.Contents | length' $CMDOUT) == 3 ]
|
||||||
|
[ $(jq '.CommonPrefixes | length' $CMDOUT) == 1 ]
|
||||||
|
aws s3api list-objects-v2 --bucket eprouvette --delimiter '/' --page-size 1 >$CMDOUT
|
||||||
|
[ $(jq '.Contents | length' $CMDOUT) == 3 ]
|
||||||
|
[ $(jq '.CommonPrefixes | length' $CMDOUT) == 1 ]
|
||||||
|
aws s3api list-objects-v2 --bucket eprouvette --prefix 'a/' >$CMDOUT
|
||||||
|
[ $(jq '.Contents | length' $CMDOUT) == 5 ]
|
||||||
|
[ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ]
|
||||||
|
aws s3api list-objects-v2 --bucket eprouvette --prefix 'a/' --delimiter '/' >$CMDOUT
|
||||||
|
[ $(jq '.Contents | length' $CMDOUT) == 4 ]
|
||||||
|
[ $(jq '.CommonPrefixes | length' $CMDOUT) == 1 ]
|
||||||
|
aws s3api list-objects-v2 --bucket eprouvette --prefix 'a/' --page-size 1 >$CMDOUT
|
||||||
|
[ $(jq '.Contents | length' $CMDOUT) == 5 ]
|
||||||
|
[ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ]
|
||||||
|
aws s3api list-objects-v2 --bucket eprouvette --prefix 'a/' --delimiter '/' --page-size 1 >$CMDOUT
|
||||||
|
[ $(jq '.Contents | length' $CMDOUT) == 4 ]
|
||||||
|
[ $(jq '.CommonPrefixes | length' $CMDOUT) == 1 ]
|
||||||
|
aws s3api list-objects-v2 --bucket eprouvette --start-after 'Z' >$CMDOUT
|
||||||
|
[ $(jq '.Contents | length' $CMDOUT) == 8 ]
|
||||||
|
[ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ]
|
||||||
|
aws s3api list-objects-v2 --bucket eprouvette --start-after 'c' >$CMDOUT
|
||||||
|
! [ -s $CMDOUT ]
|
||||||
|
|
||||||
|
|
||||||
|
aws s3api list-objects --bucket eprouvette >$CMDOUT
|
||||||
|
[ $(jq '.Contents | length' $CMDOUT) == 8 ]
|
||||||
|
[ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ]
|
||||||
|
aws s3api list-objects --bucket eprouvette --page-size 1 >$CMDOUT
|
||||||
|
[ $(jq '.Contents | length' $CMDOUT) == 8 ]
|
||||||
|
[ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ]
|
||||||
|
aws s3api list-objects --bucket eprouvette --delimiter '/' >$CMDOUT
|
||||||
|
[ $(jq '.Contents | length' $CMDOUT) == 3 ]
|
||||||
|
[ $(jq '.CommonPrefixes | length' $CMDOUT) == 1 ]
|
||||||
|
# @FIXME it does not work as expected but might be a limitation of aws s3api
|
||||||
|
# The problem is the conjunction of a delimiter + pagination + v1 of listobjects
|
||||||
|
#aws s3api list-objects --bucket eprouvette --delimiter '/' --page-size 1 >$CMDOUT
|
||||||
|
#[ $(jq '.Contents | length' $CMDOUT) == 3 ]
|
||||||
|
#[ $(jq '.CommonPrefixes | length' $CMDOUT) == 1 ]
|
||||||
|
aws s3api list-objects --bucket eprouvette --prefix 'a/' >$CMDOUT
|
||||||
|
[ $(jq '.Contents | length' $CMDOUT) == 5 ]
|
||||||
|
[ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ]
|
||||||
|
aws s3api list-objects --bucket eprouvette --prefix 'a/' --delimiter '/' >$CMDOUT
|
||||||
|
[ $(jq '.Contents | length' $CMDOUT) == 4 ]
|
||||||
|
[ $(jq '.CommonPrefixes | length' $CMDOUT) == 1 ]
|
||||||
|
aws s3api list-objects --bucket eprouvette --prefix 'a/' --page-size 1 >$CMDOUT
|
||||||
|
[ $(jq '.Contents | length' $CMDOUT) == 5 ]
|
||||||
|
[ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ]
|
||||||
|
# @FIXME idem
|
||||||
|
#aws s3api list-objects --bucket eprouvette --prefix 'a/' --delimiter '/' --page-size 1 >$CMDOUT
|
||||||
|
#[ $(jq '.Contents | length' $CMDOUT) == 4 ]
|
||||||
|
#[ $(jq '.CommonPrefixes | length' $CMDOUT) == 1 ]
|
||||||
|
aws s3api list-objects --bucket eprouvette --starting-token 'Z' >$CMDOUT
|
||||||
|
[ $(jq '.Contents | length' $CMDOUT) == 8 ]
|
||||||
|
[ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ]
|
||||||
|
aws s3api list-objects --bucket eprouvette --starting-token 'c' >$CMDOUT
|
||||||
|
! [ -s $CMDOUT ]
|
||||||
|
|
||||||
|
aws s3api list-objects-v2 --bucket eprouvette | \
|
||||||
|
jq -c '. | {Objects: [.Contents[] | {Key: .Key}], Quiet: true}' | \
|
||||||
|
aws s3api delete-objects --bucket eprouvette --delete file:///dev/stdin
|
||||||
|
|
||||||
|
|
||||||
|
echo "Test Multipart Upload"
|
||||||
|
aws s3api create-multipart-upload --bucket eprouvette --key a
|
||||||
|
aws s3api create-multipart-upload --bucket eprouvette --key a
|
||||||
|
aws s3api create-multipart-upload --bucket eprouvette --key c
|
||||||
|
aws s3api create-multipart-upload --bucket eprouvette --key c/a
|
||||||
|
aws s3api create-multipart-upload --bucket eprouvette --key c/b
|
||||||
|
|
||||||
|
aws s3api list-multipart-uploads --bucket eprouvette >$CMDOUT
|
||||||
|
[ $(jq '.Uploads | length' $CMDOUT) == 5 ]
|
||||||
|
[ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ]
|
||||||
|
aws s3api list-multipart-uploads --bucket eprouvette --page-size 1 >$CMDOUT
|
||||||
|
[ $(jq '.Uploads | length' $CMDOUT) == 5 ]
|
||||||
|
[ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ]
|
||||||
|
aws s3api list-multipart-uploads --bucket eprouvette --delimiter '/' >$CMDOUT
|
||||||
|
[ $(jq '.Uploads | length' $CMDOUT) == 3 ]
|
||||||
|
[ $(jq '.CommonPrefixes | length' $CMDOUT) == 1 ]
|
||||||
|
aws s3api list-multipart-uploads --bucket eprouvette --delimiter '/' --page-size 1 >$CMDOUT
|
||||||
|
[ $(jq '.Uploads | length' $CMDOUT) == 3 ]
|
||||||
|
[ $(jq '.CommonPrefixes | length' $CMDOUT) == 1 ]
|
||||||
|
aws s3api list-multipart-uploads --bucket eprouvette --prefix 'c' >$CMDOUT
|
||||||
|
[ $(jq '.Uploads | length' $CMDOUT) == 3 ]
|
||||||
|
[ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ]
|
||||||
|
aws s3api list-multipart-uploads --bucket eprouvette --prefix 'c' --page-size 1 >$CMDOUT
|
||||||
|
[ $(jq '.Uploads | length' $CMDOUT) == 3 ]
|
||||||
|
[ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ]
|
||||||
|
aws s3api list-multipart-uploads --bucket eprouvette --prefix 'c' --delimiter '/' >$CMDOUT
|
||||||
|
[ $(jq '.Uploads | length' $CMDOUT) == 1 ]
|
||||||
|
[ $(jq '.CommonPrefixes | length' $CMDOUT) == 1 ]
|
||||||
|
aws s3api list-multipart-uploads --bucket eprouvette --prefix 'c' --delimiter '/' --page-size 1 >$CMDOUT
|
||||||
|
[ $(jq '.Uploads | length' $CMDOUT) == 1 ]
|
||||||
|
[ $(jq '.CommonPrefixes | length' $CMDOUT) == 1 ]
|
||||||
|
aws s3api list-multipart-uploads --bucket eprouvette --starting-token 'ZZZZZ' >$CMDOUT
|
||||||
|
[ $(jq '.Uploads | length' $CMDOUT) == 5 ]
|
||||||
|
[ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ]
|
||||||
|
aws s3api list-multipart-uploads --bucket eprouvette --starting-token 'd' >$CMDOUT
|
||||||
|
! [ -s $CMDOUT ]
|
||||||
|
|
||||||
|
aws s3api list-multipart-uploads --bucket eprouvette | \
|
||||||
|
jq -r '.Uploads[] | "\(.Key) \(.UploadId)"' | \
|
||||||
|
while read r; do
|
||||||
|
key=$(echo $r|cut -d' ' -f 1);
|
||||||
|
uid=$(echo $r|cut -d' ' -f 2);
|
||||||
|
aws s3api abort-multipart-upload --bucket eprouvette --key $key --upload-id $uid;
|
||||||
|
echo "Deleted ${key}:${uid}"
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
if [ -z "$SKIP_AWS" ]; then
|
if [ -z "$SKIP_AWS" ]; then
|
||||||
echo "🪣 Test bucket logic "
|
echo "🪣 Test bucket logic "
|
||||||
AWS_ACCESS_KEY_ID=`cat /tmp/garage.s3 |cut -d' ' -f1`
|
AWS_ACCESS_KEY_ID=`cat /tmp/garage.s3 |cut -d' ' -f1`
|
||||||
|
@ -151,5 +288,6 @@ AWS_SECRET_ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2`
|
||||||
garage -c /tmp/config.1.toml bucket deny --read --write eprouvette --key $AWS_ACCESS_KEY_ID
|
garage -c /tmp/config.1.toml bucket deny --read --write eprouvette --key $AWS_ACCESS_KEY_ID
|
||||||
garage -c /tmp/config.1.toml bucket delete --yes eprouvette
|
garage -c /tmp/config.1.toml bucket delete --yes eprouvette
|
||||||
garage -c /tmp/config.1.toml key delete --yes $AWS_ACCESS_KEY_ID
|
garage -c /tmp/config.1.toml key delete --yes $AWS_ACCESS_KEY_ID
|
||||||
|
exec 3>&-
|
||||||
|
|
||||||
echo "✅ Success"
|
echo "✅ Success"
|
||||||
|
|
|
@ -83,6 +83,7 @@ function refresh_toolchain {
|
||||||
pkgs.which
|
pkgs.which
|
||||||
pkgs.openssl
|
pkgs.openssl
|
||||||
pkgs.curl
|
pkgs.curl
|
||||||
|
pkgs.jq
|
||||||
] else [])
|
] else [])
|
||||||
++
|
++
|
||||||
(if release then [
|
(if release then [
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
|
use std::cmp::{max, min};
|
||||||
use std::net::SocketAddr;
|
use std::net::SocketAddr;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
@ -217,16 +218,18 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
|
||||||
handle_list(
|
handle_list(
|
||||||
garage,
|
garage,
|
||||||
&ListObjectsQuery {
|
&ListObjectsQuery {
|
||||||
is_v2: false,
|
common: ListQueryCommon {
|
||||||
bucket_name: bucket,
|
bucket_name: bucket,
|
||||||
bucket_id,
|
bucket_id,
|
||||||
delimiter: delimiter.map(|d| d.to_string()),
|
delimiter: delimiter.map(|d| d.to_string()),
|
||||||
max_keys: max_keys.unwrap_or(1000),
|
page_size: max_keys.map(|p| min(1000, max(1, p))).unwrap_or(1000),
|
||||||
prefix: prefix.unwrap_or_default(),
|
prefix: prefix.unwrap_or_default(),
|
||||||
|
urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false),
|
||||||
|
},
|
||||||
|
is_v2: false,
|
||||||
marker,
|
marker,
|
||||||
continuation_token: None,
|
continuation_token: None,
|
||||||
start_after: None,
|
start_after: None,
|
||||||
urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false),
|
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
|
@ -246,16 +249,18 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
|
||||||
handle_list(
|
handle_list(
|
||||||
garage,
|
garage,
|
||||||
&ListObjectsQuery {
|
&ListObjectsQuery {
|
||||||
is_v2: true,
|
common: ListQueryCommon {
|
||||||
bucket_name: bucket,
|
bucket_name: bucket,
|
||||||
bucket_id,
|
bucket_id,
|
||||||
delimiter: delimiter.map(|d| d.to_string()),
|
delimiter: delimiter.map(|d| d.to_string()),
|
||||||
max_keys: max_keys.unwrap_or(1000),
|
page_size: max_keys.map(|p| min(1000, max(1, p))).unwrap_or(1000),
|
||||||
|
urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false),
|
||||||
prefix: prefix.unwrap_or_default(),
|
prefix: prefix.unwrap_or_default(),
|
||||||
|
},
|
||||||
|
is_v2: true,
|
||||||
marker: None,
|
marker: None,
|
||||||
continuation_token,
|
continuation_token,
|
||||||
start_after,
|
start_after,
|
||||||
urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false),
|
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
|
@ -266,6 +271,32 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
|
||||||
)))
|
)))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Endpoint::ListMultipartUploads {
|
||||||
|
bucket,
|
||||||
|
delimiter,
|
||||||
|
encoding_type,
|
||||||
|
key_marker,
|
||||||
|
max_uploads,
|
||||||
|
prefix,
|
||||||
|
upload_id_marker,
|
||||||
|
} => {
|
||||||
|
handle_list_multipart_upload(
|
||||||
|
garage,
|
||||||
|
&ListMultipartUploadsQuery {
|
||||||
|
common: ListQueryCommon {
|
||||||
|
bucket_name: bucket,
|
||||||
|
bucket_id,
|
||||||
|
delimiter: delimiter.map(|d| d.to_string()),
|
||||||
|
page_size: max_uploads.map(|p| min(1000, max(1, p))).unwrap_or(1000),
|
||||||
|
prefix: prefix.unwrap_or_default(),
|
||||||
|
urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false),
|
||||||
|
},
|
||||||
|
key_marker,
|
||||||
|
upload_id_marker,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
Endpoint::DeleteObjects { .. } => {
|
Endpoint::DeleteObjects { .. } => {
|
||||||
handle_delete_objects(garage, bucket_id, req, content_sha256).await
|
handle_delete_objects(garage, bucket_id, req, content_sha256).await
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,6 +7,7 @@ use garage_model::bucket_alias_table::*;
|
||||||
use garage_model::bucket_table::Bucket;
|
use garage_model::bucket_table::Bucket;
|
||||||
use garage_model::garage::Garage;
|
use garage_model::garage::Garage;
|
||||||
use garage_model::key_table::Key;
|
use garage_model::key_table::Key;
|
||||||
|
use garage_model::object_table::ObjectFilter;
|
||||||
use garage_model::permission::BucketKeyPerm;
|
use garage_model::permission::BucketKeyPerm;
|
||||||
use garage_table::util::*;
|
use garage_table::util::*;
|
||||||
use garage_util::crdt::*;
|
use garage_util::crdt::*;
|
||||||
|
@ -226,7 +227,7 @@ pub async fn handle_delete_bucket(
|
||||||
// Check bucket is empty
|
// Check bucket is empty
|
||||||
let objects = garage
|
let objects = garage
|
||||||
.object_table
|
.object_table
|
||||||
.get_range(&bucket_id, None, Some(DeletedFilter::NotDeleted), 10)
|
.get_range(&bucket_id, None, Some(ObjectFilter::IsData), 10)
|
||||||
.await?;
|
.await?;
|
||||||
if !objects.is_empty() {
|
if !objects.is_empty() {
|
||||||
return Err(Error::BucketNotEmpty);
|
return Err(Error::BucketNotEmpty);
|
||||||
|
|
1149
src/api/s3_list.rs
1149
src/api/s3_list.rs
File diff suppressed because it is too large
Load diff
|
@ -610,7 +610,7 @@ pub(crate) fn get_headers(req: &Request<Body>) -> Result<ObjectVersionHeaders, E
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn decode_upload_id(id: &str) -> Result<Uuid, Error> {
|
pub fn decode_upload_id(id: &str) -> Result<Uuid, Error> {
|
||||||
let id_bin = hex::decode(id).map_err(|_| Error::NoSuchUpload)?;
|
let id_bin = hex::decode(id).map_err(|_| Error::NoSuchUpload)?;
|
||||||
if id_bin.len() != 32 {
|
if id_bin.len() != 32 {
|
||||||
return Err(Error::NoSuchUpload);
|
return Err(Error::NoSuchUpload);
|
||||||
|
|
|
@ -350,7 +350,7 @@ pub enum Endpoint {
|
||||||
delimiter: Option<char>,
|
delimiter: Option<char>,
|
||||||
encoding_type: Option<String>,
|
encoding_type: Option<String>,
|
||||||
key_marker: Option<String>,
|
key_marker: Option<String>,
|
||||||
max_uploads: Option<u64>,
|
max_uploads: Option<usize>,
|
||||||
prefix: Option<String>,
|
prefix: Option<String>,
|
||||||
upload_id_marker: Option<String>,
|
upload_id_marker: Option<String>,
|
||||||
},
|
},
|
||||||
|
|
|
@ -141,6 +141,60 @@ pub struct CompleteMultipartUploadResult {
|
||||||
pub etag: Value,
|
pub etag: Value,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, PartialEq)]
|
||||||
|
pub struct Initiator {
|
||||||
|
#[serde(rename = "DisplayName")]
|
||||||
|
pub display_name: Value,
|
||||||
|
#[serde(rename = "ID")]
|
||||||
|
pub id: Value,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, PartialEq)]
|
||||||
|
pub struct ListMultipartItem {
|
||||||
|
#[serde(rename = "Initiated")]
|
||||||
|
pub initiated: Value,
|
||||||
|
#[serde(rename = "Initiator")]
|
||||||
|
pub initiator: Initiator,
|
||||||
|
#[serde(rename = "Key")]
|
||||||
|
pub key: Value,
|
||||||
|
#[serde(rename = "UploadId")]
|
||||||
|
pub upload_id: Value,
|
||||||
|
#[serde(rename = "Owner")]
|
||||||
|
pub owner: Owner,
|
||||||
|
#[serde(rename = "StorageClass")]
|
||||||
|
pub storage_class: Value,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, PartialEq)]
|
||||||
|
pub struct ListMultipartUploadsResult {
|
||||||
|
#[serde(serialize_with = "xmlns_tag")]
|
||||||
|
pub xmlns: (),
|
||||||
|
#[serde(rename = "Bucket")]
|
||||||
|
pub bucket: Value,
|
||||||
|
#[serde(rename = "KeyMarker")]
|
||||||
|
pub key_marker: Option<Value>,
|
||||||
|
#[serde(rename = "UploadIdMarker")]
|
||||||
|
pub upload_id_marker: Option<Value>,
|
||||||
|
#[serde(rename = "NextKeyMarker")]
|
||||||
|
pub next_key_marker: Option<Value>,
|
||||||
|
#[serde(rename = "NextUploadIdMarker")]
|
||||||
|
pub next_upload_id_marker: Option<Value>,
|
||||||
|
#[serde(rename = "Prefix")]
|
||||||
|
pub prefix: Value,
|
||||||
|
#[serde(rename = "Delimiter")]
|
||||||
|
pub delimiter: Option<Value>,
|
||||||
|
#[serde(rename = "MaxUploads")]
|
||||||
|
pub max_uploads: IntValue,
|
||||||
|
#[serde(rename = "IsTruncated")]
|
||||||
|
pub is_truncated: Value,
|
||||||
|
#[serde(rename = "Upload")]
|
||||||
|
pub upload: Vec<ListMultipartItem>,
|
||||||
|
#[serde(rename = "CommonPrefixes")]
|
||||||
|
pub common_prefixes: Vec<CommonPrefix>,
|
||||||
|
#[serde(rename = "EncodingType")]
|
||||||
|
pub encoding_type: Option<Value>,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize, PartialEq)]
|
#[derive(Debug, Serialize, PartialEq)]
|
||||||
pub struct ListBucketItem {
|
pub struct ListBucketItem {
|
||||||
#[serde(rename = "Key")]
|
#[serde(rename = "Key")]
|
||||||
|
@ -432,6 +486,58 @@ mod tests {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn list_multipart_uploads_result() -> Result<(), ApiError> {
|
||||||
|
let result = ListMultipartUploadsResult {
|
||||||
|
xmlns: (),
|
||||||
|
bucket: Value("example-bucket".to_string()),
|
||||||
|
key_marker: None,
|
||||||
|
next_key_marker: None,
|
||||||
|
upload_id_marker: None,
|
||||||
|
encoding_type: None,
|
||||||
|
next_upload_id_marker: None,
|
||||||
|
upload: vec![],
|
||||||
|
delimiter: Some(Value("/".to_string())),
|
||||||
|
prefix: Value("photos/2006/".to_string()),
|
||||||
|
max_uploads: IntValue(1000),
|
||||||
|
is_truncated: Value("false".to_string()),
|
||||||
|
common_prefixes: vec![
|
||||||
|
CommonPrefix {
|
||||||
|
prefix: Value("photos/2006/February/".to_string()),
|
||||||
|
},
|
||||||
|
CommonPrefix {
|
||||||
|
prefix: Value("photos/2006/January/".to_string()),
|
||||||
|
},
|
||||||
|
CommonPrefix {
|
||||||
|
prefix: Value("photos/2006/March/".to_string()),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
to_xml_with_header(&result)?,
|
||||||
|
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
|
||||||
|
<ListMultipartUploadsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
|
||||||
|
<Bucket>example-bucket</Bucket>\
|
||||||
|
<Prefix>photos/2006/</Prefix>\
|
||||||
|
<Delimiter>/</Delimiter>\
|
||||||
|
<MaxUploads>1000</MaxUploads>\
|
||||||
|
<IsTruncated>false</IsTruncated>\
|
||||||
|
<CommonPrefixes>\
|
||||||
|
<Prefix>photos/2006/February/</Prefix>\
|
||||||
|
</CommonPrefixes>\
|
||||||
|
<CommonPrefixes>\
|
||||||
|
<Prefix>photos/2006/January/</Prefix>\
|
||||||
|
</CommonPrefixes>\
|
||||||
|
<CommonPrefixes>\
|
||||||
|
<Prefix>photos/2006/March/</Prefix>\
|
||||||
|
</CommonPrefixes>\
|
||||||
|
</ListMultipartUploadsResult>"
|
||||||
|
);
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn list_objects_v1_1() -> Result<(), ApiError> {
|
fn list_objects_v1_1() -> Result<(), ApiError> {
|
||||||
let result = ListBucketResult {
|
let result = ListBucketResult {
|
||||||
|
|
|
@ -21,6 +21,7 @@ use garage_model::garage::Garage;
|
||||||
use garage_model::helper::error::{Error, OkOrBadRequest};
|
use garage_model::helper::error::{Error, OkOrBadRequest};
|
||||||
use garage_model::key_table::*;
|
use garage_model::key_table::*;
|
||||||
use garage_model::migrate::Migrate;
|
use garage_model::migrate::Migrate;
|
||||||
|
use garage_model::object_table::ObjectFilter;
|
||||||
use garage_model::permission::*;
|
use garage_model::permission::*;
|
||||||
|
|
||||||
use crate::cli::*;
|
use crate::cli::*;
|
||||||
|
@ -209,7 +210,7 @@ impl AdminRpcHandler {
|
||||||
let objects = self
|
let objects = self
|
||||||
.garage
|
.garage
|
||||||
.object_table
|
.object_table
|
||||||
.get_range(&bucket_id, None, Some(DeletedFilter::NotDeleted), 10)
|
.get_range(&bucket_id, None, Some(ObjectFilter::IsData), 10)
|
||||||
.await?;
|
.await?;
|
||||||
if !objects.is_empty() {
|
if !objects.is_empty() {
|
||||||
return Err(Error::BadRequest(format!(
|
return Err(Error::BadRequest(format!(
|
||||||
|
|
|
@ -218,13 +218,19 @@ pub struct ObjectTable {
|
||||||
pub version_table: Arc<Table<VersionTable, TableShardedReplication>>,
|
pub version_table: Arc<Table<VersionTable, TableShardedReplication>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, Serialize, Deserialize)]
|
||||||
|
pub enum ObjectFilter {
|
||||||
|
IsData,
|
||||||
|
IsUploading,
|
||||||
|
}
|
||||||
|
|
||||||
impl TableSchema for ObjectTable {
|
impl TableSchema for ObjectTable {
|
||||||
const TABLE_NAME: &'static str = "object";
|
const TABLE_NAME: &'static str = "object";
|
||||||
|
|
||||||
type P = Uuid;
|
type P = Uuid;
|
||||||
type S = String;
|
type S = String;
|
||||||
type E = Object;
|
type E = Object;
|
||||||
type Filter = DeletedFilter;
|
type Filter = ObjectFilter;
|
||||||
|
|
||||||
fn updated(&self, old: Option<Self::E>, new: Option<Self::E>) {
|
fn updated(&self, old: Option<Self::E>, new: Option<Self::E>) {
|
||||||
let version_table = self.version_table.clone();
|
let version_table = self.version_table.clone();
|
||||||
|
@ -254,8 +260,10 @@ impl TableSchema for ObjectTable {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn matches_filter(entry: &Self::E, filter: &Self::Filter) -> bool {
|
fn matches_filter(entry: &Self::E, filter: &Self::Filter) -> bool {
|
||||||
let deleted = !entry.versions.iter().any(|v| v.is_data());
|
match filter {
|
||||||
filter.apply(deleted)
|
ObjectFilter::IsData => entry.versions.iter().any(|v| v.is_data()),
|
||||||
|
ObjectFilter::IsUploading => entry.versions.iter().any(|v| v.is_uploading()),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn try_migrate(bytes: &[u8]) -> Option<Self::E> {
|
fn try_migrate(bytes: &[u8]) -> Option<Self::E> {
|
||||||
|
|
Loading…
Reference in a new issue