Deuxfleurs/garage
41
56
Fork 73
Code Issues 98 Pull requests 14 Releases 30 Wiki Activity

Implement ListBuckets #70

Merged
lx merged 1 commit from feature/s3/list-buckets into main 2021-05-03 19:55:33 +00:00
Conversation 0 Commits 1 Files changed 6 +115 -2
6 changed files with 115 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
Cargo.lock generated
View file

@ -397,7 +397,9 @@ dependencies = [
"log", "log",
"md-5", "md-5",
"percent-encoding", "percent-encoding",
"quick-xml",
"roxmltree", "roxmltree",
"serde",
"sha2", "sha2",
"tokio", "tokio",
"url", "url",
@ -1034,6 +1036,16 @@ version = "1.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
[[package]]
name = "quick-xml"
version = "0.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0452695941410a58c8ce4391707ba9bad26a247173bd9886a05a5e8a8babec75"
dependencies = [
"memchr",
"serde",
]
[[package]] [[package]]
name = "quote" name = "quote"
version = "1.0.9" version = "1.0.9"

4
script/test-smoke.sh
View file

@ -32,6 +32,7 @@ echo "🧪 S3 API testing..."
if [ -z "$SKIP_AWS" ]; then if [ -z "$SKIP_AWS" ]; then
echo "🛠️ Testing with awscli" echo "🛠️ Testing with awscli"
source ${SCRIPT_FOLDER}/dev-env-aws.sh source ${SCRIPT_FOLDER}/dev-env-aws.sh
aws s3 ls
for idx in $(seq 1 3); do for idx in $(seq 1 3); do
aws s3 cp "/tmp/garage.$idx.rnd" "s3://eprouvette/&+-é\"/garage.$idx.aws" aws s3 cp "/tmp/garage.$idx.rnd" "s3://eprouvette/&+-é\"/garage.$idx.aws"
aws s3 ls s3://eprouvette aws s3 ls s3://eprouvette
@ -46,6 +47,7 @@ fi
if [ -z "$SKIP_S3CMD" ]; then if [ -z "$SKIP_S3CMD" ]; then
echo "🛠️ Testing with s3cmd" echo "🛠️ Testing with s3cmd"
source ${SCRIPT_FOLDER}/dev-env-s3cmd.sh source ${SCRIPT_FOLDER}/dev-env-s3cmd.sh
s3cmd ls
for idx in $(seq 1 3); do for idx in $(seq 1 3); do
s3cmd put "/tmp/garage.$idx.rnd" "s3://eprouvette/&+-é\"/garage.$idx.s3cmd" s3cmd put "/tmp/garage.$idx.rnd" "s3://eprouvette/&+-é\"/garage.$idx.s3cmd"
s3cmd ls s3://eprouvette s3cmd ls s3://eprouvette
@ -60,6 +62,7 @@ fi
if [ -z "$SKIP_MC" ]; then if [ -z "$SKIP_MC" ]; then
echo "🛠️ Testing with mc (minio client)" echo "🛠️ Testing with mc (minio client)"
source ${SCRIPT_FOLDER}/dev-env-mc.sh source ${SCRIPT_FOLDER}/dev-env-mc.sh
mc ls garage/
for idx in $(seq 1 3); do for idx in $(seq 1 3); do
mc cp "/tmp/garage.$idx.rnd" "garage/eprouvette/&+-é\"/garage.$idx.mc" mc cp "/tmp/garage.$idx.rnd" "garage/eprouvette/&+-é\"/garage.$idx.mc"
mc ls garage/eprouvette mc ls garage/eprouvette
@ -74,6 +77,7 @@ fi
if [ -z "$SKIP_RCLONE" ]; then if [ -z "$SKIP_RCLONE" ]; then
echo "🛠️ Testing with rclone" echo "🛠️ Testing with rclone"
source ${SCRIPT_FOLDER}/dev-env-rclone.sh source ${SCRIPT_FOLDER}/dev-env-rclone.sh
rclone lsd garage:
for idx in $(seq 1 3); do for idx in $(seq 1 3); do
cp /tmp/garage.$idx.rnd /tmp/garage.$idx.dl cp /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
rclone copy "/tmp/garage.$idx.dl" "garage:eprouvette/&+-é\"/" rclone copy "/tmp/garage.$idx.dl" "garage:eprouvette/&+-é\"/"

2
src/api/Cargo.toml
View file

@ -38,4 +38,6 @@ http-range = "0.1"
hyper = "0.14" hyper = "0.14"
percent-encoding = "2.1.0" percent-encoding = "2.1.0"
roxmltree = "0.14" roxmltree = "0.14"
serde = { version = "1.0", features = ["derive"] }
quick-xml = { version = "0.21", features = [ "serialize" ] }
url = "2.1" url = "2.1"

6
src/api/api_server.rs
View file

@ -81,10 +81,12 @@ async fn handler(
async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Response<Body>, Error> { async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Response<Body>, Error> {
let path = req.uri().path().to_string(); let path = req.uri().path().to_string();
let path = percent_encoding::percent_decode_str(&path).decode_utf8()?; let path = percent_encoding::percent_decode_str(&path).decode_utf8()?;
let (api_key, content_sha256) = check_signature(&garage, &req).await?;
if path == "/" {
return handle_list_buckets(&api_key);
}
let (bucket, key) = parse_bucket_key(&path)?; let (bucket, key) = parse_bucket_key(&path)?;
let (api_key, content_sha256) = check_signature(&garage, &req).await?;
let allowed = match req.method() { let allowed = match req.method() {
&Method::HEAD | &Method::GET => api_key.allow_read(&bucket), &Method::HEAD | &Method::GET => api_key.allow_read(&bucket),
_ => api_key.allow_write(&bucket), _ => api_key.allow_write(&bucket),

6
src/api/error.rs
View file

@ -72,6 +72,12 @@ impl From<roxmltree::Error> for Error {
} }
} }
impl From<quick_xml::de::DeError> for Error {
fn from(err: quick_xml::de::DeError) -> Self {
Self::InvalidXML(format!("{}", err))
}
}
impl Error { impl Error {
/// Get the HTTP status code that best represents the meaning of the error for the client /// Get the HTTP status code that best represents the meaning of the error for the client
pub fn http_status_code(&self) -> StatusCode { pub fn http_status_code(&self) -> StatusCode {

87
src/api/s3_bucket.rs
View file

@ -2,11 +2,62 @@ use std::fmt::Write;
use std::sync::Arc; use std::sync::Arc;
use hyper::{Body, Response}; use hyper::{Body, Response};
use quick_xml::se::to_string;
use serde::Serialize;
use garage_model::garage::Garage; use garage_model::garage::Garage;
use garage_model::key_table::Key;
use garage_util::time::*;
use crate::error::*; use crate::error::*;
#[derive(Debug, Serialize, PartialEq)]
struct CreationDate {
#[serde(rename = "$value")]
pub body: String,
}
#[derive(Debug, Serialize, PartialEq)]
struct Name {
#[serde(rename = "$value")]
pub body: String,
}
#[derive(Debug, Serialize, PartialEq)]
struct Bucket {
#[serde(rename = "CreationDate")]
pub creation_date: CreationDate,
#[serde(rename = "Name")]
pub name: Name,
}
#[derive(Debug, Serialize, PartialEq)]
struct DisplayName {
#[serde(rename = "$value")]
pub body: String,
}
#[derive(Debug, Serialize, PartialEq)]
struct ID {
#[serde(rename = "$value")]
pub body: String,
}
#[derive(Debug, Serialize, PartialEq)]
struct Owner {
#[serde(rename = "DisplayName")]
display_name: DisplayName,
#[serde(rename = "ID")]
id: ID,
}
#[derive(Debug, Serialize, PartialEq)]
struct BucketList {
#[serde(rename = "Bucket")]
pub entries: Vec<Bucket>,
}
#[derive(Debug, Serialize, PartialEq)]
struct ListAllMyBucketsResult {
#[serde(rename = "Buckets")]
buckets: BucketList,
#[serde(rename = "Owner")]
owner: Owner,
}
pub fn handle_get_bucket_location(garage: Arc<Garage>) -> Result<Response<Body>, Error> { pub fn handle_get_bucket_location(garage: Arc<Garage>) -> Result<Response<Body>, Error> {
let mut xml = String::new(); let mut xml = String::new();
@ -22,3 +73,39 @@ pub fn handle_get_bucket_location(garage: Arc<Garage>) -> Result<Response<Body>,
.header("Content-Type", "application/xml") .header("Content-Type", "application/xml")
.body(Body::from(xml.into_bytes()))?) .body(Body::from(xml.into_bytes()))?)
} }
pub fn handle_list_buckets(api_key: &Key) -> Result<Response<Body>, Error> {
let list_buckets = ListAllMyBucketsResult {
owner: Owner {
display_name: DisplayName {
body: api_key.name.get().to_string(),
},
id: ID {
body: api_key.key_id.to_string(),
},
},
buckets: BucketList {
entries: api_key
.authorized_buckets
.items()
.iter()
.map(|(name, ts, _)| Bucket {
creation_date: CreationDate {
body: msec_to_rfc3339(*ts),
},
name: Name {
body: name.to_string(),
},
})
.collect(),
},
};
let mut xml = r#"<?xml version="1.0" encoding="UTF-8"?>"#.to_string();
xml.push_str(&to_string(&list_buckets)?);
trace!("xml: {}", xml);
Ok(Response::builder()
.header("Content-Type", "application/xml")
.body(Body::from(xml))?)
}