Merge branch 'master' into feature/website
This commit is contained in:
commit
aa320aa04a
3 changed files with 75 additions and 45 deletions
9
Cargo.lock
generated
9
Cargo.lock
generated
|
@ -59,6 +59,12 @@ version = "0.11.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7"
|
checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "base64"
|
||||||
|
version = "0.13.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "bitflags"
|
name = "bitflags"
|
||||||
version = "1.2.1"
|
version = "1.2.1"
|
||||||
|
@ -437,6 +443,7 @@ dependencies = [
|
||||||
name = "garage_api"
|
name = "garage_api"
|
||||||
version = "0.1.1"
|
version = "0.1.1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"base64 0.13.0",
|
||||||
"bytes 0.4.12",
|
"bytes 0.4.12",
|
||||||
"chrono",
|
"chrono",
|
||||||
"crypto-mac",
|
"crypto-mac",
|
||||||
|
@ -1410,7 +1417,7 @@ version = "0.17.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "c0d4a31f5d68413404705d6982529b0e11a9aacd4839d1d6222ee3b8cb4015e1"
|
checksum = "c0d4a31f5d68413404705d6982529b0e11a9aacd4839d1d6222ee3b8cb4015e1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"base64",
|
"base64 0.11.0",
|
||||||
"log",
|
"log",
|
||||||
"ring",
|
"ring",
|
||||||
"sct",
|
"sct",
|
||||||
|
|
|
@ -20,6 +20,7 @@ garage_model = { version = "0.1.1", path = "../model" }
|
||||||
err-derive = "0.2.3"
|
err-derive = "0.2.3"
|
||||||
bytes = "0.4"
|
bytes = "0.4"
|
||||||
hex = "0.3"
|
hex = "0.3"
|
||||||
|
base64 = "0.13"
|
||||||
log = "0.4"
|
log = "0.4"
|
||||||
chrono = "0.4"
|
chrono = "0.4"
|
||||||
md-5 = "0.9.1"
|
md-5 = "0.9.1"
|
||||||
|
|
|
@ -4,7 +4,7 @@ use std::sync::Arc;
|
||||||
|
|
||||||
use futures::stream::*;
|
use futures::stream::*;
|
||||||
use hyper::{Body, Request, Response};
|
use hyper::{Body, Request, Response};
|
||||||
use md5::{Digest as Md5Digest, Md5};
|
use md5::{digest::generic_array::*, Digest as Md5Digest, Md5};
|
||||||
use sha2::{Digest as Sha256Digest, Sha256};
|
use sha2::{Digest as Sha256Digest, Sha256};
|
||||||
|
|
||||||
use garage_table::*;
|
use garage_table::*;
|
||||||
|
@ -48,20 +48,35 @@ pub async fn handle_put(
|
||||||
if first_block.len() < INLINE_THRESHOLD {
|
if first_block.len() < INLINE_THRESHOLD {
|
||||||
let mut md5sum = Md5::new();
|
let mut md5sum = Md5::new();
|
||||||
md5sum.update(&first_block[..]);
|
md5sum.update(&first_block[..]);
|
||||||
let etag = hex::encode(md5sum.finalize());
|
let md5sum_arr = md5sum.finalize();
|
||||||
|
let md5sum_hex = hex::encode(md5sum_arr);
|
||||||
|
|
||||||
|
let mut sha256sum = Sha256::new();
|
||||||
|
sha256sum.input(&first_block[..]);
|
||||||
|
let sha256sum_arr = sha256sum.result();
|
||||||
|
let mut hash = [0u8; 32];
|
||||||
|
hash.copy_from_slice(&sha256sum_arr[..]);
|
||||||
|
let sha256sum_hash = Hash::from(hash);
|
||||||
|
|
||||||
|
ensure_checksum_matches(
|
||||||
|
md5sum_arr.as_slice(),
|
||||||
|
sha256sum_hash,
|
||||||
|
content_md5.as_deref(),
|
||||||
|
content_sha256,
|
||||||
|
)?;
|
||||||
|
|
||||||
object_version.state = ObjectVersionState::Complete(ObjectVersionData::Inline(
|
object_version.state = ObjectVersionState::Complete(ObjectVersionData::Inline(
|
||||||
ObjectVersionMeta {
|
ObjectVersionMeta {
|
||||||
headers,
|
headers,
|
||||||
size: first_block.len() as u64,
|
size: first_block.len() as u64,
|
||||||
etag: etag.clone(),
|
etag: md5sum_hex.clone(),
|
||||||
},
|
},
|
||||||
first_block,
|
first_block,
|
||||||
));
|
));
|
||||||
|
|
||||||
let object = Object::new(bucket.into(), key.into(), vec![object_version]);
|
let object = Object::new(bucket.into(), key.into(), vec![object_version]);
|
||||||
garage.object_table.insert(&object).await?;
|
garage.object_table.insert(&object).await?;
|
||||||
return Ok(put_response(version_uuid, etag));
|
return Ok(put_response(version_uuid, md5sum_hex));
|
||||||
}
|
}
|
||||||
|
|
||||||
let version = Version::new(version_uuid, bucket.into(), key.into(), false, vec![]);
|
let version = Version::new(version_uuid, bucket.into(), key.into(), false, vec![]);
|
||||||
|
@ -70,7 +85,7 @@ pub async fn handle_put(
|
||||||
let object = Object::new(bucket.into(), key.into(), vec![object_version.clone()]);
|
let object = Object::new(bucket.into(), key.into(), vec![object_version.clone()]);
|
||||||
garage.object_table.insert(&object).await?;
|
garage.object_table.insert(&object).await?;
|
||||||
|
|
||||||
let (total_size, md5sum, sha256sum) = read_and_put_blocks(
|
let (total_size, md5sum_arr, sha256sum) = read_and_put_blocks(
|
||||||
&garage,
|
&garage,
|
||||||
version,
|
version,
|
||||||
1,
|
1,
|
||||||
|
@ -80,7 +95,40 @@ pub async fn handle_put(
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
// Validate MD5 sum against content-md5 header and sha256sum against signed content-sha256
|
ensure_checksum_matches(
|
||||||
|
md5sum_arr.as_slice(),
|
||||||
|
sha256sum,
|
||||||
|
content_md5.as_deref(),
|
||||||
|
content_sha256,
|
||||||
|
)?;
|
||||||
|
|
||||||
|
// TODO: if at any step we have an error, we should undo everything we did
|
||||||
|
|
||||||
|
let md5sum_hex = hex::encode(md5sum_arr);
|
||||||
|
|
||||||
|
object_version.state = ObjectVersionState::Complete(ObjectVersionData::FirstBlock(
|
||||||
|
ObjectVersionMeta {
|
||||||
|
headers,
|
||||||
|
size: total_size,
|
||||||
|
etag: md5sum_hex.clone(),
|
||||||
|
},
|
||||||
|
first_block_hash,
|
||||||
|
));
|
||||||
|
|
||||||
|
let object = Object::new(bucket.into(), key.into(), vec![object_version]);
|
||||||
|
garage.object_table.insert(&object).await?;
|
||||||
|
|
||||||
|
Ok(put_response(version_uuid, md5sum_hex))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Validate MD5 sum against content-md5 header
|
||||||
|
/// and sha256sum against signed content-sha256
|
||||||
|
fn ensure_checksum_matches(
|
||||||
|
md5sum: &[u8],
|
||||||
|
sha256sum: garage_util::data::FixedBytes32,
|
||||||
|
content_md5: Option<&str>,
|
||||||
|
content_sha256: Option<garage_util::data::FixedBytes32>,
|
||||||
|
) -> Result<(), Error> {
|
||||||
if let Some(expected_sha256) = content_sha256 {
|
if let Some(expected_sha256) = content_sha256 {
|
||||||
if expected_sha256 != sha256sum {
|
if expected_sha256 != sha256sum {
|
||||||
return Err(Error::BadRequest(format!(
|
return Err(Error::BadRequest(format!(
|
||||||
|
@ -91,28 +139,13 @@ pub async fn handle_put(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if let Some(expected_md5) = content_md5 {
|
if let Some(expected_md5) = content_md5 {
|
||||||
if expected_md5.trim_matches('"') != md5sum {
|
if expected_md5.trim_matches('"') != base64::encode(md5sum) {
|
||||||
return Err(Error::BadRequest(format!("Unable to validate content-md5")));
|
return Err(Error::BadRequest(format!("Unable to validate content-md5")));
|
||||||
} else {
|
} else {
|
||||||
trace!("Successfully validated content-md5");
|
trace!("Successfully validated content-md5");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Ok(())
|
||||||
// TODO: if at any step we have an error, we should undo everything we did
|
|
||||||
|
|
||||||
object_version.state = ObjectVersionState::Complete(ObjectVersionData::FirstBlock(
|
|
||||||
ObjectVersionMeta {
|
|
||||||
headers,
|
|
||||||
size: total_size,
|
|
||||||
etag: md5sum.clone(),
|
|
||||||
},
|
|
||||||
first_block_hash,
|
|
||||||
));
|
|
||||||
|
|
||||||
let object = Object::new(bucket.into(), key.into(), vec![object_version]);
|
|
||||||
garage.object_table.insert(&object).await?;
|
|
||||||
|
|
||||||
Ok(put_response(version_uuid, md5sum))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn read_and_put_blocks(
|
async fn read_and_put_blocks(
|
||||||
|
@ -122,7 +155,7 @@ async fn read_and_put_blocks(
|
||||||
first_block: Vec<u8>,
|
first_block: Vec<u8>,
|
||||||
first_block_hash: Hash,
|
first_block_hash: Hash,
|
||||||
chunker: &mut BodyChunker,
|
chunker: &mut BodyChunker,
|
||||||
) -> Result<(u64, String, Hash), Error> {
|
) -> Result<(u64, GenericArray<u8, typenum::U16>, Hash), Error> {
|
||||||
let mut md5sum = Md5::new();
|
let mut md5sum = Md5::new();
|
||||||
let mut sha256sum = Sha256::new();
|
let mut sha256sum = Sha256::new();
|
||||||
md5sum.update(&first_block[..]);
|
md5sum.update(&first_block[..]);
|
||||||
|
@ -165,14 +198,14 @@ async fn read_and_put_blocks(
|
||||||
}
|
}
|
||||||
|
|
||||||
let total_size = next_offset as u64;
|
let total_size = next_offset as u64;
|
||||||
let md5sum = hex::encode(md5sum.finalize());
|
let md5sum_arr = md5sum.finalize();
|
||||||
|
|
||||||
let sha256sum = sha256sum.result();
|
let sha256sum = sha256sum.result();
|
||||||
let mut hash = [0u8; 32];
|
let mut hash = [0u8; 32];
|
||||||
hash.copy_from_slice(&sha256sum[..]);
|
hash.copy_from_slice(&sha256sum[..]);
|
||||||
let sha256sum = Hash::from(hash);
|
let sha256sum = Hash::from(hash);
|
||||||
|
|
||||||
Ok((total_size, md5sum, sha256sum))
|
Ok((total_size, md5sum_arr, sha256sum))
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn put_block_meta(
|
async fn put_block_meta(
|
||||||
|
@ -338,7 +371,7 @@ pub async fn handle_put_part(
|
||||||
// Copy block to store
|
// Copy block to store
|
||||||
let version = Version::new(version_uuid, bucket.into(), key.into(), false, vec![]);
|
let version = Version::new(version_uuid, bucket.into(), key.into(), false, vec![]);
|
||||||
let first_block_hash = hash(&first_block[..]);
|
let first_block_hash = hash(&first_block[..]);
|
||||||
let (_, md5sum, sha256sum) = read_and_put_blocks(
|
let (_, md5sum_arr, sha256sum) = read_and_put_blocks(
|
||||||
&garage,
|
&garage,
|
||||||
version,
|
version,
|
||||||
part_number,
|
part_number,
|
||||||
|
@ -348,23 +381,12 @@ pub async fn handle_put_part(
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
// Validate MD5 sum against content-md5 header and sha256sum against signed content-sha256
|
ensure_checksum_matches(
|
||||||
if let Some(expected_sha256) = content_sha256 {
|
md5sum_arr.as_slice(),
|
||||||
if expected_sha256 != sha256sum {
|
sha256sum,
|
||||||
return Err(Error::BadRequest(format!(
|
content_md5.as_deref(),
|
||||||
"Unable to validate x-amz-content-sha256"
|
content_sha256,
|
||||||
)));
|
)?;
|
||||||
} else {
|
|
||||||
trace!("Successfully validated x-amz-content-sha256");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if let Some(expected_md5) = content_md5 {
|
|
||||||
if expected_md5.trim_matches('"') != md5sum {
|
|
||||||
return Err(Error::BadRequest(format!("Unable to validate content-md5")));
|
|
||||||
} else {
|
|
||||||
trace!("Successfully validated content-md5");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(Response::new(Body::from(vec![])))
|
Ok(Response::new(Body::from(vec![])))
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue