Merge pull request 'Fix checksum handling' (#21) from bug/checksums into master

Reviewed-on: Deuxfleurs/garage#21
This commit is contained in:
Alex 2020-11-22 15:29:02 +01:00
commit ba1f8060fe
3 changed files with 75 additions and 45 deletions

9
Cargo.lock generated
View file

@ -59,6 +59,12 @@ version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7"
[[package]]
name = "base64"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
[[package]] [[package]]
name = "bitflags" name = "bitflags"
version = "1.2.1" version = "1.2.1"
@ -436,6 +442,7 @@ dependencies = [
name = "garage_api" name = "garage_api"
version = "0.1.1" version = "0.1.1"
dependencies = [ dependencies = [
"base64 0.13.0",
"bytes 0.4.12", "bytes 0.4.12",
"chrono", "chrono",
"crypto-mac", "crypto-mac",
@ -1378,7 +1385,7 @@ version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0d4a31f5d68413404705d6982529b0e11a9aacd4839d1d6222ee3b8cb4015e1" checksum = "c0d4a31f5d68413404705d6982529b0e11a9aacd4839d1d6222ee3b8cb4015e1"
dependencies = [ dependencies = [
"base64", "base64 0.11.0",
"log", "log",
"ring", "ring",
"sct", "sct",

View file

@ -20,6 +20,7 @@ garage_model = { version = "0.1.1", path = "../model" }
err-derive = "0.2.3" err-derive = "0.2.3"
bytes = "0.4" bytes = "0.4"
hex = "0.3" hex = "0.3"
base64 = "0.13"
log = "0.4" log = "0.4"
chrono = "0.4" chrono = "0.4"
md-5 = "0.9.1" md-5 = "0.9.1"

View file

@ -4,7 +4,7 @@ use std::sync::Arc;
use futures::stream::*; use futures::stream::*;
use hyper::{Body, Request, Response}; use hyper::{Body, Request, Response};
use md5::{Digest as Md5Digest, Md5}; use md5::{digest::generic_array::*, Digest as Md5Digest, Md5};
use sha2::{Digest as Sha256Digest, Sha256}; use sha2::{Digest as Sha256Digest, Sha256};
use garage_table::*; use garage_table::*;
@ -48,20 +48,35 @@ pub async fn handle_put(
if first_block.len() < INLINE_THRESHOLD { if first_block.len() < INLINE_THRESHOLD {
let mut md5sum = Md5::new(); let mut md5sum = Md5::new();
md5sum.update(&first_block[..]); md5sum.update(&first_block[..]);
let etag = hex::encode(md5sum.finalize()); let md5sum_arr = md5sum.finalize();
let md5sum_hex = hex::encode(md5sum_arr);
let mut sha256sum = Sha256::new();
sha256sum.input(&first_block[..]);
let sha256sum_arr = sha256sum.result();
let mut hash = [0u8; 32];
hash.copy_from_slice(&sha256sum_arr[..]);
let sha256sum_hash = Hash::from(hash);
ensure_checksum_matches(
md5sum_arr.as_slice(),
sha256sum_hash,
content_md5.as_deref(),
content_sha256,
)?;
object_version.state = ObjectVersionState::Complete(ObjectVersionData::Inline( object_version.state = ObjectVersionState::Complete(ObjectVersionData::Inline(
ObjectVersionMeta { ObjectVersionMeta {
headers, headers,
size: first_block.len() as u64, size: first_block.len() as u64,
etag: etag.clone(), etag: md5sum_hex.clone(),
}, },
first_block, first_block,
)); ));
let object = Object::new(bucket.into(), key.into(), vec![object_version]); let object = Object::new(bucket.into(), key.into(), vec![object_version]);
garage.object_table.insert(&object).await?; garage.object_table.insert(&object).await?;
return Ok(put_response(version_uuid, etag)); return Ok(put_response(version_uuid, md5sum_hex));
} }
let version = Version::new(version_uuid, bucket.into(), key.into(), false, vec![]); let version = Version::new(version_uuid, bucket.into(), key.into(), false, vec![]);
@ -70,7 +85,7 @@ pub async fn handle_put(
let object = Object::new(bucket.into(), key.into(), vec![object_version.clone()]); let object = Object::new(bucket.into(), key.into(), vec![object_version.clone()]);
garage.object_table.insert(&object).await?; garage.object_table.insert(&object).await?;
let (total_size, md5sum, sha256sum) = read_and_put_blocks( let (total_size, md5sum_arr, sha256sum) = read_and_put_blocks(
&garage, &garage,
version, version,
1, 1,
@ -80,7 +95,40 @@ pub async fn handle_put(
) )
.await?; .await?;
// Validate MD5 sum against content-md5 header and sha256sum against signed content-sha256 ensure_checksum_matches(
md5sum_arr.as_slice(),
sha256sum,
content_md5.as_deref(),
content_sha256,
)?;
// TODO: if at any step we have an error, we should undo everything we did
let md5sum_hex = hex::encode(md5sum_arr);
object_version.state = ObjectVersionState::Complete(ObjectVersionData::FirstBlock(
ObjectVersionMeta {
headers,
size: total_size,
etag: md5sum_hex.clone(),
},
first_block_hash,
));
let object = Object::new(bucket.into(), key.into(), vec![object_version]);
garage.object_table.insert(&object).await?;
Ok(put_response(version_uuid, md5sum_hex))
}
/// Validate MD5 sum against content-md5 header
/// and sha256sum against signed content-sha256
fn ensure_checksum_matches(
md5sum: &[u8],
sha256sum: garage_util::data::FixedBytes32,
content_md5: Option<&str>,
content_sha256: Option<garage_util::data::FixedBytes32>,
) -> Result<(), Error> {
if let Some(expected_sha256) = content_sha256 { if let Some(expected_sha256) = content_sha256 {
if expected_sha256 != sha256sum { if expected_sha256 != sha256sum {
return Err(Error::BadRequest(format!( return Err(Error::BadRequest(format!(
@ -91,28 +139,13 @@ pub async fn handle_put(
} }
} }
if let Some(expected_md5) = content_md5 { if let Some(expected_md5) = content_md5 {
if expected_md5.trim_matches('"') != md5sum { if expected_md5.trim_matches('"') != base64::encode(md5sum) {
return Err(Error::BadRequest(format!("Unable to validate content-md5"))); return Err(Error::BadRequest(format!("Unable to validate content-md5")));
} else { } else {
trace!("Successfully validated content-md5"); trace!("Successfully validated content-md5");
} }
} }
Ok(())
// TODO: if at any step we have an error, we should undo everything we did
object_version.state = ObjectVersionState::Complete(ObjectVersionData::FirstBlock(
ObjectVersionMeta {
headers,
size: total_size,
etag: md5sum.clone(),
},
first_block_hash,
));
let object = Object::new(bucket.into(), key.into(), vec![object_version]);
garage.object_table.insert(&object).await?;
Ok(put_response(version_uuid, md5sum))
} }
async fn read_and_put_blocks( async fn read_and_put_blocks(
@ -122,7 +155,7 @@ async fn read_and_put_blocks(
first_block: Vec<u8>, first_block: Vec<u8>,
first_block_hash: Hash, first_block_hash: Hash,
chunker: &mut BodyChunker, chunker: &mut BodyChunker,
) -> Result<(u64, String, Hash), Error> { ) -> Result<(u64, GenericArray<u8, typenum::U16>, Hash), Error> {
let mut md5sum = Md5::new(); let mut md5sum = Md5::new();
let mut sha256sum = Sha256::new(); let mut sha256sum = Sha256::new();
md5sum.update(&first_block[..]); md5sum.update(&first_block[..]);
@ -165,14 +198,14 @@ async fn read_and_put_blocks(
} }
let total_size = next_offset as u64; let total_size = next_offset as u64;
let md5sum = hex::encode(md5sum.finalize()); let md5sum_arr = md5sum.finalize();
let sha256sum = sha256sum.result(); let sha256sum = sha256sum.result();
let mut hash = [0u8; 32]; let mut hash = [0u8; 32];
hash.copy_from_slice(&sha256sum[..]); hash.copy_from_slice(&sha256sum[..]);
let sha256sum = Hash::from(hash); let sha256sum = Hash::from(hash);
Ok((total_size, md5sum, sha256sum)) Ok((total_size, md5sum_arr, sha256sum))
} }
async fn put_block_meta( async fn put_block_meta(
@ -338,7 +371,7 @@ pub async fn handle_put_part(
// Copy block to store // Copy block to store
let version = Version::new(version_uuid, bucket.into(), key.into(), false, vec![]); let version = Version::new(version_uuid, bucket.into(), key.into(), false, vec![]);
let first_block_hash = hash(&first_block[..]); let first_block_hash = hash(&first_block[..]);
let (_, md5sum, sha256sum) = read_and_put_blocks( let (_, md5sum_arr, sha256sum) = read_and_put_blocks(
&garage, &garage,
version, version,
part_number, part_number,
@ -348,23 +381,12 @@ pub async fn handle_put_part(
) )
.await?; .await?;
// Validate MD5 sum against content-md5 header and sha256sum against signed content-sha256 ensure_checksum_matches(
if let Some(expected_sha256) = content_sha256 { md5sum_arr.as_slice(),
if expected_sha256 != sha256sum { sha256sum,
return Err(Error::BadRequest(format!( content_md5.as_deref(),
"Unable to validate x-amz-content-sha256" content_sha256,
))); )?;
} else {
trace!("Successfully validated x-amz-content-sha256");
}
}
if let Some(expected_md5) = content_md5 {
if expected_md5.trim_matches('"') != md5sum {
return Err(Error::BadRequest(format!("Unable to validate content-md5")));
} else {
trace!("Successfully validated content-md5");
}
}
Ok(Response::new(Body::from(vec![]))) Ok(Response::new(Body::from(vec![])))
} }