From cee7560fc1c3e885dc80dfee233211f54ac9db7d Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Sun, 16 Feb 2025 16:44:34 +0100 Subject: [PATCH] api: refactor: move checksum algorithms to common --- Cargo.lock | 5 + src/api/common/Cargo.toml | 5 + src/api/common/signature/checksum.rs | 181 +++++++++++++++++++++++++++ src/api/common/signature/error.rs | 4 + src/api/common/signature/mod.rs | 1 + src/api/k2v/error.rs | 7 ++ src/api/s3/checksum.rs | 171 +------------------------ src/api/s3/copy.rs | 1 + src/api/s3/encryption.rs | 2 +- src/api/s3/error.rs | 3 +- src/api/s3/get.rs | 3 +- src/api/s3/multipart.rs | 1 + src/api/s3/post_object.rs | 1 + src/api/s3/put.rs | 1 + 14 files changed, 215 insertions(+), 171 deletions(-) create mode 100644 src/api/common/signature/checksum.rs diff --git a/Cargo.lock b/Cargo.lock index ad5d098d..26f6ea1d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1301,8 +1301,11 @@ dependencies = [ name = "garage_api_common" version = "1.0.1" dependencies = [ + "base64 0.21.7", "bytes", "chrono", + "crc32c", + "crc32fast", "crypto-common", "err-derive", "futures", @@ -1316,11 +1319,13 @@ dependencies = [ "hyper 1.6.0", "hyper-util", "idna 0.5.0", + "md-5", "nom", "opentelemetry", "pin-project", "serde", "serde_json", + "sha1", "sha2", "tokio", "tracing", diff --git a/src/api/common/Cargo.toml b/src/api/common/Cargo.toml index 5b9cf479..c33d585d 100644 --- a/src/api/common/Cargo.toml +++ b/src/api/common/Cargo.toml @@ -18,16 +18,21 @@ garage_model.workspace = true garage_table.workspace = true garage_util.workspace = true +base64.workspace = true bytes.workspace = true chrono.workspace = true +crc32fast.workspace = true +crc32c.workspace = true crypto-common.workspace = true err-derive.workspace = true hex.workspace = true hmac.workspace = true +md-5.workspace = true idna.workspace = true tracing.workspace = true nom.workspace = true pin-project.workspace = true +sha1.workspace = true sha2.workspace = true futures.workspace = true diff --git a/src/api/common/signature/checksum.rs b/src/api/common/signature/checksum.rs new file mode 100644 index 00000000..c6beb33f --- /dev/null +++ b/src/api/common/signature/checksum.rs @@ -0,0 +1,181 @@ +use std::convert::{TryFrom, TryInto}; +use std::hash::Hasher; + +use base64::prelude::*; +use crc32c::Crc32cHasher as Crc32c; +use crc32fast::Hasher as Crc32; +use md5::{Digest, Md5}; +use sha1::Sha1; +use sha2::Sha256; + +use http::HeaderName; + +use garage_util::data::*; + +use garage_model::s3::object_table::*; + +use super::error::*; + +pub const X_AMZ_CHECKSUM_ALGORITHM: HeaderName = + HeaderName::from_static("x-amz-checksum-algorithm"); +pub const X_AMZ_CHECKSUM_MODE: HeaderName = HeaderName::from_static("x-amz-checksum-mode"); +pub const X_AMZ_CHECKSUM_CRC32: HeaderName = HeaderName::from_static("x-amz-checksum-crc32"); +pub const X_AMZ_CHECKSUM_CRC32C: HeaderName = HeaderName::from_static("x-amz-checksum-crc32c"); +pub const X_AMZ_CHECKSUM_SHA1: HeaderName = HeaderName::from_static("x-amz-checksum-sha1"); +pub const X_AMZ_CHECKSUM_SHA256: HeaderName = HeaderName::from_static("x-amz-checksum-sha256"); + +pub type Crc32Checksum = [u8; 4]; +pub type Crc32cChecksum = [u8; 4]; +pub type Md5Checksum = [u8; 16]; +pub type Sha1Checksum = [u8; 20]; +pub type Sha256Checksum = [u8; 32]; + +#[derive(Debug, Default)] +pub struct ExpectedChecksums { + // base64-encoded md5 (content-md5 header) + pub md5: Option, + // content_sha256 (as a Hash / FixedBytes32) + pub sha256: Option, + // extra x-amz-checksum-* header + pub extra: Option, +} + +pub struct Checksummer { + pub crc32: Option, + pub crc32c: Option, + pub md5: Option, + pub sha1: Option, + pub sha256: Option, +} + +#[derive(Default)] +pub struct Checksums { + pub crc32: Option, + pub crc32c: Option, + pub md5: Option, + pub sha1: Option, + pub sha256: Option, +} + +impl Checksummer { + pub fn init(expected: &ExpectedChecksums, require_md5: bool) -> Self { + let mut ret = Self { + crc32: None, + crc32c: None, + md5: None, + sha1: None, + sha256: None, + }; + + if expected.md5.is_some() || require_md5 { + ret.md5 = Some(Md5::new()); + } + if expected.sha256.is_some() || matches!(&expected.extra, Some(ChecksumValue::Sha256(_))) { + ret.sha256 = Some(Sha256::new()); + } + if matches!(&expected.extra, Some(ChecksumValue::Crc32(_))) { + ret.crc32 = Some(Crc32::new()); + } + if matches!(&expected.extra, Some(ChecksumValue::Crc32c(_))) { + ret.crc32c = Some(Crc32c::default()); + } + if matches!(&expected.extra, Some(ChecksumValue::Sha1(_))) { + ret.sha1 = Some(Sha1::new()); + } + ret + } + + pub fn add(mut self, algo: Option) -> Self { + match algo { + Some(ChecksumAlgorithm::Crc32) => { + self.crc32 = Some(Crc32::new()); + } + Some(ChecksumAlgorithm::Crc32c) => { + self.crc32c = Some(Crc32c::default()); + } + Some(ChecksumAlgorithm::Sha1) => { + self.sha1 = Some(Sha1::new()); + } + Some(ChecksumAlgorithm::Sha256) => { + self.sha256 = Some(Sha256::new()); + } + None => (), + } + self + } + + pub fn update(&mut self, bytes: &[u8]) { + if let Some(crc32) = &mut self.crc32 { + crc32.update(bytes); + } + if let Some(crc32c) = &mut self.crc32c { + crc32c.write(bytes); + } + if let Some(md5) = &mut self.md5 { + md5.update(bytes); + } + if let Some(sha1) = &mut self.sha1 { + sha1.update(bytes); + } + if let Some(sha256) = &mut self.sha256 { + sha256.update(bytes); + } + } + + pub fn finalize(self) -> Checksums { + Checksums { + crc32: self.crc32.map(|x| u32::to_be_bytes(x.finalize())), + crc32c: self + .crc32c + .map(|x| u32::to_be_bytes(u32::try_from(x.finish()).unwrap())), + md5: self.md5.map(|x| x.finalize()[..].try_into().unwrap()), + sha1: self.sha1.map(|x| x.finalize()[..].try_into().unwrap()), + sha256: self.sha256.map(|x| x.finalize()[..].try_into().unwrap()), + } + } +} + +impl Checksums { + pub fn verify(&self, expected: &ExpectedChecksums) -> Result<(), Error> { + if let Some(expected_md5) = &expected.md5 { + match self.md5 { + Some(md5) if BASE64_STANDARD.encode(&md5) == expected_md5.trim_matches('"') => (), + _ => { + return Err(Error::InvalidDigest( + "MD5 checksum verification failed (from content-md5)".into(), + )) + } + } + } + if let Some(expected_sha256) = &expected.sha256 { + match self.sha256 { + Some(sha256) if &sha256[..] == expected_sha256.as_slice() => (), + _ => { + return Err(Error::InvalidDigest( + "SHA256 checksum verification failed (from x-amz-content-sha256)".into(), + )) + } + } + } + if let Some(extra) = expected.extra { + let algo = extra.algorithm(); + if self.extract(Some(algo)) != Some(extra) { + return Err(Error::InvalidDigest(format!( + "Failed to validate checksum for algorithm {:?}", + algo + ))); + } + } + Ok(()) + } + + pub fn extract(&self, algo: Option) -> Option { + match algo { + None => None, + Some(ChecksumAlgorithm::Crc32) => Some(ChecksumValue::Crc32(self.crc32.unwrap())), + Some(ChecksumAlgorithm::Crc32c) => Some(ChecksumValue::Crc32c(self.crc32c.unwrap())), + Some(ChecksumAlgorithm::Sha1) => Some(ChecksumValue::Sha1(self.sha1.unwrap())), + Some(ChecksumAlgorithm::Sha256) => Some(ChecksumValue::Sha256(self.sha256.unwrap())), + } + } +} diff --git a/src/api/common/signature/error.rs b/src/api/common/signature/error.rs index 2d92a072..b2f396b5 100644 --- a/src/api/common/signature/error.rs +++ b/src/api/common/signature/error.rs @@ -18,6 +18,10 @@ pub enum Error { /// The request contained an invalid UTF-8 sequence in its path or in other parameters #[error(display = "Invalid UTF-8: {}", _0)] InvalidUtf8Str(#[error(source)] std::str::Utf8Error), + + /// The provided digest (checksum) value was invalid + #[error(display = "Invalid digest: {}", _0)] + InvalidDigest(String), } impl From for Error diff --git a/src/api/common/signature/mod.rs b/src/api/common/signature/mod.rs index 27082168..08b0aa7e 100644 --- a/src/api/common/signature/mod.rs +++ b/src/api/common/signature/mod.rs @@ -11,6 +11,7 @@ use garage_util::data::{sha256sum, Hash}; use error::*; +pub mod checksum; pub mod error; pub mod payload; pub mod streaming; diff --git a/src/api/k2v/error.rs b/src/api/k2v/error.rs index 3cd0e6f7..b7ca5aa4 100644 --- a/src/api/k2v/error.rs +++ b/src/api/k2v/error.rs @@ -23,6 +23,10 @@ pub enum Error { #[error(display = "Authorization header malformed, unexpected scope: {}", _0)] AuthorizationHeaderMalformed(String), + /// The provided digest (checksum) value was invalid + #[error(display = "Invalid digest: {}", _0)] + InvalidDigest(String), + /// The object requested don't exists #[error(display = "Key not found")] NoSuchKey, @@ -54,6 +58,7 @@ impl From for Error { Self::AuthorizationHeaderMalformed(c) } SignatureError::InvalidUtf8Str(i) => Self::InvalidUtf8Str(i), + SignatureError::InvalidDigest(d) => Self::InvalidDigest(d), } } } @@ -71,6 +76,7 @@ impl Error { Error::InvalidBase64(_) => "InvalidBase64", Error::InvalidUtf8Str(_) => "InvalidUtf8String", Error::InvalidCausalityToken => "CausalityToken", + Error::InvalidDigest(_) => "InvalidDigest", } } } @@ -85,6 +91,7 @@ impl ApiError for Error { Error::AuthorizationHeaderMalformed(_) | Error::InvalidBase64(_) | Error::InvalidUtf8Str(_) + | Error::InvalidDigest(_) | Error::InvalidCausalityToken => StatusCode::BAD_REQUEST, } } diff --git a/src/api/s3/checksum.rs b/src/api/s3/checksum.rs index 02fb55ec..a720a82f 100644 --- a/src/api/s3/checksum.rs +++ b/src/api/s3/checksum.rs @@ -8,181 +8,16 @@ use md5::{Digest, Md5}; use sha1::Sha1; use sha2::Sha256; -use http::{HeaderMap, HeaderName, HeaderValue}; +use http::{HeaderMap, HeaderValue}; -use garage_util::data::*; use garage_util::error::OkOrMessage; use garage_model::s3::object_table::*; +use garage_api_common::signature::checksum::*; + use crate::error::*; -pub const X_AMZ_CHECKSUM_ALGORITHM: HeaderName = - HeaderName::from_static("x-amz-checksum-algorithm"); -pub const X_AMZ_CHECKSUM_MODE: HeaderName = HeaderName::from_static("x-amz-checksum-mode"); -pub const X_AMZ_CHECKSUM_CRC32: HeaderName = HeaderName::from_static("x-amz-checksum-crc32"); -pub const X_AMZ_CHECKSUM_CRC32C: HeaderName = HeaderName::from_static("x-amz-checksum-crc32c"); -pub const X_AMZ_CHECKSUM_SHA1: HeaderName = HeaderName::from_static("x-amz-checksum-sha1"); -pub const X_AMZ_CHECKSUM_SHA256: HeaderName = HeaderName::from_static("x-amz-checksum-sha256"); - -pub type Crc32Checksum = [u8; 4]; -pub type Crc32cChecksum = [u8; 4]; -pub type Md5Checksum = [u8; 16]; -pub type Sha1Checksum = [u8; 20]; -pub type Sha256Checksum = [u8; 32]; - -#[derive(Debug, Default)] -pub(crate) struct ExpectedChecksums { - // base64-encoded md5 (content-md5 header) - pub md5: Option, - // content_sha256 (as a Hash / FixedBytes32) - pub sha256: Option, - // extra x-amz-checksum-* header - pub extra: Option, -} - -pub(crate) struct Checksummer { - pub crc32: Option, - pub crc32c: Option, - pub md5: Option, - pub sha1: Option, - pub sha256: Option, -} - -#[derive(Default)] -pub(crate) struct Checksums { - pub crc32: Option, - pub crc32c: Option, - pub md5: Option, - pub sha1: Option, - pub sha256: Option, -} - -impl Checksummer { - pub(crate) fn init(expected: &ExpectedChecksums, require_md5: bool) -> Self { - let mut ret = Self { - crc32: None, - crc32c: None, - md5: None, - sha1: None, - sha256: None, - }; - - if expected.md5.is_some() || require_md5 { - ret.md5 = Some(Md5::new()); - } - if expected.sha256.is_some() || matches!(&expected.extra, Some(ChecksumValue::Sha256(_))) { - ret.sha256 = Some(Sha256::new()); - } - if matches!(&expected.extra, Some(ChecksumValue::Crc32(_))) { - ret.crc32 = Some(Crc32::new()); - } - if matches!(&expected.extra, Some(ChecksumValue::Crc32c(_))) { - ret.crc32c = Some(Crc32c::default()); - } - if matches!(&expected.extra, Some(ChecksumValue::Sha1(_))) { - ret.sha1 = Some(Sha1::new()); - } - ret - } - - pub(crate) fn add(mut self, algo: Option) -> Self { - match algo { - Some(ChecksumAlgorithm::Crc32) => { - self.crc32 = Some(Crc32::new()); - } - Some(ChecksumAlgorithm::Crc32c) => { - self.crc32c = Some(Crc32c::default()); - } - Some(ChecksumAlgorithm::Sha1) => { - self.sha1 = Some(Sha1::new()); - } - Some(ChecksumAlgorithm::Sha256) => { - self.sha256 = Some(Sha256::new()); - } - None => (), - } - self - } - - pub(crate) fn update(&mut self, bytes: &[u8]) { - if let Some(crc32) = &mut self.crc32 { - crc32.update(bytes); - } - if let Some(crc32c) = &mut self.crc32c { - crc32c.write(bytes); - } - if let Some(md5) = &mut self.md5 { - md5.update(bytes); - } - if let Some(sha1) = &mut self.sha1 { - sha1.update(bytes); - } - if let Some(sha256) = &mut self.sha256 { - sha256.update(bytes); - } - } - - pub(crate) fn finalize(self) -> Checksums { - Checksums { - crc32: self.crc32.map(|x| u32::to_be_bytes(x.finalize())), - crc32c: self - .crc32c - .map(|x| u32::to_be_bytes(u32::try_from(x.finish()).unwrap())), - md5: self.md5.map(|x| x.finalize()[..].try_into().unwrap()), - sha1: self.sha1.map(|x| x.finalize()[..].try_into().unwrap()), - sha256: self.sha256.map(|x| x.finalize()[..].try_into().unwrap()), - } - } -} - -impl Checksums { - pub fn verify(&self, expected: &ExpectedChecksums) -> Result<(), Error> { - if let Some(expected_md5) = &expected.md5 { - match self.md5 { - Some(md5) if BASE64_STANDARD.encode(&md5) == expected_md5.trim_matches('"') => (), - _ => { - return Err(Error::InvalidDigest( - "MD5 checksum verification failed (from content-md5)".into(), - )) - } - } - } - if let Some(expected_sha256) = &expected.sha256 { - match self.sha256 { - Some(sha256) if &sha256[..] == expected_sha256.as_slice() => (), - _ => { - return Err(Error::InvalidDigest( - "SHA256 checksum verification failed (from x-amz-content-sha256)".into(), - )) - } - } - } - if let Some(extra) = expected.extra { - let algo = extra.algorithm(); - if self.extract(Some(algo)) != Some(extra) { - return Err(Error::InvalidDigest(format!( - "Failed to validate checksum for algorithm {:?}", - algo - ))); - } - } - Ok(()) - } - - pub fn extract(&self, algo: Option) -> Option { - match algo { - None => None, - Some(ChecksumAlgorithm::Crc32) => Some(ChecksumValue::Crc32(self.crc32.unwrap())), - Some(ChecksumAlgorithm::Crc32c) => Some(ChecksumValue::Crc32c(self.crc32c.unwrap())), - Some(ChecksumAlgorithm::Sha1) => Some(ChecksumValue::Sha1(self.sha1.unwrap())), - Some(ChecksumAlgorithm::Sha256) => Some(ChecksumValue::Sha256(self.sha256.unwrap())), - } - } -} - -// ---- - #[derive(Default)] pub(crate) struct MultipartChecksummer { pub md5: Md5, diff --git a/src/api/s3/copy.rs b/src/api/s3/copy.rs index 07d50ea5..4bf68406 100644 --- a/src/api/s3/copy.rs +++ b/src/api/s3/copy.rs @@ -21,6 +21,7 @@ use garage_model::s3::object_table::*; use garage_model::s3::version_table::*; use garage_api_common::helpers::*; +use garage_api_common::signature::checksum::*; use crate::api_server::{ReqBody, ResBody}; use crate::checksum::*; diff --git a/src/api/s3/encryption.rs b/src/api/s3/encryption.rs index b38d7792..fa7285ca 100644 --- a/src/api/s3/encryption.rs +++ b/src/api/s3/encryption.rs @@ -29,8 +29,8 @@ use garage_model::garage::Garage; use garage_model::s3::object_table::{ObjectVersionEncryption, ObjectVersionMetaInner}; use garage_api_common::common_error::*; +use garage_api_common::signature::checksum::Md5Checksum; -use crate::checksum::Md5Checksum; use crate::error::Error; const X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM: HeaderName = diff --git a/src/api/s3/error.rs b/src/api/s3/error.rs index 1bb8909c..6d4b7a11 100644 --- a/src/api/s3/error.rs +++ b/src/api/s3/error.rs @@ -80,7 +80,7 @@ pub enum Error { #[error(display = "Invalid encryption algorithm: {:?}, should be AES256", _0)] InvalidEncryptionAlgorithm(String), - /// The client sent invalid XML data + /// The provided digest (checksum) value was invalid #[error(display = "Invalid digest: {}", _0)] InvalidDigest(String), @@ -119,6 +119,7 @@ impl From for Error { Self::AuthorizationHeaderMalformed(c) } SignatureError::InvalidUtf8Str(i) => Self::InvalidUtf8Str(i), + SignatureError::InvalidDigest(d) => Self::InvalidDigest(d), } } } diff --git a/src/api/s3/get.rs b/src/api/s3/get.rs index c2393a51..6627cf4a 100644 --- a/src/api/s3/get.rs +++ b/src/api/s3/get.rs @@ -26,9 +26,10 @@ use garage_model::s3::object_table::*; use garage_model::s3::version_table::*; use garage_api_common::helpers::*; +use garage_api_common::signature::checksum::X_AMZ_CHECKSUM_MODE; use crate::api_server::ResBody; -use crate::checksum::{add_checksum_response_headers, X_AMZ_CHECKSUM_MODE}; +use crate::checksum::add_checksum_response_headers; use crate::encryption::EncryptionParams; use crate::error::*; diff --git a/src/api/s3/multipart.rs b/src/api/s3/multipart.rs index fa053df2..7f8d6440 100644 --- a/src/api/s3/multipart.rs +++ b/src/api/s3/multipart.rs @@ -16,6 +16,7 @@ use garage_model::s3::object_table::*; use garage_model::s3::version_table::*; use garage_api_common::helpers::*; +use garage_api_common::signature::checksum::*; use garage_api_common::signature::verify_signed_content; use crate::api_server::{ReqBody, ResBody}; diff --git a/src/api/s3/post_object.rs b/src/api/s3/post_object.rs index 6c0e73d4..908ee9f3 100644 --- a/src/api/s3/post_object.rs +++ b/src/api/s3/post_object.rs @@ -18,6 +18,7 @@ use garage_model::s3::object_table::*; use garage_api_common::cors::*; use garage_api_common::helpers::*; +use garage_api_common::signature::checksum::*; use garage_api_common::signature::payload::{verify_v4, Authorization}; use crate::api_server::ResBody; diff --git a/src/api/s3/put.rs b/src/api/s3/put.rs index 530b4e7b..834be6f1 100644 --- a/src/api/s3/put.rs +++ b/src/api/s3/put.rs @@ -31,6 +31,7 @@ use garage_model::s3::object_table::*; use garage_model::s3::version_table::*; use garage_api_common::helpers::*; +use garage_api_common::signature::checksum::*; use crate::api_server::{ReqBody, ResBody}; use crate::checksum::*;