From a826c361a9f9adb45ae7499da07f71c966897e82 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 19 Mar 2025 15:51:06 +0100 Subject: [PATCH] add crc64nvme checksumming algorithm (fix #963) --- Cargo.lock | 26 +++++++++++++++++++ Cargo.toml | 1 + src/api/common/Cargo.toml | 1 + src/api/common/signature/checksum.rs | 39 ++++++++++++++++++++++++++++ src/api/s3/Cargo.toml | 1 + src/api/s3/list.rs | 6 +++++ src/api/s3/multipart.rs | 27 +++++++++++++++++++ src/api/s3/xml.rs | 7 +++++ src/model/s3/object_table.rs | 3 +++ 9 files changed, 111 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index f64c50dc..01fee410 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -812,6 +812,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + [[package]] name = "crc32c" version = "0.6.8" @@ -830,6 +845,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crc64fast-nvme" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4955638f00a809894c947f85a024020a20815b65a5eea633798ea7924edab2b3" +dependencies = [ + "crc", +] + [[package]] name = "crossbeam-channel" version = "0.5.14" @@ -1332,6 +1356,7 @@ dependencies = [ "chrono", "crc32c", "crc32fast", + "crc64fast-nvme", "crypto-common", "err-derive", "futures", @@ -1392,6 +1417,7 @@ dependencies = [ "chrono", "crc32c", "crc32fast", + "crc64fast-nvme", "err-derive", "form_urlencoded", "futures", diff --git a/Cargo.toml b/Cargo.toml index ab35f757..42deb99b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,6 +51,7 @@ cfg-if = "1.0" chrono = { version = "0.4", features = ["serde"] } crc32fast = "1.4" crc32c = "0.6" +crc64fast-nvme = "1.2" crypto-common = "0.1" err-derive = "0.3" gethostname = "0.4" diff --git a/src/api/common/Cargo.toml b/src/api/common/Cargo.toml index 6d906423..5608a5e3 100644 --- a/src/api/common/Cargo.toml +++ b/src/api/common/Cargo.toml @@ -23,6 +23,7 @@ bytes.workspace = true chrono.workspace = true crc32fast.workspace = true crc32c.workspace = true +crc64fast-nvme.workspace = true crypto-common.workspace = true err-derive.workspace = true hex.workspace = true diff --git a/src/api/common/signature/checksum.rs b/src/api/common/signature/checksum.rs index 3c5e7c53..0fb66ce5 100644 --- a/src/api/common/signature/checksum.rs +++ b/src/api/common/signature/checksum.rs @@ -4,6 +4,7 @@ use std::hash::Hasher; use base64::prelude::*; use crc32c::Crc32cHasher as Crc32c; use crc32fast::Hasher as Crc32; +use crc64fast_nvme::Digest as Crc64Nvme; use md5::{Digest, Md5}; use sha1::Sha1; use sha2::Sha256; @@ -23,11 +24,14 @@ pub const X_AMZ_CHECKSUM_ALGORITHM: HeaderName = pub const X_AMZ_CHECKSUM_MODE: HeaderName = HeaderName::from_static("x-amz-checksum-mode"); pub const X_AMZ_CHECKSUM_CRC32: HeaderName = HeaderName::from_static("x-amz-checksum-crc32"); pub const X_AMZ_CHECKSUM_CRC32C: HeaderName = HeaderName::from_static("x-amz-checksum-crc32c"); +pub const X_AMZ_CHECKSUM_CRC64NVME: HeaderName = + HeaderName::from_static("x-amz-checksum-crc64nvme"); pub const X_AMZ_CHECKSUM_SHA1: HeaderName = HeaderName::from_static("x-amz-checksum-sha1"); pub const X_AMZ_CHECKSUM_SHA256: HeaderName = HeaderName::from_static("x-amz-checksum-sha256"); pub type Crc32Checksum = [u8; 4]; pub type Crc32cChecksum = [u8; 4]; +pub type Crc64NvmeChecksum = [u8; 8]; pub type Md5Checksum = [u8; 16]; pub type Sha1Checksum = [u8; 20]; pub type Sha256Checksum = [u8; 32]; @@ -45,6 +49,7 @@ pub struct ExpectedChecksums { pub struct Checksummer { pub crc32: Option, pub crc32c: Option, + pub crc64nvme: Option, pub md5: Option, pub sha1: Option, pub sha256: Option, @@ -54,6 +59,7 @@ pub struct Checksummer { pub struct Checksums { pub crc32: Option, pub crc32c: Option, + pub crc64nvme: Option, pub md5: Option, pub sha1: Option, pub sha256: Option, @@ -64,6 +70,7 @@ impl Checksummer { Self { crc32: None, crc32c: None, + crc64nvme: None, md5: None, sha1: None, sha256: None, @@ -96,6 +103,9 @@ impl Checksummer { if matches!(&expected.extra, Some(ChecksumValue::Crc32c(_))) { self.crc32c = Some(Crc32c::default()); } + if matches!(&expected.extra, Some(ChecksumValue::Crc64Nvme(_))) { + self.crc64nvme = Some(Crc64Nvme::default()); + } if matches!(&expected.extra, Some(ChecksumValue::Sha1(_))) { self.sha1 = Some(Sha1::new()); } @@ -109,6 +119,9 @@ impl Checksummer { Some(ChecksumAlgorithm::Crc32c) => { self.crc32c = Some(Crc32c::default()); } + Some(ChecksumAlgorithm::Crc64Nvme) => { + self.crc64nvme = Some(Crc64Nvme::default()); + } Some(ChecksumAlgorithm::Sha1) => { self.sha1 = Some(Sha1::new()); } @@ -127,6 +140,9 @@ impl Checksummer { if let Some(crc32c) = &mut self.crc32c { crc32c.write(bytes); } + if let Some(crc64nvme) = &mut self.crc64nvme { + crc64nvme.write(bytes); + } if let Some(md5) = &mut self.md5 { md5.update(bytes); } @@ -144,6 +160,7 @@ impl Checksummer { crc32c: self .crc32c .map(|x| u32::to_be_bytes(u32::try_from(x.finish()).unwrap())), + crc64nvme: self.crc64nvme.map(|x| u64::to_be_bytes(x.sum64())), md5: self.md5.map(|x| x.finalize()[..].try_into().unwrap()), sha1: self.sha1.map(|x| x.finalize()[..].try_into().unwrap()), sha256: self.sha256.map(|x| x.finalize()[..].try_into().unwrap()), @@ -190,6 +207,9 @@ impl Checksums { None => None, Some(ChecksumAlgorithm::Crc32) => Some(ChecksumValue::Crc32(self.crc32.unwrap())), Some(ChecksumAlgorithm::Crc32c) => Some(ChecksumValue::Crc32c(self.crc32c.unwrap())), + Some(ChecksumAlgorithm::Crc64Nvme) => { + Some(ChecksumValue::Crc64Nvme(self.crc64nvme.unwrap())) + } Some(ChecksumAlgorithm::Sha1) => Some(ChecksumValue::Sha1(self.sha1.unwrap())), Some(ChecksumAlgorithm::Sha256) => Some(ChecksumValue::Sha256(self.sha256.unwrap())), } @@ -202,6 +222,7 @@ pub fn parse_checksum_algorithm(algo: &str) -> Result match algo { "CRC32" => Ok(ChecksumAlgorithm::Crc32), "CRC32C" => Ok(ChecksumAlgorithm::Crc32c), + "CRC64NVME" => Ok(ChecksumAlgorithm::Crc64Nvme), "SHA1" => Ok(ChecksumAlgorithm::Sha1), "SHA256" => Ok(ChecksumAlgorithm::Sha256), _ => Err(Error::bad_request("invalid checksum algorithm")), @@ -225,6 +246,7 @@ pub fn request_trailer_checksum_algorithm( None => Ok(None), Some(x) if x == X_AMZ_CHECKSUM_CRC32 => Ok(Some(ChecksumAlgorithm::Crc32)), Some(x) if x == X_AMZ_CHECKSUM_CRC32C => Ok(Some(ChecksumAlgorithm::Crc32c)), + Some(x) if x == X_AMZ_CHECKSUM_CRC64NVME => Ok(Some(ChecksumAlgorithm::Crc64Nvme)), Some(x) if x == X_AMZ_CHECKSUM_SHA1 => Ok(Some(ChecksumAlgorithm::Sha1)), Some(x) if x == X_AMZ_CHECKSUM_SHA256 => Ok(Some(ChecksumAlgorithm::Sha256)), _ => Err(Error::bad_request("invalid checksum algorithm")), @@ -243,6 +265,12 @@ pub fn request_checksum_value( if headers.contains_key(X_AMZ_CHECKSUM_CRC32C) { ret.push(extract_checksum_value(headers, ChecksumAlgorithm::Crc32c)?); } + if headers.contains_key(X_AMZ_CHECKSUM_CRC64NVME) { + ret.push(extract_checksum_value( + headers, + ChecksumAlgorithm::Crc64Nvme, + )?); + } if headers.contains_key(X_AMZ_CHECKSUM_SHA1) { ret.push(extract_checksum_value(headers, ChecksumAlgorithm::Sha1)?); } @@ -281,6 +309,14 @@ pub fn extract_checksum_value( .ok_or_bad_request("invalid x-amz-checksum-crc32c header")?; Ok(ChecksumValue::Crc32c(crc32c)) } + ChecksumAlgorithm::Crc64Nvme => { + let crc64nvme = headers + .get(X_AMZ_CHECKSUM_CRC64NVME) + .and_then(|x| BASE64_STANDARD.decode(&x).ok()) + .and_then(|x| x.try_into().ok()) + .ok_or_bad_request("invalid x-amz-checksum-crc64nvme header")?; + Ok(ChecksumValue::Crc64Nvme(crc64nvme)) + } ChecksumAlgorithm::Sha1 => { let sha1 = headers .get(X_AMZ_CHECKSUM_SHA1) @@ -311,6 +347,9 @@ pub fn add_checksum_response_headers( Some(ChecksumValue::Crc32c(crc32c)) => { resp = resp.header(X_AMZ_CHECKSUM_CRC32C, BASE64_STANDARD.encode(&crc32c)); } + Some(ChecksumValue::Crc64Nvme(crc64nvme)) => { + resp = resp.header(X_AMZ_CHECKSUM_CRC64NVME, BASE64_STANDARD.encode(&crc64nvme)); + } Some(ChecksumValue::Sha1(sha1)) => { resp = resp.header(X_AMZ_CHECKSUM_SHA1, BASE64_STANDARD.encode(&sha1)); } diff --git a/src/api/s3/Cargo.toml b/src/api/s3/Cargo.toml index 47aaab8c..e236729f 100644 --- a/src/api/s3/Cargo.toml +++ b/src/api/s3/Cargo.toml @@ -29,6 +29,7 @@ bytes.workspace = true chrono.workspace = true crc32fast.workspace = true crc32c.workspace = true +crc64fast-nvme.workspace = true err-derive.workspace = true hex.workspace = true hmac.workspace = true diff --git a/src/api/s3/list.rs b/src/api/s3/list.rs index ff5ca383..797fdec0 100644 --- a/src/api/s3/list.rs +++ b/src/api/s3/list.rs @@ -334,6 +334,12 @@ pub async fn handle_list_parts( } _ => None, }, + checksum_crc64nvme: match &checksum { + Some(ChecksumValue::Crc64Nvme(x)) => { + Some(s3_xml::Value(BASE64_STANDARD.encode(&x))) + } + _ => None, + }, checksum_sha1: match &checksum { Some(ChecksumValue::Sha1(x)) => { Some(s3_xml::Value(BASE64_STANDARD.encode(&x))) diff --git a/src/api/s3/multipart.rs b/src/api/s3/multipart.rs index 52ea90e8..2758c273 100644 --- a/src/api/s3/multipart.rs +++ b/src/api/s3/multipart.rs @@ -6,6 +6,7 @@ use std::sync::Arc; use base64::prelude::*; use crc32c::Crc32cHasher as Crc32c; use crc32fast::Hasher as Crc32; +use crc64fast_nvme::Digest as Crc64Nvme; use futures::prelude::*; use hyper::{Request, Response}; use md5::{Digest, Md5}; @@ -481,6 +482,10 @@ pub async fn handle_complete_multipart_upload( Some(ChecksumValue::Crc32c(x)) => Some(s3_xml::Value(BASE64_STANDARD.encode(&x))), _ => None, }, + checksum_crc64nvme: match &checksum_extra { + Some(ChecksumValue::Crc64Nvme(x)) => Some(s3_xml::Value(BASE64_STANDARD.encode(&x))), + _ => None, + }, checksum_sha1: match &checksum_extra { Some(ChecksumValue::Sha1(x)) => Some(s3_xml::Value(BASE64_STANDARD.encode(&x))), _ => None, @@ -604,6 +609,15 @@ fn parse_complete_multipart_upload_body( .try_into() .ok()?, )) + } else if let Some(crc64nvme) = item + .children() + .find(|e| e.has_tag_name("ChecksumCRC64NVME")) + { + Some(ChecksumValue::Crc64Nvme( + BASE64_STANDARD.decode(crc64nvme.text()?).ok()?[..] + .try_into() + .ok()?, + )) } else if let Some(sha1) = item.children().find(|e| e.has_tag_name("ChecksumSHA1")) { Some(ChecksumValue::Sha1( BASE64_STANDARD.decode(sha1.text()?).ok()?[..] @@ -644,6 +658,7 @@ pub(crate) struct MultipartChecksummer { pub(crate) enum MultipartExtraChecksummer { Crc32(Crc32), Crc32c(Crc32c), + Crc64Nvme(Crc64Nvme), Sha1(Sha1), Sha256(Sha256), } @@ -660,6 +675,9 @@ impl MultipartChecksummer { Some(ChecksumAlgorithm::Crc32c) => { Some(MultipartExtraChecksummer::Crc32c(Crc32c::default())) } + Some(ChecksumAlgorithm::Crc64Nvme) => { + Some(MultipartExtraChecksummer::Crc64Nvme(Crc64Nvme::default())) + } Some(ChecksumAlgorithm::Sha1) => Some(MultipartExtraChecksummer::Sha1(Sha1::new())), Some(ChecksumAlgorithm::Sha256) => { Some(MultipartExtraChecksummer::Sha256(Sha256::new())) @@ -689,6 +707,12 @@ impl MultipartChecksummer { ) => { crc32c.write(&x); } + ( + Some(MultipartExtraChecksummer::Crc64Nvme(ref mut crc64nvme)), + Some(ChecksumValue::Crc64Nvme(x)), + ) => { + crc64nvme.write(&x); + } (Some(MultipartExtraChecksummer::Sha1(ref mut sha1)), Some(ChecksumValue::Sha1(x))) => { sha1.update(&x); } @@ -718,6 +742,9 @@ impl MultipartChecksummer { Some(MultipartExtraChecksummer::Crc32c(crc32c)) => Some(ChecksumValue::Crc32c( u32::to_be_bytes(u32::try_from(crc32c.finish()).unwrap()), )), + Some(MultipartExtraChecksummer::Crc64Nvme(crc64nvme)) => Some( + ChecksumValue::Crc64Nvme(u64::to_be_bytes(crc64nvme.sum64())), + ), Some(MultipartExtraChecksummer::Sha1(sha1)) => { Some(ChecksumValue::Sha1(sha1.finalize()[..].try_into().unwrap())) } diff --git a/src/api/s3/xml.rs b/src/api/s3/xml.rs index e8af3ec0..7dea3d1c 100644 --- a/src/api/s3/xml.rs +++ b/src/api/s3/xml.rs @@ -135,6 +135,8 @@ pub struct CompleteMultipartUploadResult { pub checksum_crc32: Option, #[serde(rename = "ChecksumCRC32C")] pub checksum_crc32c: Option, + #[serde(rename = "ChecksumCR64NVME")] + pub checksum_crc64nvme: Option, #[serde(rename = "ChecksumSHA1")] pub checksum_sha1: Option, #[serde(rename = "ChecksumSHA256")] @@ -209,6 +211,8 @@ pub struct PartItem { pub checksum_crc32: Option, #[serde(rename = "ChecksumCRC32C")] pub checksum_crc32c: Option, + #[serde(rename = "ChecksumCRC64NVME")] + pub checksum_crc64nvme: Option, #[serde(rename = "ChecksumSHA1")] pub checksum_sha1: Option, #[serde(rename = "ChecksumSHA256")] @@ -518,6 +522,7 @@ mod tests { etag: Value("\"3858f62230ac3c915f300c664312c11f-9\"".to_string()), checksum_crc32: None, checksum_crc32c: None, + checksum_crc64nvme: None, checksum_sha1: Some(Value("ZJAnHyG8PeKz9tI8UTcHrJos39A=".into())), checksum_sha256: None, }; @@ -803,6 +808,7 @@ mod tests { size: IntValue(10485760), checksum_crc32: None, checksum_crc32c: None, + checksum_crc64nvme: None, checksum_sha256: Some(Value( "5RQ3A5uk0w7ojNjvegohch4JRBBGN/cLhsNrPzfv/hA=".into(), )), @@ -816,6 +822,7 @@ mod tests { checksum_sha256: None, checksum_crc32c: None, checksum_crc32: Some(Value("ZJAnHyG8=".into())), + checksum_crc64nvme: None, checksum_sha1: None, }, ], diff --git a/src/model/s3/object_table.rs b/src/model/s3/object_table.rs index f6204766..3d805d1a 100644 --- a/src/model/s3/object_table.rs +++ b/src/model/s3/object_table.rs @@ -282,6 +282,7 @@ mod v010 { pub enum ChecksumAlgorithm { Crc32, Crc32c, + Crc64Nvme, Sha1, Sha256, } @@ -291,6 +292,7 @@ mod v010 { pub enum ChecksumValue { Crc32(#[serde(with = "serde_bytes")] [u8; 4]), Crc32c(#[serde(with = "serde_bytes")] [u8; 4]), + Crc64Nvme(#[serde(with = "serde_bytes")] [u8; 8]), Sha1(#[serde(with = "serde_bytes")] [u8; 20]), Sha256(#[serde(with = "serde_bytes")] [u8; 32]), } @@ -492,6 +494,7 @@ impl ChecksumValue { match self { ChecksumValue::Crc32(_) => ChecksumAlgorithm::Crc32, ChecksumValue::Crc32c(_) => ChecksumAlgorithm::Crc32c, + ChecksumValue::Crc64Nvme(_) => ChecksumAlgorithm::Crc64Nvme, ChecksumValue::Sha1(_) => ChecksumAlgorithm::Sha1, ChecksumValue::Sha256(_) => ChecksumAlgorithm::Sha256, }