Deuxfleurs/garage
34
65
Fork 83
Code Issues 95 Pull requests 16 Releases 32 Wiki Activity

add crc64nvme checksumming algorithm (fix #963)
All checks were successful
ci/woodpecker/push/debug Pipeline was successful
Details
ci/woodpecker/pr/debug Pipeline was successful
Details

Browse source
This commit is contained in:
Alex 2025-03-19 15:51:06 +01:00
parent fb6db494cc
commit a826c361a9
9 changed files with 111 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
Cargo.lock generated
View file

@ -812,6 +812,21 @@ dependencies = [
"libc",
]
[[package]]
name = "crc"
version = "3.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636"
dependencies = [
"crc-catalog",
]
[[package]]
name = "crc-catalog"
version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"
[[package]]
name = "crc32c"
version = "0.6.8"
@ -830,6 +845,15 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "crc64fast-nvme"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4955638f00a809894c947f85a024020a20815b65a5eea633798ea7924edab2b3"
dependencies = [
"crc",
]
[[package]]
name = "crossbeam-channel"
version = "0.5.14"
@ -1332,6 +1356,7 @@ dependencies = [
"chrono",
"crc32c",
"crc32fast",
"crc64fast-nvme",
"crypto-common",
"err-derive",
"futures",
@ -1392,6 +1417,7 @@ dependencies = [
"chrono",
"crc32c",
"crc32fast",
"crc64fast-nvme",
"err-derive",
"form_urlencoded",
"futures",

1
Cargo.toml
View file

@ -51,6 +51,7 @@ cfg-if = "1.0"
chrono = { version = "0.4", features = ["serde"] }
crc32fast = "1.4"
crc32c = "0.6"
crc64fast-nvme = "1.2"
crypto-common = "0.1"
err-derive = "0.3"
gethostname = "0.4"

1
src/api/common/Cargo.toml
View file

@ -23,6 +23,7 @@ bytes.workspace = true
chrono.workspace = true
crc32fast.workspace = true
crc32c.workspace = true
crc64fast-nvme.workspace = true
crypto-common.workspace = true
err-derive.workspace = true
hex.workspace = true

39
src/api/common/signature/checksum.rs
View file

@ -4,6 +4,7 @@ use std::hash::Hasher;
use base64::prelude::*;
use crc32c::Crc32cHasher as Crc32c;
use crc32fast::Hasher as Crc32;
use crc64fast_nvme::Digest as Crc64Nvme;
use md5::{Digest, Md5};
use sha1::Sha1;
use sha2::Sha256;
@ -23,11 +24,14 @@ pub const X_AMZ_CHECKSUM_ALGORITHM: HeaderName =
pub const X_AMZ_CHECKSUM_MODE: HeaderName = HeaderName::from_static("x-amz-checksum-mode");
pub const X_AMZ_CHECKSUM_CRC32: HeaderName = HeaderName::from_static("x-amz-checksum-crc32");
pub const X_AMZ_CHECKSUM_CRC32C: HeaderName = HeaderName::from_static("x-amz-checksum-crc32c");
pub const X_AMZ_CHECKSUM_CRC64NVME: HeaderName =
HeaderName::from_static("x-amz-checksum-crc64nvme");
pub const X_AMZ_CHECKSUM_SHA1: HeaderName = HeaderName::from_static("x-amz-checksum-sha1");
pub const X_AMZ_CHECKSUM_SHA256: HeaderName = HeaderName::from_static("x-amz-checksum-sha256");
pub type Crc32Checksum = [u8; 4];
pub type Crc32cChecksum = [u8; 4];
pub type Crc64NvmeChecksum = [u8; 8];
pub type Md5Checksum = [u8; 16];
pub type Sha1Checksum = [u8; 20];
pub type Sha256Checksum = [u8; 32];
@ -45,6 +49,7 @@ pub struct ExpectedChecksums {
pub struct Checksummer {
pub crc32: Option<Crc32>,
pub crc32c: Option<Crc32c>,
pub crc64nvme: Option<Crc64Nvme>,
pub md5: Option<Md5>,
pub sha1: Option<Sha1>,
pub sha256: Option<Sha256>,
@ -54,6 +59,7 @@ pub struct Checksummer {
pub struct Checksums {
pub crc32: Option<Crc32Checksum>,
pub crc32c: Option<Crc32cChecksum>,
pub crc64nvme: Option<Crc64NvmeChecksum>,
pub md5: Option<Md5Checksum>,
pub sha1: Option<Sha1Checksum>,
pub sha256: Option<Sha256Checksum>,
@ -64,6 +70,7 @@ impl Checksummer {
Self {
crc32: None,
crc32c: None,
crc64nvme: None,
md5: None,
sha1: None,
sha256: None,
@ -96,6 +103,9 @@ impl Checksummer {
if matches!(&expected.extra, Some(ChecksumValue::Crc32c(_))) {
self.crc32c = Some(Crc32c::default());
}
if matches!(&expected.extra, Some(ChecksumValue::Crc64Nvme(_))) {
self.crc64nvme = Some(Crc64Nvme::default());
}
if matches!(&expected.extra, Some(ChecksumValue::Sha1(_))) {
self.sha1 = Some(Sha1::new());
}
@ -109,6 +119,9 @@ impl Checksummer {
Some(ChecksumAlgorithm::Crc32c) => {
self.crc32c = Some(Crc32c::default());
}
Some(ChecksumAlgorithm::Crc64Nvme) => {
self.crc64nvme = Some(Crc64Nvme::default());
}
Some(ChecksumAlgorithm::Sha1) => {
self.sha1 = Some(Sha1::new());
}
@ -127,6 +140,9 @@ impl Checksummer {
if let Some(crc32c) = &mut self.crc32c {
crc32c.write(bytes);
}
if let Some(crc64nvme) = &mut self.crc64nvme {
crc64nvme.write(bytes);
}
if let Some(md5) = &mut self.md5 {
md5.update(bytes);
}
@ -144,6 +160,7 @@ impl Checksummer {
crc32c: self
.crc32c
.map(|x| u32::to_be_bytes(u32::try_from(x.finish()).unwrap())),
crc64nvme: self.crc64nvme.map(|x| u64::to_be_bytes(x.sum64())),
md5: self.md5.map(|x| x.finalize()[..].try_into().unwrap()),
sha1: self.sha1.map(|x| x.finalize()[..].try_into().unwrap()),
sha256: self.sha256.map(|x| x.finalize()[..].try_into().unwrap()),
@ -190,6 +207,9 @@ impl Checksums {
None => None,
Some(ChecksumAlgorithm::Crc32) => Some(ChecksumValue::Crc32(self.crc32.unwrap())),
Some(ChecksumAlgorithm::Crc32c) => Some(ChecksumValue::Crc32c(self.crc32c.unwrap())),
Some(ChecksumAlgorithm::Crc64Nvme) => {
Some(ChecksumValue::Crc64Nvme(self.crc64nvme.unwrap()))
}
Some(ChecksumAlgorithm::Sha1) => Some(ChecksumValue::Sha1(self.sha1.unwrap())),
Some(ChecksumAlgorithm::Sha256) => Some(ChecksumValue::Sha256(self.sha256.unwrap())),
}
@ -202,6 +222,7 @@ pub fn parse_checksum_algorithm(algo: &str) -> Result<ChecksumAlgorithm, Error>
match algo {
"CRC32" => Ok(ChecksumAlgorithm::Crc32),
"CRC32C" => Ok(ChecksumAlgorithm::Crc32c),
"CRC64NVME" => Ok(ChecksumAlgorithm::Crc64Nvme),
"SHA1" => Ok(ChecksumAlgorithm::Sha1),
"SHA256" => Ok(ChecksumAlgorithm::Sha256),
_ => Err(Error::bad_request("invalid checksum algorithm")),
@ -225,6 +246,7 @@ pub fn request_trailer_checksum_algorithm(
None => Ok(None),
Some(x) if x == X_AMZ_CHECKSUM_CRC32 => Ok(Some(ChecksumAlgorithm::Crc32)),
Some(x) if x == X_AMZ_CHECKSUM_CRC32C => Ok(Some(ChecksumAlgorithm::Crc32c)),
Some(x) if x == X_AMZ_CHECKSUM_CRC64NVME => Ok(Some(ChecksumAlgorithm::Crc64Nvme)),
Some(x) if x == X_AMZ_CHECKSUM_SHA1 => Ok(Some(ChecksumAlgorithm::Sha1)),
Some(x) if x == X_AMZ_CHECKSUM_SHA256 => Ok(Some(ChecksumAlgorithm::Sha256)),
_ => Err(Error::bad_request("invalid checksum algorithm")),
@ -243,6 +265,12 @@ pub fn request_checksum_value(
if headers.contains_key(X_AMZ_CHECKSUM_CRC32C) {
ret.push(extract_checksum_value(headers, ChecksumAlgorithm::Crc32c)?);
}
if headers.contains_key(X_AMZ_CHECKSUM_CRC64NVME) {
ret.push(extract_checksum_value(
headers,
ChecksumAlgorithm::Crc64Nvme,
)?);
}
if headers.contains_key(X_AMZ_CHECKSUM_SHA1) {
ret.push(extract_checksum_value(headers, ChecksumAlgorithm::Sha1)?);
}
@ -281,6 +309,14 @@ pub fn extract_checksum_value(
.ok_or_bad_request("invalid x-amz-checksum-crc32c header")?;
Ok(ChecksumValue::Crc32c(crc32c))
}
ChecksumAlgorithm::Crc64Nvme => {
let crc64nvme = headers
.get(X_AMZ_CHECKSUM_CRC64NVME)
.and_then(|x| BASE64_STANDARD.decode(&x).ok())
.and_then(|x| x.try_into().ok())
.ok_or_bad_request("invalid x-amz-checksum-crc64nvme header")?;
Ok(ChecksumValue::Crc64Nvme(crc64nvme))
}
ChecksumAlgorithm::Sha1 => {
let sha1 = headers
.get(X_AMZ_CHECKSUM_SHA1)
@ -311,6 +347,9 @@ pub fn add_checksum_response_headers(
Some(ChecksumValue::Crc32c(crc32c)) => {
resp = resp.header(X_AMZ_CHECKSUM_CRC32C, BASE64_STANDARD.encode(&crc32c));
}
Some(ChecksumValue::Crc64Nvme(crc64nvme)) => {
resp = resp.header(X_AMZ_CHECKSUM_CRC64NVME, BASE64_STANDARD.encode(&crc64nvme));
}
Some(ChecksumValue::Sha1(sha1)) => {
resp = resp.header(X_AMZ_CHECKSUM_SHA1, BASE64_STANDARD.encode(&sha1));
}

1
src/api/s3/Cargo.toml
View file

@ -29,6 +29,7 @@ bytes.workspace = true
chrono.workspace = true
crc32fast.workspace = true
crc32c.workspace = true
crc64fast-nvme.workspace = true
err-derive.workspace = true
hex.workspace = true
hmac.workspace = true

6
src/api/s3/list.rs
View file

@ -334,6 +334,12 @@ pub async fn handle_list_parts(
}
_ => None,
},
checksum_crc64nvme: match &checksum {
Some(ChecksumValue::Crc64Nvme(x)) => {
Some(s3_xml::Value(BASE64_STANDARD.encode(&x)))
}
_ => None,
},
checksum_sha1: match &checksum {
Some(ChecksumValue::Sha1(x)) => {
Some(s3_xml::Value(BASE64_STANDARD.encode(&x)))

27
src/api/s3/multipart.rs
View file

@ -6,6 +6,7 @@ use std::sync::Arc;
use base64::prelude::*;
use crc32c::Crc32cHasher as Crc32c;
use crc32fast::Hasher as Crc32;
use crc64fast_nvme::Digest as Crc64Nvme;
use futures::prelude::*;
use hyper::{Request, Response};
use md5::{Digest, Md5};
@ -481,6 +482,10 @@ pub async fn handle_complete_multipart_upload(
Some(ChecksumValue::Crc32c(x)) => Some(s3_xml::Value(BASE64_STANDARD.encode(&x))),
_ => None,
},
checksum_crc64nvme: match &checksum_extra {
Some(ChecksumValue::Crc64Nvme(x)) => Some(s3_xml::Value(BASE64_STANDARD.encode(&x))),
_ => None,
},
checksum_sha1: match &checksum_extra {
Some(ChecksumValue::Sha1(x)) => Some(s3_xml::Value(BASE64_STANDARD.encode(&x))),
_ => None,
@ -604,6 +609,15 @@ fn parse_complete_multipart_upload_body(
.try_into()
.ok()?,
))
} else if let Some(crc64nvme) = item
.children()
.find(|e| e.has_tag_name("ChecksumCRC64NVME"))
{
Some(ChecksumValue::Crc64Nvme(
BASE64_STANDARD.decode(crc64nvme.text()?).ok()?[..]
.try_into()
.ok()?,
))
} else if let Some(sha1) = item.children().find(|e| e.has_tag_name("ChecksumSHA1")) {
Some(ChecksumValue::Sha1(
BASE64_STANDARD.decode(sha1.text()?).ok()?[..]
@ -644,6 +658,7 @@ pub(crate) struct MultipartChecksummer {
pub(crate) enum MultipartExtraChecksummer {
Crc32(Crc32),
Crc32c(Crc32c),
Crc64Nvme(Crc64Nvme),
Sha1(Sha1),
Sha256(Sha256),
}
@ -660,6 +675,9 @@ impl MultipartChecksummer {
Some(ChecksumAlgorithm::Crc32c) => {
Some(MultipartExtraChecksummer::Crc32c(Crc32c::default()))
}
Some(ChecksumAlgorithm::Crc64Nvme) => {
Some(MultipartExtraChecksummer::Crc64Nvme(Crc64Nvme::default()))
}
Some(ChecksumAlgorithm::Sha1) => Some(MultipartExtraChecksummer::Sha1(Sha1::new())),
Some(ChecksumAlgorithm::Sha256) => {
Some(MultipartExtraChecksummer::Sha256(Sha256::new()))
@ -689,6 +707,12 @@ impl MultipartChecksummer {
) => {
crc32c.write(&x);
}
(
Some(MultipartExtraChecksummer::Crc64Nvme(ref mut crc64nvme)),
Some(ChecksumValue::Crc64Nvme(x)),
) => {
crc64nvme.write(&x);
}
(Some(MultipartExtraChecksummer::Sha1(ref mut sha1)), Some(ChecksumValue::Sha1(x))) => {
sha1.update(&x);
}
@ -718,6 +742,9 @@ impl MultipartChecksummer {
Some(MultipartExtraChecksummer::Crc32c(crc32c)) => Some(ChecksumValue::Crc32c(
u32::to_be_bytes(u32::try_from(crc32c.finish()).unwrap()),
)),
Some(MultipartExtraChecksummer::Crc64Nvme(crc64nvme)) => Some(
ChecksumValue::Crc64Nvme(u64::to_be_bytes(crc64nvme.sum64())),
),
Some(MultipartExtraChecksummer::Sha1(sha1)) => {
Some(ChecksumValue::Sha1(sha1.finalize()[..].try_into().unwrap()))
}

7
src/api/s3/xml.rs
View file

@ -135,6 +135,8 @@ pub struct CompleteMultipartUploadResult {
pub checksum_crc32: Option<Value>,
#[serde(rename = "ChecksumCRC32C")]
pub checksum_crc32c: Option<Value>,
#[serde(rename = "ChecksumCR64NVME")]
pub checksum_crc64nvme: Option<Value>,
#[serde(rename = "ChecksumSHA1")]
pub checksum_sha1: Option<Value>,
#[serde(rename = "ChecksumSHA256")]
@ -209,6 +211,8 @@ pub struct PartItem {
pub checksum_crc32: Option<Value>,
#[serde(rename = "ChecksumCRC32C")]
pub checksum_crc32c: Option<Value>,
#[serde(rename = "ChecksumCRC64NVME")]
pub checksum_crc64nvme: Option<Value>,
#[serde(rename = "ChecksumSHA1")]
pub checksum_sha1: Option<Value>,
#[serde(rename = "ChecksumSHA256")]
@ -518,6 +522,7 @@ mod tests {
etag: Value("\"3858f62230ac3c915f300c664312c11f-9\"".to_string()),
checksum_crc32: None,
checksum_crc32c: None,
checksum_crc64nvme: None,
checksum_sha1: Some(Value("ZJAnHyG8PeKz9tI8UTcHrJos39A=".into())),
checksum_sha256: None,
};
@ -803,6 +808,7 @@ mod tests {
size: IntValue(10485760),
checksum_crc32: None,
checksum_crc32c: None,
checksum_crc64nvme: None,
checksum_sha256: Some(Value(
"5RQ3A5uk0w7ojNjvegohch4JRBBGN/cLhsNrPzfv/hA=".into(),
)),
@ -816,6 +822,7 @@ mod tests {
checksum_sha256: None,
checksum_crc32c: None,
checksum_crc32: Some(Value("ZJAnHyG8=".into())),
checksum_crc64nvme: None,
checksum_sha1: None,
},
],

3
src/model/s3/object_table.rs
View file

@ -282,6 +282,7 @@ mod v010 {
pub enum ChecksumAlgorithm {
Crc32,
Crc32c,
Crc64Nvme,
Sha1,
Sha256,
}
@ -291,6 +292,7 @@ mod v010 {
pub enum ChecksumValue {
Crc32(#[serde(with = "serde_bytes")] [u8; 4]),
Crc32c(#[serde(with = "serde_bytes")] [u8; 4]),
Crc64Nvme(#[serde(with = "serde_bytes")] [u8; 8]),
Sha1(#[serde(with = "serde_bytes")] [u8; 20]),
Sha256(#[serde(with = "serde_bytes")] [u8; 32]),
}
@ -492,6 +494,7 @@ impl ChecksumValue {
match self {
ChecksumValue::Crc32(_) => ChecksumAlgorithm::Crc32,
ChecksumValue::Crc32c(_) => ChecksumAlgorithm::Crc32c,
ChecksumValue::Crc64Nvme(_) => ChecksumAlgorithm::Crc64Nvme,
ChecksumValue::Sha1(_) => ChecksumAlgorithm::Sha1,
ChecksumValue::Sha256(_) => ChecksumAlgorithm::Sha256,
}