forked from Deuxfleurs/garage
Implement correct ETag for objects created with PutObject
This commit is contained in:
parent
9305e5e87f
commit
6c7f9704ea
3 changed files with 67 additions and 8 deletions
46
Cargo.lock
generated
46
Cargo.lock
generated
|
|
@ -59,6 +59,14 @@ dependencies = [
|
|||
"generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "block-buffer"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"generic-array 0.14.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "block-padding"
|
||||
version = "0.1.5"
|
||||
|
|
@ -186,6 +194,14 @@ dependencies = [
|
|||
"generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "digest"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"generic-array 0.14.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "env_logger"
|
||||
version = "0.7.1"
|
||||
|
|
@ -382,6 +398,7 @@ dependencies = [
|
|||
"httpdate 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hyper 0.13.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"md-5 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"roxmltree 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"sha2 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
|
|
@ -602,6 +619,15 @@ dependencies = [
|
|||
"typenum 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"typenum 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"version_check 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gethostname"
|
||||
version = "0.2.1"
|
||||
|
|
@ -843,6 +869,16 @@ name = "maybe-uninit"
|
|||
version = "2.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "md-5"
|
||||
version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"block-buffer 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"digest 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"opaque-debug 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.3.3"
|
||||
|
|
@ -941,6 +977,11 @@ name = "opaque-debug"
|
|||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "opaque-debug"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot"
|
||||
version = "0.10.2"
|
||||
|
|
@ -1695,6 +1736,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
"checksum base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7"
|
||||
"checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
|
||||
"checksum block-buffer 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b"
|
||||
"checksum block-buffer 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4"
|
||||
"checksum block-padding 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5"
|
||||
"checksum bumpalo 3.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820"
|
||||
"checksum byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7"
|
||||
|
|
@ -1711,6 +1753,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
"checksum crossbeam-utils 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8"
|
||||
"checksum crypto-mac 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4434400df11d95d556bac068ddfedd482915eb18fe8bea89bc80b6e4b1c179e5"
|
||||
"checksum digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5"
|
||||
"checksum digest 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066"
|
||||
"checksum env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36"
|
||||
"checksum err-derive 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "22deed3a8124cff5fa835713fa105621e43bbdc46690c3a6b68328a012d350d4"
|
||||
"checksum fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed"
|
||||
|
|
@ -1734,6 +1777,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
"checksum garage_util 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "77de76a4167c041094f3f3415c6d3d773373e0326668fbce70dfd3b024788800"
|
||||
"checksum generator 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)" = "add72f17bb81521258fcc8a7a3245b1e184e916bfbe34f0ea89558f440df5c68"
|
||||
"checksum generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec"
|
||||
"checksum generic-array 0.14.3 (registry+https://github.com/rust-lang/crates.io-index)" = "60fb4bb6bba52f78a471264d9a3b7d026cc0af47b22cd2cffbc0b787ca003e63"
|
||||
"checksum gethostname 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e692e296bfac1d2533ef168d0b60ff5897b8b70a4009276834014dd8924cc028"
|
||||
"checksum getrandom 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb"
|
||||
"checksum h2 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "79b7246d7e4b979c03fa093da39cfb3617a96bbeee6310af63991668d7e843ff"
|
||||
|
|
@ -1762,6 +1806,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
"checksum loom 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4ecc775857611e1df29abba5c41355cdf540e7e9d4acfdf0f355eefee82330b7"
|
||||
"checksum matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"
|
||||
"checksum maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00"
|
||||
"checksum md-5 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15"
|
||||
"checksum memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400"
|
||||
"checksum memoffset 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "c198b026e1bbf08a937e94c6c60f9ec4a2267f5b0d2eec9c1b21b061ce2be55f"
|
||||
"checksum mio 0.6.22 (registry+https://github.com/rust-lang/crates.io-index)" = "fce347092656428bc8eaf6201042cb551b8d67855af7374542a92a0fbfcac430"
|
||||
|
|
@ -1773,6 +1818,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
"checksum num_cpus 1.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3"
|
||||
"checksum once_cell 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0b631f7e854af39a1739f401cf34a8a013dfe09eac4fa4dba91e9768bd28168d"
|
||||
"checksum opaque-debug 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c"
|
||||
"checksum opaque-debug 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
|
||||
"checksum parking_lot 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e"
|
||||
"checksum parking_lot_core 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3"
|
||||
"checksum percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ bytes = "0.4"
|
|||
hex = "0.3"
|
||||
log = "0.4"
|
||||
chrono = "0.4"
|
||||
md-5 = "0.9.1"
|
||||
sha2 = "0.8"
|
||||
hmac = "0.7"
|
||||
crypto-mac = "0.7"
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ use std::collections::{BTreeMap, VecDeque};
|
|||
use std::fmt::Write;
|
||||
use std::sync::Arc;
|
||||
|
||||
use md5::{Md5, Digest};
|
||||
use futures::stream::*;
|
||||
use hyper::{Body, Request, Response};
|
||||
|
||||
|
|
@ -41,18 +42,22 @@ pub async fn handle_put(
|
|||
};
|
||||
|
||||
if first_block.len() < INLINE_THRESHOLD {
|
||||
let mut md5sum = Md5::new();
|
||||
md5sum.update(&first_block[..]);
|
||||
let etag = hex::encode(md5sum.finalize());
|
||||
|
||||
object_version.state = ObjectVersionState::Complete(ObjectVersionData::Inline(
|
||||
ObjectVersionMeta {
|
||||
headers,
|
||||
size: first_block.len() as u64,
|
||||
etag: "".to_string(), // TODO
|
||||
etag: etag.clone(),
|
||||
},
|
||||
first_block,
|
||||
));
|
||||
|
||||
let object = Object::new(bucket.into(), key.into(), vec![object_version]);
|
||||
garage.object_table.insert(&object).await?;
|
||||
return Ok(put_response(version_uuid));
|
||||
return Ok(put_response(version_uuid, etag));
|
||||
}
|
||||
|
||||
let version = Version::new(version_uuid, bucket.into(), key.into(), false, vec![]);
|
||||
|
|
@ -61,7 +66,7 @@ pub async fn handle_put(
|
|||
let object = Object::new(bucket.into(), key.into(), vec![object_version.clone()]);
|
||||
garage.object_table.insert(&object).await?;
|
||||
|
||||
let total_size = read_and_put_blocks(
|
||||
let (total_size, etag) = read_and_put_blocks(
|
||||
&garage,
|
||||
version,
|
||||
1,
|
||||
|
|
@ -77,7 +82,7 @@ pub async fn handle_put(
|
|||
ObjectVersionMeta {
|
||||
headers,
|
||||
size: total_size,
|
||||
etag: "".to_string(), // TODO
|
||||
etag: etag.clone(),
|
||||
},
|
||||
first_block_hash,
|
||||
));
|
||||
|
|
@ -85,7 +90,7 @@ pub async fn handle_put(
|
|||
let object = Object::new(bucket.into(), key.into(), vec![object_version]);
|
||||
garage.object_table.insert(&object).await?;
|
||||
|
||||
Ok(put_response(version_uuid))
|
||||
Ok(put_response(version_uuid, etag))
|
||||
}
|
||||
|
||||
async fn read_and_put_blocks(
|
||||
|
|
@ -95,7 +100,10 @@ async fn read_and_put_blocks(
|
|||
first_block: Vec<u8>,
|
||||
first_block_hash: Hash,
|
||||
chunker: &mut BodyChunker,
|
||||
) -> Result<u64, Error> {
|
||||
) -> Result<(u64, String), Error> {
|
||||
let mut md5sum = Md5::new();
|
||||
md5sum.update(&first_block[..]);
|
||||
|
||||
let mut next_offset = first_block.len();
|
||||
let mut put_curr_version_block = put_block_meta(
|
||||
garage.clone(),
|
||||
|
|
@ -113,6 +121,7 @@ async fn read_and_put_blocks(
|
|||
let (_, _, next_block) =
|
||||
futures::try_join!(put_curr_block, put_curr_version_block, chunker.next())?;
|
||||
if let Some(block) = next_block {
|
||||
md5sum.update(&block[..]);
|
||||
let block_hash = hash(&block[..]);
|
||||
let block_len = block.len();
|
||||
put_curr_version_block = put_block_meta(
|
||||
|
|
@ -130,7 +139,9 @@ async fn read_and_put_blocks(
|
|||
}
|
||||
}
|
||||
|
||||
Ok(next_offset as u64)
|
||||
let total_size = next_offset as u64;
|
||||
let md5sum = hex::encode(md5sum.finalize());
|
||||
Ok((total_size, md5sum))
|
||||
}
|
||||
|
||||
async fn put_block_meta(
|
||||
|
|
@ -203,9 +214,10 @@ impl BodyChunker {
|
|||
}
|
||||
}
|
||||
|
||||
pub fn put_response(version_uuid: UUID) -> Response<Body> {
|
||||
pub fn put_response(version_uuid: UUID, etag: String) -> Response<Body> {
|
||||
Response::builder()
|
||||
.header("x-amz-version-id", hex::encode(version_uuid))
|
||||
.header("ETag", etag)
|
||||
// TODO ETag
|
||||
.body(Body::from(vec![]))
|
||||
.unwrap()
|
||||
|
|
|
|||
Loading…
Reference in a new issue