Prepare for multipart uploads

This commit is contained in:
Alex 2020-04-26 18:55:13 +00:00
parent ea7e4748ed
commit 9cb870f950
7 changed files with 61 additions and 33 deletions

4
TODO
View file

@ -8,8 +8,10 @@ We will have to introduce lots of dummy data and then add/remove nodes many time
Attaining S3 compatibility
--------------------------
- api_server following the S3 semantics for head/get/put/list/delete: verify more that it works as intended
- multipart uploads
- fix sync not working in some cases ? (when starting from empty?)
- api_server following the S3 semantics for head/get/put/list/delete: verify more that it works as intended
- PUT requests: verify content-md5 if provided
- possibly other necessary endpoints ?

View file

@ -134,10 +134,9 @@ async fn handler_inner(
.body(empty_body)
.unwrap();
Ok(response)
},
}
&Method::DELETE => Err(Error::Forbidden(
"Cannot delete buckets using S3 api, please talk to Garage directly"
.into(),
"Cannot delete buckets using S3 api, please talk to Garage directly".into(),
)),
&Method::GET => {
let mut params = HashMap::new();

View file

@ -43,7 +43,7 @@ pub async fn handle_head(
.versions()
.iter()
.rev()
.filter(|v| v.is_complete && v.data != ObjectVersionData::DeleteMarker)
.filter(|v| v.is_data())
.next()
{
Some(v) => v,
@ -76,7 +76,7 @@ pub async fn handle_get(
.versions()
.iter()
.rev()
.filter(|v| v.is_complete)
.filter(|v| v.is_complete())
.next()
{
Some(v) => v,

View file

@ -8,7 +8,6 @@ use hyper::Response;
use garage_util::error::Error;
use garage_core::garage::Garage;
use garage_core::object_table::*;
use crate::api_server::BodyType;
use crate::http_util::*;
@ -44,11 +43,7 @@ pub async fn handle_list(
)
.await?;
for object in objects.iter() {
if let Some(version) = object
.versions()
.iter()
.find(|x| x.is_complete && x.data != ObjectVersionData::DeleteMarker)
{
if let Some(version) = object.versions().iter().find(|x| x.is_data()) {
if !object.key.starts_with(prefix) {
truncated = false;
break;
@ -56,7 +51,7 @@ pub async fn handle_list(
let common_prefix = if delimiter.len() > 0 {
let relative_key = &object.key[prefix.len()..];
match relative_key.find(delimiter) {
Some(i) => Some(&object.key[..prefix.len()+i+delimiter.len()]),
Some(i) => Some(&object.key[..prefix.len() + i + delimiter.len()]),
None => None,
}
} else {
@ -70,7 +65,9 @@ pub async fn handle_list(
last_modified: version.timestamp,
size: version.size,
},
Some(_lri) => return Err(Error::Message(format!("Duplicate key?? {}", object.key))),
Some(_lri) => {
return Err(Error::Message(format!("Duplicate key?? {}", object.key)))
}
};
result_keys.insert(object.key.clone(), info);
};

View file

@ -33,13 +33,13 @@ pub async fn handle_put(
timestamp: now_msec(),
mime_type: mime_type.to_string(),
size: first_block.len() as u64,
is_complete: false,
state: ObjectVersionState::Uploading,
data: ObjectVersionData::DeleteMarker,
};
if first_block.len() < INLINE_THRESHOLD {
object_version.data = ObjectVersionData::Inline(first_block);
object_version.is_complete = true;
object_version.state = ObjectVersionState::Complete;
let object = Object::new(bucket.into(), key.into(), vec![object_version]);
garage.object_table.insert(&object).await?;
@ -54,7 +54,8 @@ pub async fn handle_put(
garage.object_table.insert(&object).await?;
let mut next_offset = first_block.len();
let mut put_curr_version_block = put_block_meta(garage.clone(), &version, 0, first_block_hash);
let mut put_curr_version_block =
put_block_meta(garage.clone(), &version, 0, 0, first_block_hash);
let mut put_curr_block = garage
.block_manager
.rpc_put_block(first_block_hash, first_block);
@ -66,7 +67,7 @@ pub async fn handle_put(
let block_hash = hash(&block[..]);
let block_len = block.len();
put_curr_version_block =
put_block_meta(garage.clone(), &version, next_offset as u64, block_hash);
put_block_meta(garage.clone(), &version, 0, next_offset as u64, block_hash);
put_curr_block = garage.block_manager.rpc_put_block(block_hash, block);
next_offset += block_len;
} else {
@ -76,7 +77,7 @@ pub async fn handle_put(
// TODO: if at any step we have an error, we should undo everything we did
object_version.is_complete = true;
object_version.state = ObjectVersionState::Complete;
object_version.size = next_offset as u64;
let object = Object::new(bucket.into(), key.into(), vec![object_version]);
@ -88,12 +89,19 @@ pub async fn handle_put(
async fn put_block_meta(
garage: Arc<Garage>,
version: &Version,
part_number: u64,
offset: u64,
hash: Hash,
) -> Result<(), Error> {
// TODO: don't clone, restart from empty block list ??
let mut version = version.clone();
version.add_block(VersionBlock { offset, hash }).unwrap();
version
.add_block(VersionBlock {
part_number,
offset,
hash,
})
.unwrap();
let block_ref = BlockRef {
block: hash,
@ -180,7 +188,7 @@ pub async fn handle_delete(garage: Arc<Garage>, bucket: &str, key: &str) -> Resu
timestamp: now_msec(),
mime_type: "application/x-delete-marker".into(),
size: 0,
is_complete: true,
state: ObjectVersionState::Complete,
data: ObjectVersionData::DeleteMarker,
}],
);

View file

@ -61,11 +61,31 @@ pub struct ObjectVersion {
pub mime_type: String,
pub size: u64,
pub is_complete: bool,
pub state: ObjectVersionState,
pub data: ObjectVersionData,
}
#[derive(PartialEq, Clone, Copy, Debug, Serialize, Deserialize)]
pub enum ObjectVersionState {
Uploading,
Complete,
Aborted,
}
impl ObjectVersionState {
fn max(self, other: Self) -> Self {
use ObjectVersionState::*;
if self == Aborted || other == Aborted {
Aborted
} else if self == Complete || other == Complete {
Complete
} else {
Uploading
}
}
}
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
pub enum ObjectVersionData {
DeleteMarker,
@ -74,8 +94,14 @@ pub enum ObjectVersionData {
}
impl ObjectVersion {
fn cmp_key(&self) -> (u64, &UUID) {
(self.timestamp, &self.uuid)
fn cmp_key(&self) -> (u64, UUID) {
(self.timestamp, self.uuid)
}
pub fn is_complete(&self) -> bool {
self.state == ObjectVersionState::Complete
}
pub fn is_data(&self) -> bool {
self.state == ObjectVersionState::Complete && self.data != ObjectVersionData::DeleteMarker
}
}
@ -98,9 +124,7 @@ impl Entry<String, String> for Object {
if other_v.size > v.size {
v.size = other_v.size;
}
if other_v.is_complete && !v.is_complete {
v.is_complete = true;
}
v.state = v.state.max(other_v.state);
}
Err(i) => {
self.versions.insert(i, other_v.clone());
@ -112,7 +136,7 @@ impl Entry<String, String> for Object {
.iter()
.enumerate()
.rev()
.filter(|(_, v)| v.is_complete)
.filter(|(_, v)| v.is_complete())
.next()
.map(|(vi, _)| vi);
@ -159,9 +183,6 @@ impl TableSchema for ObjectTable {
}
fn matches_filter(entry: &Self::E, _filter: &Self::Filter) -> bool {
entry
.versions
.iter()
.any(|x| x.is_complete && x.data != ObjectVersionData::DeleteMarker)
entry.versions.iter().any(|v| v.is_data())
}
}

View file

@ -64,6 +64,7 @@ impl Version {
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
pub struct VersionBlock {
pub part_number: u64,
pub offset: u64,
pub hash: Hash,
}