garage/src/api/s3/delete.rs

171 lines
4 KiB
Rust
Raw Normal View History

2020-05-04 13:09:23 +00:00
use std::sync::Arc;
2020-05-01 15:52:35 +00:00
use hyper::{Body, Request, Response, StatusCode};
2020-04-28 10:18:14 +00:00
use garage_util::data::*;
2021-03-15 15:21:41 +00:00
use garage_util::time::*;
2020-04-28 10:18:14 +00:00
2020-07-07 11:59:22 +00:00
use garage_model::garage::Garage;
First implementation of K2V (#293) **Specification:** View spec at [this URL](https://git.deuxfleurs.fr/Deuxfleurs/garage/src/branch/k2v/doc/drafts/k2v-spec.md) - [x] Specify the structure of K2V triples - [x] Specify the DVVS format used for causality detection - [x] Specify the K2V index (just a counter of number of values per partition key) - [x] Specify single-item endpoints: ReadItem, InsertItem, DeleteItem - [x] Specify index endpoint: ReadIndex - [x] Specify multi-item endpoints: InsertBatch, ReadBatch, DeleteBatch - [x] Move to JSON objects instead of tuples - [x] Specify endpoints for polling for updates on single values (PollItem) **Implementation:** - [x] Table for K2V items, causal contexts - [x] Indexing mechanism and table for K2V index - [x] Make API handlers a bit more generic - [x] K2V API endpoint - [x] K2V API router - [x] ReadItem - [x] InsertItem - [x] DeleteItem - [x] PollItem - [x] ReadIndex - [x] InsertBatch - [x] ReadBatch - [x] DeleteBatch **Testing:** - [x] Just a simple Python script that does some requests to check visually that things are going right (does not contain parsing of results or assertions on returned values) - [x] Actual tests: - [x] Adapt testing framework - [x] Simple test with InsertItem + ReadItem - [x] Test with several Insert/Read/DeleteItem + ReadIndex - [x] Test all combinations of return formats for ReadItem - [x] Test with ReadBatch, InsertBatch, DeleteBatch - [x] Test with PollItem - [x] Test error codes - [ ] Fix most broken stuff - [x] test PollItem broken randomly - [x] when invalid causality tokens are given, errors should be 4xx not 5xx **Improvements:** - [x] Descending range queries - [x] Specify - [x] Implement - [x] Add test - [x] Batch updates to index counter - [x] Put K2V behind `k2v` feature flag Co-authored-by: Alex Auvolat <alex@adnab.me> Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/293 Co-authored-by: Alex <alex@adnab.me> Co-committed-by: Alex <alex@adnab.me>
2022-05-10 11:16:57 +00:00
use garage_model::s3::object_table::*;
2020-04-28 10:18:14 +00:00
First version of admin API (#298) **Spec:** - [x] Start writing - [x] Specify all layout endpoints - [x] Specify all endpoints for operations on keys - [x] Specify all endpoints for operations on key/bucket permissions - [x] Specify all endpoints for operations on buckets - [x] Specify all endpoints for operations on bucket aliases View rendered spec at <https://git.deuxfleurs.fr/Deuxfleurs/garage/src/branch/admin-api/doc/drafts/admin-api.md> **Code:** - [x] Refactor code for admin api to use common api code that was created for K2V **General endpoints:** - [x] Metrics - [x] GetClusterStatus - [x] ConnectClusterNodes - [x] GetClusterLayout - [x] UpdateClusterLayout - [x] ApplyClusterLayout - [x] RevertClusterLayout **Key-related endpoints:** - [x] ListKeys - [x] CreateKey - [x] ImportKey - [x] GetKeyInfo - [x] UpdateKey - [x] DeleteKey **Bucket-related endpoints:** - [x] ListBuckets - [x] CreateBucket - [x] GetBucketInfo - [x] DeleteBucket - [x] PutBucketWebsite - [x] DeleteBucketWebsite **Operations on key/bucket permissions:** - [x] BucketAllowKey - [x] BucketDenyKey **Operations on bucket aliases:** - [x] GlobalAliasBucket - [x] GlobalUnaliasBucket - [x] LocalAliasBucket - [x] LocalUnaliasBucket **And also:** - [x] Separate error type for the admin API (this PR includes a quite big refactoring of error handling) - [x] Add management of website access - [ ] Check that nothing is missing wrt what can be done using the CLI - [ ] Improve formatting of the spec - [x] Make sure everyone is cool with the API design Fix #231 Fix #295 Co-authored-by: Alex Auvolat <alex@adnab.me> Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/298 Co-authored-by: Alex <alex@adnab.me> Co-committed-by: Alex <alex@adnab.me>
2022-05-24 10:16:39 +00:00
use crate::s3::error::*;
First implementation of K2V (#293) **Specification:** View spec at [this URL](https://git.deuxfleurs.fr/Deuxfleurs/garage/src/branch/k2v/doc/drafts/k2v-spec.md) - [x] Specify the structure of K2V triples - [x] Specify the DVVS format used for causality detection - [x] Specify the K2V index (just a counter of number of values per partition key) - [x] Specify single-item endpoints: ReadItem, InsertItem, DeleteItem - [x] Specify index endpoint: ReadIndex - [x] Specify multi-item endpoints: InsertBatch, ReadBatch, DeleteBatch - [x] Move to JSON objects instead of tuples - [x] Specify endpoints for polling for updates on single values (PollItem) **Implementation:** - [x] Table for K2V items, causal contexts - [x] Indexing mechanism and table for K2V index - [x] Make API handlers a bit more generic - [x] K2V API endpoint - [x] K2V API router - [x] ReadItem - [x] InsertItem - [x] DeleteItem - [x] PollItem - [x] ReadIndex - [x] InsertBatch - [x] ReadBatch - [x] DeleteBatch **Testing:** - [x] Just a simple Python script that does some requests to check visually that things are going right (does not contain parsing of results or assertions on returned values) - [x] Actual tests: - [x] Adapt testing framework - [x] Simple test with InsertItem + ReadItem - [x] Test with several Insert/Read/DeleteItem + ReadIndex - [x] Test all combinations of return formats for ReadItem - [x] Test with ReadBatch, InsertBatch, DeleteBatch - [x] Test with PollItem - [x] Test error codes - [ ] Fix most broken stuff - [x] test PollItem broken randomly - [x] when invalid causality tokens are given, errors should be 4xx not 5xx **Improvements:** - [x] Descending range queries - [x] Specify - [x] Implement - [x] Add test - [x] Batch updates to index counter - [x] Put K2V behind `k2v` feature flag Co-authored-by: Alex Auvolat <alex@adnab.me> Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/293 Co-authored-by: Alex <alex@adnab.me> Co-committed-by: Alex <alex@adnab.me>
2022-05-10 11:16:57 +00:00
use crate::s3::xml as s3_xml;
use crate::signature::verify_signed_content;
2020-05-01 15:52:35 +00:00
2020-05-04 13:09:23 +00:00
async fn handle_delete_internal(
garage: &Garage,
2021-12-14 12:55:11 +00:00
bucket_id: Uuid,
2020-05-04 13:09:23 +00:00
key: &str,
2021-05-02 21:13:08 +00:00
) -> Result<(Uuid, Uuid), Error> {
2020-11-11 15:12:42 +00:00
let object = garage
2020-04-28 10:18:14 +00:00
.object_table
2021-12-14 12:55:11 +00:00
.get(&bucket_id, &key.to_string())
2020-04-28 10:18:14 +00:00
.await?
2022-01-05 16:07:36 +00:00
.ok_or(Error::NoSuchKey)?; // No need to delete
2020-04-28 10:18:14 +00:00
2021-04-23 20:18:00 +00:00
let interesting_versions = object.versions().iter().filter(|v| {
!matches!(
v.state,
ObjectVersionState::Aborted
| ObjectVersionState::Complete(ObjectVersionData::DeleteMarker)
)
2020-04-28 10:18:14 +00:00
});
2021-03-15 14:26:29 +00:00
let mut version_to_delete = None;
2020-04-28 10:18:14 +00:00
let mut timestamp = now_msec();
for v in interesting_versions {
2021-03-15 14:26:29 +00:00
if v.timestamp + 1 > timestamp || version_to_delete.is_none() {
version_to_delete = Some(v.uuid);
2020-05-04 13:09:23 +00:00
}
2020-04-28 10:18:14 +00:00
timestamp = std::cmp::max(timestamp, v.timestamp + 1);
}
2022-01-05 16:07:36 +00:00
let deleted_version = version_to_delete.ok_or(Error::NoSuchKey)?;
2020-04-28 10:18:14 +00:00
let version_uuid = gen_uuid();
let object = Object::new(
2021-12-14 12:55:11 +00:00
bucket_id,
2020-04-28 10:18:14 +00:00
key.into(),
vec![ObjectVersion {
uuid: version_uuid,
2021-03-15 18:51:16 +00:00
timestamp,
2020-07-08 15:33:24 +00:00
state: ObjectVersionState::Complete(ObjectVersionData::DeleteMarker),
2020-04-28 10:18:14 +00:00
}],
);
garage.object_table.insert(&object).await?;
2021-10-26 08:20:05 +00:00
Ok((deleted_version, version_uuid))
2020-05-01 15:52:35 +00:00
}
2020-05-04 13:09:23 +00:00
pub async fn handle_delete(
garage: Arc<Garage>,
2021-12-14 12:55:11 +00:00
bucket_id: Uuid,
2020-05-04 13:09:23 +00:00
key: &str,
) -> Result<Response<Body>, Error> {
2020-05-04 13:09:23 +00:00
let (_deleted_version, delete_marker_version) =
2021-12-14 12:55:11 +00:00
handle_delete_internal(&garage, bucket_id, key).await?;
2020-05-01 15:52:35 +00:00
Ok(Response::builder()
.header("x-amz-version-id", hex::encode(delete_marker_version))
.status(StatusCode::NO_CONTENT)
.body(Body::from(vec![]))
2020-05-01 15:52:35 +00:00
.unwrap())
}
2020-05-04 13:09:23 +00:00
pub async fn handle_delete_objects(
garage: Arc<Garage>,
2021-12-14 12:55:11 +00:00
bucket_id: Uuid,
2020-05-04 13:09:23 +00:00
req: Request<Body>,
content_sha256: Option<Hash>,
) -> Result<Response<Body>, Error> {
2020-05-04 13:09:23 +00:00
let body = hyper::body::to_bytes(req.into_body()).await?;
if let Some(content_sha256) = content_sha256 {
verify_signed_content(content_sha256, &body[..])?;
}
2021-10-26 08:20:05 +00:00
let cmd_xml = roxmltree::Document::parse(std::str::from_utf8(&body)?)?;
2020-11-08 14:04:30 +00:00
let cmd = parse_delete_objects_xml(&cmd_xml).ok_or_bad_request("Invalid delete XML query")?;
2020-05-04 13:09:23 +00:00
let mut ret_deleted = Vec::new();
let mut ret_errors = Vec::new();
2020-05-01 15:52:35 +00:00
2020-05-04 13:09:23 +00:00
for obj in cmd.objects.iter() {
2021-12-14 12:55:11 +00:00
match handle_delete_internal(&garage, bucket_id, &obj.key).await {
2020-05-04 13:09:23 +00:00
Ok((deleted_version, delete_marker_version)) => {
if cmd.quiet {
continue;
}
ret_deleted.push(s3_xml::Deleted {
key: s3_xml::Value(obj.key.clone()),
version_id: s3_xml::Value(hex::encode(deleted_version)),
delete_marker_version_id: s3_xml::Value(hex::encode(delete_marker_version)),
});
2020-05-04 13:09:23 +00:00
}
Err(e) => {
ret_errors.push(s3_xml::DeleteError {
code: s3_xml::Value(e.aws_code().to_string()),
key: Some(s3_xml::Value(obj.key.clone())),
message: s3_xml::Value(format!("{}", e)),
version_id: None,
});
2020-05-04 13:09:23 +00:00
}
}
}
2020-05-01 15:52:35 +00:00
let xml = s3_xml::to_xml_with_header(&s3_xml::DeleteResult {
xmlns: (),
deleted: ret_deleted,
errors: ret_errors,
})?;
2020-05-01 15:52:35 +00:00
Ok(Response::builder()
2021-02-23 17:46:25 +00:00
.header("Content-Type", "application/xml")
.body(Body::from(xml))?)
2020-05-01 15:52:35 +00:00
}
struct DeleteRequest {
quiet: bool,
2020-05-04 13:09:23 +00:00
objects: Vec<DeleteObject>,
2020-04-28 10:18:14 +00:00
}
2020-05-01 15:52:35 +00:00
struct DeleteObject {
2020-05-04 13:09:23 +00:00
key: String,
2020-05-01 15:52:35 +00:00
}
fn parse_delete_objects_xml(xml: &roxmltree::Document) -> Option<DeleteRequest> {
let mut ret = DeleteRequest {
quiet: false,
objects: vec![],
};
2020-05-04 13:09:23 +00:00
let root = xml.root();
let delete = root.first_child()?;
2020-11-11 15:12:42 +00:00
2020-05-04 13:09:23 +00:00
if !delete.has_tag_name("Delete") {
return None;
2020-05-04 13:09:23 +00:00
}
2020-05-01 15:52:35 +00:00
2020-05-04 13:09:23 +00:00
for item in delete.children() {
if item.has_tag_name("Object") {
let key = item.children().find(|e| e.has_tag_name("Key"))?;
let key_str = key.text()?;
ret.objects.push(DeleteObject {
key: key_str.to_string(),
});
} else if item.has_tag_name("Quiet") {
if item.text()? == "true" {
ret.quiet = true;
} else {
ret.quiet = false;
}
2020-05-04 13:09:23 +00:00
} else {
return None;
2020-05-04 13:09:23 +00:00
}
}
Some(ret)
2020-05-04 13:09:23 +00:00
}