K2V client improvements #307

Merged
lx merged 4 commits from improve-k2v-client into main 2022-05-24 10:48:06 +00:00
6 changed files with 65 additions and 6 deletions

1
Cargo.lock generated
View file

@ -1584,6 +1584,7 @@ dependencies = [
"clap 3.1.18", "clap 3.1.18",
"garage_util 0.7.0", "garage_util 0.7.0",
"http", "http",
"log",
"rusoto_core", "rusoto_core",
"rusoto_credential", "rusoto_credential",
"rusoto_signature", "rusoto_signature",

View file

@ -688,7 +688,7 @@ in
registry = "registry+https://github.com/rust-lang/crates.io-index"; registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b"; }; src = fetchCratesIo { inherit name version; sha256 = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b"; };
dependencies = { dependencies = {
${ if hostPlatform.config == "aarch64-linux-android" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" || hostPlatform.config == "aarch64-apple-darwin" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; }; ${ if hostPlatform.config == "aarch64-linux-android" || hostPlatform.config == "aarch64-apple-darwin" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; };
}; };
}); });
@ -2117,6 +2117,7 @@ in
clap = rustPackages."registry+https://github.com/rust-lang/crates.io-index".clap."3.1.18" { inherit profileName; }; clap = rustPackages."registry+https://github.com/rust-lang/crates.io-index".clap."3.1.18" { inherit profileName; };
garage_util = rustPackages."unknown".garage_util."0.7.0" { inherit profileName; }; garage_util = rustPackages."unknown".garage_util."0.7.0" { inherit profileName; };
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; }; http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.16" { inherit profileName; };
rusoto_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rusoto_core."0.48.0" { inherit profileName; }; rusoto_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rusoto_core."0.48.0" { inherit profileName; };
rusoto_credential = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rusoto_credential."0.48.0" { inherit profileName; }; rusoto_credential = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rusoto_credential."0.48.0" { inherit profileName; };
rusoto_signature = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rusoto_signature."0.48.0" { inherit profileName; }; rusoto_signature = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rusoto_signature."0.48.0" { inherit profileName; };
@ -5029,7 +5030,7 @@ in
[ "default" ] [ "default" ]
]; ];
dependencies = { dependencies = {
${ if hostPlatform.config == "aarch64-uwp-windows-msvc" || hostPlatform.config == "aarch64-pc-windows-msvc" then "windows_aarch64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_msvc."0.32.0" { inherit profileName; }; ${ if hostPlatform.config == "aarch64-pc-windows-msvc" || hostPlatform.config == "aarch64-uwp-windows-msvc" then "windows_aarch64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_msvc."0.32.0" { inherit profileName; };
${ if hostPlatform.config == "i686-pc-windows-gnu" || hostPlatform.config == "i686-uwp-windows-gnu" then "windows_i686_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnu."0.32.0" { inherit profileName; }; ${ if hostPlatform.config == "i686-pc-windows-gnu" || hostPlatform.config == "i686-uwp-windows-gnu" then "windows_i686_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnu."0.32.0" { inherit profileName; };
${ if hostPlatform.config == "i686-uwp-windows-msvc" || hostPlatform.config == "i686-pc-windows-msvc" then "windows_i686_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_msvc."0.32.0" { inherit profileName; }; ${ if hostPlatform.config == "i686-uwp-windows-msvc" || hostPlatform.config == "i686-pc-windows-msvc" then "windows_i686_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_msvc."0.32.0" { inherit profileName; };
${ if hostPlatform.config == "x86_64-pc-windows-gnu" || hostPlatform.config == "x86_64-uwp-windows-gnu" then "windows_x86_64_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnu."0.32.0" { inherit profileName; }; ${ if hostPlatform.config == "x86_64-pc-windows-gnu" || hostPlatform.config == "x86_64-uwp-windows-gnu" then "windows_x86_64_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnu."0.32.0" { inherit profileName; };

View file

@ -6,6 +6,7 @@ edition = "2018"
[dependencies] [dependencies]
base64 = "0.13.0" base64 = "0.13.0"
http = "0.2.6" http = "0.2.6"
log = "0.4"
rusoto_core = "0.48.0" rusoto_core = "0.48.0"
rusoto_credential = "0.48.0" rusoto_credential = "0.48.0"
rusoto_signature = "0.48.0" rusoto_signature = "0.48.0"
@ -22,6 +23,10 @@ garage_util = { path = "../util", optional = true }
[features] [features]
cli = ["clap", "tokio/fs", "tokio/io-std", "garage_util"] cli = ["clap", "tokio/fs", "tokio/io-std", "garage_util"]
[lib]
path = "lib.rs"
[[bin]] [[bin]]
name = "k2v-cli" name = "k2v-cli"
path = "bin/k2v-cli.rs"
required-features = ["cli"] required-features = ["cli"]

View file

@ -5,6 +5,13 @@ use thiserror::Error;
/// Errors returned by this crate /// Errors returned by this crate
#[derive(Error, Debug)] #[derive(Error, Debug)]
pub enum Error { pub enum Error {
#[error("{0}, {1}: {2} (path = {3})")]
Remote(
http::StatusCode,
Cow<'static, str>,
Cow<'static, str>,
Cow<'static, str>,
),
#[error("received invalid response: {0}")] #[error("received invalid response: {0}")]
InvalidResponse(Cow<'static, str>), InvalidResponse(Cow<'static, str>),
#[error("not found")] #[error("not found")]

View file

@ -4,6 +4,7 @@ use std::time::Duration;
use http::header::{ACCEPT, CONTENT_LENGTH, CONTENT_TYPE}; use http::header::{ACCEPT, CONTENT_LENGTH, CONTENT_TYPE};
use http::status::StatusCode; use http::status::StatusCode;
use http::HeaderMap; use http::HeaderMap;
use log::{debug, error};
use rusoto_core::{ByteStream, DispatchSignedRequest, HttpClient}; use rusoto_core::{ByteStream, DispatchSignedRequest, HttpClient};
use rusoto_credential::AwsCredentials; use rusoto_credential::AwsCredentials;
@ -310,12 +311,47 @@ impl K2vClient {
StatusCode::NO_CONTENT => Vec::new(), StatusCode::NO_CONTENT => Vec::new(),
StatusCode::NOT_FOUND => return Err(Error::NotFound), StatusCode::NOT_FOUND => return Err(Error::NotFound),
StatusCode::NOT_MODIFIED => Vec::new(), StatusCode::NOT_MODIFIED => Vec::new(),
_ => { s => {
return Err(Error::InvalidResponse( let err_body = read_body(&mut res.headers, res.body)
format!("invalid error code: {}", res.status).into(), .await
)) .unwrap_or_default();
let err_body_str = std::str::from_utf8(&err_body)
.map(String::from)
.unwrap_or_else(|_| base64::encode(&err_body));
if s.is_client_error() || s.is_server_error() {
error!("Error response {}: {}", res.status, err_body_str);
let err = match serde_json::from_slice::<ErrorResponse>(&err_body) {
Ok(err) => Error::Remote(
res.status,
err.code.into(),
err.message.into(),
err.path.into(),
),
Err(_) => Error::Remote(
res.status,
"unknown".into(),
err_body_str.into(),
"?".into(),
),
};
return Err(err);
} else {
let msg = format!(
"Unexpected response code {}. Response body: {}",
res.status, err_body_str
);
error!("{}", msg);
return Err(Error::InvalidResponse(msg.into()));
}
} }
}; };
debug!(
"Response body: {}",
std::str::from_utf8(&body)
.map(String::from)
.unwrap_or_else(|_| base64::encode(&body))
);
Ok(Response { Ok(Response {
body, body,
@ -558,6 +594,15 @@ struct BatchDeleteResponse<'a> {
deleted_items: u64, deleted_items: u64,
} }
#[derive(Deserialize)]
struct ErrorResponse {
code: String,
message: String,
#[allow(dead_code)]
region: String,
path: String,
}
struct Response { struct Response {
body: Vec<u8>, body: Vec<u8>,
status: StatusCode, status: StatusCode,