forked from Deuxfleurs/garage
Compare commits
No commits in common. "aee0d97f2265bf6fd1461b3dd388aa4c5c910d3c" and "ee494f5aa2d1da4f1a0d1a7a27e5666183362cee" have entirely different histories.
aee0d97f22
...
ee494f5aa2
14 changed files with 2031 additions and 1928 deletions
1202
Cargo.lock
generated
1202
Cargo.lock
generated
File diff suppressed because it is too large
Load diff
|
@ -47,7 +47,7 @@ http-range = "0.1"
|
||||||
hyper = { version = "0.14", features = ["server", "http1", "runtime", "tcp", "stream"] }
|
hyper = { version = "0.14", features = ["server", "http1", "runtime", "tcp", "stream"] }
|
||||||
multer = "2.0"
|
multer = "2.0"
|
||||||
percent-encoding = "2.1.0"
|
percent-encoding = "2.1.0"
|
||||||
roxmltree = "0.18"
|
roxmltree = "0.14"
|
||||||
serde = { version = "1.0", features = ["derive"] }
|
serde = { version = "1.0", features = ["derive"] }
|
||||||
serde_bytes = "0.11"
|
serde_bytes = "0.11"
|
||||||
serde_json = "1.0"
|
serde_json = "1.0"
|
||||||
|
|
|
@ -28,7 +28,7 @@ hex = "0.4"
|
||||||
tracing = "0.1"
|
tracing = "0.1"
|
||||||
rand = "0.8"
|
rand = "0.8"
|
||||||
|
|
||||||
async-compression = { version = "0.4", features = ["tokio", "zstd"] }
|
async-compression = { version = "0.3", features = ["tokio", "zstd"] }
|
||||||
zstd = { version = "0.12", default-features = false }
|
zstd = { version = "0.12", default-features = false }
|
||||||
|
|
||||||
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
|
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
|
||||||
|
|
|
@ -61,8 +61,7 @@ opentelemetry-otlp = { version = "0.10", optional = true }
|
||||||
prometheus = { version = "0.13", optional = true }
|
prometheus = { version = "0.13", optional = true }
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
aws-config = "0.55.2"
|
aws-sdk-s3 = "0.19"
|
||||||
aws-sdk-s3 = "0.28"
|
|
||||||
chrono = "0.4"
|
chrono = "0.4"
|
||||||
http = "0.2"
|
http = "0.2"
|
||||||
hmac = "0.12"
|
hmac = "0.12"
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
use crate::common;
|
use crate::common;
|
||||||
use crate::common::ext::CommandExt;
|
use crate::common::ext::CommandExt;
|
||||||
use aws_sdk_s3::operation::delete_bucket::DeleteBucketOutput;
|
use aws_sdk_s3::model::BucketLocationConstraint;
|
||||||
|
use aws_sdk_s3::output::DeleteBucketOutput;
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_bucket_all() {
|
async fn test_bucket_all() {
|
||||||
|
@ -62,7 +63,10 @@ async fn test_bucket_all() {
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
assert_eq!(r.location_constraint.unwrap().as_str(), "garage-integ-test");
|
match r.location_constraint.unwrap() {
|
||||||
|
BucketLocationConstraint::Unknown(v) if v.as_str() == "garage-integ-test" => (),
|
||||||
|
_ => unreachable!("wrong region"),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
// (Stub) check GetVersioning
|
// (Stub) check GetVersioning
|
||||||
|
|
|
@ -1,16 +1,15 @@
|
||||||
use aws_sdk_s3::config::Credentials;
|
use aws_sdk_s3::{Client, Config, Credentials, Endpoint};
|
||||||
use aws_sdk_s3::{Client, Config};
|
|
||||||
|
|
||||||
use super::garage::Key;
|
use super::garage::{Instance, Key};
|
||||||
use crate::common::garage::DEFAULT_PORT;
|
|
||||||
|
|
||||||
pub fn build_client(key: &Key) -> Client {
|
pub fn build_client(instance: &Instance, key: &Key) -> Client {
|
||||||
let credentials = Credentials::new(&key.id, &key.secret, None, None, "garage-integ-test");
|
let credentials = Credentials::new(&key.id, &key.secret, None, None, "garage-integ-test");
|
||||||
|
let endpoint = Endpoint::immutable(instance.s3_uri());
|
||||||
|
|
||||||
let config = Config::builder()
|
let config = Config::builder()
|
||||||
.endpoint_url(format!("http://127.0.0.1:{}", DEFAULT_PORT))
|
|
||||||
.region(super::REGION)
|
.region(super::REGION)
|
||||||
.credentials_provider(credentials)
|
.credentials_provider(credentials)
|
||||||
|
.endpoint_resolver(endpoint)
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
Client::from_conf(config)
|
Client::from_conf(config)
|
||||||
|
|
|
@ -1,5 +1,4 @@
|
||||||
use aws_sdk_s3::config::Region;
|
use aws_sdk_s3::{Client, Region};
|
||||||
use aws_sdk_s3::Client;
|
|
||||||
use ext::*;
|
use ext::*;
|
||||||
use k2v_client::K2vClient;
|
use k2v_client::K2vClient;
|
||||||
|
|
||||||
|
@ -33,7 +32,7 @@ impl Context {
|
||||||
fn new() -> Self {
|
fn new() -> Self {
|
||||||
let garage = garage::instance();
|
let garage = garage::instance();
|
||||||
let key = garage.key(None);
|
let key = garage.key(None);
|
||||||
let client = client::build_client(&key);
|
let client = client::build_client(garage, &key);
|
||||||
let custom_request = CustomRequester::new_s3(garage, &key);
|
let custom_request = CustomRequester::new_s3(garage, &key);
|
||||||
let k2v_request = CustomRequester::new_k2v(garage, &key);
|
let k2v_request = CustomRequester::new_k2v(garage, &key);
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
use crate::common;
|
use crate::common;
|
||||||
use aws_sdk_s3::primitives::ByteStream;
|
use aws_sdk_s3::model::{CompletedMultipartUpload, CompletedPart};
|
||||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
use aws_sdk_s3::types::ByteStream;
|
||||||
|
|
||||||
const SZ_5MB: usize = 5 * 1024 * 1024;
|
const SZ_5MB: usize = 5 * 1024 * 1024;
|
||||||
const SZ_10MB: usize = 10 * 1024 * 1024;
|
const SZ_10MB: usize = 10 * 1024 * 1024;
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
use crate::common;
|
use crate::common;
|
||||||
use aws_sdk_s3::primitives::ByteStream;
|
use aws_sdk_s3::model::{Delete, ObjectIdentifier};
|
||||||
use aws_sdk_s3::types::{Delete, ObjectIdentifier};
|
use aws_sdk_s3::types::ByteStream;
|
||||||
|
|
||||||
const STD_KEY: &str = "hello world";
|
const STD_KEY: &str = "hello world";
|
||||||
const CTRL_KEY: &str = "\x00\x01\x02\x00";
|
const CTRL_KEY: &str = "\x00\x01\x02\x00";
|
||||||
|
|
|
@ -2,7 +2,7 @@ use crate::common;
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_simple() {
|
async fn test_simple() {
|
||||||
use aws_sdk_s3::primitives::ByteStream;
|
use aws_sdk_s3::types::ByteStream;
|
||||||
|
|
||||||
let ctx = common::context();
|
let ctx = common::context();
|
||||||
let bucket = ctx.create_bucket("test-simple");
|
let bucket = ctx.create_bucket("test-simple");
|
||||||
|
|
|
@ -4,8 +4,8 @@ use crate::k2v::json_body;
|
||||||
|
|
||||||
use assert_json_diff::assert_json_eq;
|
use assert_json_diff::assert_json_eq;
|
||||||
use aws_sdk_s3::{
|
use aws_sdk_s3::{
|
||||||
primitives::ByteStream,
|
model::{CorsConfiguration, CorsRule, ErrorDocument, IndexDocument, WebsiteConfiguration},
|
||||||
types::{CorsConfiguration, CorsRule, ErrorDocument, IndexDocument, WebsiteConfiguration},
|
types::ByteStream,
|
||||||
};
|
};
|
||||||
use http::{Request, StatusCode};
|
use http::{Request, StatusCode};
|
||||||
use hyper::{
|
use hyper::{
|
||||||
|
|
|
@ -311,19 +311,23 @@ impl BatchOutputKind {
|
||||||
.collect::<Vec<_>>()
|
.collect::<Vec<_>>()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn display_poll_range_output(&self, poll_range: PollRangeResult) -> ! {
|
fn display_poll_range_output(
|
||||||
|
&self,
|
||||||
|
seen_marker: String,
|
||||||
|
values: BTreeMap<String, CausalValue>,
|
||||||
|
) -> ! {
|
||||||
if self.json {
|
if self.json {
|
||||||
let json = serde_json::json!({
|
let json = serde_json::json!({
|
||||||
"values": self.values_json(poll_range.items),
|
"values": self.values_json(values),
|
||||||
"seen_marker": poll_range.seen_marker,
|
"seen_marker": seen_marker,
|
||||||
});
|
});
|
||||||
|
|
||||||
let stdout = std::io::stdout();
|
let stdout = std::io::stdout();
|
||||||
serde_json::to_writer_pretty(stdout, &json).unwrap();
|
serde_json::to_writer_pretty(stdout, &json).unwrap();
|
||||||
exit(0)
|
exit(0)
|
||||||
} else {
|
} else {
|
||||||
println!("seen marker: {}", poll_range.seen_marker);
|
println!("seen marker: {}", seen_marker);
|
||||||
self.display_human_output(poll_range.items)
|
self.display_human_output(values)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -497,8 +501,8 @@ async fn main() -> Result<(), Error> {
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
match res {
|
match res {
|
||||||
Some(poll_range_output) => {
|
Some((items, seen_marker)) => {
|
||||||
output_kind.display_poll_range_output(poll_range_output);
|
output_kind.display_poll_range_output(seen_marker, items);
|
||||||
}
|
}
|
||||||
None => {
|
None => {
|
||||||
if output_kind.json {
|
if output_kind.json {
|
||||||
|
|
|
@ -182,7 +182,7 @@ impl K2vClient {
|
||||||
filter: Option<PollRangeFilter<'_>>,
|
filter: Option<PollRangeFilter<'_>>,
|
||||||
seen_marker: Option<&str>,
|
seen_marker: Option<&str>,
|
||||||
timeout: Option<Duration>,
|
timeout: Option<Duration>,
|
||||||
) -> Result<Option<PollRangeResult>, Error> {
|
) -> Result<Option<(BTreeMap<String, CausalValue>, String)>, Error> {
|
||||||
let timeout = timeout.unwrap_or(DEFAULT_POLL_TIMEOUT);
|
let timeout = timeout.unwrap_or(DEFAULT_POLL_TIMEOUT);
|
||||||
|
|
||||||
let request = PollRangeRequest {
|
let request = PollRangeRequest {
|
||||||
|
@ -217,10 +217,7 @@ impl K2vClient {
|
||||||
})
|
})
|
||||||
.collect::<BTreeMap<_, _>>();
|
.collect::<BTreeMap<_, _>>();
|
||||||
|
|
||||||
Ok(Some(PollRangeResult {
|
Ok(Some((items, resp.seen_marker)))
|
||||||
items,
|
|
||||||
seen_marker: resp.seen_marker,
|
|
||||||
}))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Perform an InsertItem request, inserting a value for a single pk+sk.
|
/// Perform an InsertItem request, inserting a value for a single pk+sk.
|
||||||
|
@ -573,7 +570,6 @@ pub struct Filter<'a> {
|
||||||
pub reverse: bool,
|
pub reverse: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Filter for a poll range operations.
|
|
||||||
#[derive(Debug, Default, Clone, Serialize)]
|
#[derive(Debug, Default, Clone, Serialize)]
|
||||||
pub struct PollRangeFilter<'a> {
|
pub struct PollRangeFilter<'a> {
|
||||||
pub start: Option<&'a str>,
|
pub start: Option<&'a str>,
|
||||||
|
@ -581,15 +577,6 @@ pub struct PollRangeFilter<'a> {
|
||||||
pub prefix: Option<&'a str>,
|
pub prefix: Option<&'a str>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Response to a poll_range query
|
|
||||||
#[derive(Debug, Default, Clone, Serialize)]
|
|
||||||
pub struct PollRangeResult {
|
|
||||||
/// List of items that have changed since last PollRange call.
|
|
||||||
pub items: BTreeMap<String, CausalValue>,
|
|
||||||
/// opaque string representing items already seen for future PollRange calls.
|
|
||||||
pub seen_marker: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize)]
|
#[derive(Debug, Clone, Serialize)]
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
struct PollRangeRequest<'a> {
|
struct PollRangeRequest<'a> {
|
||||||
|
|
Loading…
Reference in a new issue