Merge pull request 'Increase Garage tests robustness' (#526) from tests/increase-robustness into main

Reviewed-on: #526
Reviewed-by: Alex <alex@adnab.me>
Reviewed-by: trinity-1686a <trinity.pointard@gmail.com>

src/garage/tests/admin.rs

@@ -21,14 +21,7 @@ async fn test_admin_bucket_perms() {
 
 	ctx.garage
 		.command()
-		.args([
-			"bucket",
-			"allow",
-			"--read",
-			"--key",
-			&ctx.garage.key.id,
-			BCKT_NAME,
-		])
+		.args(["bucket", "allow", "--read", "--key", &ctx.key.id, BCKT_NAME])
 		.quiet()
 		.expect_success_status("Could not create bucket");
 
@@ -36,14 +29,7 @@ async fn test_admin_bucket_perms() {
 
 	ctx.garage
 		.command()
-		.args([
-			"bucket",
-			"deny",
-			"--read",
-			"--key",
-			&ctx.garage.key.name,
-			BCKT_NAME,
-		])
+		.args(["bucket", "deny", "--read", "--key", &ctx.key.id, BCKT_NAME])
 		.quiet()
 		.expect_success_status("Could not create bucket");
 
@@ -51,14 +37,7 @@ async fn test_admin_bucket_perms() {
 
 	ctx.garage
 		.command()
-		.args([
-			"bucket",
-			"allow",
-			"--read",
-			"--key",
-			&ctx.garage.key.name,
-			BCKT_NAME,
-		])
+		.args(["bucket", "allow", "--read", "--key", &ctx.key.id, BCKT_NAME])
 		.quiet()
 		.expect_success_status("Could not create bucket");
 

src/garage/tests/bucket.rs

@@ -13,7 +13,7 @@ async fn test_bucket_all() {
 		ctx.garage
 			.command()
 			.args(["key", "deny"])
-			.args(["--create-bucket", &ctx.garage.key.id])
+			.args(["--create-bucket", &ctx.key.id])
 			.quiet()
 			.expect_success_output("Could not deny key to create buckets");
 
@@ -26,7 +26,7 @@ async fn test_bucket_all() {
 		ctx.garage
 			.command()
 			.args(["key", "allow"])
-			.args(["--create-bucket", &ctx.garage.key.id])
+			.args(["--create-bucket", &ctx.key.id])
 			.quiet()
 			.expect_success_output("Could not deny key to create buckets");
 

src/garage/tests/common/client.rs

@@ -1,15 +1,9 @@
 use aws_sdk_s3::{Client, Config, Credentials, Endpoint};
 
-use super::garage::Instance;
+use super::garage::{Instance, Key};
 
-pub fn build_client(instance: &Instance) -> Client {
-	let credentials = Credentials::new(
-		&instance.key.id,
-		&instance.key.secret,
-		None,
-		None,
-		"garage-integ-test",
-	);
+pub fn build_client(instance: &Instance, key: &Key) -> Client {
+	let credentials = Credentials::new(&key.id, &key.secret, None, None, "garage-integ-test");
 	let endpoint = Endpoint::immutable(instance.s3_uri());
 
 	let config = Config::builder()

src/garage/tests/common/custom_requester.rs

@@ -14,6 +14,7 @@ use garage_api::signature;
 /// You should ever only use this to send requests AWS sdk won't send,
 /// like to reproduce behavior of unusual implementations found to be
 /// problematic.
+#[derive(Clone)]
 pub struct CustomRequester {
 	key: Key,
 	uri: Uri,
@@ -22,18 +23,18 @@ pub struct CustomRequester {
 }
 
 impl CustomRequester {
-	pub fn new_s3(instance: &Instance) -> Self {
+	pub fn new_s3(instance: &Instance, key: &Key) -> Self {
 		CustomRequester {
-			key: instance.key.clone(),
+			key: key.clone(),
 			uri: instance.s3_uri(),
 			service: "s3",
 			client: Client::new(),
 		}
 	}
 
-	pub fn new_k2v(instance: &Instance) -> Self {
+	pub fn new_k2v(instance: &Instance, key: &Key) -> Self {
 		CustomRequester {
-			key: instance.key.clone(),
+			key: key.clone(),
 			uri: instance.k2v_uri(),
 			service: "k2v",
 			client: Client::new(),

src/garage/tests/common/garage.rs

@@ -13,7 +13,7 @@ static GARAGE_TEST_SECRET: &str =
 
 #[derive(Debug, Default, Clone)]
 pub struct Key {
-	pub name: String,
+	pub name: Option<String>,
 	pub id: String,
 	pub secret: String,
 }
@@ -21,7 +21,7 @@ pub struct Key {
 pub struct Instance {
 	process: process::Child,
 	pub path: PathBuf,
-	pub key: Key,
+	pub default_key: Key,
 	pub s3_port: u16,
 	pub k2v_port: u16,
 	pub web_port: u16,
@@ -102,7 +102,7 @@ api_bind_addr = "127.0.0.1:{admin_port}"
 		Instance {
 			process: child,
 			path,
-			key: Key::default(),
+			default_key: Key::default(),
 			s3_port: port,
 			k2v_port: port + 1,
 			web_port: port + 3,
@@ -111,14 +111,27 @@ api_bind_addr = "127.0.0.1:{admin_port}"
 	}
 
 	fn setup(&mut self) {
+		self.wait_for_boot();
+		self.setup_layout();
+		self.default_key = self.key(Some("garage_test"));
+	}
+
+	fn wait_for_boot(&mut self) {
 		use std::{thread, time::Duration};
 
-		// Wait for node to be ready
-		thread::sleep(Duration::from_secs(2));
-
-		self.setup_layout();
-
-		self.key = self.new_key("garage_test");
+		// 60 * 2 seconds = 120 seconds = 2min
+		for _ in 0..60 {
+			let termination = self
+				.command()
+				.args(["status"])
+				.quiet()
+				.status()
+				.expect("Unable to run command");
+			if termination.success() {
+				break;
+			}
+			thread::sleep(Duration::from_secs(2));
+		}
 	}
 
 	fn setup_layout(&self) {
@@ -169,14 +182,17 @@ api_bind_addr = "127.0.0.1:{admin_port}"
 			.expect("Could not build garage endpoint URI")
 	}
 
-	pub fn new_key(&self, name: &str) -> Key {
+	pub fn key(&self, maybe_name: Option<&str>) -> Key {
 		let mut key = Key::default();
 
-		let output = self
-			.command()
-			.args(["key", "new"])
-			.args(["--name", name])
-			.expect_success_output("Could not create key");
+		let mut cmd = self.command();
+		let base = cmd.args(["key", "new"]);
+		let with_name = match maybe_name {
+			Some(name) => base.args(["--name", name]),
+			None => base,
+		};
+
+		let output = with_name.expect_success_output("Could not create key");
 		let stdout = String::from_utf8(output.stdout).unwrap();
 
 		for line in stdout.lines() {
@@ -193,7 +209,7 @@ api_bind_addr = "127.0.0.1:{admin_port}"
 		assert!(!key.secret.is_empty(), "Invalid key: Key secret is empty");
 
 		Key {
-			name: name.to_owned(),
+			name: maybe_name.map(String::from),
 			..key
 		}
 	}

src/garage/tests/common/mod.rs

@@ -13,13 +13,16 @@ use custom_requester::CustomRequester;
 
 const REGION: Region = Region::from_static("garage-integ-test");
 
+#[derive(Clone)]
 pub struct Context {
 	pub garage: &'static garage::Instance,
+	pub key: garage::Key,
 	pub client: Client,
 	pub custom_request: CustomRequester,
 	pub k2v: K2VContext,
 }
 
+#[derive(Clone)]
 pub struct K2VContext {
 	pub request: CustomRequester,
 }
@@ -27,13 +30,15 @@ pub struct K2VContext {
 impl Context {
 	fn new() -> Self {
 		let garage = garage::instance();
-		let client = client::build_client(garage);
-		let custom_request = CustomRequester::new_s3(garage);
-		let k2v_request = CustomRequester::new_k2v(garage);
+		let key = garage.key(None);
+		let client = client::build_client(garage, &key);
+		let custom_request = CustomRequester::new_s3(garage, &key);
+		let k2v_request = CustomRequester::new_k2v(garage, &key);
 
 		Context {
 			garage,
 			client,
+			key,
 			custom_request,
 			k2v: K2VContext {
 				request: k2v_request,
@@ -57,7 +62,7 @@ impl Context {
 			.args(["bucket", "allow"])
 			.args(["--owner", "--read", "--write"])
 			.arg(&bucket_name)
-			.args(["--key", &self.garage.key.name])
+			.args(["--key", &self.key.id])
 			.quiet()
 			.expect_success_status("Could not allow key for bucket");
 

src/garage/tests/k2v/poll.rs

@@ -57,8 +57,8 @@ async fn test_poll_item() {
 	let poll = {
 		let bucket = bucket.clone();
 		let ct = ct.clone();
+		let ctx = ctx.clone();
 		tokio::spawn(async move {
-			let ctx = common::context();
 			ctx.k2v
 				.request
 				.builder(bucket.clone())
@@ -171,8 +171,8 @@ async fn test_poll_range() {
 	// Second poll range, which will complete later
 	let poll = {
 		let bucket = bucket.clone();
+		let ctx = ctx.clone();
 		tokio::spawn(async move {
-			let ctx = common::context();
 			ctx.k2v
 				.request
 				.builder(bucket.clone())
@@ -220,8 +220,8 @@ async fn test_poll_range() {
 	// Start a new poll operation
 	let poll = {
 		let bucket = bucket.clone();
+		let ctx = ctx.clone();
 		tokio::spawn(async move {
-			let ctx = common::context();
 			ctx.k2v
 				.request
 				.builder(bucket.clone())

src/garage/tests/s3/streaming_signature.rs

@@ -109,7 +109,7 @@ async fn test_create_bucket_streaming() {
 	ctx.garage
 		.command()
 		.args(["key", "allow"])
-		.args(["--create-bucket", &ctx.garage.key.id])
+		.args(["--create-bucket", &ctx.key.id])
 		.quiet()
 		.expect_success_output("Could not allow key to create buckets");