From 0951b5db75e2576a004718b5cdcfe66ce7d70028 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 6 Mar 2025 18:49:56 +0100 Subject: [PATCH] admin api: implement ClusterLayoutSkipDeadNodes and use it in CLI --- doc/api/garage-admin-v2.json | 72 +++++++++++++++++++++++ src/api/admin/api.rs | 17 ++++++ src/api/admin/cluster.rs | 61 ++++++++++++++++++++ src/api/admin/openapi.rs | 13 +++++ src/api/admin/router_v2.rs | 1 + src/garage/cli/layout.rs | 109 ----------------------------------- src/garage/cli/mod.rs | 2 - src/garage/cli_v2/layout.rs | 37 ++++++++---- src/garage/cli_v2/mod.rs | 2 - src/garage/main.rs | 2 - 10 files changed, 190 insertions(+), 126 deletions(-) delete mode 100644 src/garage/cli/layout.rs diff --git a/doc/api/garage-admin-v2.json b/doc/api/garage-admin-v2.json index 598f82a3..921d8d4c 100644 --- a/doc/api/garage-admin-v2.json +++ b/doc/api/garage-admin-v2.json @@ -157,6 +157,40 @@ } } }, + "/v2/ClusterLayoutSkipDeadNodes": { + "post": { + "tags": [ + "Cluster layout" + ], + "description": "Force progress in layout update trackers", + "operationId": "ClusterLayoutSkipDeadNodes", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ClusterLayoutSkipDeadNodesRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Request has been taken into account", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ClusterLayoutSkipDeadNodesResponse" + } + } + } + }, + "500": { + "description": "Internal server error" + } + } + } + }, "/v2/ConnectClusterNodes": { "post": { "tags": [ @@ -1624,6 +1658,44 @@ } } }, + "ClusterLayoutSkipDeadNodesRequest": { + "type": "object", + "required": [ + "version", + "allowMissingData" + ], + "properties": { + "allowMissingData": { + "type": "boolean" + }, + "version": { + "type": "integer", + "format": "int64", + "minimum": 0 + } + } + }, + "ClusterLayoutSkipDeadNodesResponse": { + "type": "object", + "required": [ + "ackUpdated", + "syncUpdated" + ], + "properties": { + "ackUpdated": { + "type": "array", + "items": { + "type": "string" + } + }, + "syncUpdated": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, "ClusterLayoutVersion": { "type": "object", "required": [ diff --git a/src/api/admin/api.rs b/src/api/admin/api.rs index ea017f7b..ec0a9e3c 100644 --- a/src/api/admin/api.rs +++ b/src/api/admin/api.rs @@ -56,6 +56,7 @@ admin_endpoints![ PreviewClusterLayoutChanges, ApplyClusterLayout, RevertClusterLayout, + ClusterLayoutSkipDeadNodes, // Access key operations ListKeys, @@ -422,6 +423,22 @@ pub struct RevertClusterLayoutRequest; #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] pub struct RevertClusterLayoutResponse(pub GetClusterLayoutResponse); +// ---- ClusterLayoutSkipDeadNodes ---- + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct ClusterLayoutSkipDeadNodesRequest { + pub version: u64, + pub allow_missing_data: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct ClusterLayoutSkipDeadNodesResponse { + pub ack_updated: Vec, + pub sync_updated: Vec, +} + // ********************************************** // Access key operations // ********************************************** diff --git a/src/api/admin/cluster.rs b/src/api/admin/cluster.rs index 3c076064..8171aa98 100644 --- a/src/api/admin/cluster.rs +++ b/src/api/admin/cluster.rs @@ -465,6 +465,67 @@ impl RequestHandler for RevertClusterLayoutRequest { } } +impl RequestHandler for ClusterLayoutSkipDeadNodesRequest { + type Response = ClusterLayoutSkipDeadNodesResponse; + + async fn handle( + self, + garage: &Arc, + _admin: &Admin, + ) -> Result { + let status = garage.system.get_known_nodes(); + + let mut layout = garage.system.cluster_layout().inner().clone(); + let mut ack_updated = vec![]; + let mut sync_updated = vec![]; + + if layout.versions.len() == 1 { + return Err(Error::bad_request( + "This command cannot be called when there is only one live cluster layout version", + )); + } + + let min_v = layout.min_stored(); + if self.version <= min_v || self.version > layout.current().version { + return Err(Error::bad_request(format!( + "Invalid version, you may use the following version numbers: {}", + (min_v + 1..=layout.current().version) + .map(|x| x.to_string()) + .collect::>() + .join(" ") + ))); + } + + let all_nodes = layout.get_all_nodes(); + for node in all_nodes.iter() { + // Update ACK tracker for dead nodes or for all nodes if --allow-missing-data + if self.allow_missing_data || !status.iter().any(|x| x.id == *node && x.is_up) { + if layout.update_trackers.ack_map.set_max(*node, self.version) { + ack_updated.push(hex::encode(node)); + } + } + + // If --allow-missing-data, update SYNC tracker for all nodes. + if self.allow_missing_data { + if layout.update_trackers.sync_map.set_max(*node, self.version) { + sync_updated.push(hex::encode(node)); + } + } + } + + garage + .system + .layout_manager + .update_cluster_layout(&layout) + .await?; + + Ok(ClusterLayoutSkipDeadNodesResponse { + ack_updated, + sync_updated, + }) + } +} + // ---- impl From for ZoneRedundancy { diff --git a/src/api/admin/openapi.rs b/src/api/admin/openapi.rs index 0a31449b..01a694e5 100644 --- a/src/api/admin/openapi.rs +++ b/src/api/admin/openapi.rs @@ -172,6 +172,18 @@ fn ApplyClusterLayout() -> () {} )] fn RevertClusterLayout() -> () {} +#[utoipa::path(post, + path = "/v2/ClusterLayoutSkipDeadNodes", + tag = "Cluster layout", + description = "Force progress in layout update trackers", + request_body = ClusterLayoutSkipDeadNodesRequest, + responses( + (status = 200, description = "Request has been taken into account", body = ClusterLayoutSkipDeadNodesResponse), + (status = 500, description = "Internal server error") + ), +)] +fn ClusterLayoutSkipDeadNodes() -> () {} + // ********************************************** // Access key operations // ********************************************** @@ -718,6 +730,7 @@ impl Modify for SecurityAddon { PreviewClusterLayoutChanges, ApplyClusterLayout, RevertClusterLayout, + ClusterLayoutSkipDeadNodes, // Key operations ListKeys, GetKeyInfo, diff --git a/src/api/admin/router_v2.rs b/src/api/admin/router_v2.rs index 318e7173..9f6106e5 100644 --- a/src/api/admin/router_v2.rs +++ b/src/api/admin/router_v2.rs @@ -41,6 +41,7 @@ impl AdminApiRequest { POST PreviewClusterLayoutChanges (), POST ApplyClusterLayout (body), POST RevertClusterLayout (), + POST ClusterLayoutSkipDeadNodes (body), // API key endpoints GET GetKeyInfo (query_opt::id, query_opt::search, parse_default(false)::show_secret_key), POST UpdateKey (body_field, query::id), diff --git a/src/garage/cli/layout.rs b/src/garage/cli/layout.rs deleted file mode 100644 index 352f792b..00000000 --- a/src/garage/cli/layout.rs +++ /dev/null @@ -1,109 +0,0 @@ -use garage_util::error::*; - -use garage_rpc::layout::*; -use garage_rpc::system::*; -use garage_rpc::*; - -use crate::cli::structs::*; - -pub async fn cmd_layout_skip_dead_nodes( - rpc_cli: &Endpoint, - rpc_host: NodeID, - opt: SkipDeadNodesOpt, -) -> Result<(), Error> { - let status = fetch_status(rpc_cli, rpc_host).await?; - let mut layout = fetch_layout(rpc_cli, rpc_host).await?; - - if layout.versions.len() == 1 { - return Err(Error::Message( - "This command cannot be called when there is only one live cluster layout version" - .into(), - )); - } - - let min_v = layout.min_stored(); - if opt.version <= min_v || opt.version > layout.current().version { - return Err(Error::Message(format!( - "Invalid version, you may use the following version numbers: {}", - (min_v + 1..=layout.current().version) - .map(|x| x.to_string()) - .collect::>() - .join(" ") - ))); - } - - let all_nodes = layout.get_all_nodes(); - let mut did_something = false; - for node in all_nodes.iter() { - // Update ACK tracker for dead nodes or for all nodes if --allow-missing-data - if opt.allow_missing_data || !status.iter().any(|x| x.id == *node && x.is_up) { - if layout.update_trackers.ack_map.set_max(*node, opt.version) { - println!("Increased the ACK tracker for node {:?}", node); - did_something = true; - } - } - - // If --allow-missing-data, update SYNC tracker for all nodes. - if opt.allow_missing_data { - if layout.update_trackers.sync_map.set_max(*node, opt.version) { - println!("Increased the SYNC tracker for node {:?}", node); - did_something = true; - } - } - } - - if did_something { - send_layout(rpc_cli, rpc_host, layout).await?; - println!("Success."); - Ok(()) - } else if !opt.allow_missing_data { - Err(Error::Message("Nothing was done, try passing the `--allow-missing-data` flag to force progress even when not enough nodes can complete a metadata sync.".into())) - } else { - Err(Error::Message( - "Sorry, there is nothing I can do for you. Please wait patiently. If you ask for help, please send the output of the `garage layout history` command.".into(), - )) - } -} - -// --- utility --- - -async fn fetch_status( - rpc_cli: &Endpoint, - rpc_host: NodeID, -) -> Result, Error> { - match rpc_cli - .call(&rpc_host, SystemRpc::GetKnownNodes, PRIO_NORMAL) - .await?? - { - SystemRpc::ReturnKnownNodes(nodes) => Ok(nodes), - resp => Err(Error::unexpected_rpc_message(resp)), - } -} - -async fn fetch_layout( - rpc_cli: &Endpoint, - rpc_host: NodeID, -) -> Result { - match rpc_cli - .call(&rpc_host, SystemRpc::PullClusterLayout, PRIO_NORMAL) - .await?? - { - SystemRpc::AdvertiseClusterLayout(t) => Ok(t), - resp => Err(Error::unexpected_rpc_message(resp)), - } -} - -async fn send_layout( - rpc_cli: &Endpoint, - rpc_host: NodeID, - layout: LayoutHistory, -) -> Result<(), Error> { - rpc_cli - .call( - &rpc_host, - SystemRpc::AdvertiseClusterLayout(layout), - PRIO_NORMAL, - ) - .await??; - Ok(()) -} diff --git a/src/garage/cli/mod.rs b/src/garage/cli/mod.rs index e007808b..146fac56 100644 --- a/src/garage/cli/mod.rs +++ b/src/garage/cli/mod.rs @@ -3,5 +3,3 @@ pub(crate) mod structs; pub(crate) mod convert_db; pub(crate) mod init; pub(crate) mod repair; - -pub(crate) mod layout; diff --git a/src/garage/cli_v2/layout.rs b/src/garage/cli_v2/layout.rs index 250cd0b0..bab6f28e 100644 --- a/src/garage/cli_v2/layout.rs +++ b/src/garage/cli_v2/layout.rs @@ -6,7 +6,6 @@ use garage_util::error::*; use garage_api_admin::api::*; use garage_rpc::layout; -use crate::cli::layout as cli_v1; use crate::cli::structs::*; use crate::cli_v2::*; @@ -20,16 +19,7 @@ impl Cli { LayoutOperation::Apply(apply_opt) => self.cmd_apply_layout(apply_opt).await, LayoutOperation::Revert(revert_opt) => self.cmd_revert_layout(revert_opt).await, LayoutOperation::History => self.cmd_layout_history().await, - - // TODO - LayoutOperation::SkipDeadNodes(assume_sync_opt) => { - cli_v1::cmd_layout_skip_dead_nodes( - &self.system_rpc_endpoint, - self.rpc_host, - assume_sync_opt, - ) - .await - } + LayoutOperation::SkipDeadNodes(opt) => self.cmd_skip_dead_nodes(opt).await, } } @@ -304,6 +294,31 @@ To know the correct value of the new layout version, invoke `garage layout show` Ok(()) } + + pub async fn cmd_skip_dead_nodes(&self, opt: SkipDeadNodesOpt) -> Result<(), Error> { + let res = self + .api_request(ClusterLayoutSkipDeadNodesRequest { + version: opt.version, + allow_missing_data: opt.allow_missing_data, + }) + .await?; + + if !res.sync_updated.is_empty() || !res.ack_updated.is_empty() { + for node in res.ack_updated.iter() { + println!("Increased the ACK tracker for node {:.16}", node); + } + for node in res.sync_updated.iter() { + println!("Increased the SYNC tracker for node {:.16}", node); + } + Ok(()) + } else if !opt.allow_missing_data { + Err(Error::Message("Nothing was done, try passing the `--allow-missing-data` flag to force progress even when not enough nodes can complete a metadata sync.".into())) + } else { + Err(Error::Message( + "Sorry, there is nothing I can do for you. Please wait patiently. If you ask for help, please send the output of the `garage layout history` command.".into(), + )) + } + } } // -------------------------- diff --git a/src/garage/cli_v2/mod.rs b/src/garage/cli_v2/mod.rs index 28c7c824..40673b91 100644 --- a/src/garage/cli_v2/mod.rs +++ b/src/garage/cli_v2/mod.rs @@ -13,7 +13,6 @@ use std::time::Duration; use garage_util::error::*; -use garage_rpc::system::*; use garage_rpc::*; use garage_api_admin::api::*; @@ -23,7 +22,6 @@ use garage_api_admin::RequestHandler; use crate::cli::structs::*; pub struct Cli { - pub system_rpc_endpoint: Arc>, pub proxy_rpc_endpoint: Arc>, pub rpc_host: NodeID, } diff --git a/src/garage/main.rs b/src/garage/main.rs index 683042d9..5d392c44 100644 --- a/src/garage/main.rs +++ b/src/garage/main.rs @@ -289,11 +289,9 @@ async fn cli_command(opt: Opt) -> Result<(), Error> { Err(e).err_context("Unable to connect to destination RPC host. Check that you are using the same value of rpc_secret as them, and that you have their correct full-length node ID (public key).")?; } - let system_rpc_endpoint = netapp.endpoint::(SYSTEM_RPC_PATH.into()); let proxy_rpc_endpoint = netapp.endpoint::(PROXY_RPC_PATH.into()); let cli = cli_v2::Cli { - system_rpc_endpoint, proxy_rpc_endpoint, rpc_host: id, };