2020-04-24 10:10:01 +00:00
|
|
|
use std::path::PathBuf;
|
|
|
|
|
|
|
|
use tokio::sync::watch;
|
|
|
|
|
|
|
|
use garage_util::background::*;
|
|
|
|
use garage_util::config::*;
|
|
|
|
use garage_util::error::Error;
|
|
|
|
|
2022-05-24 10:16:39 +00:00
|
|
|
use garage_api::admin::api_server::AdminApiServer;
|
2022-05-10 11:16:57 +00:00
|
|
|
use garage_api::s3::api_server::S3ApiServer;
|
2020-07-07 11:59:22 +00:00
|
|
|
use garage_model::garage::Garage;
|
2022-09-07 15:54:16 +00:00
|
|
|
use garage_web::WebServer;
|
2020-04-24 10:10:01 +00:00
|
|
|
|
2022-05-10 11:16:57 +00:00
|
|
|
#[cfg(feature = "k2v")]
|
|
|
|
use garage_api::k2v::api_server::K2VApiServer;
|
|
|
|
|
2021-10-19 14:16:10 +00:00
|
|
|
use crate::admin::*;
|
2022-09-03 21:40:44 +00:00
|
|
|
#[cfg(feature = "telemetry-otlp")]
|
2022-05-24 10:16:39 +00:00
|
|
|
use crate::tracing_setup::*;
|
2023-02-03 14:27:39 +00:00
|
|
|
use crate::{fill_secrets, Secrets};
|
2020-04-24 10:10:01 +00:00
|
|
|
|
2021-04-23 20:41:24 +00:00
|
|
|
async fn wait_from(mut chan: watch::Receiver<bool>) {
|
2021-03-15 21:36:41 +00:00
|
|
|
while !*chan.borrow() {
|
|
|
|
if chan.changed().await.is_err() {
|
2020-04-24 10:10:01 +00:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-02-03 14:27:39 +00:00
|
|
|
pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Error> {
|
2020-04-24 10:10:01 +00:00
|
|
|
info!("Loading configuration...");
|
2023-02-03 14:27:39 +00:00
|
|
|
let config = fill_secrets(read_config(config_file)?, secrets);
|
2020-04-24 10:10:01 +00:00
|
|
|
|
2022-09-07 15:54:16 +00:00
|
|
|
// ---- Initialize Garage internals ----
|
|
|
|
|
2022-09-20 15:45:18 +00:00
|
|
|
#[cfg(feature = "metrics")]
|
|
|
|
let metrics_exporter = opentelemetry_prometheus::exporter().init();
|
|
|
|
|
2022-12-14 11:51:16 +00:00
|
|
|
info!("Initializing Garage main data store...");
|
|
|
|
let garage = Garage::new(config.clone())?;
|
|
|
|
|
2020-04-24 10:10:01 +00:00
|
|
|
info!("Initializing background runner...");
|
2022-09-28 08:41:59 +00:00
|
|
|
let watch_cancel = watch_shutdown_signal();
|
2022-12-14 11:51:16 +00:00
|
|
|
let (background, await_background_done) = BackgroundRunner::new(watch_cancel.clone());
|
2021-10-14 09:50:12 +00:00
|
|
|
|
2022-12-14 11:28:07 +00:00
|
|
|
info!("Spawning Garage workers...");
|
2022-12-14 11:51:16 +00:00
|
|
|
garage.spawn_workers(&background);
|
2022-12-14 11:28:07 +00:00
|
|
|
|
2022-09-06 13:43:50 +00:00
|
|
|
if config.admin.trace_sink.is_some() {
|
2022-09-03 21:40:44 +00:00
|
|
|
info!("Initialize tracing...");
|
|
|
|
|
|
|
|
#[cfg(feature = "telemetry-otlp")]
|
2022-09-06 13:43:50 +00:00
|
|
|
init_tracing(config.admin.trace_sink.as_ref().unwrap(), garage.system.id)?;
|
2022-09-03 21:40:44 +00:00
|
|
|
|
|
|
|
#[cfg(not(feature = "telemetry-otlp"))]
|
2022-09-07 15:54:16 +00:00
|
|
|
error!("Garage was built without OTLP exporter, admin.trace_sink is ignored.");
|
2022-02-17 22:28:23 +00:00
|
|
|
}
|
|
|
|
|
2022-05-24 10:16:39 +00:00
|
|
|
info!("Initialize Admin API server and metrics collector...");
|
2022-09-20 15:45:18 +00:00
|
|
|
let admin_server = AdminApiServer::new(
|
|
|
|
garage.clone(),
|
|
|
|
#[cfg(feature = "metrics")]
|
|
|
|
metrics_exporter,
|
|
|
|
);
|
2022-05-24 10:16:39 +00:00
|
|
|
|
2022-06-15 18:20:28 +00:00
|
|
|
info!("Launching internal Garage cluster communications...");
|
2021-10-14 09:50:12 +00:00
|
|
|
let run_system = tokio::spawn(garage.system.clone().run(watch_cancel.clone()));
|
2020-04-24 10:10:01 +00:00
|
|
|
|
2021-09-28 06:57:20 +00:00
|
|
|
info!("Create admin RPC handler...");
|
2022-12-14 11:51:16 +00:00
|
|
|
AdminRpcHandler::new(garage.clone(), background.clone());
|
2021-10-14 09:50:12 +00:00
|
|
|
|
2022-09-07 15:54:16 +00:00
|
|
|
// ---- Launch public-facing API servers ----
|
|
|
|
|
|
|
|
let mut servers = vec![];
|
|
|
|
|
|
|
|
if let Some(s3_bind_addr) = &config.s3_api.api_bind_addr {
|
|
|
|
info!("Initializing S3 API server...");
|
|
|
|
servers.push((
|
|
|
|
"S3 API",
|
|
|
|
tokio::spawn(S3ApiServer::run(
|
|
|
|
garage.clone(),
|
2023-09-29 16:41:00 +00:00
|
|
|
s3_bind_addr.clone(),
|
2022-09-07 15:54:16 +00:00
|
|
|
config.s3_api.s3_region.clone(),
|
|
|
|
wait_from(watch_cancel.clone()),
|
|
|
|
)),
|
|
|
|
));
|
|
|
|
}
|
2021-10-14 09:50:12 +00:00
|
|
|
|
2022-09-07 15:54:16 +00:00
|
|
|
if config.k2v_api.is_some() {
|
|
|
|
#[cfg(feature = "k2v")]
|
|
|
|
{
|
|
|
|
info!("Initializing K2V API server...");
|
|
|
|
servers.push((
|
|
|
|
"K2V API",
|
|
|
|
tokio::spawn(K2VApiServer::run(
|
|
|
|
garage.clone(),
|
2023-09-29 16:41:00 +00:00
|
|
|
config.k2v_api.as_ref().unwrap().api_bind_addr.clone(),
|
2022-09-07 15:54:16 +00:00
|
|
|
config.s3_api.s3_region.clone(),
|
|
|
|
wait_from(watch_cancel.clone()),
|
|
|
|
)),
|
|
|
|
));
|
|
|
|
}
|
|
|
|
#[cfg(not(feature = "k2v"))]
|
|
|
|
error!("K2V is not enabled in this build, cannot start K2V API server");
|
2022-05-10 11:16:57 +00:00
|
|
|
}
|
2022-09-07 15:54:16 +00:00
|
|
|
|
|
|
|
if let Some(web_config) = &config.s3_web {
|
|
|
|
info!("Initializing web server...");
|
|
|
|
servers.push((
|
|
|
|
"Web",
|
|
|
|
tokio::spawn(WebServer::run(
|
|
|
|
garage.clone(),
|
2023-09-29 16:41:00 +00:00
|
|
|
web_config.bind_addr.clone(),
|
2022-09-07 15:54:16 +00:00
|
|
|
web_config.root_domain.clone(),
|
|
|
|
wait_from(watch_cancel.clone()),
|
|
|
|
)),
|
|
|
|
));
|
2021-10-14 09:50:12 +00:00
|
|
|
}
|
2022-09-07 15:54:16 +00:00
|
|
|
|
|
|
|
if let Some(admin_bind_addr) = &config.admin.api_bind_addr {
|
|
|
|
info!("Launching Admin API server...");
|
|
|
|
servers.push((
|
|
|
|
"Admin",
|
2023-09-29 16:41:00 +00:00
|
|
|
tokio::spawn(
|
|
|
|
admin_server.run(admin_bind_addr.clone(), wait_from(watch_cancel.clone())),
|
|
|
|
),
|
2022-09-07 15:54:16 +00:00
|
|
|
));
|
2021-10-14 09:50:12 +00:00
|
|
|
}
|
2022-09-07 15:54:16 +00:00
|
|
|
|
2022-09-07 16:02:13 +00:00
|
|
|
#[cfg(not(feature = "metrics"))]
|
2022-09-07 16:10:19 +00:00
|
|
|
if config.admin.metrics_token.is_some() {
|
2022-09-07 16:02:13 +00:00
|
|
|
warn!("This Garage version is built without the metrics feature");
|
|
|
|
}
|
|
|
|
|
2023-09-12 12:35:19 +00:00
|
|
|
if servers.is_empty() {
|
|
|
|
// Nothing runs except netapp (not in servers)
|
|
|
|
// Await shutdown signal before proceeding to shutting down netapp
|
|
|
|
wait_from(watch_cancel).await;
|
|
|
|
} else {
|
|
|
|
// Stuff runs
|
|
|
|
|
|
|
|
// When a cancel signal is sent, stuff stops
|
|
|
|
|
|
|
|
// Collect stuff
|
|
|
|
for (desc, join_handle) in servers {
|
|
|
|
if let Err(e) = join_handle.await? {
|
|
|
|
error!("{} server exited with error: {}", desc, e);
|
|
|
|
} else {
|
|
|
|
info!("{} server exited without error.", desc);
|
|
|
|
}
|
2022-09-07 15:54:16 +00:00
|
|
|
}
|
2021-09-28 06:57:20 +00:00
|
|
|
}
|
2021-10-14 09:50:12 +00:00
|
|
|
|
|
|
|
// Remove RPC handlers for system to break reference cycles
|
2023-09-12 12:35:19 +00:00
|
|
|
info!("Deregistering RPC handlers for shutdown...");
|
2021-10-14 09:50:12 +00:00
|
|
|
garage.system.netapp.drop_all_handlers();
|
2022-05-24 10:16:39 +00:00
|
|
|
opentelemetry::global::shutdown_tracer_provider();
|
2021-10-14 09:50:12 +00:00
|
|
|
|
2021-10-15 09:05:09 +00:00
|
|
|
// Await for netapp RPC system to end
|
2021-10-14 09:50:12 +00:00
|
|
|
run_system.await?;
|
2022-05-24 10:16:39 +00:00
|
|
|
info!("Netapp exited");
|
2021-10-15 09:05:09 +00:00
|
|
|
|
2022-03-15 11:04:12 +00:00
|
|
|
// Drop all references so that stuff can terminate properly
|
2021-10-15 09:05:09 +00:00
|
|
|
drop(garage);
|
|
|
|
|
|
|
|
// Await for all background tasks to end
|
2021-10-14 09:50:12 +00:00
|
|
|
await_background_done.await?;
|
2020-04-24 10:10:01 +00:00
|
|
|
|
|
|
|
info!("Cleaning up...");
|
|
|
|
|
|
|
|
Ok(())
|
|
|
|
}
|
2022-09-28 08:41:59 +00:00
|
|
|
|
|
|
|
#[cfg(unix)]
|
|
|
|
fn watch_shutdown_signal() -> watch::Receiver<bool> {
|
|
|
|
use tokio::signal::unix::*;
|
|
|
|
|
|
|
|
let (send_cancel, watch_cancel) = watch::channel(false);
|
|
|
|
tokio::spawn(async move {
|
|
|
|
let mut sigint = signal(SignalKind::interrupt()).expect("Failed to install SIGINT handler");
|
|
|
|
let mut sigterm =
|
|
|
|
signal(SignalKind::terminate()).expect("Failed to install SIGTERM handler");
|
|
|
|
let mut sighup = signal(SignalKind::hangup()).expect("Failed to install SIGHUP handler");
|
|
|
|
tokio::select! {
|
|
|
|
_ = sigint.recv() => info!("Received SIGINT, shutting down."),
|
|
|
|
_ = sigterm.recv() => info!("Received SIGTERM, shutting down."),
|
|
|
|
_ = sighup.recv() => info!("Received SIGHUP, shutting down."),
|
|
|
|
}
|
|
|
|
send_cancel.send(true).unwrap();
|
|
|
|
});
|
|
|
|
watch_cancel
|
|
|
|
}
|
|
|
|
|
|
|
|
#[cfg(windows)]
|
|
|
|
fn watch_shutdown_signal() -> watch::Receiver<bool> {
|
|
|
|
use tokio::signal::windows::*;
|
|
|
|
|
|
|
|
let (send_cancel, watch_cancel) = watch::channel(false);
|
|
|
|
tokio::spawn(async move {
|
|
|
|
let mut sigint = ctrl_c().expect("Failed to install Ctrl-C handler");
|
|
|
|
let mut sigclose = ctrl_close().expect("Failed to install Ctrl-Close handler");
|
|
|
|
let mut siglogoff = ctrl_logoff().expect("Failed to install Ctrl-Logoff handler");
|
|
|
|
let mut sigsdown = ctrl_shutdown().expect("Failed to install Ctrl-Shutdown handler");
|
|
|
|
tokio::select! {
|
|
|
|
_ = sigint.recv() => info!("Received Ctrl-C, shutting down."),
|
|
|
|
_ = sigclose.recv() => info!("Received Ctrl-Close, shutting down."),
|
|
|
|
_ = siglogoff.recv() => info!("Received Ctrl-Logoff, shutting down."),
|
|
|
|
_ = sigsdown.recv() => info!("Received Ctrl-Shutdown, shutting down."),
|
|
|
|
}
|
|
|
|
send_cancel.send(true).unwrap();
|
|
|
|
});
|
|
|
|
watch_cancel
|
|
|
|
}
|