diff --git a/README.md b/README.md index ddba6d09a..4e9cdaf65 100644 --- a/README.md +++ b/README.md @@ -60,8 +60,9 @@ it the node advertises `0.0.0.0` and logs a warning. It still dials out and follows the chain, but peers cannot reach it until discovery learns the address from their replies. The Beacon API binds `127.0.0.1:5052` by default, so `--http-address` is only needed to reach it from another host. Stop the node -with `Ctrl+C`; it shuts down gracefully on SIGINT only, so a systemd unit needs -`KillSignal=SIGINT`. +with `Ctrl+C`. v1.0.0 shuts down gracefully on SIGINT only, so a systemd unit +running it needs `KillSignal=SIGINT`; later versions handle SIGTERM, systemd's +default, too. - `ethlambda-linux-x86_64` targets x86-64-v3, so it needs a CPU with AVX2 (Haswell or later), and glibc 2.35 or newer. @@ -150,7 +151,8 @@ docker run -d --name ethrex --network ethereum \ --authrpc.addr 0.0.0.0 \ --authrpc.jwtsecret /jwt.hex -# The node shuts down gracefully on SIGINT only, so `--stop-signal` makes `docker stop` send it. +# v1.0.0 shuts down gracefully on SIGINT only (later versions handle SIGTERM too), so +# `--stop-signal` makes `docker stop` send SIGINT rather than SIGTERM. # Flushing its state can outlast Docker's default 10 s, after which Docker kills it and the # unflushed state is lost, so `--stop-timeout` gives it a minute. docker run -d --name ethlambda-beacon --network ethereum --stop-signal SIGINT --stop-timeout 60 \ @@ -181,7 +183,8 @@ mkdir -p beacon-data # This host's public address, published in the node's ENR (see above) PUBLIC_IP=$(curl -s https://ifconfig.me) -# The node shuts down gracefully on SIGINT only, so `--stop-signal` makes `docker stop` send it. +# v1.0.0 shuts down gracefully on SIGINT only (later versions handle SIGTERM too), so +# `--stop-signal` makes `docker stop` send SIGINT rather than SIGTERM. # Flushing its state can outlast Docker's default 10 s, after which Docker kills it and the # unflushed state is lost, so `--stop-timeout` gives it a minute. docker run -d --name ethlambda-beacon --stop-signal SIGINT --stop-timeout 60 \ diff --git a/bin/ethlambda/src/main.rs b/bin/ethlambda/src/main.rs index 6eaead20e..767f2dfa5 100644 --- a/bin/ethlambda/src/main.rs +++ b/bin/ethlambda/src/main.rs @@ -8,6 +8,7 @@ mod era; mod fd_limit; mod keygen; mod network; +mod shutdown; mod slashing_protection; mod validator; mod version; @@ -43,6 +44,7 @@ use tokio_util::sync::CancellationToken; use cli::{Network, Options}; use command::Command; +use shutdown::ShutdownSignals; use ethlambda_blockchain::block_builder::ProposerConfig; use ethlambda_blockchain::key_manager::{KeyRole, ValidatorKeyPair}; @@ -1019,36 +1021,32 @@ fn check_custody_set(custody_columns: &[u64]) -> eyre::Result<()> { Ok(()) } -/// Wait for ctrl-c, then stop and join whatever is running. +/// Wait for ctrl-c or SIGTERM, then stop and join whatever is running. /// -/// A 2nd, 3rd and 4th ctrl-c escalate to `std::process::exit(1)` rather than -/// leaving shutdown stuck on an actor that hangs in `stop()`/`join()`. +/// A 2nd, 3rd and 4th signal, ctrl-c or SIGTERM in any mix, escalate to +/// `std::process::exit(1)` rather than leaving shutdown stuck on an actor that +/// hangs in `stop()`/`join()` (see [`ShutdownSignals::force_exit_on_repeats`]). +/// +/// Both signals are listened for before `Node initialized` is logged, so a +/// signal sent once that line is out is never lost or fatal. One sent earlier, +/// during startup, still has its default action: see the `shutdown` module. /// /// Shared by both chains. Mainnet used to park on `pending()` here, so a /// follower killed mid-write left recovery to do; it now tears down the same /// way the lean node does. async fn wait_for_shutdown(node: RunningNode) { + let mut signals = ShutdownSignals::listen(); info!("Node initialized"); - // 1st ctrl+c: start graceful shutdown - tokio::signal::ctrl_c().await.ok(); + // 1st ctrl+c or SIGTERM: start graceful shutdown + let signal = signals.recv().await; - info!("Shutdown signal received, stopping actors and servers..."); + info!( + signal, + "Shutdown signal received, stopping actors and servers..." + ); - tokio::spawn(async move { - // This can be turned into a loop - tokio::signal::ctrl_c().await.ok(); - warn!( - "Graceful shutdown in progress. Press ctrl+C 2 more times to force ungraceful shutdown" - ); - tokio::signal::ctrl_c().await.ok(); - warn!( - "Graceful shutdown in progress. Press ctrl+C 1 more times to force ungraceful shutdown" - ); - tokio::signal::ctrl_c().await.ok(); - info!("Forced ungraceful shutdown..."); - std::process::exit(1); - }); + signals.force_exit_on_repeats(); let blockchain_ref = node.blockchain.actor_ref().clone(); let p2p_ref = node.p2p.actor_ref().clone(); @@ -1158,6 +1156,10 @@ fn init_shadow_cost(shadow: &cli::ShadowOptions) { async fn run_test_driver(rpc_config: RpcConfig) -> eyre::Result<()> { use tokio::sync::RwLock; + // Before `Test-driver RPC ready`, so a signal sent once that line is out + // is never lost or fatal; `tests/shutdown.rs` relies on it. + let mut signals = ShutdownSignals::listen(); + let driver: ethlambda_rpc::test_driver::DriverState = Arc::new(RwLock::new(ethlambda_rpc::test_driver::empty_driver_store())); @@ -1174,8 +1176,12 @@ async fn run_test_driver(rpc_config: RpcConfig) -> eyre::Result<()> { info!("Test-driver RPC ready"); - tokio::signal::ctrl_c().await.ok(); - info!("Shutdown signal received, stopping test-driver RPC..."); + let signal = signals.recv().await; + info!( + signal, + "Shutdown signal received, stopping test-driver RPC..." + ); + signals.force_exit_on_repeats(); shutdown_token.cancel(); let _ = rpc_handle.await; info!("Shutdown complete"); diff --git a/bin/ethlambda/src/shutdown.rs b/bin/ethlambda/src/shutdown.rs new file mode 100644 index 000000000..48c8e5392 --- /dev/null +++ b/bin/ethlambda/src/shutdown.rs @@ -0,0 +1,113 @@ +//! The signals that stop the node: SIGINT (ctrl-c) and, on Unix, SIGTERM, +//! which is what `docker stop`, systemd and Kubernetes send. +//! +//! The first one starts a graceful shutdown. Three more, of either kind, end +//! the process with `exit(1)` rather than leave it stuck on an actor that +//! hangs in `stop()`/`join()`. +//! +//! Listening starts once the node has started (`wait_for_shutdown`, +//! `run_test_driver`), not at the top of `run_node`. Until then both signals +//! keep their default action: they end the process at once, except as PID 1 +//! in a container, where they are ignored. Listening from the top would latch +//! a signal sent during startup, but it would also hold a ctrl-c sent during a +//! long checkpoint sync until startup finished, since nothing in startup gives +//! up on a signal. + +#[cfg(unix)] +use tokio::signal::unix::{Signal, SignalKind, signal}; +use tracing::{info, warn}; + +/// How many signals after the first one force the process to exit. +const SIGNALS_TO_FORCE_EXIT: u32 = 3; + +/// A listener for SIGINT and SIGTERM. +/// +/// Both are listened for from [`ShutdownSignals::listen`] on, and one that +/// arrives before [`ShutdownSignals::recv`] runs is held until it does. From +/// then on neither signal ends the process by itself, for the life of the +/// process: tokio never uninstalls a handler. That is why the repeats that +/// force an exit are counted here ([`ShutdownSignals::force_exit_on_repeats`]) +/// rather than left to the signal's default action. +pub(crate) struct ShutdownSignals { + #[cfg(unix)] + interrupt: Option, + #[cfg(unix)] + terminate: Option, +} + +impl ShutdownSignals { + /// Start listening. A signal that cannot be listened for is warned about + /// once, here, and keeps its default action. + pub(crate) fn listen() -> Self { + Self { + #[cfg(unix)] + interrupt: listen_for(SignalKind::interrupt(), "SIGINT"), + #[cfg(unix)] + terminate: listen_for(SignalKind::terminate(), "SIGTERM"), + } + } + + /// Wait for the next SIGINT or SIGTERM, and name it. + /// + /// Only a delivered signal returns. A signal that is not listened for, or + /// whose stream has ended (its runtime is shutting down), never does. + #[cfg(unix)] + pub(crate) async fn recv(&mut self) -> &'static str { + tokio::select! { + Some(()) = next(&mut self.interrupt) => "SIGINT", + Some(()) = next(&mut self.terminate) => "SIGTERM", + else => std::future::pending().await, + } + } + + /// Wait for the next ctrl-c, the only one of the two outside Unix. + #[cfg(not(unix))] + pub(crate) async fn recv(&mut self) -> &'static str { + if let Err(err) = tokio::signal::ctrl_c().await { + warn!(%err, "Cannot listen for ctrl-c"); + std::future::pending::<()>().await; + } + "SIGINT" + } + + /// Spawn a task that counts the signals still to come, once a graceful + /// shutdown has started, and calls `exit(1)` on the + /// [`SIGNALS_TO_FORCE_EXIT`]th. + /// + /// Either signal counts, so `kill ` after a ctrl-c that hung counts + /// toward the exit just as another ctrl-c does. + pub(crate) fn force_exit_on_repeats(mut self) { + tokio::spawn(async move { + for remaining in (1..SIGNALS_TO_FORCE_EXIT).rev() { + let signal = self.recv().await; + warn!( + signal, + "Graceful shutdown in progress. Send {remaining} more ctrl-c or SIGTERM to force ungraceful shutdown" + ); + } + let signal = self.recv().await; + info!(signal, "Forced ungraceful shutdown..."); + std::process::exit(1); + }); + } +} + +#[cfg(unix)] +fn listen_for(kind: SignalKind, name: &'static str) -> Option { + signal(kind) + .inspect_err(|err| { + warn!( + %err, + signal = name, + "Cannot listen for this signal; it will not shut the node down gracefully" + ) + }) + .ok() +} + +/// The next delivery of `signal`, or `None` when it is not listened for or +/// its stream has ended. +#[cfg(unix)] +async fn next(signal: &mut Option) -> Option<()> { + signal.as_mut()?.recv().await +} diff --git a/bin/ethlambda/tests/shutdown.rs b/bin/ethlambda/tests/shutdown.rs new file mode 100644 index 000000000..84f324fc8 --- /dev/null +++ b/bin/ethlambda/tests/shutdown.rs @@ -0,0 +1,182 @@ +//! The binary shuts down gracefully on SIGTERM, the signal `docker stop`, +//! systemd and Kubernetes send. +//! +//! Runs the real binary in Hive test-driver mode (`HIVE_LEAN_TEST_DRIVER=1`), +//! which reaches the shutdown wait without keys, a genesis or a database, and +//! listens for signals before it logs `Test-driver RPC ready`. + +#![cfg(unix)] + +use std::io::{BufRead, BufReader, Read}; +use std::process::{Child, Command, ExitStatus, Stdio}; +use std::sync::mpsc::{self, Receiver, RecvTimeoutError, Sender}; +use std::time::{Duration, Instant}; + +/// Startup compiles leanVM's aggregation bytecode before the test-driver +/// branch, so this is generous. +const READY_TIMEOUT: Duration = Duration::from_secs(120); +/// A test-driver shutdown has only its HTTP server to stop. +const EXIT_TIMEOUT: Duration = Duration::from_secs(30); + +/// Kills the child if the test ends before it exits, so a failure leaves no +/// process behind. +struct Node(Child); + +impl Drop for Node { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +#[test] +fn sigterm_shuts_the_node_down_gracefully() { + let dir = tempfile::tempdir().expect("temp dir"); + // `node`'s required flags. Test-driver mode reads none of them, so the + // paths need not exist. + let required = [ + "--genesis", + "config.yaml", + "--validators", + "validators.yaml", + "--bootnodes", + "nodes.yaml", + "--validator-config", + "validator-config.yaml", + "--hash-sig-keys-dir", + "keys", + "--node-key", + "node.key", + "--node-id", + "ethlambda_0", + ]; + let mut node = Node( + Command::new(env!("CARGO_BIN_EXE_ethlambda")) + .arg("node") + .args(required) + // The OS picks the port, so parallel runs never collide. + .args(["--api-port", "0"]) + .env("HIVE_LEAN_TEST_DRIVER", "1") + .env("NO_COLOR", "1") + .env_remove("RUST_LOG") + .current_dir(dir.path()) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn ethlambda"), + ); + let lines = read_lines(&mut node.0); + + let mut seen = Vec::new(); + wait_for_line(&lines, &mut seen, "Test-driver RPC ready", READY_TIMEOUT); + + let pid = libc::pid_t::try_from(node.0.id()).expect("pid fits pid_t"); + // SAFETY: `kill` takes plain integers and touches no memory of ours. + let sent = unsafe { libc::kill(pid, libc::SIGTERM) }; + assert_eq!(sent, 0, "kill(SIGTERM) failed"); + + let status = wait_for_exit(&mut node.0, EXIT_TIMEOUT); + drain(&lines, &mut seen, EXIT_TIMEOUT); + let log = seen.join("\n"); + + assert_eq!( + status.code(), + Some(0), + "expected a clean exit, got {status}; output:\n{log}" + ); + assert!( + seen.iter() + .any(|line| line.contains("Shutdown signal received") && line.contains("\"SIGTERM\"")), + "no shutdown line naming SIGTERM; output:\n{log}" + ); + assert!( + seen.iter().any(|line| line.contains("Shutdown complete")), + "no `Shutdown complete`; output:\n{log}" + ); +} + +/// Every line the child writes, stdout (its logs) and stderr (an error that +/// ends it) alike, each read on a thread of its own so a full pipe never +/// blocks the child. The channel disconnects once both reach end of file. +fn read_lines(child: &mut Child) -> Receiver { + let stdout = child.stdout.take().expect("stdout is piped"); + let stderr = child.stderr.take().expect("stderr is piped"); + let (tx, rx) = mpsc::channel(); + forward_lines(stdout, tx.clone()); + forward_lines(stderr, tx); + rx +} + +fn forward_lines(pipe: impl Read + Send + 'static, tx: Sender) { + std::thread::spawn(move || { + for line in BufReader::new(pipe).lines().map_while(Result::ok) { + if tx.send(line).is_err() { + break; + } + } + }); +} + +/// Read what is left once the child has exited, until both pipes close. +fn drain(lines: &Receiver, seen: &mut Vec, timeout: Duration) { + let deadline = Instant::now() + timeout; + loop { + let left = deadline.saturating_duration_since(Instant::now()); + match lines.recv_timeout(left) { + Ok(line) => seen.push(line), + Err(RecvTimeoutError::Disconnected) => return, + Err(RecvTimeoutError::Timeout) => { + panic!("the node's output did not close within {timeout:?} of its exit") + } + } + } +} + +/// Wait for a line containing `needle`, keeping every line read in `seen`. +fn wait_for_line( + lines: &Receiver, + seen: &mut Vec, + needle: &str, + timeout: Duration, +) { + let deadline = Instant::now() + timeout; + loop { + let left = deadline.saturating_duration_since(Instant::now()); + match lines.recv_timeout(left) { + Ok(line) => { + let found = line.contains(needle); + seen.push(line); + if found { + return; + } + } + Err(RecvTimeoutError::Timeout) => { + panic!( + "no `{needle}` within {timeout:?}; output:\n{}", + seen.join("\n") + ) + } + Err(RecvTimeoutError::Disconnected) => { + panic!( + "the node exited before `{needle}`; output:\n{}", + seen.join("\n") + ) + } + } + } +} + +fn wait_for_exit(child: &mut Child, timeout: Duration) -> ExitStatus { + let deadline = Instant::now() + timeout; + loop { + if let Some(status) = child.try_wait().expect("try_wait") { + return status; + } + assert!( + Instant::now() < deadline, + "the node did not exit within {timeout:?} of SIGTERM" + ); + std::thread::sleep(Duration::from_millis(50)); + } +} diff --git a/docs/architecture.md b/docs/architecture.md index 99b6f7fdb..a6f040ddb 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -259,8 +259,8 @@ spawn is fail-fast, so a misconfigured node stops at boot instead of hours later swarm, spawn `P2PServer`, and wire the two together with `InitP2P` and `InitBlockChain`. 6. Spawn the API and metrics servers. -Shutdown runs in reverse: the first ctrl+c stops both actors and cancels the servers' -shutdown token, and three more force the process to exit if a graceful stop hangs. +Shutdown runs in reverse: the first ctrl+c or SIGTERM stops both actors and cancels the servers' +shutdown token, and three more, of either kind, force the process to exit if a graceful stop hangs. > Note: booting with `HIVE_LEAN_TEST_DRIVER=1` short-circuits everything from step 2 on and > exposes only the Hive test-driver endpoints, so a driver run never touches the node key, diff --git a/docs/cli.md b/docs/cli.md index b04b4d89e..51c1fcae1 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -249,7 +249,7 @@ not chain-specific, branches once, and shares the shutdown: | build the swarm, spawn P2P, start discv5 | shared; discv5 only with `--discovery.enable` | shared | | start the HTTP server | shared | shared | | spawn the chain actor and wire it to P2P | yes | yes: `BlockChain::spawn_beacon` | -| ctrl-c, stop and join the actors | shared | shared | +| ctrl-c or SIGTERM, stop and join the actors | shared | shared | `ChainSetup` is what the `match` produces: the wire configuration, the ENR entries that describe it, the store the req/resp handlers answer from, the