From 8d7470104b390f1b2b5fd5bd87b6db74bc71a3a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 18 Nov 2026 10:36:57 -0300 Subject: [PATCH 1/6] fix: shut down gracefully on SIGTERM The node only listened for SIGINT, so SIGTERM (what `docker stop`, systemd and Kubernetes send) killed it at once with exit 143: no shutdown log, actors and servers not stopped, nothing flushed. `shutdown_signal()` resolves on the first SIGINT or SIGTERM and names which one. Both the node's `wait_for_shutdown` and the Hive test driver wait on it. It is the same function #783 adds, so that PR rebases over this one. The 2nd-4th ctrl-c escalation stays ctrl-c only: once SIGTERM has a listener a repeated one no longer kills the process, and a hung shutdown ends with the SIGKILL that `docker stop` and systemd send after their timeout. The README keeps `--stop-signal SIGINT` on its `docker run` lines, since `:latest` is v1.0.0, which still exits on SIGTERM; the comments now say so. --- README.md | 9 +++++--- bin/ethlambda/src/main.rs | 45 +++++++++++++++++++++++++++++++++------ docs/architecture.md | 2 +- docs/cli.md | 2 +- 4 files changed, 46 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index ddba6d09a..de1f577e0 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,8 @@ it the node advertises `0.0.0.0` and logs a warning. It still dials out and follows the chain, but peers cannot reach it until discovery learns the address from their replies. The Beacon API binds `127.0.0.1:5052` by default, so `--http-address` is only needed to reach it from another host. Stop the node -with `Ctrl+C`; it shuts down gracefully on SIGINT only, so a systemd unit needs +with `Ctrl+C` or SIGTERM, which is what systemd sends by default. v1.0.0 shuts +down gracefully on SIGINT only, so a systemd unit running that release needs `KillSignal=SIGINT`. - `ethlambda-linux-x86_64` targets x86-64-v3, so it needs a CPU with AVX2 @@ -150,7 +151,8 @@ docker run -d --name ethrex --network ethereum \ --authrpc.addr 0.0.0.0 \ --authrpc.jwtsecret /jwt.hex -# The node shuts down gracefully on SIGINT only, so `--stop-signal` makes `docker stop` send it. +# `docker stop` sends SIGTERM, which ends v1.0.0 without a graceful shutdown (later images handle +# it), so `--stop-signal` makes it send SIGINT instead. # Flushing its state can outlast Docker's default 10 s, after which Docker kills it and the # unflushed state is lost, so `--stop-timeout` gives it a minute. docker run -d --name ethlambda-beacon --network ethereum --stop-signal SIGINT --stop-timeout 60 \ @@ -181,7 +183,8 @@ mkdir -p beacon-data # This host's public address, published in the node's ENR (see above) PUBLIC_IP=$(curl -s https://ifconfig.me) -# The node shuts down gracefully on SIGINT only, so `--stop-signal` makes `docker stop` send it. +# `docker stop` sends SIGTERM, which ends v1.0.0 without a graceful shutdown (later images handle +# it), so `--stop-signal` makes it send SIGINT instead. # Flushing its state can outlast Docker's default 10 s, after which Docker kills it and the # unflushed state is lost, so `--stop-timeout` gives it a minute. docker run -d --name ethlambda-beacon --stop-signal SIGINT --stop-timeout 60 \ diff --git a/bin/ethlambda/src/main.rs b/bin/ethlambda/src/main.rs index 6eaead20e..0bddaf030 100644 --- a/bin/ethlambda/src/main.rs +++ b/bin/ethlambda/src/main.rs @@ -1019,10 +1019,13 @@ fn check_custody_set(custody_columns: &[u64]) -> eyre::Result<()> { Ok(()) } -/// Wait for ctrl-c, then stop and join whatever is running. +/// Wait for ctrl-c or SIGTERM, then stop and join whatever is running. /// /// A 2nd, 3rd and 4th ctrl-c escalate to `std::process::exit(1)` rather than -/// leaving shutdown stuck on an actor that hangs in `stop()`/`join()`. +/// leaving shutdown stuck on an actor that hangs in `stop()`/`join()`. A +/// repeated SIGTERM does not: once [`shutdown_signal`] listens for it, it no +/// longer kills the process, so a supervisor that wants the process gone +/// sends SIGKILL, as `docker stop` and systemd do after their timeout. /// /// Shared by both chains. Mainnet used to park on `pending()` here, so a /// follower killed mid-write left recovery to do; it now tears down the same @@ -1030,10 +1033,13 @@ fn check_custody_set(custody_columns: &[u64]) -> eyre::Result<()> { async fn wait_for_shutdown(node: RunningNode) { info!("Node initialized"); - // 1st ctrl+c: start graceful shutdown - tokio::signal::ctrl_c().await.ok(); + // 1st ctrl+c or SIGTERM: start graceful shutdown + let signal = shutdown_signal().await; - info!("Shutdown signal received, stopping actors and servers..."); + info!( + signal, + "Shutdown signal received, stopping actors and servers..." + ); tokio::spawn(async move { // This can be turned into a loop @@ -1064,6 +1070,28 @@ async fn wait_for_shutdown(node: RunningNode) { info!("Shutdown complete"); } +/// Resolve on the first SIGINT (ctrl-c) or SIGTERM, naming which one. +/// +/// SIGTERM is what `docker stop`, systemd and Kubernetes send, so a container +/// no longer needs `--stop-signal SIGINT` to shut down gracefully. +async fn shutdown_signal() -> &'static str { + #[cfg(unix)] + { + use tokio::signal::unix::{SignalKind, signal}; + match signal(SignalKind::terminate()) { + Ok(mut terminate) => { + return tokio::select! { + _ = tokio::signal::ctrl_c() => "SIGINT", + _ = terminate.recv() => "SIGTERM", + }; + } + Err(err) => warn!(%err, "Cannot listen for SIGTERM; only ctrl-c shuts down gracefully"), + } + } + tokio::signal::ctrl_c().await.ok(); + "SIGINT" +} + /// The ENRs to start from when `--bootnodes` was not given. /// /// A resolved network (built-in mainnet, or a loaded directory) publishes its @@ -1174,8 +1202,11 @@ async fn run_test_driver(rpc_config: RpcConfig) -> eyre::Result<()> { info!("Test-driver RPC ready"); - tokio::signal::ctrl_c().await.ok(); - info!("Shutdown signal received, stopping test-driver RPC..."); + let signal = shutdown_signal().await; + info!( + signal, + "Shutdown signal received, stopping test-driver RPC..." + ); shutdown_token.cancel(); let _ = rpc_handle.await; info!("Shutdown complete"); diff --git a/docs/architecture.md b/docs/architecture.md index 99b6f7fdb..ecc98648b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -259,7 +259,7 @@ spawn is fail-fast, so a misconfigured node stops at boot instead of hours later swarm, spawn `P2PServer`, and wire the two together with `InitP2P` and `InitBlockChain`. 6. Spawn the API and metrics servers. -Shutdown runs in reverse: the first ctrl+c stops both actors and cancels the servers' +Shutdown runs in reverse: the first ctrl+c or SIGTERM stops both actors and cancels the servers' shutdown token, and three more force the process to exit if a graceful stop hangs. > Note: booting with `HIVE_LEAN_TEST_DRIVER=1` short-circuits everything from step 2 on and diff --git a/docs/cli.md b/docs/cli.md index a317bf32a..3c6362607 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -249,7 +249,7 @@ not chain-specific, branches once, and shares the shutdown: | build the swarm, spawn P2P, start discv5 | shared; discv5 only with `--discovery.enable` | shared | | start the HTTP server | shared | shared | | spawn the chain actor and wire it to P2P | yes | yes: `BlockChain::spawn_beacon` | -| ctrl-c, stop and join the actors | shared | shared | +| ctrl-c or SIGTERM, stop and join the actors | shared | shared | `ChainSetup` is what the `match` produces: the wire configuration, the ENR entries that describe it, the store the req/resp handlers answer from, the From a1d9cfdd5db6c693646cb0c73233c7e8c1d840f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 18 Nov 2026 10:41:53 -0300 Subject: [PATCH 2/6] docs(readme): drop --stop-signal and --stop-timeout from the docker run lines --- README.md | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index de1f577e0..dfb511e61 100644 --- a/README.md +++ b/README.md @@ -151,11 +151,7 @@ docker run -d --name ethrex --network ethereum \ --authrpc.addr 0.0.0.0 \ --authrpc.jwtsecret /jwt.hex -# `docker stop` sends SIGTERM, which ends v1.0.0 without a graceful shutdown (later images handle -# it), so `--stop-signal` makes it send SIGINT instead. -# Flushing its state can outlast Docker's default 10 s, after which Docker kills it and the -# unflushed state is lost, so `--stop-timeout` gives it a minute. -docker run -d --name ethlambda-beacon --network ethereum --stop-signal SIGINT --stop-timeout 60 \ +docker run -d --name ethlambda-beacon --network ethereum \ -p 9000:9000/udp -p 9001:9001/udp -p 9001:9001/tcp \ -p 127.0.0.1:5052:5052 \ -v "$PWD/beacon-data:/data" \ @@ -183,11 +179,7 @@ mkdir -p beacon-data # This host's public address, published in the node's ENR (see above) PUBLIC_IP=$(curl -s https://ifconfig.me) -# `docker stop` sends SIGTERM, which ends v1.0.0 without a graceful shutdown (later images handle -# it), so `--stop-signal` makes it send SIGINT instead. -# Flushing its state can outlast Docker's default 10 s, after which Docker kills it and the -# unflushed state is lost, so `--stop-timeout` gives it a minute. -docker run -d --name ethlambda-beacon --stop-signal SIGINT --stop-timeout 60 \ +docker run -d --name ethlambda-beacon \ -p 9000:9000/udp -p 9001:9001/udp -p 9001:9001/tcp \ -p 127.0.0.1:5052:5052 \ -v "$PWD/beacon-data:/data" \ From 6ad6599651fd8824703cbd8c55786d5fe1506046 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 18 Nov 2026 10:46:48 -0300 Subject: [PATCH 3/6] docs(readme): drop the v1.0.0 KillSignal note --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index dfb511e61..840c0ec93 100644 --- a/README.md +++ b/README.md @@ -60,9 +60,7 @@ it the node advertises `0.0.0.0` and logs a warning. It still dials out and follows the chain, but peers cannot reach it until discovery learns the address from their replies. The Beacon API binds `127.0.0.1:5052` by default, so `--http-address` is only needed to reach it from another host. Stop the node -with `Ctrl+C` or SIGTERM, which is what systemd sends by default. v1.0.0 shuts -down gracefully on SIGINT only, so a systemd unit running that release needs -`KillSignal=SIGINT`. +with `Ctrl+C` or SIGTERM. - `ethlambda-linux-x86_64` targets x86-64-v3, so it needs a CPU with AVX2 (Haswell or later), and glibc 2.35 or newer. From e3069eaee2b2abcdbdb4bbd1d4f85f88f8cf4776 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 18 Nov 2026 21:56:54 -0300 Subject: [PATCH 4/6] docs(readme): keep --stop-signal SIGINT and --stop-timeout 60 for v1.0.0 `latest` (the image and the release download) is still v1.0.0, which has no SIGTERM handler. In the image the binary is PID 1, so `docker stop`'s SIGTERM is discarded and the node is SIGKILLed at the timeout; a systemd unit without `KillSignal=SIGINT` kills it at once. Restore `--stop-signal SIGINT` on both `docker run` lines and the `KillSignal=SIGINT` note, saying that v1.0.0 shuts down gracefully on SIGINT only and later versions handle SIGTERM too. `--stop-timeout 60` stays for its own reason: a clean shutdown during catch-up can outlast Docker's 10 s default whichever signal starts it. --- README.md | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 840c0ec93..4e9cdaf65 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,9 @@ it the node advertises `0.0.0.0` and logs a warning. It still dials out and follows the chain, but peers cannot reach it until discovery learns the address from their replies. The Beacon API binds `127.0.0.1:5052` by default, so `--http-address` is only needed to reach it from another host. Stop the node -with `Ctrl+C` or SIGTERM. +with `Ctrl+C`. v1.0.0 shuts down gracefully on SIGINT only, so a systemd unit +running it needs `KillSignal=SIGINT`; later versions handle SIGTERM, systemd's +default, too. - `ethlambda-linux-x86_64` targets x86-64-v3, so it needs a CPU with AVX2 (Haswell or later), and glibc 2.35 or newer. @@ -149,7 +151,11 @@ docker run -d --name ethrex --network ethereum \ --authrpc.addr 0.0.0.0 \ --authrpc.jwtsecret /jwt.hex -docker run -d --name ethlambda-beacon --network ethereum \ +# v1.0.0 shuts down gracefully on SIGINT only (later versions handle SIGTERM too), so +# `--stop-signal` makes `docker stop` send SIGINT rather than SIGTERM. +# Flushing its state can outlast Docker's default 10 s, after which Docker kills it and the +# unflushed state is lost, so `--stop-timeout` gives it a minute. +docker run -d --name ethlambda-beacon --network ethereum --stop-signal SIGINT --stop-timeout 60 \ -p 9000:9000/udp -p 9001:9001/udp -p 9001:9001/tcp \ -p 127.0.0.1:5052:5052 \ -v "$PWD/beacon-data:/data" \ @@ -177,7 +183,11 @@ mkdir -p beacon-data # This host's public address, published in the node's ENR (see above) PUBLIC_IP=$(curl -s https://ifconfig.me) -docker run -d --name ethlambda-beacon \ +# v1.0.0 shuts down gracefully on SIGINT only (later versions handle SIGTERM too), so +# `--stop-signal` makes `docker stop` send SIGINT rather than SIGTERM. +# Flushing its state can outlast Docker's default 10 s, after which Docker kills it and the +# unflushed state is lost, so `--stop-timeout` gives it a minute. +docker run -d --name ethlambda-beacon --stop-signal SIGINT --stop-timeout 60 \ -p 9000:9000/udp -p 9001:9001/udp -p 9001:9001/tcp \ -p 127.0.0.1:5052:5052 \ -v "$PWD/beacon-data:/data" \ From 386f5260a6861d59c2015544ba7eb189bda3f231 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:08:38 -0300 Subject: [PATCH 5/6] fix: let SIGTERM force the exit too, and only on a delivered signal The escalation task counted ctrl-c only. Once tokio listens for SIGTERM it no longer kills the process, so after a ctrl-c whose shutdown hung, `kill ` did nothing and logged nothing, where on main it killed the process. And `shutdown_signal()` matched `_` in its `select!`, so a failed SIGINT registration resolved at once and was logged as a received SIGINT; repeated, that would have been three instant "signals" and `exit(1)`. `ShutdownSignals` (new `shutdown` module) registers SIGINT and SIGTERM once, up front, and warns once for a signal it cannot listen for. Its `recv` returns only on a delivered signal (`Some(())` arms, with an `else` that never resolves), and a signal that arrives between two calls is held for the next. `force_exit_on_repeats` moves it into the escalation task, so the 2nd, 3rd and 4th signal of either kind force `exit(1)`, and no signal is lost between the first wait and the task. Both `wait_for_shutdown` and `run_test_driver` build it before their ready line (`Node initialized`, `Test-driver RPC ready`), so a signal sent once that line is out is handled. The test driver gets the same escalation; its HTTP drain is already bounded by `HTTP_DRAIN_TIMEOUT` since it goes through `serve()`. Listening still starts there, not at the top of `run_node`: latching a signal during startup would also hold a ctrl-c sent during a long checkpoint sync until startup finished, since nothing in startup gives up on a signal. The module doc says so. docs/architecture.md: three more signals "of either kind" force the exit. --- bin/ethlambda/src/main.rs | 61 ++++++------------ bin/ethlambda/src/shutdown.rs | 113 ++++++++++++++++++++++++++++++++++ docs/architecture.md | 2 +- 3 files changed, 132 insertions(+), 44 deletions(-) create mode 100644 bin/ethlambda/src/shutdown.rs diff --git a/bin/ethlambda/src/main.rs b/bin/ethlambda/src/main.rs index 0bddaf030..5d9ce7139 100644 --- a/bin/ethlambda/src/main.rs +++ b/bin/ethlambda/src/main.rs @@ -8,6 +8,7 @@ mod era; mod fd_limit; mod keygen; mod network; +mod shutdown; mod slashing_protection; mod validator; mod version; @@ -43,6 +44,7 @@ use tokio_util::sync::CancellationToken; use cli::{Network, Options}; use command::Command; +use shutdown::ShutdownSignals; use ethlambda_blockchain::block_builder::ProposerConfig; use ethlambda_blockchain::key_manager::{KeyRole, ValidatorKeyPair}; @@ -1021,40 +1023,30 @@ fn check_custody_set(custody_columns: &[u64]) -> eyre::Result<()> { /// Wait for ctrl-c or SIGTERM, then stop and join whatever is running. /// -/// A 2nd, 3rd and 4th ctrl-c escalate to `std::process::exit(1)` rather than -/// leaving shutdown stuck on an actor that hangs in `stop()`/`join()`. A -/// repeated SIGTERM does not: once [`shutdown_signal`] listens for it, it no -/// longer kills the process, so a supervisor that wants the process gone -/// sends SIGKILL, as `docker stop` and systemd do after their timeout. +/// A 2nd, 3rd and 4th signal, ctrl-c or SIGTERM in any mix, escalate to +/// `std::process::exit(1)` rather than leaving shutdown stuck on an actor that +/// hangs in `stop()`/`join()` (see [`ShutdownSignals::force_exit_on_repeats`]). +/// +/// Both signals are listened for before `Node initialized` is logged, so a +/// signal sent once that line is out is never lost or fatal. One sent earlier, +/// during startup, still has its default action: see the `shutdown` module. /// /// Shared by both chains. Mainnet used to park on `pending()` here, so a /// follower killed mid-write left recovery to do; it now tears down the same /// way the lean node does. async fn wait_for_shutdown(node: RunningNode) { + let mut signals = ShutdownSignals::listen(); info!("Node initialized"); // 1st ctrl+c or SIGTERM: start graceful shutdown - let signal = shutdown_signal().await; + let signal = signals.recv().await; info!( signal, "Shutdown signal received, stopping actors and servers..." ); - tokio::spawn(async move { - // This can be turned into a loop - tokio::signal::ctrl_c().await.ok(); - warn!( - "Graceful shutdown in progress. Press ctrl+C 2 more times to force ungraceful shutdown" - ); - tokio::signal::ctrl_c().await.ok(); - warn!( - "Graceful shutdown in progress. Press ctrl+C 1 more times to force ungraceful shutdown" - ); - tokio::signal::ctrl_c().await.ok(); - info!("Forced ungraceful shutdown..."); - std::process::exit(1); - }); + signals.force_exit_on_repeats(); let blockchain_ref = node.blockchain.actor_ref().clone(); let p2p_ref = node.p2p.actor_ref().clone(); @@ -1070,28 +1062,6 @@ async fn wait_for_shutdown(node: RunningNode) { info!("Shutdown complete"); } -/// Resolve on the first SIGINT (ctrl-c) or SIGTERM, naming which one. -/// -/// SIGTERM is what `docker stop`, systemd and Kubernetes send, so a container -/// no longer needs `--stop-signal SIGINT` to shut down gracefully. -async fn shutdown_signal() -> &'static str { - #[cfg(unix)] - { - use tokio::signal::unix::{SignalKind, signal}; - match signal(SignalKind::terminate()) { - Ok(mut terminate) => { - return tokio::select! { - _ = tokio::signal::ctrl_c() => "SIGINT", - _ = terminate.recv() => "SIGTERM", - }; - } - Err(err) => warn!(%err, "Cannot listen for SIGTERM; only ctrl-c shuts down gracefully"), - } - } - tokio::signal::ctrl_c().await.ok(); - "SIGINT" -} - /// The ENRs to start from when `--bootnodes` was not given. /// /// A resolved network (built-in mainnet, or a loaded directory) publishes its @@ -1186,6 +1156,10 @@ fn init_shadow_cost(shadow: &cli::ShadowOptions) { async fn run_test_driver(rpc_config: RpcConfig) -> eyre::Result<()> { use tokio::sync::RwLock; + // Before `Test-driver RPC ready`, so a signal sent once that line is out + // is never lost or fatal. + let mut signals = ShutdownSignals::listen(); + let driver: ethlambda_rpc::test_driver::DriverState = Arc::new(RwLock::new(ethlambda_rpc::test_driver::empty_driver_store())); @@ -1202,11 +1176,12 @@ async fn run_test_driver(rpc_config: RpcConfig) -> eyre::Result<()> { info!("Test-driver RPC ready"); - let signal = shutdown_signal().await; + let signal = signals.recv().await; info!( signal, "Shutdown signal received, stopping test-driver RPC..." ); + signals.force_exit_on_repeats(); shutdown_token.cancel(); let _ = rpc_handle.await; info!("Shutdown complete"); diff --git a/bin/ethlambda/src/shutdown.rs b/bin/ethlambda/src/shutdown.rs new file mode 100644 index 000000000..48c8e5392 --- /dev/null +++ b/bin/ethlambda/src/shutdown.rs @@ -0,0 +1,113 @@ +//! The signals that stop the node: SIGINT (ctrl-c) and, on Unix, SIGTERM, +//! which is what `docker stop`, systemd and Kubernetes send. +//! +//! The first one starts a graceful shutdown. Three more, of either kind, end +//! the process with `exit(1)` rather than leave it stuck on an actor that +//! hangs in `stop()`/`join()`. +//! +//! Listening starts once the node has started (`wait_for_shutdown`, +//! `run_test_driver`), not at the top of `run_node`. Until then both signals +//! keep their default action: they end the process at once, except as PID 1 +//! in a container, where they are ignored. Listening from the top would latch +//! a signal sent during startup, but it would also hold a ctrl-c sent during a +//! long checkpoint sync until startup finished, since nothing in startup gives +//! up on a signal. + +#[cfg(unix)] +use tokio::signal::unix::{Signal, SignalKind, signal}; +use tracing::{info, warn}; + +/// How many signals after the first one force the process to exit. +const SIGNALS_TO_FORCE_EXIT: u32 = 3; + +/// A listener for SIGINT and SIGTERM. +/// +/// Both are listened for from [`ShutdownSignals::listen`] on, and one that +/// arrives before [`ShutdownSignals::recv`] runs is held until it does. From +/// then on neither signal ends the process by itself, for the life of the +/// process: tokio never uninstalls a handler. That is why the repeats that +/// force an exit are counted here ([`ShutdownSignals::force_exit_on_repeats`]) +/// rather than left to the signal's default action. +pub(crate) struct ShutdownSignals { + #[cfg(unix)] + interrupt: Option, + #[cfg(unix)] + terminate: Option, +} + +impl ShutdownSignals { + /// Start listening. A signal that cannot be listened for is warned about + /// once, here, and keeps its default action. + pub(crate) fn listen() -> Self { + Self { + #[cfg(unix)] + interrupt: listen_for(SignalKind::interrupt(), "SIGINT"), + #[cfg(unix)] + terminate: listen_for(SignalKind::terminate(), "SIGTERM"), + } + } + + /// Wait for the next SIGINT or SIGTERM, and name it. + /// + /// Only a delivered signal returns. A signal that is not listened for, or + /// whose stream has ended (its runtime is shutting down), never does. + #[cfg(unix)] + pub(crate) async fn recv(&mut self) -> &'static str { + tokio::select! { + Some(()) = next(&mut self.interrupt) => "SIGINT", + Some(()) = next(&mut self.terminate) => "SIGTERM", + else => std::future::pending().await, + } + } + + /// Wait for the next ctrl-c, the only one of the two outside Unix. + #[cfg(not(unix))] + pub(crate) async fn recv(&mut self) -> &'static str { + if let Err(err) = tokio::signal::ctrl_c().await { + warn!(%err, "Cannot listen for ctrl-c"); + std::future::pending::<()>().await; + } + "SIGINT" + } + + /// Spawn a task that counts the signals still to come, once a graceful + /// shutdown has started, and calls `exit(1)` on the + /// [`SIGNALS_TO_FORCE_EXIT`]th. + /// + /// Either signal counts, so `kill ` after a ctrl-c that hung counts + /// toward the exit just as another ctrl-c does. + pub(crate) fn force_exit_on_repeats(mut self) { + tokio::spawn(async move { + for remaining in (1..SIGNALS_TO_FORCE_EXIT).rev() { + let signal = self.recv().await; + warn!( + signal, + "Graceful shutdown in progress. Send {remaining} more ctrl-c or SIGTERM to force ungraceful shutdown" + ); + } + let signal = self.recv().await; + info!(signal, "Forced ungraceful shutdown..."); + std::process::exit(1); + }); + } +} + +#[cfg(unix)] +fn listen_for(kind: SignalKind, name: &'static str) -> Option { + signal(kind) + .inspect_err(|err| { + warn!( + %err, + signal = name, + "Cannot listen for this signal; it will not shut the node down gracefully" + ) + }) + .ok() +} + +/// The next delivery of `signal`, or `None` when it is not listened for or +/// its stream has ended. +#[cfg(unix)] +async fn next(signal: &mut Option) -> Option<()> { + signal.as_mut()?.recv().await +} diff --git a/docs/architecture.md b/docs/architecture.md index ecc98648b..a6f040ddb 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -260,7 +260,7 @@ spawn is fail-fast, so a misconfigured node stops at boot instead of hours later 6. Spawn the API and metrics servers. Shutdown runs in reverse: the first ctrl+c or SIGTERM stops both actors and cancels the servers' -shutdown token, and three more force the process to exit if a graceful stop hangs. +shutdown token, and three more, of either kind, force the process to exit if a graceful stop hangs. > Note: booting with `HIVE_LEAN_TEST_DRIVER=1` short-circuits everything from step 2 on and > exposes only the Hive test-driver endpoints, so a driver run never touches the node key, From 0f9e2508bb1d6d10dc65d131579fb525806a1d2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:08:51 -0300 Subject: [PATCH 6/6] test: shut the binary down with SIGTERM in test-driver mode `tests/shutdown.rs` spawns the built binary as `HIVE_LEAN_TEST_DRIVER=1 ethlambda node`, which reaches the shutdown wait without keys, a genesis or a database. Once it logs `Test-driver RPC ready` the test sends SIGTERM and asserts exit code 0, a `Shutdown signal received` line naming "SIGTERM", and `Shutdown complete`. It is race-free because `run_test_driver` registers its signals before that line. `--api-port 0` lets the OS pick the port, so parallel runs never collide, and a guard kills the child if the test fails first. Run against bf66e86c's `main.rs`, which listens for ctrl-c only, it fails: the child dies by signal 15 and has no exit code. This is the bin crate's first integration test, so the test run now links the binary itself as well as its unit-test harness. --- bin/ethlambda/src/main.rs | 2 +- bin/ethlambda/tests/shutdown.rs | 182 ++++++++++++++++++++++++++++++++ 2 files changed, 183 insertions(+), 1 deletion(-) create mode 100644 bin/ethlambda/tests/shutdown.rs diff --git a/bin/ethlambda/src/main.rs b/bin/ethlambda/src/main.rs index 5d9ce7139..767f2dfa5 100644 --- a/bin/ethlambda/src/main.rs +++ b/bin/ethlambda/src/main.rs @@ -1157,7 +1157,7 @@ async fn run_test_driver(rpc_config: RpcConfig) -> eyre::Result<()> { use tokio::sync::RwLock; // Before `Test-driver RPC ready`, so a signal sent once that line is out - // is never lost or fatal. + // is never lost or fatal; `tests/shutdown.rs` relies on it. let mut signals = ShutdownSignals::listen(); let driver: ethlambda_rpc::test_driver::DriverState = diff --git a/bin/ethlambda/tests/shutdown.rs b/bin/ethlambda/tests/shutdown.rs new file mode 100644 index 000000000..84f324fc8 --- /dev/null +++ b/bin/ethlambda/tests/shutdown.rs @@ -0,0 +1,182 @@ +//! The binary shuts down gracefully on SIGTERM, the signal `docker stop`, +//! systemd and Kubernetes send. +//! +//! Runs the real binary in Hive test-driver mode (`HIVE_LEAN_TEST_DRIVER=1`), +//! which reaches the shutdown wait without keys, a genesis or a database, and +//! listens for signals before it logs `Test-driver RPC ready`. + +#![cfg(unix)] + +use std::io::{BufRead, BufReader, Read}; +use std::process::{Child, Command, ExitStatus, Stdio}; +use std::sync::mpsc::{self, Receiver, RecvTimeoutError, Sender}; +use std::time::{Duration, Instant}; + +/// Startup compiles leanVM's aggregation bytecode before the test-driver +/// branch, so this is generous. +const READY_TIMEOUT: Duration = Duration::from_secs(120); +/// A test-driver shutdown has only its HTTP server to stop. +const EXIT_TIMEOUT: Duration = Duration::from_secs(30); + +/// Kills the child if the test ends before it exits, so a failure leaves no +/// process behind. +struct Node(Child); + +impl Drop for Node { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +#[test] +fn sigterm_shuts_the_node_down_gracefully() { + let dir = tempfile::tempdir().expect("temp dir"); + // `node`'s required flags. Test-driver mode reads none of them, so the + // paths need not exist. + let required = [ + "--genesis", + "config.yaml", + "--validators", + "validators.yaml", + "--bootnodes", + "nodes.yaml", + "--validator-config", + "validator-config.yaml", + "--hash-sig-keys-dir", + "keys", + "--node-key", + "node.key", + "--node-id", + "ethlambda_0", + ]; + let mut node = Node( + Command::new(env!("CARGO_BIN_EXE_ethlambda")) + .arg("node") + .args(required) + // The OS picks the port, so parallel runs never collide. + .args(["--api-port", "0"]) + .env("HIVE_LEAN_TEST_DRIVER", "1") + .env("NO_COLOR", "1") + .env_remove("RUST_LOG") + .current_dir(dir.path()) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn ethlambda"), + ); + let lines = read_lines(&mut node.0); + + let mut seen = Vec::new(); + wait_for_line(&lines, &mut seen, "Test-driver RPC ready", READY_TIMEOUT); + + let pid = libc::pid_t::try_from(node.0.id()).expect("pid fits pid_t"); + // SAFETY: `kill` takes plain integers and touches no memory of ours. + let sent = unsafe { libc::kill(pid, libc::SIGTERM) }; + assert_eq!(sent, 0, "kill(SIGTERM) failed"); + + let status = wait_for_exit(&mut node.0, EXIT_TIMEOUT); + drain(&lines, &mut seen, EXIT_TIMEOUT); + let log = seen.join("\n"); + + assert_eq!( + status.code(), + Some(0), + "expected a clean exit, got {status}; output:\n{log}" + ); + assert!( + seen.iter() + .any(|line| line.contains("Shutdown signal received") && line.contains("\"SIGTERM\"")), + "no shutdown line naming SIGTERM; output:\n{log}" + ); + assert!( + seen.iter().any(|line| line.contains("Shutdown complete")), + "no `Shutdown complete`; output:\n{log}" + ); +} + +/// Every line the child writes, stdout (its logs) and stderr (an error that +/// ends it) alike, each read on a thread of its own so a full pipe never +/// blocks the child. The channel disconnects once both reach end of file. +fn read_lines(child: &mut Child) -> Receiver { + let stdout = child.stdout.take().expect("stdout is piped"); + let stderr = child.stderr.take().expect("stderr is piped"); + let (tx, rx) = mpsc::channel(); + forward_lines(stdout, tx.clone()); + forward_lines(stderr, tx); + rx +} + +fn forward_lines(pipe: impl Read + Send + 'static, tx: Sender) { + std::thread::spawn(move || { + for line in BufReader::new(pipe).lines().map_while(Result::ok) { + if tx.send(line).is_err() { + break; + } + } + }); +} + +/// Read what is left once the child has exited, until both pipes close. +fn drain(lines: &Receiver, seen: &mut Vec, timeout: Duration) { + let deadline = Instant::now() + timeout; + loop { + let left = deadline.saturating_duration_since(Instant::now()); + match lines.recv_timeout(left) { + Ok(line) => seen.push(line), + Err(RecvTimeoutError::Disconnected) => return, + Err(RecvTimeoutError::Timeout) => { + panic!("the node's output did not close within {timeout:?} of its exit") + } + } + } +} + +/// Wait for a line containing `needle`, keeping every line read in `seen`. +fn wait_for_line( + lines: &Receiver, + seen: &mut Vec, + needle: &str, + timeout: Duration, +) { + let deadline = Instant::now() + timeout; + loop { + let left = deadline.saturating_duration_since(Instant::now()); + match lines.recv_timeout(left) { + Ok(line) => { + let found = line.contains(needle); + seen.push(line); + if found { + return; + } + } + Err(RecvTimeoutError::Timeout) => { + panic!( + "no `{needle}` within {timeout:?}; output:\n{}", + seen.join("\n") + ) + } + Err(RecvTimeoutError::Disconnected) => { + panic!( + "the node exited before `{needle}`; output:\n{}", + seen.join("\n") + ) + } + } + } +} + +fn wait_for_exit(child: &mut Child, timeout: Duration) -> ExitStatus { + let deadline = Instant::now() + timeout; + loop { + if let Some(status) = child.try_wait().expect("try_wait") { + return status; + } + assert!( + Instant::now() < deadline, + "the node did not exit within {timeout:?} of SIGTERM" + ); + std::thread::sleep(Duration::from_millis(50)); + } +}