Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,9 @@ it the node advertises `0.0.0.0` and logs a warning. It still dials out and
follows the chain, but peers cannot reach it until discovery learns the address
from their replies. The Beacon API binds `127.0.0.1:5052` by default, so
`--http-address` is only needed to reach it from another host. Stop the node
with `Ctrl+C`; it shuts down gracefully on SIGINT only, so a systemd unit needs
`KillSignal=SIGINT`.
with `Ctrl+C`. v1.0.0 shuts down gracefully on SIGINT only, so a systemd unit
running it needs `KillSignal=SIGINT`; later versions handle SIGTERM, systemd's
default, too.

- `ethlambda-linux-x86_64` targets x86-64-v3, so it needs a CPU with AVX2
(Haswell or later), and glibc 2.35 or newer.
Expand Down Expand Up @@ -150,7 +151,8 @@ docker run -d --name ethrex --network ethereum \
--authrpc.addr 0.0.0.0 \
--authrpc.jwtsecret /jwt.hex

# The node shuts down gracefully on SIGINT only, so `--stop-signal` makes `docker stop` send it.
# v1.0.0 shuts down gracefully on SIGINT only (later versions handle SIGTERM too), so
# `--stop-signal` makes `docker stop` send SIGINT rather than SIGTERM.
# Flushing its state can outlast Docker's default 10 s, after which Docker kills it and the
# unflushed state is lost, so `--stop-timeout` gives it a minute.
docker run -d --name ethlambda-beacon --network ethereum --stop-signal SIGINT --stop-timeout 60 \
Expand Down Expand Up @@ -181,7 +183,8 @@ mkdir -p beacon-data
# This host's public address, published in the node's ENR (see above)
PUBLIC_IP=$(curl -s https://ifconfig.me)

# The node shuts down gracefully on SIGINT only, so `--stop-signal` makes `docker stop` send it.
# v1.0.0 shuts down gracefully on SIGINT only (later versions handle SIGTERM too), so
# `--stop-signal` makes `docker stop` send SIGINT rather than SIGTERM.
# Flushing its state can outlast Docker's default 10 s, after which Docker kills it and the
# unflushed state is lost, so `--stop-timeout` gives it a minute.
docker run -d --name ethlambda-beacon --stop-signal SIGINT --stop-timeout 60 \
Expand Down
50 changes: 28 additions & 22 deletions bin/ethlambda/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ mod era;
mod fd_limit;
mod keygen;
mod network;
mod shutdown;
mod slashing_protection;
mod validator;
mod version;
Expand Down Expand Up @@ -43,6 +44,7 @@ use tokio_util::sync::CancellationToken;

use cli::{Network, Options};
use command::Command;
use shutdown::ShutdownSignals;

use ethlambda_blockchain::block_builder::ProposerConfig;
use ethlambda_blockchain::key_manager::{KeyRole, ValidatorKeyPair};
Expand Down Expand Up @@ -1019,36 +1021,32 @@ fn check_custody_set(custody_columns: &[u64]) -> eyre::Result<()> {
Ok(())
}

/// Wait for ctrl-c, then stop and join whatever is running.
/// Wait for ctrl-c or SIGTERM, then stop and join whatever is running.
///
/// A 2nd, 3rd and 4th ctrl-c escalate to `std::process::exit(1)` rather than
/// leaving shutdown stuck on an actor that hangs in `stop()`/`join()`.
/// A 2nd, 3rd and 4th signal, ctrl-c or SIGTERM in any mix, escalate to
/// `std::process::exit(1)` rather than leaving shutdown stuck on an actor that
/// hangs in `stop()`/`join()` (see [`ShutdownSignals::force_exit_on_repeats`]).
///
/// Both signals are listened for before `Node initialized` is logged, so a
/// signal sent once that line is out is never lost or fatal. One sent earlier,
/// during startup, still has its default action: see the `shutdown` module.
///
/// Shared by both chains. Mainnet used to park on `pending()` here, so a
/// follower killed mid-write left recovery to do; it now tears down the same
/// way the lean node does.
async fn wait_for_shutdown(node: RunningNode) {
let mut signals = ShutdownSignals::listen();
info!("Node initialized");

// 1st ctrl+c: start graceful shutdown
tokio::signal::ctrl_c().await.ok();
// 1st ctrl+c or SIGTERM: start graceful shutdown
let signal = signals.recv().await;

info!("Shutdown signal received, stopping actors and servers...");
info!(
signal,
"Shutdown signal received, stopping actors and servers..."
);

tokio::spawn(async move {
// This can be turned into a loop
tokio::signal::ctrl_c().await.ok();
warn!(
"Graceful shutdown in progress. Press ctrl+C 2 more times to force ungraceful shutdown"
);
tokio::signal::ctrl_c().await.ok();
warn!(
"Graceful shutdown in progress. Press ctrl+C 1 more times to force ungraceful shutdown"
);
tokio::signal::ctrl_c().await.ok();
info!("Forced ungraceful shutdown...");
std::process::exit(1);
});
signals.force_exit_on_repeats();

let blockchain_ref = node.blockchain.actor_ref().clone();
let p2p_ref = node.p2p.actor_ref().clone();
Expand Down Expand Up @@ -1158,6 +1156,10 @@ fn init_shadow_cost(shadow: &cli::ShadowOptions) {
async fn run_test_driver(rpc_config: RpcConfig) -> eyre::Result<()> {
use tokio::sync::RwLock;

// Before `Test-driver RPC ready`, so a signal sent once that line is out
// is never lost or fatal; `tests/shutdown.rs` relies on it.
let mut signals = ShutdownSignals::listen();

let driver: ethlambda_rpc::test_driver::DriverState =
Arc::new(RwLock::new(ethlambda_rpc::test_driver::empty_driver_store()));

Expand All @@ -1174,8 +1176,12 @@ async fn run_test_driver(rpc_config: RpcConfig) -> eyre::Result<()> {

info!("Test-driver RPC ready");

tokio::signal::ctrl_c().await.ok();
info!("Shutdown signal received, stopping test-driver RPC...");
let signal = signals.recv().await;
info!(
signal,
"Shutdown signal received, stopping test-driver RPC..."
);
signals.force_exit_on_repeats();
shutdown_token.cancel();
let _ = rpc_handle.await;
info!("Shutdown complete");
Expand Down
113 changes: 113 additions & 0 deletions bin/ethlambda/src/shutdown.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
//! The signals that stop the node: SIGINT (ctrl-c) and, on Unix, SIGTERM,
//! which is what `docker stop`, systemd and Kubernetes send.
//!
//! The first one starts a graceful shutdown. Three more, of either kind, end
//! the process with `exit(1)` rather than leave it stuck on an actor that
//! hangs in `stop()`/`join()`.
//!
//! Listening starts once the node has started (`wait_for_shutdown`,
//! `run_test_driver`), not at the top of `run_node`. Until then both signals
//! keep their default action: they end the process at once, except as PID 1
//! in a container, where they are ignored. Listening from the top would latch
//! a signal sent during startup, but it would also hold a ctrl-c sent during a
//! long checkpoint sync until startup finished, since nothing in startup gives
//! up on a signal.

#[cfg(unix)]
use tokio::signal::unix::{Signal, SignalKind, signal};
use tracing::{info, warn};

/// How many signals after the first one force the process to exit.
const SIGNALS_TO_FORCE_EXIT: u32 = 3;

/// A listener for SIGINT and SIGTERM.
///
/// Both are listened for from [`ShutdownSignals::listen`] on, and one that
/// arrives before [`ShutdownSignals::recv`] runs is held until it does. From
/// then on neither signal ends the process by itself, for the life of the
/// process: tokio never uninstalls a handler. That is why the repeats that
/// force an exit are counted here ([`ShutdownSignals::force_exit_on_repeats`])
/// rather than left to the signal's default action.
pub(crate) struct ShutdownSignals {
#[cfg(unix)]
interrupt: Option<Signal>,
#[cfg(unix)]
terminate: Option<Signal>,
}

impl ShutdownSignals {
/// Start listening. A signal that cannot be listened for is warned about
/// once, here, and keeps its default action.
pub(crate) fn listen() -> Self {
Self {
#[cfg(unix)]
interrupt: listen_for(SignalKind::interrupt(), "SIGINT"),
#[cfg(unix)]
terminate: listen_for(SignalKind::terminate(), "SIGTERM"),
}
}

/// Wait for the next SIGINT or SIGTERM, and name it.
///
/// Only a delivered signal returns. A signal that is not listened for, or
/// whose stream has ended (its runtime is shutting down), never does.
#[cfg(unix)]
pub(crate) async fn recv(&mut self) -> &'static str {
tokio::select! {
Some(()) = next(&mut self.interrupt) => "SIGINT",
Some(()) = next(&mut self.terminate) => "SIGTERM",
else => std::future::pending().await,
}
}

/// Wait for the next ctrl-c, the only one of the two outside Unix.
#[cfg(not(unix))]
pub(crate) async fn recv(&mut self) -> &'static str {
if let Err(err) = tokio::signal::ctrl_c().await {
warn!(%err, "Cannot listen for ctrl-c");
std::future::pending::<()>().await;
}
"SIGINT"
}

/// Spawn a task that counts the signals still to come, once a graceful
/// shutdown has started, and calls `exit(1)` on the
/// [`SIGNALS_TO_FORCE_EXIT`]th.
///
/// Either signal counts, so `kill <pid>` after a ctrl-c that hung counts
/// toward the exit just as another ctrl-c does.
pub(crate) fn force_exit_on_repeats(mut self) {
tokio::spawn(async move {
for remaining in (1..SIGNALS_TO_FORCE_EXIT).rev() {
let signal = self.recv().await;
warn!(
signal,
"Graceful shutdown in progress. Send {remaining} more ctrl-c or SIGTERM to force ungraceful shutdown"
);
}
let signal = self.recv().await;
info!(signal, "Forced ungraceful shutdown...");
std::process::exit(1);
});
}
}

#[cfg(unix)]
fn listen_for(kind: SignalKind, name: &'static str) -> Option<Signal> {
signal(kind)
.inspect_err(|err| {
warn!(
%err,
signal = name,
"Cannot listen for this signal; it will not shut the node down gracefully"
)
})
.ok()
}

/// The next delivery of `signal`, or `None` when it is not listened for or
/// its stream has ended.
#[cfg(unix)]
async fn next(signal: &mut Option<Signal>) -> Option<()> {
signal.as_mut()?.recv().await
}
Loading
Loading