diff --git a/crates/edgezero-adapter-axum/tests/fixtures/blocking_accept.rs b/crates/edgezero-adapter-axum/tests/fixtures/blocking_accept.rs new file mode 100644 index 000000000..2eb24b00c --- /dev/null +++ b/crates/edgezero-adapter-axum/tests/fixtures/blocking_accept.rs @@ -0,0 +1,68 @@ +//! Accept helper for test fixtures that poll a non-blocking std listener. + +use std::io; +use std::net::{SocketAddr, TcpListener, TcpStream}; + +/// Accepts one connection and returns its stream in blocking mode. +/// +/// Fixtures make the listener non-blocking so their accept loop can poll a deadline or stop +/// flag. macOS `accept(2)` copies `O_NONBLOCK` to the accepted socket (Linux does not), so a +/// read on the returned stream would otherwise fail with `WouldBlock` at once instead of +/// honouring its read timeout. `WouldBlock` from `accept` itself is still returned to the caller. +#[inline] +pub(crate) fn accept_blocking(listener: &TcpListener) -> io::Result<(TcpStream, SocketAddr)> { + let (stream, peer) = listener.accept()?; + stream.set_nonblocking(false)?; + Ok((stream, peer)) +} + +#[cfg(test)] +mod tests { + use super::accept_blocking; + use std::io::{ErrorKind, Read as _}; + use std::net::{TcpListener, TcpStream}; + use std::thread; + use std::time::{Duration, Instant}; + + #[test] + fn stream_accepted_from_nonblocking_listener_waits_for_read_timeout() { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind loopback"); + listener + .set_nonblocking(true) + .expect("nonblocking listener"); + let address = listener.local_addr().expect("listener address"); + let _client = TcpStream::connect(address).expect("connect loopback client"); + let deadline = Instant::now() + .checked_add(Duration::from_secs(2)) + .expect("deadline fits the monotonic clock"); + let (mut stream, _peer) = loop { + match accept_blocking(&listener) { + Ok(accepted) => break accepted, + Err(error) if error.kind() == ErrorKind::WouldBlock => { + assert!( + Instant::now() < deadline, + "loopback connection was not queued" + ); + thread::sleep(Duration::from_millis(5)); + } + Err(error) => panic!("accept failed: {error}"), + } + }; + stream + .set_read_timeout(Some(Duration::from_millis(200))) + .expect("read timeout"); + let started = Instant::now(); + let error = stream + .read(&mut [0_u8; 1]) + .expect_err("the client sent nothing"); + let waited = started.elapsed(); + assert!( + matches!(error.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut), + "unexpected read error: {error}" + ); + assert!( + waited >= Duration::from_millis(150), + "read returned after {waited:?}, before its 200 ms timeout" + ); + } +} diff --git a/crates/edgezero-adapter-axum/tests/native_aws_startup.rs b/crates/edgezero-adapter-axum/tests/native_aws_startup.rs index a0b6283ce..849f268c8 100644 --- a/crates/edgezero-adapter-axum/tests/native_aws_startup.rs +++ b/crates/edgezero-adapter-axum/tests/native_aws_startup.rs @@ -4,12 +4,17 @@ //! They exercise the public runner and real pinned SDK, not a provider seam. No AWS account, //! DNS hostname, external socket, image, or real cloud credential is used. +#[cfg(test)] +#[path = "fixtures/blocking_accept.rs"] +mod blocking_accept; + #[cfg(test)] #[expect( clippy::arbitrary_source_item_ordering, reason = "loopback fixture helpers remain grouped before startup scenarios" )] mod tests { + use crate::blocking_accept::accept_blocking; use std::fmt::Write as _; use std::fs; use std::io::{ErrorKind, Read, Write as _}; @@ -83,7 +88,7 @@ mod tests { .checked_add(watch) .expect("fixture deadline fits the monotonic clock"); loop { - match listener.accept() { + match accept_blocking(&listener) { Ok((mut stream, peer)) => { assert!(peer.ip().is_loopback(), "fixture peer was not loopback"); stream diff --git a/crates/edgezero-adapter-axum/tests/native_celld_startup.rs b/crates/edgezero-adapter-axum/tests/native_celld_startup.rs index a9dbe9c54..8d057f909 100644 --- a/crates/edgezero-adapter-axum/tests/native_celld_startup.rs +++ b/crates/edgezero-adapter-axum/tests/native_celld_startup.rs @@ -8,8 +8,13 @@ reason = "fixture helpers precede startup scenarios" )] +#[cfg(test)] +#[path = "fixtures/blocking_accept.rs"] +mod blocking_accept; + #[cfg(test)] mod tests { + use crate::blocking_accept::accept_blocking; use std::fs; use std::io::{ErrorKind, Read, Write as _}; use std::net::{SocketAddr, TcpListener, TcpStream}; @@ -47,7 +52,7 @@ mod tests { thread::spawn(move || { let deadline = Instant::now().checked_add(WAIT).expect("fixture deadline"); while !worker_stop.load(Ordering::Relaxed) && Instant::now() < deadline { - match listener.accept() { + match accept_blocking(&listener) { Ok((mut stream, peer)) => { assert!(peer.ip().is_loopback()); stream diff --git a/crates/edgezero-adapter-axum/tests/production_hosting.rs b/crates/edgezero-adapter-axum/tests/production_hosting.rs index c74a86c0f..2bf70337f 100644 --- a/crates/edgezero-adapter-axum/tests/production_hosting.rs +++ b/crates/edgezero-adapter-axum/tests/production_hosting.rs @@ -20,13 +20,14 @@ mod tests { use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _, symlink}; use std::os::unix::process::ExitStatusExt as _; use std::process::{Child, Command, ExitStatus, Stdio}; - use std::sync::{Arc, Mutex, mpsc}; + use std::sync::{Arc, Mutex, MutexGuard, PoisonError, mpsc}; use std::thread::{self, JoinHandle}; use std::time::{Duration, Instant}; use edgezero_core::blob_envelope::BlobEnvelope; use serde_json::{Value, json}; use tempfile::TempDir; + use tokio::net::TcpSocket; const WAIT: Duration = Duration::from_secs(5); @@ -64,11 +65,43 @@ mod tests { .expect("envelope") } + /// macOS marks a new socket close-on-exec only after `socket()` returns, so a fixture + /// process spawned by a parallel test in that gap inherits it. An inherited reservation + /// stays bound in that process, so the fixture server that later binds the port fails with + /// `AddrInUse`, and an inherited client stream stays open after the test drops it. Port + /// reservations, client streams and spawns hold this lock so they never overlap. + static RESERVE_OR_SPAWN: Mutex<()> = Mutex::new(()); + + fn reserve_or_spawn() -> MutexGuard<'static, ()> { + RESERVE_OR_SPAWN + .lock() + .unwrap_or_else(PoisonError::into_inner) + } + + /// Opens a client stream without letting a concurrently spawned fixture inherit it. + fn connect(addr: SocketAddr, timeout: Duration) -> io::Result { + let _connecting = reserve_or_spawn(); + TcpStream::connect_timeout(&addr, timeout) + } + + /// Reserves a loopback port for a fixture server that binds it itself. fn address() -> SocketAddr { - TcpListener::bind("127.0.0.1:0") - .expect("port reservation") - .local_addr() - .expect("address") + reserved_address().0 + } + + /// Reserves a loopback port and keeps it bound, but not listening, until the holder drops. + /// No parallel test can bind the port meanwhile and nothing accepts a connection to it, so + /// a "nothing answers" check cannot reach a server that reused a released reservation. + /// While held, the check cannot fail by itself: callers also assert that the server never + /// reached its listener bind. + fn reserved_address() -> (SocketAddr, TcpSocket) { + let _reserving = reserve_or_spawn(); + let socket = TcpSocket::new_v4().expect("port reservation socket"); + socket + .bind(SocketAddr::from((Ipv4Addr::LOCALHOST, 0))) + .expect("port reservation"); + let addr = socket.local_addr().expect("reserved address"); + (addr, socket) } fn command(mode: &str, roots: &Roots, addr: SocketAddr) -> Command { @@ -101,11 +134,13 @@ mod tests { impl Server { fn spawn(mut command: Command, addr: SocketAddr) -> Self { + let spawning = reserve_or_spawn(); let mut child = command .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() .expect("fixture process"); + drop(spawning); let output = Arc::new(Mutex::new(String::new())); let (sender, events) = mpsc::channel(); let stdout = child.stdout.take().expect("stdout"); @@ -173,18 +208,18 @@ mod tests { } fn signal(&self, name: &str) { - assert!( - Command::new("kill") - .arg(format!("-{name}")) - .arg(self.child.id().to_string()) - .status() - .expect("send signal") - .success() - ); + let spawning = reserve_or_spawn(); + let status = Command::new("kill") + .arg(format!("-{name}")) + .arg(self.child.id().to_string()) + .status() + .expect("send signal"); + drop(spawning); + assert!(status.success()); } fn socket(&self) -> io::Result { - let socket = TcpStream::connect_timeout(&self.addr, Duration::from_millis(50))?; + let socket = connect(self.addr, Duration::from_millis(50))?; socket.set_read_timeout(Some(WAIT))?; socket.set_write_timeout(Some(WAIT))?; Ok(socket) @@ -246,6 +281,10 @@ mod tests { "startup diagnostic leaked fixture contents: {log}" ); assert!(!log.contains("INITIALIZED")); + assert!( + !log.contains("startup bind"), + "failed startup reached bind: {log}" + ); TcpStream::connect_timeout(&addr, Duration::from_millis(50)) .expect_err("failed startup never accepts connections"); log @@ -377,7 +416,7 @@ mod tests { } fn raw_request(addr: SocketAddr, request: &str) -> String { - let mut socket = TcpStream::connect_timeout(&addr, WAIT).expect("wire connection"); + let mut socket = connect(addr, WAIT).expect("wire connection"); socket.set_read_timeout(Some(WAIT)).expect("read timeout"); socket.set_write_timeout(Some(WAIT)).expect("write timeout"); socket.write_all(request.as_bytes()).expect("wire request"); @@ -567,6 +606,10 @@ mod tests { } #[test] + #[cfg_attr( + not(target_os = "linux"), + ignore = "binds extra loopback source addresses (127.0.0.2, 127.0.0.3), which only Linux routes by default (macOS lo0 has only 127.0.0.1); on macOS run with --ignored after `sudo ifconfig lo0 alias 127.0.0.2 up` and `sudo ifconfig lo0 alias 127.0.0.3 up`" + )] fn native_source_bound_single_proxy_and_mapped_cidr_match_only_owned_peers() { for networks in [ "127.0.0.2/32", @@ -627,6 +670,10 @@ mod tests { } #[test] + #[cfg_attr( + not(target_os = "linux"), + ignore = "binds extra loopback source addresses (127.0.0.2, 127.0.0.3), which only Linux routes by default (macOS lo0 has only 127.0.0.1); on macOS run with --ignored after `sudo ifconfig lo0 alias 127.0.0.2 up` and `sudo ifconfig lo0 alias 127.0.0.3 up`" + )] fn native_two_proxy_chain_stops_at_untrusted_hop_and_gateway_replaces_prefix() { for (networks, boundary) in [ ("127.0.0.2/32,127.0.0.3/32", "127.0.0.1"), @@ -773,6 +820,10 @@ mod tests { } #[test] + #[cfg_attr( + not(target_os = "linux"), + ignore = "binds extra loopback source addresses (127.0.0.2, 127.0.0.3), which only Linux routes by default (macOS lo0 has only 127.0.0.1); on macOS run with --ignored after `sudo ifconfig lo0 alias 127.0.0.2 up` and `sudo ifconfig lo0 alias 127.0.0.3 up`" + )] fn native_selected_families_conflicts_malformed_sets_gaps_and_exhaustion() { for family in ["forwarded", "x-forwarded"] { let roots = Roots::new(); @@ -847,6 +898,10 @@ mod tests { } #[test] + #[cfg_attr( + not(target_os = "linux"), + ignore = "binds extra loopback source addresses (127.0.0.2, 127.0.0.3), which only Linux routes by default (macOS lo0 has only 127.0.0.1); on macOS run with --ignored after `sudo ifconfig lo0 alias 127.0.0.2 up` and `sudo ifconfig lo0 alias 127.0.0.3 up`" + )] fn native_verified_tls_ingress_overwrites_public_assertions_and_rejects_wrong_name() { let roots = Roots::new(); let addr = address(); @@ -1644,7 +1699,7 @@ mod tests { (None, None, Some("RECORDS_SENTINEL"), "REQUEST_RECORDS"), (None, None, Some("FALSE"), "REQUEST_RECORDS"), ] { - let addr = address(); + let (addr, _addr_held) = reserved_address(); let mut launch = command("diagnostics", &roots, addr); if let Some(value) = networks { launch.env("EDGEZERO__ADAPTER__TRUSTED_PROXY_CIDRS", value); @@ -1685,7 +1740,7 @@ mod tests { ), ("EDGEZERO__LOGGING__REQUEST_RECORDS", "REQUEST_RECORDS"), ] { - let addr = address(); + let (addr, _addr_held) = reserved_address(); let mut launch = command("diagnostics", &roots, addr); let mut raw = b"NON_UNICODE_SENTINEL".to_vec(); raw.push(0xff); @@ -1828,7 +1883,7 @@ mod tests { OsString::from_vec(vec![0xff]), ] { let roots = Roots::new(); - let addr = address(); + let (addr, _addr_held) = reserved_address(); let mut launch = command(mode, &roots, addr); launch.env("EDGEZERO__ADAPTER__JSON_BODY_LIMIT_BYTES", value); let log = failed(launch, addr, "JSON_BODY_LIMIT_BYTES"); @@ -1932,7 +1987,7 @@ mod tests { #[test] fn stop_during_startup_never_binds_or_becomes_ready() { let roots = Roots::new(); - let addr = address(); + let (addr, _addr_held) = reserved_address(); let mut server = Server::spawn(command("starting", &roots, addr), addr); server.wait_line("INITIALIZING"); TcpStream::connect_timeout(&addr, Duration::from_millis(50)) @@ -1966,7 +2021,7 @@ mod tests { "settings", ), ] { - let addr = address(); + let (addr, _addr_held) = reserved_address(); let mut cmd = command("initialized", &roots, addr); cmd.env(key, value); failed(cmd, addr, category); @@ -1975,15 +2030,15 @@ mod tests { ("hook-error", "application configuration"), ("initializer-error", "initializer"), ] { - let addr = address(); + let (addr, _addr_held) = reserved_address(); failed(command(mode, &roots, addr), addr, category); } - let addr = address(); + let (addr, _addr_held) = reserved_address(); let mut cmd = command("initialized", &roots, addr); cmd.env_remove("EDGEZERO__ADAPTER__CONFIG_DIR"); failed(cmd, addr, "CONFIG_DIR"); let missing = roots.cwd.path().join("missing-data-SENTINEL"); - let missing_addr = address(); + let (missing_addr, _missing_addr_held) = reserved_address(); let mut missing_cmd = command("initialized", &roots, missing_addr); missing_cmd.env("EDGEZERO__ADAPTER__DATA_DIR", &missing); failed(missing_cmd, missing_addr, "DATA_DIR"); @@ -2006,10 +2061,10 @@ mod tests { let roots = Roots::new(); let file = roots.config.path().join("local-config-settings.json"); fs::remove_file(&file).expect("missing file fixture"); - let addr = address(); + let (addr, _addr_held) = reserved_address(); failed(command("initialized", &roots, addr), addr, "missing"); fs::create_dir_all(&file).expect("non-regular fixture"); - let directory_addr = address(); + let (directory_addr, _directory_addr_held) = reserved_address(); failed( command("initialized", &roots, directory_addr), directory_addr, @@ -2017,7 +2072,7 @@ mod tests { ); fs::remove_dir(&file).expect("remove fixture directory"); roots.map(&json!({"settings": {"CONFIG_SENTINEL": true}})); - let malformed_addr = address(); + let (malformed_addr, _malformed_addr_held) = reserved_address(); failed( command("initialized", &roots, malformed_addr), malformed_addr, @@ -2036,7 +2091,7 @@ mod tests { ), ] { roots.map(&map); - let typed_addr = address(); + let (typed_addr, _typed_addr_held) = reserved_address(); failed( command("initialized", &roots, typed_addr), typed_addr, @@ -2047,7 +2102,7 @@ mod tests { serde_json::from_str(&blob("valid-greeting", "fixture_token")).expect("envelope"); envelope["data"]["greeting"] = json!("CHANGED_CONFIG_SENTINEL"); roots.map(&json!({"settings": serde_json::to_string(&envelope).expect("envelope")})); - let integrity_addr = address(); + let (integrity_addr, _integrity_addr_held) = reserved_address(); failed( command("initialized", &roots, integrity_addr), integrity_addr, @@ -2121,7 +2176,7 @@ mod tests { .expect("snapshot") .ends_with("selected-greeting") ); - let second_addr = address(); + let (second_addr, _second_addr_held) = reserved_address(); failed( command("initialized", &roots, second_addr), second_addr, @@ -2273,7 +2328,7 @@ mod tests { #[cfg(not(feature = "geo-cloudfront"))] fn geo_hosting_rejects_unavailable_cloudfront() { let cwd = tempfile::tempdir().expect("unrelated cwd"); - let addr = address(); + let (addr, _addr_held) = reserved_address(); let mut server = Server::spawn(geo_command("geo-cloudfront", &cwd, addr), addr); assert!(!server.wait().success()); let log = server.log(); @@ -2281,6 +2336,10 @@ mod tests { assert!(!log.contains("INITIALIZED")); assert!(!log.contains("HOOK_ENTERED")); assert!(!log.contains("ready")); + assert!( + !log.contains("startup bind"), + "unavailable implementation reached bind" + ); TcpStream::connect_timeout(&addr, Duration::from_millis(50)) .expect_err("unavailable implementation never binds"); assert_eq!(fs::read_dir(cwd.path()).expect("unrelated cwd").count(), 0); @@ -2349,12 +2408,16 @@ mod tests { #[test] fn geo_hosting_invalid_policy_never_initializes_or_listens() { let cwd = tempfile::tempdir().expect("unrelated cwd"); - let addr = address(); + let (addr, _addr_held) = reserved_address(); let mut server = Server::spawn(geo_command("geo-invalid-policy", &cwd, addr), addr); assert!(!server.wait().success()); assert!(!server.log().contains("INITIALIZED")); assert!(!server.log().contains("HOOK_ENTERED")); assert!(!server.log().contains("x-origin-verify-fixture")); + assert!( + !server.log().contains("startup bind"), + "invalid policy reached bind" + ); TcpStream::connect_timeout(&addr, Duration::from_millis(50)) .expect_err("invalid policy never binds"); assert_eq!(fs::read_dir(cwd.path()).expect("unrelated cwd").count(), 0); diff --git a/crates/edgezero-cli/tests/celld_generated_proof.rs b/crates/edgezero-cli/tests/celld_generated_proof.rs index 8b96613bd..fcfd83537 100644 --- a/crates/edgezero-cli/tests/celld_generated_proof.rs +++ b/crates/edgezero-cli/tests/celld_generated_proof.rs @@ -4,6 +4,11 @@ //! `EDGEZERO_CELLD_STORE_SERVICE`, Node.js and OpenSSL; missing tools are //! failures, not skips. +#[cfg(test)] +#[cfg(unix)] +#[path = "support/blocking_accept.rs"] +mod blocking_accept; + #[cfg(test)] #[cfg(unix)] mod tests { @@ -12,6 +17,7 @@ mod tests { reason = "keep fixture ownership, generated source and proof steps together" )] + use crate::blocking_accept::accept_blocking; use std::collections::BTreeMap; use std::env; use std::fmt::Write as _; @@ -76,7 +82,7 @@ mod tests { let worker = thread::spawn(move || { let end = deadline(Duration::from_mins(10)); while !worker_stop.load(Ordering::Relaxed) && Instant::now() < end { - match listener.accept() { + match accept_blocking(&listener) { Ok((mut stream, peer)) => { assert!(peer.ip().is_loopback()); stream diff --git a/crates/edgezero-cli/tests/dynamodb_generated_proof.rs b/crates/edgezero-cli/tests/dynamodb_generated_proof.rs index 3cadaabdf..aa13389d3 100644 --- a/crates/edgezero-cli/tests/dynamodb_generated_proof.rs +++ b/crates/edgezero-cli/tests/dynamodb_generated_proof.rs @@ -7,6 +7,11 @@ //! fixture with dummy credentials and a cleared child environment. Serialize builds that share //! `EDGEZERO_GENERATED_TEST_TARGET_DIR`. +#[cfg(test)] +#[cfg(unix)] +#[path = "support/blocking_accept.rs"] +mod blocking_accept; + #[cfg(test)] mod tests { #[cfg(unix)] @@ -16,6 +21,7 @@ mod tests { reason = "group test fixture types and their protocol helpers by responsibility" )] + use crate::blocking_accept::accept_blocking; use std::collections::BTreeMap; use std::env; use std::fs; @@ -64,7 +70,7 @@ mod tests { let worker = thread::spawn(move || { let deadline = deadline_after(Duration::from_mins(3)); while !worker_stop.load(Ordering::Relaxed) && Instant::now() < deadline { - match listener.accept() { + match accept_blocking(&listener) { Ok((mut stream, peer)) => { assert!(peer.ip().is_loopback(), "fixture accepts loopback only"); stream diff --git a/crates/edgezero-cli/tests/native_store_generated_proof.rs b/crates/edgezero-cli/tests/native_store_generated_proof.rs index 8675e585a..c15981573 100644 --- a/crates/edgezero-cli/tests/native_store_generated_proof.rs +++ b/crates/edgezero-cli/tests/native_store_generated_proof.rs @@ -20,10 +20,16 @@ //! `EDGEZERO_GENERATED_TEST_TARGET_DIR` can select a shared target directory; otherwise builds //! use the generated project's `target/`. Serialize builds that share a target directory. +#[cfg(test)] +#[cfg(unix)] +#[path = "support/blocking_accept.rs"] +mod blocking_accept; + #[cfg(test)] #[cfg(unix)] mod tests { + use crate::blocking_accept::accept_blocking; use std::env::var_os; use std::fs; use std::io::{ErrorKind, Read as _, Write as _}; @@ -114,7 +120,7 @@ mod tests { let mut extra = Vec::new(); let deadline = deadline_after(SERVICE_GRACE); while Instant::now() < deadline { - match listener.accept() { + match accept_blocking(&listener) { Ok((mut extra_stream, peer)) => { assert!(peer.ip().is_loopback(), "only literal loopback peers"); extra_stream @@ -304,7 +310,7 @@ mod tests { fn accept_before(listener: &TcpListener, deadline: Instant) -> (TcpStream, SocketAddr) { loop { - match listener.accept() { + match accept_blocking(listener) { Ok((stream, peer)) => { assert!(peer.ip().is_loopback(), "fixture peer must be loopback"); return (stream, peer); diff --git a/crates/edgezero-cli/tests/support/blocking_accept.rs b/crates/edgezero-cli/tests/support/blocking_accept.rs new file mode 100644 index 000000000..2eb24b00c --- /dev/null +++ b/crates/edgezero-cli/tests/support/blocking_accept.rs @@ -0,0 +1,68 @@ +//! Accept helper for test fixtures that poll a non-blocking std listener. + +use std::io; +use std::net::{SocketAddr, TcpListener, TcpStream}; + +/// Accepts one connection and returns its stream in blocking mode. +/// +/// Fixtures make the listener non-blocking so their accept loop can poll a deadline or stop +/// flag. macOS `accept(2)` copies `O_NONBLOCK` to the accepted socket (Linux does not), so a +/// read on the returned stream would otherwise fail with `WouldBlock` at once instead of +/// honouring its read timeout. `WouldBlock` from `accept` itself is still returned to the caller. +#[inline] +pub(crate) fn accept_blocking(listener: &TcpListener) -> io::Result<(TcpStream, SocketAddr)> { + let (stream, peer) = listener.accept()?; + stream.set_nonblocking(false)?; + Ok((stream, peer)) +} + +#[cfg(test)] +mod tests { + use super::accept_blocking; + use std::io::{ErrorKind, Read as _}; + use std::net::{TcpListener, TcpStream}; + use std::thread; + use std::time::{Duration, Instant}; + + #[test] + fn stream_accepted_from_nonblocking_listener_waits_for_read_timeout() { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind loopback"); + listener + .set_nonblocking(true) + .expect("nonblocking listener"); + let address = listener.local_addr().expect("listener address"); + let _client = TcpStream::connect(address).expect("connect loopback client"); + let deadline = Instant::now() + .checked_add(Duration::from_secs(2)) + .expect("deadline fits the monotonic clock"); + let (mut stream, _peer) = loop { + match accept_blocking(&listener) { + Ok(accepted) => break accepted, + Err(error) if error.kind() == ErrorKind::WouldBlock => { + assert!( + Instant::now() < deadline, + "loopback connection was not queued" + ); + thread::sleep(Duration::from_millis(5)); + } + Err(error) => panic!("accept failed: {error}"), + } + }; + stream + .set_read_timeout(Some(Duration::from_millis(200))) + .expect("read timeout"); + let started = Instant::now(); + let error = stream + .read(&mut [0_u8; 1]) + .expect_err("the client sent nothing"); + let waited = started.elapsed(); + assert!( + matches!(error.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut), + "unexpected read error: {error}" + ); + assert!( + waited >= Duration::from_millis(150), + "read returned after {waited:?}, before its 200 ms timeout" + ); + } +} diff --git a/crates/edgezero-store-aws/src/dynamodb_records_tests.rs b/crates/edgezero-store-aws/src/dynamodb_records_tests.rs index 93e431f6a..738c845c1 100644 --- a/crates/edgezero-store-aws/src/dynamodb_records_tests.rs +++ b/crates/edgezero-store-aws/src/dynamodb_records_tests.rs @@ -9,6 +9,7 @@ use super::*; use crate::dynamodb_kv::sdk_tests::{active_table, loopback_client, read_request}; use crate::settings::{AwsPreparationLimits, DynamoDbKvLimits}; +use crate::test_net::accept_blocking; use edgezero_core::record_store::{RecordKeyLayout, contract}; use edgezero_core::time::MonotonicClock; use serde_json::{Value, json}; @@ -77,7 +78,7 @@ impl Fixture { let worker_stopped = Arc::clone(&stopped); let worker = thread::spawn(move || { while !worker_stopped.load(Ordering::Acquire) { - match listener.accept() { + match accept_blocking(&listener) { Ok((mut stream, _)) => { stream .set_read_timeout(Some(Duration::from_secs(3))) diff --git a/crates/edgezero-store-aws/src/dynamodb_tests.rs b/crates/edgezero-store-aws/src/dynamodb_tests.rs index 861d054cb..522c9d8e9 100644 --- a/crates/edgezero-store-aws/src/dynamodb_tests.rs +++ b/crates/edgezero-store-aws/src/dynamodb_tests.rs @@ -12,6 +12,7 @@ use super::*; use crate::preparation::AwsPreparation; use crate::settings::{AwsPreparationLimits, DynamoDbKvLimits, DynamoDbKvSettings}; +use crate::test_net::accept_blocking; use edgezero_core::time::MonotonicClock; use std::collections::{BTreeMap, HashMap, VecDeque}; use std::io::{ErrorKind, Read as _, Write as _}; @@ -144,7 +145,7 @@ impl DynamoFixture { let worker_accepted = Arc::clone(&accepted); let worker = thread::spawn(move || { while !worker_stopped.load(Ordering::Acquire) { - match listener.accept() { + match accept_blocking(&listener) { Ok((mut stream, _)) => { worker_accepted.fetch_add(1, Ordering::Relaxed); stream diff --git a/crates/edgezero-store-aws/src/lib.rs b/crates/edgezero-store-aws/src/lib.rs index c4df04a63..24c7e3e56 100644 --- a/crates/edgezero-store-aws/src/lib.rs +++ b/crates/edgezero-store-aws/src/lib.rs @@ -53,6 +53,12 @@ mod secrets_manager; #[cfg(test)] #[cfg(all(not(target_arch = "wasm32"), feature = "secrets-manager"))] mod snapshot_tests; +#[cfg(test)] +#[cfg(all( + not(target_arch = "wasm32"), + any(feature = "secrets-manager", feature = "dynamodb-kv") +))] +mod test_net; #[cfg(all( not(target_arch = "wasm32"), any( diff --git a/crates/edgezero-store-aws/src/secrets_manager.rs b/crates/edgezero-store-aws/src/secrets_manager.rs index d5fcd961e..d1adbefca 100644 --- a/crates/edgezero-store-aws/src/secrets_manager.rs +++ b/crates/edgezero-store-aws/src/secrets_manager.rs @@ -323,6 +323,7 @@ fn classify_sdk_error( mod tests { use super::{classify_operation_error, materialize_payload}; use crate::PreparationError; + use crate::test_net::accept_blocking; use aws_sdk_secretsmanager::error::{ConnectorError, SdkError}; use aws_sdk_secretsmanager::operation::get_secret_value::GetSecretValueOutput; use aws_sdk_secretsmanager::primitives::Blob; @@ -331,13 +332,27 @@ mod tests { use std::io::{Read as _, Write as _}; use std::net::{TcpListener, TcpStream}; use std::process::{Command, Stdio}; - use std::sync::atomic::{AtomicU64, Ordering}; + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::thread; use std::time::{Duration, Instant}; + /// How long the holding fixture waits for the client to abandon an unanswered attempt. + /// The timeout child's attempt deadline is 100 ms and its operation budget is 10 s, so only + /// the attempt deadline can close a held connection inside this bound. + const HOLD_BOUND: Duration = Duration::from_secs(5); + const TEST_ARN: &str = "arn:aws:secretsmanager:us-east-1:111122223333:secret:fixture-AbCdEf"; static NEXT_FIXTURE: AtomicU64 = AtomicU64::new(0); + /// One connection the SDK opened against `holding_fixture`. + struct HeldConnection { + /// The client closed the connection within `HOLD_BOUND`. + closed_by_client: bool, + /// Lowercased request head and body, when the complete request arrived first. + request: Option<(String, String)>, + } + struct LoopbackFixture { listener: TcpListener, worker: thread::JoinHandle>, @@ -358,7 +373,6 @@ mod tests { #[derive(Clone)] enum FixtureResponse { Decryption, - Delay(Duration), Denied, InternalError, Json(String), @@ -420,21 +434,9 @@ mod tests { body.contains(TEST_ARN), "request must contain only the fictional fixture ARN" ); - if let FixtureResponse::Delay(delay) = response { - thread::sleep(delay); - let payload = b"{\"SecretString\":\"VALUE_SENTINEL\"}"; - let _header_write = write!( - stream, - "HTTP/1.1 200 OK\r\nContent-Type: application/x-amz-json-1.1\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", - payload.len() - ); - let _body_write = stream.write_all(payload); - return (headers, body); - } let (status, code, payload) = match response { FixtureResponse::Decryption => (400_u16, "Bad Request", r#"{"__type":"DecryptionFailure","message":"VALUE_SENTINEL"}"#.to_owned()), FixtureResponse::Denied => (400_u16, "Bad Request", r#"{"__type":"AccessDeniedException","message":"ARN_SENTINEL CREDENTIAL_SENTINEL"}"#.to_owned()), - FixtureResponse::Delay(_) => return (headers, body), FixtureResponse::InternalError => (500_u16, "Internal Server Error", r#"{"__type":"InternalServiceError","message":"SDK_ERROR_SENTINEL"}"#.to_owned()), FixtureResponse::Json(payload) => (200_u16, "OK", payload), FixtureResponse::Missing => (400_u16, "Bad Request", r#"{"__type":"ResourceNotFoundException","message":"ARN_SENTINEL"}"#.to_owned()), @@ -449,6 +451,95 @@ mod tests { (headers, body) } + /// Accepts every connection the SDK opens and never answers, so the client's attempt + /// deadline is the only thing that ends each attempt. The deadline runs from the start of + /// the attempt, so a descheduled child can abandon a connection before writing its request; + /// that is recorded as a client-side close, not treated as a fixture failure. + fn holding_fixture() -> ( + String, + Arc, + thread::JoinHandle>, + ) { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind literal loopback only"); + listener + .set_nonblocking(true) + .expect("nonblocking listener"); + let endpoint = format!( + "http://{}", + listener.local_addr().expect("loopback address") + ); + let stop = Arc::new(AtomicBool::new(false)); + let worker_stop = Arc::clone(&stop); + let worker = thread::spawn(move || { + let deadline = Instant::now() + .checked_add(Duration::from_secs(30)) + .expect("fixture deadline fits the monotonic clock"); + let mut held = Vec::new(); + loop { + // Read the flag before accepting: once the child has exited, every connection it + // opened is already queued, so draining until `WouldBlock` counts all of them. + let stopping = worker_stop.load(Ordering::Acquire); + match accept_blocking(&listener) { + Ok((stream, peer)) => { + assert!(peer.ip().is_loopback(), "fixture peer must be loopback"); + held.push(hold(stream)); + } + Err(error) if error.kind() == ErrorKind::WouldBlock => { + if stopping { + break; + } + assert!(Instant::now() < deadline, "SDK child did not finish"); + thread::sleep(Duration::from_millis(5)); + } + Err(error) => panic!("fixture accept failed: {error}"), + } + } + held + }); + (endpoint, stop, worker) + } + + fn hold(mut stream: TcpStream) -> HeldConnection { + stream + .set_read_timeout(Some(HOLD_BOUND)) + .expect("held connection bound"); + let mut received = Vec::new(); + let mut chunk = [0_u8; 4096]; + let closed_by_client = loop { + match stream.read(&mut chunk) { + Ok(0) => break true, + Ok(read) => { + received.extend_from_slice(&chunk[..read]); + assert!(received.len() < 64 * 1024, "bounded held request"); + } + Err(error) if error.kind() == ErrorKind::ConnectionReset => break true, + Err(error) + if matches!(error.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut) => + { + break false; + } + Err(error) => panic!("held connection read failed: {error}"), + } + }; + let request = received + .windows(4) + .position(|bytes| bytes == b"\r\n\r\n") + .and_then(|end| { + let head = String::from_utf8_lossy(&received[..end]).to_ascii_lowercase(); + let length = head + .lines() + .find_map(|line| line.strip_prefix("content-length:").map(str::trim)) + .and_then(|value| value.parse::().ok())?; + let body = received.get(end.checked_add(4)?..)?; + (body.len() >= length) + .then(|| (head, String::from_utf8_lossy(&body[..length]).into_owned())) + }); + HeldConnection { + closed_by_client, + request, + } + } + fn loopback_fixture(responses: Vec) -> (String, LoopbackFixture) { let listener = TcpListener::bind("127.0.0.1:0").expect("bind literal loopback only"); listener @@ -465,7 +556,7 @@ mod tests { let mut bodies = Vec::new(); for response in responses { let stream = loop { - match worker_listener.accept() { + match accept_blocking(&worker_listener) { Ok((stream, peer)) => { assert!(peer.ip().is_loopback(), "fixture peer must be loopback"); break stream; @@ -772,20 +863,34 @@ mod tests { #[test] fn real_sdk_attempt_timeout_is_observed_without_an_outer_retry_loop() { - let (endpoint, fixture) = - loopback_fixture(vec![FixtureResponse::Delay(Duration::from_millis(350))]); + let (endpoint, stop, fixture) = holding_fixture(); let child = isolated_sdk_child(&endpoint, "timeout"); - let requests = fixture.join().expect("fixture thread"); + stop.store(true, Ordering::Release); + let held = fixture.join().expect("fixture thread"); assert!( child.status.success(), "timeout child failed: {}", String::from_utf8_lossy(&child.stderr) ); - assert_eq!( - requests.len(), - 1, - "SDK timeout behavior is inside the one provider operation" + assert!( + (1..=3).contains(&held.len()), + "only the SDK's own attempts (at most 3) reach the fixture: {} connections", + held.len() + ); + assert!( + held.iter().all(|connection| connection.closed_by_client), + "the attempt deadline, not the 10 s operation budget, ends every unanswered attempt" ); + for (head, body) in held + .iter() + .filter_map(|connection| connection.request.as_ref()) + { + assert!(head.contains("x-amz-target: secretsmanager.getsecretvalue")); + assert!( + body.contains(TEST_ARN), + "request must name only the fixture ARN" + ); + } } /// Invoked only by an env-cleared child process launched by the parent fixture. @@ -820,8 +925,10 @@ mod tests { crate::settings::AwsPreparationLimits { connect_timeout_ms: 50, attempt_timeout_ms: 100, - operation_timeout_ms: 500, - preparation_timeout_ms: 2_000, + // Far beyond the attempt deadline, so only the attempt deadline can end an + // unanswered attempt. + operation_timeout_ms: 10_000, + preparation_timeout_ms: 12_000, ..crate::settings::AwsPreparationLimits::default() } } else { diff --git a/crates/edgezero-store-aws/src/test_net.rs b/crates/edgezero-store-aws/src/test_net.rs new file mode 100644 index 000000000..2eb24b00c --- /dev/null +++ b/crates/edgezero-store-aws/src/test_net.rs @@ -0,0 +1,68 @@ +//! Accept helper for test fixtures that poll a non-blocking std listener. + +use std::io; +use std::net::{SocketAddr, TcpListener, TcpStream}; + +/// Accepts one connection and returns its stream in blocking mode. +/// +/// Fixtures make the listener non-blocking so their accept loop can poll a deadline or stop +/// flag. macOS `accept(2)` copies `O_NONBLOCK` to the accepted socket (Linux does not), so a +/// read on the returned stream would otherwise fail with `WouldBlock` at once instead of +/// honouring its read timeout. `WouldBlock` from `accept` itself is still returned to the caller. +#[inline] +pub(crate) fn accept_blocking(listener: &TcpListener) -> io::Result<(TcpStream, SocketAddr)> { + let (stream, peer) = listener.accept()?; + stream.set_nonblocking(false)?; + Ok((stream, peer)) +} + +#[cfg(test)] +mod tests { + use super::accept_blocking; + use std::io::{ErrorKind, Read as _}; + use std::net::{TcpListener, TcpStream}; + use std::thread; + use std::time::{Duration, Instant}; + + #[test] + fn stream_accepted_from_nonblocking_listener_waits_for_read_timeout() { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind loopback"); + listener + .set_nonblocking(true) + .expect("nonblocking listener"); + let address = listener.local_addr().expect("listener address"); + let _client = TcpStream::connect(address).expect("connect loopback client"); + let deadline = Instant::now() + .checked_add(Duration::from_secs(2)) + .expect("deadline fits the monotonic clock"); + let (mut stream, _peer) = loop { + match accept_blocking(&listener) { + Ok(accepted) => break accepted, + Err(error) if error.kind() == ErrorKind::WouldBlock => { + assert!( + Instant::now() < deadline, + "loopback connection was not queued" + ); + thread::sleep(Duration::from_millis(5)); + } + Err(error) => panic!("accept failed: {error}"), + } + }; + stream + .set_read_timeout(Some(Duration::from_millis(200))) + .expect("read timeout"); + let started = Instant::now(); + let error = stream + .read(&mut [0_u8; 1]) + .expect_err("the client sent nothing"); + let waited = started.elapsed(); + assert!( + matches!(error.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut), + "unexpected read error: {error}" + ); + assert!( + waited >= Duration::from_millis(150), + "read returned after {waited:?}, before its 200 ms timeout" + ); + } +}