From 450122063e89ef6c8905d306e68482f855417ca0 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 20 Nov 2026 15:13:47 +0530 Subject: [PATCH 1/5] test: restore blocking mode on streams accepted by std fixtures Fixtures that poll a non-blocking std TcpListener read the accepted stream with set_read_timeout. macOS accept(2) copies O_NONBLOCK to the accepted socket (Linux does not), so those reads failed with WouldBlock at once and tests failed at random on macOS under parallel load. Add a small accept_blocking helper per crate (store-aws, adapter-axum tests, cli tests) that restores blocking mode, route all nine accept sites through it, and cover it with a test that fails on macOS when the accepted stream does not wait for its read timeout. --- .../tests/fixtures/blocking_accept.rs | 68 +++++++++++++++++++ .../tests/native_aws_startup.rs | 7 +- .../tests/native_celld_startup.rs | 7 +- .../tests/celld_generated_proof.rs | 8 ++- .../tests/dynamodb_generated_proof.rs | 8 ++- .../tests/native_store_generated_proof.rs | 10 ++- .../tests/support/blocking_accept.rs | 68 +++++++++++++++++++ .../src/dynamodb_records_tests.rs | 3 +- .../edgezero-store-aws/src/dynamodb_tests.rs | 3 +- crates/edgezero-store-aws/src/lib.rs | 6 ++ .../edgezero-store-aws/src/secrets_manager.rs | 3 +- crates/edgezero-store-aws/src/test_net.rs | 68 +++++++++++++++++++ 12 files changed, 250 insertions(+), 9 deletions(-) create mode 100644 crates/edgezero-adapter-axum/tests/fixtures/blocking_accept.rs create mode 100644 crates/edgezero-cli/tests/support/blocking_accept.rs create mode 100644 crates/edgezero-store-aws/src/test_net.rs diff --git a/crates/edgezero-adapter-axum/tests/fixtures/blocking_accept.rs b/crates/edgezero-adapter-axum/tests/fixtures/blocking_accept.rs new file mode 100644 index 000000000..2eb24b00c --- /dev/null +++ b/crates/edgezero-adapter-axum/tests/fixtures/blocking_accept.rs @@ -0,0 +1,68 @@ +//! Accept helper for test fixtures that poll a non-blocking std listener. + +use std::io; +use std::net::{SocketAddr, TcpListener, TcpStream}; + +/// Accepts one connection and returns its stream in blocking mode. +/// +/// Fixtures make the listener non-blocking so their accept loop can poll a deadline or stop +/// flag. macOS `accept(2)` copies `O_NONBLOCK` to the accepted socket (Linux does not), so a +/// read on the returned stream would otherwise fail with `WouldBlock` at once instead of +/// honouring its read timeout. `WouldBlock` from `accept` itself is still returned to the caller. +#[inline] +pub(crate) fn accept_blocking(listener: &TcpListener) -> io::Result<(TcpStream, SocketAddr)> { + let (stream, peer) = listener.accept()?; + stream.set_nonblocking(false)?; + Ok((stream, peer)) +} + +#[cfg(test)] +mod tests { + use super::accept_blocking; + use std::io::{ErrorKind, Read as _}; + use std::net::{TcpListener, TcpStream}; + use std::thread; + use std::time::{Duration, Instant}; + + #[test] + fn stream_accepted_from_nonblocking_listener_waits_for_read_timeout() { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind loopback"); + listener + .set_nonblocking(true) + .expect("nonblocking listener"); + let address = listener.local_addr().expect("listener address"); + let _client = TcpStream::connect(address).expect("connect loopback client"); + let deadline = Instant::now() + .checked_add(Duration::from_secs(2)) + .expect("deadline fits the monotonic clock"); + let (mut stream, _peer) = loop { + match accept_blocking(&listener) { + Ok(accepted) => break accepted, + Err(error) if error.kind() == ErrorKind::WouldBlock => { + assert!( + Instant::now() < deadline, + "loopback connection was not queued" + ); + thread::sleep(Duration::from_millis(5)); + } + Err(error) => panic!("accept failed: {error}"), + } + }; + stream + .set_read_timeout(Some(Duration::from_millis(200))) + .expect("read timeout"); + let started = Instant::now(); + let error = stream + .read(&mut [0_u8; 1]) + .expect_err("the client sent nothing"); + let waited = started.elapsed(); + assert!( + matches!(error.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut), + "unexpected read error: {error}" + ); + assert!( + waited >= Duration::from_millis(150), + "read returned after {waited:?}, before its 200 ms timeout" + ); + } +} diff --git a/crates/edgezero-adapter-axum/tests/native_aws_startup.rs b/crates/edgezero-adapter-axum/tests/native_aws_startup.rs index a0b6283ce..849f268c8 100644 --- a/crates/edgezero-adapter-axum/tests/native_aws_startup.rs +++ b/crates/edgezero-adapter-axum/tests/native_aws_startup.rs @@ -4,12 +4,17 @@ //! They exercise the public runner and real pinned SDK, not a provider seam. No AWS account, //! DNS hostname, external socket, image, or real cloud credential is used. +#[cfg(test)] +#[path = "fixtures/blocking_accept.rs"] +mod blocking_accept; + #[cfg(test)] #[expect( clippy::arbitrary_source_item_ordering, reason = "loopback fixture helpers remain grouped before startup scenarios" )] mod tests { + use crate::blocking_accept::accept_blocking; use std::fmt::Write as _; use std::fs; use std::io::{ErrorKind, Read, Write as _}; @@ -83,7 +88,7 @@ mod tests { .checked_add(watch) .expect("fixture deadline fits the monotonic clock"); loop { - match listener.accept() { + match accept_blocking(&listener) { Ok((mut stream, peer)) => { assert!(peer.ip().is_loopback(), "fixture peer was not loopback"); stream diff --git a/crates/edgezero-adapter-axum/tests/native_celld_startup.rs b/crates/edgezero-adapter-axum/tests/native_celld_startup.rs index a9dbe9c54..8d057f909 100644 --- a/crates/edgezero-adapter-axum/tests/native_celld_startup.rs +++ b/crates/edgezero-adapter-axum/tests/native_celld_startup.rs @@ -8,8 +8,13 @@ reason = "fixture helpers precede startup scenarios" )] +#[cfg(test)] +#[path = "fixtures/blocking_accept.rs"] +mod blocking_accept; + #[cfg(test)] mod tests { + use crate::blocking_accept::accept_blocking; use std::fs; use std::io::{ErrorKind, Read, Write as _}; use std::net::{SocketAddr, TcpListener, TcpStream}; @@ -47,7 +52,7 @@ mod tests { thread::spawn(move || { let deadline = Instant::now().checked_add(WAIT).expect("fixture deadline"); while !worker_stop.load(Ordering::Relaxed) && Instant::now() < deadline { - match listener.accept() { + match accept_blocking(&listener) { Ok((mut stream, peer)) => { assert!(peer.ip().is_loopback()); stream diff --git a/crates/edgezero-cli/tests/celld_generated_proof.rs b/crates/edgezero-cli/tests/celld_generated_proof.rs index 8b96613bd..fcfd83537 100644 --- a/crates/edgezero-cli/tests/celld_generated_proof.rs +++ b/crates/edgezero-cli/tests/celld_generated_proof.rs @@ -4,6 +4,11 @@ //! `EDGEZERO_CELLD_STORE_SERVICE`, Node.js and OpenSSL; missing tools are //! failures, not skips. +#[cfg(test)] +#[cfg(unix)] +#[path = "support/blocking_accept.rs"] +mod blocking_accept; + #[cfg(test)] #[cfg(unix)] mod tests { @@ -12,6 +17,7 @@ mod tests { reason = "keep fixture ownership, generated source and proof steps together" )] + use crate::blocking_accept::accept_blocking; use std::collections::BTreeMap; use std::env; use std::fmt::Write as _; @@ -76,7 +82,7 @@ mod tests { let worker = thread::spawn(move || { let end = deadline(Duration::from_mins(10)); while !worker_stop.load(Ordering::Relaxed) && Instant::now() < end { - match listener.accept() { + match accept_blocking(&listener) { Ok((mut stream, peer)) => { assert!(peer.ip().is_loopback()); stream diff --git a/crates/edgezero-cli/tests/dynamodb_generated_proof.rs b/crates/edgezero-cli/tests/dynamodb_generated_proof.rs index 3cadaabdf..aa13389d3 100644 --- a/crates/edgezero-cli/tests/dynamodb_generated_proof.rs +++ b/crates/edgezero-cli/tests/dynamodb_generated_proof.rs @@ -7,6 +7,11 @@ //! fixture with dummy credentials and a cleared child environment. Serialize builds that share //! `EDGEZERO_GENERATED_TEST_TARGET_DIR`. +#[cfg(test)] +#[cfg(unix)] +#[path = "support/blocking_accept.rs"] +mod blocking_accept; + #[cfg(test)] mod tests { #[cfg(unix)] @@ -16,6 +21,7 @@ mod tests { reason = "group test fixture types and their protocol helpers by responsibility" )] + use crate::blocking_accept::accept_blocking; use std::collections::BTreeMap; use std::env; use std::fs; @@ -64,7 +70,7 @@ mod tests { let worker = thread::spawn(move || { let deadline = deadline_after(Duration::from_mins(3)); while !worker_stop.load(Ordering::Relaxed) && Instant::now() < deadline { - match listener.accept() { + match accept_blocking(&listener) { Ok((mut stream, peer)) => { assert!(peer.ip().is_loopback(), "fixture accepts loopback only"); stream diff --git a/crates/edgezero-cli/tests/native_store_generated_proof.rs b/crates/edgezero-cli/tests/native_store_generated_proof.rs index 8675e585a..c15981573 100644 --- a/crates/edgezero-cli/tests/native_store_generated_proof.rs +++ b/crates/edgezero-cli/tests/native_store_generated_proof.rs @@ -20,10 +20,16 @@ //! `EDGEZERO_GENERATED_TEST_TARGET_DIR` can select a shared target directory; otherwise builds //! use the generated project's `target/`. Serialize builds that share a target directory. +#[cfg(test)] +#[cfg(unix)] +#[path = "support/blocking_accept.rs"] +mod blocking_accept; + #[cfg(test)] #[cfg(unix)] mod tests { + use crate::blocking_accept::accept_blocking; use std::env::var_os; use std::fs; use std::io::{ErrorKind, Read as _, Write as _}; @@ -114,7 +120,7 @@ mod tests { let mut extra = Vec::new(); let deadline = deadline_after(SERVICE_GRACE); while Instant::now() < deadline { - match listener.accept() { + match accept_blocking(&listener) { Ok((mut extra_stream, peer)) => { assert!(peer.ip().is_loopback(), "only literal loopback peers"); extra_stream @@ -304,7 +310,7 @@ mod tests { fn accept_before(listener: &TcpListener, deadline: Instant) -> (TcpStream, SocketAddr) { loop { - match listener.accept() { + match accept_blocking(listener) { Ok((stream, peer)) => { assert!(peer.ip().is_loopback(), "fixture peer must be loopback"); return (stream, peer); diff --git a/crates/edgezero-cli/tests/support/blocking_accept.rs b/crates/edgezero-cli/tests/support/blocking_accept.rs new file mode 100644 index 000000000..2eb24b00c --- /dev/null +++ b/crates/edgezero-cli/tests/support/blocking_accept.rs @@ -0,0 +1,68 @@ +//! Accept helper for test fixtures that poll a non-blocking std listener. + +use std::io; +use std::net::{SocketAddr, TcpListener, TcpStream}; + +/// Accepts one connection and returns its stream in blocking mode. +/// +/// Fixtures make the listener non-blocking so their accept loop can poll a deadline or stop +/// flag. macOS `accept(2)` copies `O_NONBLOCK` to the accepted socket (Linux does not), so a +/// read on the returned stream would otherwise fail with `WouldBlock` at once instead of +/// honouring its read timeout. `WouldBlock` from `accept` itself is still returned to the caller. +#[inline] +pub(crate) fn accept_blocking(listener: &TcpListener) -> io::Result<(TcpStream, SocketAddr)> { + let (stream, peer) = listener.accept()?; + stream.set_nonblocking(false)?; + Ok((stream, peer)) +} + +#[cfg(test)] +mod tests { + use super::accept_blocking; + use std::io::{ErrorKind, Read as _}; + use std::net::{TcpListener, TcpStream}; + use std::thread; + use std::time::{Duration, Instant}; + + #[test] + fn stream_accepted_from_nonblocking_listener_waits_for_read_timeout() { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind loopback"); + listener + .set_nonblocking(true) + .expect("nonblocking listener"); + let address = listener.local_addr().expect("listener address"); + let _client = TcpStream::connect(address).expect("connect loopback client"); + let deadline = Instant::now() + .checked_add(Duration::from_secs(2)) + .expect("deadline fits the monotonic clock"); + let (mut stream, _peer) = loop { + match accept_blocking(&listener) { + Ok(accepted) => break accepted, + Err(error) if error.kind() == ErrorKind::WouldBlock => { + assert!( + Instant::now() < deadline, + "loopback connection was not queued" + ); + thread::sleep(Duration::from_millis(5)); + } + Err(error) => panic!("accept failed: {error}"), + } + }; + stream + .set_read_timeout(Some(Duration::from_millis(200))) + .expect("read timeout"); + let started = Instant::now(); + let error = stream + .read(&mut [0_u8; 1]) + .expect_err("the client sent nothing"); + let waited = started.elapsed(); + assert!( + matches!(error.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut), + "unexpected read error: {error}" + ); + assert!( + waited >= Duration::from_millis(150), + "read returned after {waited:?}, before its 200 ms timeout" + ); + } +} diff --git a/crates/edgezero-store-aws/src/dynamodb_records_tests.rs b/crates/edgezero-store-aws/src/dynamodb_records_tests.rs index 93e431f6a..738c845c1 100644 --- a/crates/edgezero-store-aws/src/dynamodb_records_tests.rs +++ b/crates/edgezero-store-aws/src/dynamodb_records_tests.rs @@ -9,6 +9,7 @@ use super::*; use crate::dynamodb_kv::sdk_tests::{active_table, loopback_client, read_request}; use crate::settings::{AwsPreparationLimits, DynamoDbKvLimits}; +use crate::test_net::accept_blocking; use edgezero_core::record_store::{RecordKeyLayout, contract}; use edgezero_core::time::MonotonicClock; use serde_json::{Value, json}; @@ -77,7 +78,7 @@ impl Fixture { let worker_stopped = Arc::clone(&stopped); let worker = thread::spawn(move || { while !worker_stopped.load(Ordering::Acquire) { - match listener.accept() { + match accept_blocking(&listener) { Ok((mut stream, _)) => { stream .set_read_timeout(Some(Duration::from_secs(3))) diff --git a/crates/edgezero-store-aws/src/dynamodb_tests.rs b/crates/edgezero-store-aws/src/dynamodb_tests.rs index 861d054cb..522c9d8e9 100644 --- a/crates/edgezero-store-aws/src/dynamodb_tests.rs +++ b/crates/edgezero-store-aws/src/dynamodb_tests.rs @@ -12,6 +12,7 @@ use super::*; use crate::preparation::AwsPreparation; use crate::settings::{AwsPreparationLimits, DynamoDbKvLimits, DynamoDbKvSettings}; +use crate::test_net::accept_blocking; use edgezero_core::time::MonotonicClock; use std::collections::{BTreeMap, HashMap, VecDeque}; use std::io::{ErrorKind, Read as _, Write as _}; @@ -144,7 +145,7 @@ impl DynamoFixture { let worker_accepted = Arc::clone(&accepted); let worker = thread::spawn(move || { while !worker_stopped.load(Ordering::Acquire) { - match listener.accept() { + match accept_blocking(&listener) { Ok((mut stream, _)) => { worker_accepted.fetch_add(1, Ordering::Relaxed); stream diff --git a/crates/edgezero-store-aws/src/lib.rs b/crates/edgezero-store-aws/src/lib.rs index c4df04a63..24c7e3e56 100644 --- a/crates/edgezero-store-aws/src/lib.rs +++ b/crates/edgezero-store-aws/src/lib.rs @@ -53,6 +53,12 @@ mod secrets_manager; #[cfg(test)] #[cfg(all(not(target_arch = "wasm32"), feature = "secrets-manager"))] mod snapshot_tests; +#[cfg(test)] +#[cfg(all( + not(target_arch = "wasm32"), + any(feature = "secrets-manager", feature = "dynamodb-kv") +))] +mod test_net; #[cfg(all( not(target_arch = "wasm32"), any( diff --git a/crates/edgezero-store-aws/src/secrets_manager.rs b/crates/edgezero-store-aws/src/secrets_manager.rs index d5fcd961e..6fe0b06bd 100644 --- a/crates/edgezero-store-aws/src/secrets_manager.rs +++ b/crates/edgezero-store-aws/src/secrets_manager.rs @@ -323,6 +323,7 @@ fn classify_sdk_error( mod tests { use super::{classify_operation_error, materialize_payload}; use crate::PreparationError; + use crate::test_net::accept_blocking; use aws_sdk_secretsmanager::error::{ConnectorError, SdkError}; use aws_sdk_secretsmanager::operation::get_secret_value::GetSecretValueOutput; use aws_sdk_secretsmanager::primitives::Blob; @@ -465,7 +466,7 @@ mod tests { let mut bodies = Vec::new(); for response in responses { let stream = loop { - match worker_listener.accept() { + match accept_blocking(&worker_listener) { Ok((stream, peer)) => { assert!(peer.ip().is_loopback(), "fixture peer must be loopback"); break stream; diff --git a/crates/edgezero-store-aws/src/test_net.rs b/crates/edgezero-store-aws/src/test_net.rs new file mode 100644 index 000000000..2eb24b00c --- /dev/null +++ b/crates/edgezero-store-aws/src/test_net.rs @@ -0,0 +1,68 @@ +//! Accept helper for test fixtures that poll a non-blocking std listener. + +use std::io; +use std::net::{SocketAddr, TcpListener, TcpStream}; + +/// Accepts one connection and returns its stream in blocking mode. +/// +/// Fixtures make the listener non-blocking so their accept loop can poll a deadline or stop +/// flag. macOS `accept(2)` copies `O_NONBLOCK` to the accepted socket (Linux does not), so a +/// read on the returned stream would otherwise fail with `WouldBlock` at once instead of +/// honouring its read timeout. `WouldBlock` from `accept` itself is still returned to the caller. +#[inline] +pub(crate) fn accept_blocking(listener: &TcpListener) -> io::Result<(TcpStream, SocketAddr)> { + let (stream, peer) = listener.accept()?; + stream.set_nonblocking(false)?; + Ok((stream, peer)) +} + +#[cfg(test)] +mod tests { + use super::accept_blocking; + use std::io::{ErrorKind, Read as _}; + use std::net::{TcpListener, TcpStream}; + use std::thread; + use std::time::{Duration, Instant}; + + #[test] + fn stream_accepted_from_nonblocking_listener_waits_for_read_timeout() { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind loopback"); + listener + .set_nonblocking(true) + .expect("nonblocking listener"); + let address = listener.local_addr().expect("listener address"); + let _client = TcpStream::connect(address).expect("connect loopback client"); + let deadline = Instant::now() + .checked_add(Duration::from_secs(2)) + .expect("deadline fits the monotonic clock"); + let (mut stream, _peer) = loop { + match accept_blocking(&listener) { + Ok(accepted) => break accepted, + Err(error) if error.kind() == ErrorKind::WouldBlock => { + assert!( + Instant::now() < deadline, + "loopback connection was not queued" + ); + thread::sleep(Duration::from_millis(5)); + } + Err(error) => panic!("accept failed: {error}"), + } + }; + stream + .set_read_timeout(Some(Duration::from_millis(200))) + .expect("read timeout"); + let started = Instant::now(); + let error = stream + .read(&mut [0_u8; 1]) + .expect_err("the client sent nothing"); + let waited = started.elapsed(); + assert!( + matches!(error.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut), + "unexpected read error: {error}" + ); + assert!( + waited >= Duration::from_millis(150), + "read returned after {waited:?}, before its 200 ms timeout" + ); + } +} From 6ac23fb7e01c80cfcb66a10488a70c16bfec90e6 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 20 Nov 2026 15:13:47 +0530 Subject: [PATCH 2/5] test: run source-bound proxy hosting tests only on Linux by default Four production_hosting tests drive the native proxy fixture, which binds its upstream socket to 127.0.0.2 or 127.0.0.3 so the server sees distinct peers. Linux routes all of 127.0.0.0/8 to lo; macOS lo0 has only 127.0.0.1, so the bind fails with EADDRNOTAVAIL, the relay drops the connection, and the tests see an empty response or TLS UnexpectedEof. Ignore them off Linux with a reason that gives the lo0 alias commands needed to run them with --ignored. --- .../tests/production_hosting.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/crates/edgezero-adapter-axum/tests/production_hosting.rs b/crates/edgezero-adapter-axum/tests/production_hosting.rs index c74a86c0f..c0e2bd8c2 100644 --- a/crates/edgezero-adapter-axum/tests/production_hosting.rs +++ b/crates/edgezero-adapter-axum/tests/production_hosting.rs @@ -567,6 +567,10 @@ mod tests { } #[test] + #[cfg_attr( + not(target_os = "linux"), + ignore = "binds extra loopback source addresses (127.0.0.2, 127.0.0.3), which only Linux routes by default (macOS lo0 has only 127.0.0.1); on macOS run with --ignored after `sudo ifconfig lo0 alias 127.0.0.2 up` and `sudo ifconfig lo0 alias 127.0.0.3 up`" + )] fn native_source_bound_single_proxy_and_mapped_cidr_match_only_owned_peers() { for networks in [ "127.0.0.2/32", @@ -627,6 +631,10 @@ mod tests { } #[test] + #[cfg_attr( + not(target_os = "linux"), + ignore = "binds extra loopback source addresses (127.0.0.2, 127.0.0.3), which only Linux routes by default (macOS lo0 has only 127.0.0.1); on macOS run with --ignored after `sudo ifconfig lo0 alias 127.0.0.2 up` and `sudo ifconfig lo0 alias 127.0.0.3 up`" + )] fn native_two_proxy_chain_stops_at_untrusted_hop_and_gateway_replaces_prefix() { for (networks, boundary) in [ ("127.0.0.2/32,127.0.0.3/32", "127.0.0.1"), @@ -773,6 +781,10 @@ mod tests { } #[test] + #[cfg_attr( + not(target_os = "linux"), + ignore = "binds extra loopback source addresses (127.0.0.2, 127.0.0.3), which only Linux routes by default (macOS lo0 has only 127.0.0.1); on macOS run with --ignored after `sudo ifconfig lo0 alias 127.0.0.2 up` and `sudo ifconfig lo0 alias 127.0.0.3 up`" + )] fn native_selected_families_conflicts_malformed_sets_gaps_and_exhaustion() { for family in ["forwarded", "x-forwarded"] { let roots = Roots::new(); @@ -847,6 +859,10 @@ mod tests { } #[test] + #[cfg_attr( + not(target_os = "linux"), + ignore = "binds extra loopback source addresses (127.0.0.2, 127.0.0.3), which only Linux routes by default (macOS lo0 has only 127.0.0.1); on macOS run with --ignored after `sudo ifconfig lo0 alias 127.0.0.2 up` and `sudo ifconfig lo0 alias 127.0.0.3 up`" + )] fn native_verified_tls_ingress_overwrites_public_assertions_and_rejects_wrong_name() { let roots = Roots::new(); let addr = address(); From ce8421a947b67299bcb092c0736801f34227e042 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 20 Nov 2026 19:03:36 +0530 Subject: [PATCH 3/5] test: make the Secrets Manager attempt-timeout proof deterministic The SDK's 100 ms attempt deadline runs from the start of the attempt. Under parallel load the child can be descheduled between the TCP connect and the request write, so the SDK correctly abandons the connection before sending anything. The fixture treated that as fatal ("SDK closed before request headers"), which failed the test about once in 20 full store-aws runs on macOS. The test now uses a fixture that accepts every attempt and never answers, recording whether the client closed each connection itself. The operation budget is 10 s, so only the attempt deadline can close a held connection within the 5 s bound. The test asserts that every attempt was closed that way, that the SDK made 1 to 3 attempts, and that any full request named the fixture ARN. Removing the attempt timeout from the provider makes the test fail. --- .../edgezero-store-aws/src/secrets_manager.rs | 146 +++++++++++++++--- 1 file changed, 123 insertions(+), 23 deletions(-) diff --git a/crates/edgezero-store-aws/src/secrets_manager.rs b/crates/edgezero-store-aws/src/secrets_manager.rs index 6fe0b06bd..4ac8358e2 100644 --- a/crates/edgezero-store-aws/src/secrets_manager.rs +++ b/crates/edgezero-store-aws/src/secrets_manager.rs @@ -332,13 +332,27 @@ mod tests { use std::io::{Read as _, Write as _}; use std::net::{TcpListener, TcpStream}; use std::process::{Command, Stdio}; - use std::sync::atomic::{AtomicU64, Ordering}; + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::thread; use std::time::{Duration, Instant}; + /// How long the holding fixture waits for the client to abandon an unanswered attempt. + /// The timeout child's attempt deadline is 100 ms and its operation budget is 10 s, so only + /// the attempt deadline can close a held connection inside this bound. + const HOLD_BOUND: Duration = Duration::from_secs(5); + const TEST_ARN: &str = "arn:aws:secretsmanager:us-east-1:111122223333:secret:fixture-AbCdEf"; static NEXT_FIXTURE: AtomicU64 = AtomicU64::new(0); + /// One connection the SDK opened against `holding_fixture`. + struct HeldConnection { + /// The client closed the connection within `HOLD_BOUND`. + closed_by_client: bool, + /// Lowercased request head and body, when the complete request arrived first. + request: Option<(String, String)>, + } + struct LoopbackFixture { listener: TcpListener, worker: thread::JoinHandle>, @@ -359,7 +373,6 @@ mod tests { #[derive(Clone)] enum FixtureResponse { Decryption, - Delay(Duration), Denied, InternalError, Json(String), @@ -421,21 +434,9 @@ mod tests { body.contains(TEST_ARN), "request must contain only the fictional fixture ARN" ); - if let FixtureResponse::Delay(delay) = response { - thread::sleep(delay); - let payload = b"{\"SecretString\":\"VALUE_SENTINEL\"}"; - let _header_write = write!( - stream, - "HTTP/1.1 200 OK\r\nContent-Type: application/x-amz-json-1.1\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", - payload.len() - ); - let _body_write = stream.write_all(payload); - return (headers, body); - } let (status, code, payload) = match response { FixtureResponse::Decryption => (400_u16, "Bad Request", r#"{"__type":"DecryptionFailure","message":"VALUE_SENTINEL"}"#.to_owned()), FixtureResponse::Denied => (400_u16, "Bad Request", r#"{"__type":"AccessDeniedException","message":"ARN_SENTINEL CREDENTIAL_SENTINEL"}"#.to_owned()), - FixtureResponse::Delay(_) => return (headers, body), FixtureResponse::InternalError => (500_u16, "Internal Server Error", r#"{"__type":"InternalServiceError","message":"SDK_ERROR_SENTINEL"}"#.to_owned()), FixtureResponse::Json(payload) => (200_u16, "OK", payload), FixtureResponse::Missing => (400_u16, "Bad Request", r#"{"__type":"ResourceNotFoundException","message":"ARN_SENTINEL"}"#.to_owned()), @@ -450,6 +451,89 @@ mod tests { (headers, body) } + /// Accepts every connection the SDK opens and never answers, so the client's attempt + /// deadline is the only thing that ends each attempt. The deadline runs from the start of + /// the attempt, so a descheduled child can abandon a connection before writing its request; + /// that is recorded as a client-side close, not treated as a fixture failure. + fn holding_fixture() -> ( + String, + Arc, + thread::JoinHandle>, + ) { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind literal loopback only"); + listener + .set_nonblocking(true) + .expect("nonblocking listener"); + let endpoint = format!( + "http://{}", + listener.local_addr().expect("loopback address") + ); + let stop = Arc::new(AtomicBool::new(false)); + let worker_stop = Arc::clone(&stop); + let worker = thread::spawn(move || { + let deadline = Instant::now() + .checked_add(Duration::from_secs(30)) + .expect("fixture deadline fits the monotonic clock"); + let mut held = Vec::new(); + while !worker_stop.load(Ordering::Acquire) { + match accept_blocking(&listener) { + Ok((stream, peer)) => { + assert!(peer.ip().is_loopback(), "fixture peer must be loopback"); + held.push(hold(stream)); + } + Err(error) if error.kind() == ErrorKind::WouldBlock => { + assert!(Instant::now() < deadline, "SDK child did not finish"); + thread::sleep(Duration::from_millis(5)); + } + Err(error) => panic!("fixture accept failed: {error}"), + } + } + held + }); + (endpoint, stop, worker) + } + + fn hold(mut stream: TcpStream) -> HeldConnection { + stream + .set_read_timeout(Some(HOLD_BOUND)) + .expect("held connection bound"); + let mut received = Vec::new(); + let mut chunk = [0_u8; 4096]; + let closed_by_client = loop { + match stream.read(&mut chunk) { + Ok(0) => break true, + Ok(read) => { + received.extend_from_slice(&chunk[..read]); + assert!(received.len() < 64 * 1024, "bounded held request"); + } + Err(error) if error.kind() == ErrorKind::ConnectionReset => break true, + Err(error) + if matches!(error.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut) => + { + break false; + } + Err(error) => panic!("held connection read failed: {error}"), + } + }; + let request = received + .windows(4) + .position(|bytes| bytes == b"\r\n\r\n") + .and_then(|end| { + let head = String::from_utf8_lossy(&received[..end]).to_ascii_lowercase(); + let length = head + .lines() + .find_map(|line| line.strip_prefix("content-length:").map(str::trim)) + .and_then(|value| value.parse::().ok())?; + let body = received.get(end.checked_add(4)?..)?; + (body.len() >= length) + .then(|| (head, String::from_utf8_lossy(&body[..length]).into_owned())) + }); + HeldConnection { + closed_by_client, + request, + } + } + fn loopback_fixture(responses: Vec) -> (String, LoopbackFixture) { let listener = TcpListener::bind("127.0.0.1:0").expect("bind literal loopback only"); listener @@ -773,20 +857,34 @@ mod tests { #[test] fn real_sdk_attempt_timeout_is_observed_without_an_outer_retry_loop() { - let (endpoint, fixture) = - loopback_fixture(vec![FixtureResponse::Delay(Duration::from_millis(350))]); + let (endpoint, stop, fixture) = holding_fixture(); let child = isolated_sdk_child(&endpoint, "timeout"); - let requests = fixture.join().expect("fixture thread"); + stop.store(true, Ordering::Release); + let held = fixture.join().expect("fixture thread"); assert!( child.status.success(), "timeout child failed: {}", String::from_utf8_lossy(&child.stderr) ); - assert_eq!( - requests.len(), - 1, - "SDK timeout behavior is inside the one provider operation" + assert!( + (1..=3).contains(&held.len()), + "only the SDK's own attempts (at most 3) reach the fixture: {} connections", + held.len() + ); + assert!( + held.iter().all(|connection| connection.closed_by_client), + "the attempt deadline, not the 10 s operation budget, ends every unanswered attempt" ); + for (head, body) in held + .iter() + .filter_map(|connection| connection.request.as_ref()) + { + assert!(head.contains("x-amz-target: secretsmanager.getsecretvalue")); + assert!( + body.contains(TEST_ARN), + "request must name only the fixture ARN" + ); + } } /// Invoked only by an env-cleared child process launched by the parent fixture. @@ -821,8 +919,10 @@ mod tests { crate::settings::AwsPreparationLimits { connect_timeout_ms: 50, attempt_timeout_ms: 100, - operation_timeout_ms: 500, - preparation_timeout_ms: 2_000, + // Far beyond the attempt deadline, so only the attempt deadline can end an + // unanswered attempt. + operation_timeout_ms: 10_000, + preparation_timeout_ms: 12_000, ..crate::settings::AwsPreparationLimits::default() } } else { From 09b09d9ee51da6e9953d3c8722f1640036740738 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 20 Nov 2026 19:12:36 +0530 Subject: [PATCH 4/5] test: keep hosting port reservations from leaking into fixture processes macOS marks a new socket close-on-exec only after socket() returns. When a parallel test spawned a fixture process in that gap, the child inherited another test's port reservation and kept it bound after the test released it. That test's fixture server then failed to bind (AddrInUse); lsof showed another fixture process holding the port as a CLOSED socket. Separately, a released reservation could be served by another test's server during a "nothing answers" check. Together these failed 6 of 20 macOS runs. Reservations and fixture spawns now hold one lock, so they never overlap. Tests that assert nothing answers keep their reservation bound, but not listening, until the check is done. --- .../tests/production_hosting.rs | 84 ++++++++++++------- 1 file changed, 55 insertions(+), 29 deletions(-) diff --git a/crates/edgezero-adapter-axum/tests/production_hosting.rs b/crates/edgezero-adapter-axum/tests/production_hosting.rs index c0e2bd8c2..1c50aaef9 100644 --- a/crates/edgezero-adapter-axum/tests/production_hosting.rs +++ b/crates/edgezero-adapter-axum/tests/production_hosting.rs @@ -20,13 +20,14 @@ mod tests { use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _, symlink}; use std::os::unix::process::ExitStatusExt as _; use std::process::{Child, Command, ExitStatus, Stdio}; - use std::sync::{Arc, Mutex, mpsc}; + use std::sync::{Arc, Mutex, MutexGuard, PoisonError, mpsc}; use std::thread::{self, JoinHandle}; use std::time::{Duration, Instant}; use edgezero_core::blob_envelope::BlobEnvelope; use serde_json::{Value, json}; use tempfile::TempDir; + use tokio::net::TcpSocket; const WAIT: Duration = Duration::from_secs(5); @@ -64,11 +65,34 @@ mod tests { .expect("envelope") } + /// macOS marks a new socket close-on-exec only after `socket()` returns, so a fixture + /// process spawned by a parallel test in that gap inherits it. An inherited reservation + /// stays bound in that process, and the fixture server that later binds the port fails + /// with `AddrInUse`. Reservations and spawns hold this lock so they never overlap. + static RESERVE_OR_SPAWN: Mutex<()> = Mutex::new(()); + + fn reserve_or_spawn() -> MutexGuard<'static, ()> { + RESERVE_OR_SPAWN + .lock() + .unwrap_or_else(PoisonError::into_inner) + } + + /// Reserves a loopback port for a fixture server that binds it itself. fn address() -> SocketAddr { - TcpListener::bind("127.0.0.1:0") - .expect("port reservation") - .local_addr() - .expect("address") + reserved_address().0 + } + + /// Reserves a loopback port and keeps it bound, but not listening, until the holder drops. + /// No parallel test can bind the port meanwhile and nothing accepts a connection to it, so + /// a "nothing answers" check cannot reach a server that reused a released reservation. + fn reserved_address() -> (SocketAddr, TcpSocket) { + let _reserving = reserve_or_spawn(); + let socket = TcpSocket::new_v4().expect("port reservation socket"); + socket + .bind(SocketAddr::from((Ipv4Addr::LOCALHOST, 0))) + .expect("port reservation"); + let addr = socket.local_addr().expect("reserved address"); + (addr, socket) } fn command(mode: &str, roots: &Roots, addr: SocketAddr) -> Command { @@ -101,11 +125,13 @@ mod tests { impl Server { fn spawn(mut command: Command, addr: SocketAddr) -> Self { + let spawning = reserve_or_spawn(); let mut child = command .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() .expect("fixture process"); + drop(spawning); let output = Arc::new(Mutex::new(String::new())); let (sender, events) = mpsc::channel(); let stdout = child.stdout.take().expect("stdout"); @@ -173,14 +199,14 @@ mod tests { } fn signal(&self, name: &str) { - assert!( - Command::new("kill") - .arg(format!("-{name}")) - .arg(self.child.id().to_string()) - .status() - .expect("send signal") - .success() - ); + let spawning = reserve_or_spawn(); + let status = Command::new("kill") + .arg(format!("-{name}")) + .arg(self.child.id().to_string()) + .status() + .expect("send signal"); + drop(spawning); + assert!(status.success()); } fn socket(&self) -> io::Result { @@ -1660,7 +1686,7 @@ mod tests { (None, None, Some("RECORDS_SENTINEL"), "REQUEST_RECORDS"), (None, None, Some("FALSE"), "REQUEST_RECORDS"), ] { - let addr = address(); + let (addr, _addr_held) = reserved_address(); let mut launch = command("diagnostics", &roots, addr); if let Some(value) = networks { launch.env("EDGEZERO__ADAPTER__TRUSTED_PROXY_CIDRS", value); @@ -1701,7 +1727,7 @@ mod tests { ), ("EDGEZERO__LOGGING__REQUEST_RECORDS", "REQUEST_RECORDS"), ] { - let addr = address(); + let (addr, _addr_held) = reserved_address(); let mut launch = command("diagnostics", &roots, addr); let mut raw = b"NON_UNICODE_SENTINEL".to_vec(); raw.push(0xff); @@ -1844,7 +1870,7 @@ mod tests { OsString::from_vec(vec![0xff]), ] { let roots = Roots::new(); - let addr = address(); + let (addr, _addr_held) = reserved_address(); let mut launch = command(mode, &roots, addr); launch.env("EDGEZERO__ADAPTER__JSON_BODY_LIMIT_BYTES", value); let log = failed(launch, addr, "JSON_BODY_LIMIT_BYTES"); @@ -1948,7 +1974,7 @@ mod tests { #[test] fn stop_during_startup_never_binds_or_becomes_ready() { let roots = Roots::new(); - let addr = address(); + let (addr, _addr_held) = reserved_address(); let mut server = Server::spawn(command("starting", &roots, addr), addr); server.wait_line("INITIALIZING"); TcpStream::connect_timeout(&addr, Duration::from_millis(50)) @@ -1982,7 +2008,7 @@ mod tests { "settings", ), ] { - let addr = address(); + let (addr, _addr_held) = reserved_address(); let mut cmd = command("initialized", &roots, addr); cmd.env(key, value); failed(cmd, addr, category); @@ -1991,15 +2017,15 @@ mod tests { ("hook-error", "application configuration"), ("initializer-error", "initializer"), ] { - let addr = address(); + let (addr, _addr_held) = reserved_address(); failed(command(mode, &roots, addr), addr, category); } - let addr = address(); + let (addr, _addr_held) = reserved_address(); let mut cmd = command("initialized", &roots, addr); cmd.env_remove("EDGEZERO__ADAPTER__CONFIG_DIR"); failed(cmd, addr, "CONFIG_DIR"); let missing = roots.cwd.path().join("missing-data-SENTINEL"); - let missing_addr = address(); + let (missing_addr, _missing_addr_held) = reserved_address(); let mut missing_cmd = command("initialized", &roots, missing_addr); missing_cmd.env("EDGEZERO__ADAPTER__DATA_DIR", &missing); failed(missing_cmd, missing_addr, "DATA_DIR"); @@ -2022,10 +2048,10 @@ mod tests { let roots = Roots::new(); let file = roots.config.path().join("local-config-settings.json"); fs::remove_file(&file).expect("missing file fixture"); - let addr = address(); + let (addr, _addr_held) = reserved_address(); failed(command("initialized", &roots, addr), addr, "missing"); fs::create_dir_all(&file).expect("non-regular fixture"); - let directory_addr = address(); + let (directory_addr, _directory_addr_held) = reserved_address(); failed( command("initialized", &roots, directory_addr), directory_addr, @@ -2033,7 +2059,7 @@ mod tests { ); fs::remove_dir(&file).expect("remove fixture directory"); roots.map(&json!({"settings": {"CONFIG_SENTINEL": true}})); - let malformed_addr = address(); + let (malformed_addr, _malformed_addr_held) = reserved_address(); failed( command("initialized", &roots, malformed_addr), malformed_addr, @@ -2052,7 +2078,7 @@ mod tests { ), ] { roots.map(&map); - let typed_addr = address(); + let (typed_addr, _typed_addr_held) = reserved_address(); failed( command("initialized", &roots, typed_addr), typed_addr, @@ -2063,7 +2089,7 @@ mod tests { serde_json::from_str(&blob("valid-greeting", "fixture_token")).expect("envelope"); envelope["data"]["greeting"] = json!("CHANGED_CONFIG_SENTINEL"); roots.map(&json!({"settings": serde_json::to_string(&envelope).expect("envelope")})); - let integrity_addr = address(); + let (integrity_addr, _integrity_addr_held) = reserved_address(); failed( command("initialized", &roots, integrity_addr), integrity_addr, @@ -2137,7 +2163,7 @@ mod tests { .expect("snapshot") .ends_with("selected-greeting") ); - let second_addr = address(); + let (second_addr, _second_addr_held) = reserved_address(); failed( command("initialized", &roots, second_addr), second_addr, @@ -2289,7 +2315,7 @@ mod tests { #[cfg(not(feature = "geo-cloudfront"))] fn geo_hosting_rejects_unavailable_cloudfront() { let cwd = tempfile::tempdir().expect("unrelated cwd"); - let addr = address(); + let (addr, _addr_held) = reserved_address(); let mut server = Server::spawn(geo_command("geo-cloudfront", &cwd, addr), addr); assert!(!server.wait().success()); let log = server.log(); @@ -2365,7 +2391,7 @@ mod tests { #[test] fn geo_hosting_invalid_policy_never_initializes_or_listens() { let cwd = tempfile::tempdir().expect("unrelated cwd"); - let addr = address(); + let (addr, _addr_held) = reserved_address(); let mut server = Server::spawn(geo_command("geo-invalid-policy", &cwd, addr), addr); assert!(!server.wait().success()); assert!(!server.log().contains("INITIALIZED")); From d86a5d326c92607c33d627533368c47c1a6c8184 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 20 Nov 2026 19:28:17 +0530 Subject: [PATCH 5/5] test: close remaining gaps in the macOS hosting and timeout fixtures production_hosting: client streams from Server::socket and raw_request also take the reservation/spawn lock, so a concurrently spawned fixture cannot inherit one and keep the connection open after the test drops it. Because a held reservation always refuses connections, "nothing answers" checks now also assert that the server never reached its listener bind. Secrets Manager: the holding fixture drains every queued connection after the child exits before counting attempts, so a starved fixture thread cannot undercount them. --- .../tests/production_hosting.rs | 29 ++++++++++++++++--- .../edgezero-store-aws/src/secrets_manager.rs | 8 ++++- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/crates/edgezero-adapter-axum/tests/production_hosting.rs b/crates/edgezero-adapter-axum/tests/production_hosting.rs index 1c50aaef9..2bf70337f 100644 --- a/crates/edgezero-adapter-axum/tests/production_hosting.rs +++ b/crates/edgezero-adapter-axum/tests/production_hosting.rs @@ -67,8 +67,9 @@ mod tests { /// macOS marks a new socket close-on-exec only after `socket()` returns, so a fixture /// process spawned by a parallel test in that gap inherits it. An inherited reservation - /// stays bound in that process, and the fixture server that later binds the port fails - /// with `AddrInUse`. Reservations and spawns hold this lock so they never overlap. + /// stays bound in that process, so the fixture server that later binds the port fails with + /// `AddrInUse`, and an inherited client stream stays open after the test drops it. Port + /// reservations, client streams and spawns hold this lock so they never overlap. static RESERVE_OR_SPAWN: Mutex<()> = Mutex::new(()); fn reserve_or_spawn() -> MutexGuard<'static, ()> { @@ -77,6 +78,12 @@ mod tests { .unwrap_or_else(PoisonError::into_inner) } + /// Opens a client stream without letting a concurrently spawned fixture inherit it. + fn connect(addr: SocketAddr, timeout: Duration) -> io::Result { + let _connecting = reserve_or_spawn(); + TcpStream::connect_timeout(&addr, timeout) + } + /// Reserves a loopback port for a fixture server that binds it itself. fn address() -> SocketAddr { reserved_address().0 @@ -85,6 +92,8 @@ mod tests { /// Reserves a loopback port and keeps it bound, but not listening, until the holder drops. /// No parallel test can bind the port meanwhile and nothing accepts a connection to it, so /// a "nothing answers" check cannot reach a server that reused a released reservation. + /// While held, the check cannot fail by itself: callers also assert that the server never + /// reached its listener bind. fn reserved_address() -> (SocketAddr, TcpSocket) { let _reserving = reserve_or_spawn(); let socket = TcpSocket::new_v4().expect("port reservation socket"); @@ -210,7 +219,7 @@ mod tests { } fn socket(&self) -> io::Result { - let socket = TcpStream::connect_timeout(&self.addr, Duration::from_millis(50))?; + let socket = connect(self.addr, Duration::from_millis(50))?; socket.set_read_timeout(Some(WAIT))?; socket.set_write_timeout(Some(WAIT))?; Ok(socket) @@ -272,6 +281,10 @@ mod tests { "startup diagnostic leaked fixture contents: {log}" ); assert!(!log.contains("INITIALIZED")); + assert!( + !log.contains("startup bind"), + "failed startup reached bind: {log}" + ); TcpStream::connect_timeout(&addr, Duration::from_millis(50)) .expect_err("failed startup never accepts connections"); log @@ -403,7 +416,7 @@ mod tests { } fn raw_request(addr: SocketAddr, request: &str) -> String { - let mut socket = TcpStream::connect_timeout(&addr, WAIT).expect("wire connection"); + let mut socket = connect(addr, WAIT).expect("wire connection"); socket.set_read_timeout(Some(WAIT)).expect("read timeout"); socket.set_write_timeout(Some(WAIT)).expect("write timeout"); socket.write_all(request.as_bytes()).expect("wire request"); @@ -2323,6 +2336,10 @@ mod tests { assert!(!log.contains("INITIALIZED")); assert!(!log.contains("HOOK_ENTERED")); assert!(!log.contains("ready")); + assert!( + !log.contains("startup bind"), + "unavailable implementation reached bind" + ); TcpStream::connect_timeout(&addr, Duration::from_millis(50)) .expect_err("unavailable implementation never binds"); assert_eq!(fs::read_dir(cwd.path()).expect("unrelated cwd").count(), 0); @@ -2397,6 +2414,10 @@ mod tests { assert!(!server.log().contains("INITIALIZED")); assert!(!server.log().contains("HOOK_ENTERED")); assert!(!server.log().contains("x-origin-verify-fixture")); + assert!( + !server.log().contains("startup bind"), + "invalid policy reached bind" + ); TcpStream::connect_timeout(&addr, Duration::from_millis(50)) .expect_err("invalid policy never binds"); assert_eq!(fs::read_dir(cwd.path()).expect("unrelated cwd").count(), 0); diff --git a/crates/edgezero-store-aws/src/secrets_manager.rs b/crates/edgezero-store-aws/src/secrets_manager.rs index 4ac8358e2..d1adbefca 100644 --- a/crates/edgezero-store-aws/src/secrets_manager.rs +++ b/crates/edgezero-store-aws/src/secrets_manager.rs @@ -475,13 +475,19 @@ mod tests { .checked_add(Duration::from_secs(30)) .expect("fixture deadline fits the monotonic clock"); let mut held = Vec::new(); - while !worker_stop.load(Ordering::Acquire) { + loop { + // Read the flag before accepting: once the child has exited, every connection it + // opened is already queued, so draining until `WouldBlock` counts all of them. + let stopping = worker_stop.load(Ordering::Acquire); match accept_blocking(&listener) { Ok((stream, peer)) => { assert!(peer.ip().is_loopback(), "fixture peer must be loopback"); held.push(hold(stream)); } Err(error) if error.kind() == ErrorKind::WouldBlock => { + if stopping { + break; + } assert!(Instant::now() < deadline, "SDK child did not finish"); thread::sleep(Duration::from_millis(5)); }