diff --git a/.github/workflows/kurtosis_assertoor.yml b/.github/workflows/kurtosis_assertoor.yml index d5ad0c553..3c329b636 100644 --- a/.github/workflows/kurtosis_assertoor.yml +++ b/.github/workflows/kurtosis_assertoor.yml @@ -17,6 +17,15 @@ on: description: "ethlambda image to test (default: ghcr.io/lambdaclass/ethlambda:unstable)" required: false type: string + battery: + description: "Which battery to run" + required: false + type: choice + options: + - assertoor + - commit-boost + - both + default: assertoor permissions: contents: read @@ -34,6 +43,13 @@ env: ETHEREUM_PACKAGE_REPO: https://github.com/lambdaclass/ethereum-package.git ETHEREUM_PACKAGE_REF: 8378c1e3030ef4d41306aea9a0ba13d19372d125 ENCLAVE: ethlambda-assertoor + # The Commit-Boost check needs the fork's `--builder` / `--builder-proposals` + # wiring for ethlambda, which the commit above lacks. It has its own pin so + # the base battery stays where it is. Fork PR #22 (that wiring) is merged as + # a15e7df; this commit adds PR #23's fix for the builder's enode bootnodes on + # top. Re-pin to #23's merge commit once it lands; see the README. + COMMIT_BOOST_PACKAGE_REF: 7fa7d1225fe5d975300ecc5584bf28ca6d42d451 + COMMIT_BOOST_ENCLAVE: ethlambda-commit-boost jobs: image: @@ -79,6 +95,9 @@ jobs: assertoor: name: Assertoor needs: image + # Every push and pull request, and every dispatch except one that asked for + # the commit-boost battery alone: `assertoor` and `both` both run this job. + if: github.event_name != 'workflow_dispatch' || inputs.battery != 'commit-boost' runs-on: ubuntu-latest timeout-minutes: 120 outputs: @@ -168,6 +187,89 @@ jobs: if: always() run: kurtosis enclave rm -f "$ENCLAVE" || true + # Dispatch-only for now: main's image does not have the builder flags until + # the builder PR stack (#694 to #705) lands. Once it is on main, drop the + # event condition so the check also runs on push and on pull requests, and at + # the same time add `commit-boost` to `notify`'s `needs` and its failed.txt to + # the Slack text: `notify` reports pushes only, so until then it can never + # see this job. + commit-boost: + name: Commit-Boost + needs: image + if: github.event_name == 'workflow_dispatch' && inputs.battery != 'assertoor' + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - uses: actions/checkout@v6.1.0 + + - name: Free up disk space + uses: $/.github/actions/free-disk + + - name: Install kurtosis + run: | + echo "deb [trusted=yes] https://sdk.kurtosis.com/kurtosis-cli-release-artifacts/ /" \ + | sudo tee /etc/apt/sources.list.d/kurtosis.list + sudo apt-get update + sudo apt-get install -y "kurtosis-cli=$KURTOSIS_VERSION" + kurtosis analytics disable + kurtosis version + + # A clone for the same reason as in the assertoor job. + - name: Fetch ethereum-package at the pinned commit + run: | + git init "$RUNNER_TEMP/ethereum-package" + cd "$RUNNER_TEMP/ethereum-package" + git fetch --depth 1 "$ETHEREUM_PACKAGE_REPO" "$COMMIT_BOOST_PACKAGE_REF" + git checkout --detach FETCH_HEAD + test "$(git rev-parse HEAD)" = "$COMMIT_BOOST_PACKAGE_REF" + + - name: Point the args at the image under test + env: + IMAGE: ${{ needs.image.outputs.image }} + run: | + cp tooling/kurtosis-assertoor/commit_boost_params.yaml "$RUNNER_TEMP/args.yaml" + yq -i ' + (.participants[] | select(.cl_type == "ethlambda") | .cl_image) = strenv(IMAGE) | + (.participants[] | select(.vc_type == "ethlambda") | .vc_image) = strenv(IMAGE) + ' "$RUNNER_TEMP/args.yaml" + cat "$RUNNER_TEMP/args.yaml" + + # pipefail, or the step takes tee's status: a Starlark error that rolled + # back a service would pass here and surface only in the next step. + - name: Start the devnet + run: | + set -o pipefail + kurtosis run "$RUNNER_TEMP/ethereum-package" \ + --enclave "$COMMIT_BOOST_ENCLAVE" --args-file "$RUNNER_TEMP/args.yaml" \ + | tee "$RUNNER_TEMP/kurtosis_output.txt" + + - name: Wait for builder blocks + run: | + tooling/kurtosis-assertoor/scripts/check-builder-blocks.sh \ + "$COMMIT_BOOST_ENCLAVE" 40 "$RUNNER_TEMP/builder-results" + + - name: Dump the enclave + if: always() + run: | + mkdir -p "$RUNNER_TEMP/dump" + cp "$RUNNER_TEMP/args.yaml" "$RUNNER_TEMP/dump/args.yaml" + cp "$RUNNER_TEMP/kurtosis_output.txt" "$RUNNER_TEMP/dump/" || true + cp -r "$RUNNER_TEMP/builder-results" "$RUNNER_TEMP/dump/" || true + cd "$RUNNER_TEMP/dump" + kurtosis enclave dump "$COMMIT_BOOST_ENCLAVE" || true + + - name: Upload the dump and results + if: always() + uses: actions/upload-artifact@v7.0.2 + with: + name: commit-boost-enclave-dump-${{ github.run_id }} + path: ${{ runner.temp }}/dump + if-no-files-found: warn + + - name: Remove the enclave + if: always() + run: kurtosis enclave rm -f "$COMMIT_BOOST_ENCLAVE" || true + notify: name: Post failure to Slack needs: [image, assertoor] diff --git a/tooling/kurtosis-assertoor/README.md b/tooling/kurtosis-assertoor/README.md index 63162928b..7dd76da6a 100644 --- a/tooling/kurtosis-assertoor/README.md +++ b/tooling/kurtosis-assertoor/README.md @@ -68,3 +68,111 @@ field by field and marks a node whose spec differs offline (`invalid node specs: spec mismatch: GloasForkEpoch`); lighthouse reports the far-future value and ethlambda omits the key. The chain itself is healthy (all nodes share a head and finalize), so this is a spec-endpoint gap, not a harness problem. + +## Commit-Boost check + +Starts the same three participants with `mev_type: commit-boost` +(`commit_boost_params.yaml`): a flashbots mev-boost-relay, a reth-rbuilder +builder with its own lighthouse beacon node, and one +[Commit-Boost](https://github.com/Commit-Boost/commit-boost-client) PBS module +per participant, which stands in for MEV-Boost. It also starts spamoor at a +light load, since rbuilder submits no bid for a block without transactions and +a heavy load slows the relay past Commit-Boost's getHeader timeout (see the +comment in the params file). The package gives the ethlambda +beacon node `--builder ` and the ethlambda validator client +`--builder-proposals`. There is no assertoor: the base battery's checks fail on +the spec-endpoint gap above, so `scripts/check-builder-blocks.sh` polls the +ethlambda services' Prometheus metrics instead, for up to the timeout it is given. + +| Service | Metric | Shows | +|---|---|---| +| `cl-3-ethlambda-geth` | `lean_beacon_builder_registrations_total{outcome="forwarded"}` >= 1 | Registrations reach the builder through the beacon node | +| `cl-3-ethlambda-geth` | `lean_beacon_builder_payload_source_total{source="builder"}` >= 1 | The beacon node's own `produceBlockV3` took a builder bid | +| `vc-3-geth-ethlambda` | `ethlambda_validator_builder_registrations_sent_total` >= 1 | The validator client registers its validators | +| `vc-3-geth-ethlambda` | `ethlambda_validator_blinded_blocks_proposed_total` >= 1 | It signs and publishes blinded blocks through ethlambda's beacon node | +| `vc-2-geth-lighthouse-ethlambda` | `ethlambda_validator_blinded_blocks_proposed_total` >= 1 | It does the same through a third-party beacon node (lighthouse) | + +Passing therefore covers the registration path, a builder block from ethlambda's +own beacon node, and ethlambda's validator client proposing blinded blocks +through lighthouse. The script writes every `lean_beacon_builder_*` and +`ethlambda_validator_*` blinded or registration series of the three services to +`/.prom`, pass or fail, and rewrites `failed.txt` with the +unmet conditions after every evaluation, so a cancelled run still says what was +missing. It stops at once, with exit status 2, if a service has no `metrics` +port: the service names depend on ethereum-package's naming and on the +participant order, and a mismatch is a setup problem, not a slow builder. + +The relay, builder, PBS and spamoor images are pinned by digest in +`commit_boost_params.yaml` (the package's defaults are floating tags). + +### Run it locally + +The image needs the builder support, so build one from a checkout of the +builder PR stack (#694 to #705). From that checkout's root, `make docker-build +DOCKER_TAG=` tags it `ghcr.io/lambdaclass/ethlambda:`; set that as the +`cl_image` and `vc_image` of the ethlambda entries in `commit_boost_params.yaml`. +Kurtosis uses an image that exists in the local Docker daemon and only pulls a +missing one (`kurtosis run --image-download`, default `missing`), so no setting +is needed; do not pass `--image-download always`. From the repository root: + +```bash +git clone https://github.com/lambdaclass/ethereum-package.git /tmp/ethereum-package +git -C /tmp/ethereum-package checkout 7fa7d1225fe5d975300ecc5584bf28ca6d42d451 + +kurtosis run /tmp/ethereum-package --enclave ethlambda-commit-boost \ + --args-file tooling/kurtosis-assertoor/commit_boost_params.yaml + +tooling/kurtosis-assertoor/scripts/check-builder-blocks.sh \ + ethlambda-commit-boost 40 /tmp/builder-results + +kurtosis enclave rm -f ethlambda-commit-boost +``` + +### In CI + +The `commit-boost` job of `.github/workflows/kurtosis_assertoor.yml` runs on +manual dispatch only. Pick `battery: commit-boost` (or `both`) and give `image` +the tag of an image built from the builder PR stack; the default `assertoor` +runs only the base battery, as before. Its pin is `COMMIT_BOOST_PACKAGE_REF`, +separate from `ETHEREUM_PACKAGE_REF`, and it uploads `commit-boost-enclave-dump-`. + +It is dispatch-only because the image that pushes and pull requests test +(`sha-<7>` of `main`, or `unstable`) does not have `--builder` or +`--builder-proposals` until the builder PR stack is on `main`, so the check +would fail there. Once it lands, drop the event condition on the job to run it +on push and on pull requests. + +### The fork pin + +lambdaclass/ethereum-package#22 added ethlambda to `apply_external_builder_flags` +(`--builder=` for the beacon node, `--builder-proposals` for the validator +client) and is merged (`a15e7df`). `COMMIT_BOOST_PACKAGE_REF` is +`7fa7d1225fe5d975300ecc5584bf28ca6d42d451`, from +lambdaclass/ethereum-package#23, which adds the builder's enode bootnode fix on +top of it. Re-pin to #23's merge commit once it lands: a squash-merge leaves the +current commit unreachable from the fork's default branch, so it may be garbage +collected. + +### Upstream workarounds + +A local run found two breakages in images and packages we do not control: + +| Breakage | Worked around in | +|---|---| +| `ethpandaops/reth-rbuilder:develop` (newest image, 2026-07-17) parses only enode URLs as bootnodes, and the package has passed ENRs since its 08-05 refactor, so the builder's execution client refused them | fork PR #23, through the `COMMIT_BOOST_PACKAGE_REF` pin | +| Commit-Boost's PBS requires `SECONDS_PER_SLOT` in the chain config, and the genesis generator now writes only `SLOT_DURATION_MS` | `mev_params.commit_boost_config` in `commit_boost_params.yaml`, which gives the chain explicitly (`chain_id` has to be a hex string) | + +Drop each once the upstream is fixed. The explicit chain must change with the +network parameters. + +### Startup noise + +The Commit-Boost PBS modules start after the consensus clients, so at startup +ethlambda's beacon node logs "Builder is not answering" and its first +registration batch fails. The validator client re-sends registrations every +epoch, so this clears within an epoch and is not a failure. + +From epoch 4 until the chain first finalizes, the beacon node builds locally and +counts `lean_beacon_builder_fallback_total{reason="no_finality"}`: like +Lighthouse, it skips the builder when the proposal is more than 3 epochs past +the finalized checkpoint, and the genesis checkpoint counts as one. diff --git a/tooling/kurtosis-assertoor/commit_boost_params.yaml b/tooling/kurtosis-assertoor/commit_boost_params.yaml new file mode 100644 index 000000000..5022ec4ae --- /dev/null +++ b/tooling/kurtosis-assertoor/commit_boost_params.yaml @@ -0,0 +1,123 @@ +# The Commit-Boost check for ethlambda: the same three participants as +# network_params.yaml, plus `mev_type: commit-boost`, so ethlambda's beacon node +# and validator client build blocks through a Commit-Boost PBS module. There is +# no assertoor: the check is scripts/check-builder-blocks.sh, which reads the +# builder metrics of every ethlambda service. See README.md ("Commit-Boost +# check") for the steps. The package comes from a clone of +# lambdaclass/ethereum-package at the commit pinned in +# .github/workflows/kurtosis_assertoor.yml (COMMIT_BOOST_PACKAGE_REF). +# +# Participants, in the order ethereum-package launches them (the order fixes the +# service names check-builder-blocks.sh reads): +# 1. lighthouse + geth, lighthouse validator. It has to come first: the first +# participant is the network's bootnode, and ethlambda does not report an +# ENR over the beacon API. +# 2. lighthouse + geth, validated by the ethlambda validator client, so the +# validator client proposes blinded blocks through a beacon node that is +# not ethlambda. +# 3. ethlambda beacon node + geth, validated by the ethlambda validator client. +# +# `mev_type: commit-boost` adds a flashbots mev-boost-relay, a reth-rbuilder +# builder (with its own lighthouse beacon node) and one Commit-Boost PBS module +# per participant. The package passes each ethlambda beacon node `--builder` and +# each ethlambda validator client `--builder-proposals`, which only an image +# built with the builder support understands. The workflow rewrites the +# `cl_image` and `vc_image` of the ethlambda entries (selected by type, not +# position) to the image under test; the values below are what a local run uses. +# If the battery ever changes `network_id`, the preset or the fork epochs, the +# explicit Commit-Boost chain under `mev_params` must change with it. +participants: + - el_type: geth + cl_type: lighthouse + - el_type: geth + cl_type: lighthouse + vc_type: ethlambda + vc_image: ghcr.io/lambdaclass/ethlambda:unstable + # Genesis keys in a fresh directory: nothing has ever signed with them, so + # registering them without history is safe. Without it the validator + # client refuses to start (no slashing-protection record), as in + # network_params.yaml. + vc_extra_params: ["--init-slashing-protection"] + - el_type: geth + cl_type: ethlambda + cl_image: ghcr.io/lambdaclass/ethlambda:unstable + vc_type: ethlambda + vc_image: ghcr.io/lambdaclass/ethlambda:unstable + vc_extra_params: ["--init-slashing-protection"] +network_params: + # ethlambda only supports the mainnet preset: 32-slot epochs of 12 s. + preset: mainnet + # Fulu is the last fork ethlambda processes; gloas stays unscheduled. + electra_fork_epoch: 0 + fulu_fork_epoch: 0 +mev_type: commit-boost +# rbuilder submits nothing for a block with no transactions, so without a +# spammer the relay never holds a bid and every getHeader answers 204. The load +# is kept light on purpose: under spamoor's defaults (a chain-load group plus +# 200 transfers and 6 blob transactions a slot) the relay's getHeader slowed +# past Commit-Boost's 950 ms timeout on a local run, so most proposals got no +# bid. The package also always adds 100 Uniswap swaps a slot, sent to the +# builder's own execution client; that is not configurable. The blob spammer +# stays so bids carry blob commitments. +additional_services: + - spamoor +spamoor_params: + # ethpandaops/spamoor:master on 2026-10-27 + image: ethpandaops/spamoor@sha256:e890785f5814218bbcb544c495820ff7b02dd554e6d714f82849a0ac8de680c5 + start_chainload: false + spammers: + - name: "EOA transfers" + description: "10 type-2 transfers a slot" + scenario: "eoatx" + config: + throughput: 10 + max_pending: 20 + max_wallets: 10 + base_fee: 20 + - name: "Blob transactions" + description: "1 type-4 transaction a slot with 1 sidecar" + scenario: "blob-combined" + config: + throughput: 1 + sidecars: 1 + max_pending: 2 + max_wallets: 2 + base_fee: 20 + blob_fee: 20 +# The package defaults are floating tags (`main`, `develop`, `latest`), so a +# run would change under us. These pin what the tags pointed to on 2026-10-26; +# bump them deliberately. To bump one, read the tag's current digest from the +# `Digest:` line of `docker buildx imagetools inspect :`, and update +# the date here with it. +mev_params: + # ethpandaops/mev-boost-relay:main + mev_relay_image: ethpandaops/mev-boost-relay@sha256:3600f04fcaf0bbb231f4737470f094fdf2314db54b4853197fa369aea90f8b19 + # ethpandaops/reth-rbuilder:develop + mev_builder_image: ethpandaops/reth-rbuilder@sha256:916ba84da2690e3e368f57da5b0c73bfd919b452de8b95f64895846e0f3a8308 + # ghcr.io/commit-boost/pbs:latest + mev_boost_image: ghcr.io/commit-boost/pbs@sha256:8612c03681d390ab7d0154f5db1581584cd265563b723947da90cd00ba7ca415 + # Commit-Boost reads the chain from the package's config.yaml by default, + # and its parser requires SECONDS_PER_SLOT, which the genesis generator no + # longer writes (it writes SLOT_DURATION_MS). So the chain is given + # explicitly: kurtosis's defaults for this network (12 s slots, genesis fork + # version 0x10000038, fulu from genesis, chain id 3151908 = 0x301824). + # chain_id must be a hex string; a bare integer does not parse. The rest is + # the package's own template (static_files/mev/commit-boost/cb-config.toml.tmpl). + commit_boost_config: | + chain = { genesis_time_secs = {{ .Timestamp }}, slot_time_secs = 12, genesis_fork_version = "0x10000038", fulu_fork_slot = 0, chain_id = "0x301824" } + + [pbs] + host = "0.0.0.0" + port = {{ .Port }} + + {{ range $index, $relay := .Relays }} + [[relays]] + id = "mev_relay_{{$index}}" + url = "{{ $relay }}" + {{- end }} + + [logs.stdout] + level = "debug" + + [logs.file] + enabled=false diff --git a/tooling/kurtosis-assertoor/scripts/check-builder-blocks.sh b/tooling/kurtosis-assertoor/scripts/check-builder-blocks.sh new file mode 100755 index 000000000..14bd6dd35 --- /dev/null +++ b/tooling/kurtosis-assertoor/scripts/check-builder-blocks.sh @@ -0,0 +1,188 @@ +#!/usr/bin/env bash +# +# Waits until ethlambda has built blocks through the builder (Commit-Boost) and +# fails if that does not happen in time. +# +# Usage: check-builder-blocks.sh +# +# Reads Prometheus metrics through `kurtosis port print +# metrics`, and succeeds once every condition below holds: +# +# ethlambda beacon node (participant 3) +# lean_beacon_builder_registrations_total{outcome="forwarded"} >= 1 +# lean_beacon_builder_payload_source_total{source="builder"} >= 1 +# ethlambda validator client of participant 3 +# ethlambda_validator_builder_registrations_sent_total >= 1 +# ethlambda_validator_blinded_blocks_proposed_total >= 1 +# ethlambda validator client of participant 2 (on a lighthouse beacon node) +# ethlambda_validator_blinded_blocks_proposed_total >= 1 +# +# After every evaluation, writes the conditions still unmet to +# /failed.txt, so a run cut short still says what was missing. On +# every exit, writes each ethlambda service's builder metric lines to +# /.prom. +# +# Exits 1 if a condition is unmet at the deadline, and 2 at once if the check +# cannot run: kurtosis or curl is missing, or a service has no `metrics` port +# (ethereum-package renamed it, or the participants were reordered). + +set -euo pipefail + +ENCLAVE="${1:?enclave name required}" +TIMEOUT_MIN="${2:?timeout in minutes required}" +OUT="${3:?output directory required}" + +# Service names follow ethereum-package: cl---, and vc--- +# with - appended when the validator client differs from the beacon node. +CL3="cl-3-ethlambda-geth" +VC2="vc-2-geth-lighthouse-ethlambda" +VC3="vc-3-geth-ethlambda" +SERVICES=("$CL3" "$VC2" "$VC3") + +# service|series (name, with its labels when the metric is labelled)|description +CONDITIONS=( + "$CL3|lean_beacon_builder_registrations_total{outcome=\"forwarded\"}|beacon node forwarded a registration" + "$CL3|lean_beacon_builder_payload_source_total{source=\"builder\"}|beacon node served a builder payload" + "$VC3|ethlambda_validator_builder_registrations_sent_total|validator client of participant 3 sent a registration" + "$VC3|ethlambda_validator_blinded_blocks_proposed_total|validator client of participant 3 proposed a blinded block" + "$VC2|ethlambda_validator_blinded_blocks_proposed_total|validator client of participant 2 proposed a blinded block" +) + +mkdir -p "$OUT" +: > "$OUT/failed.txt" + +for tool in kurtosis curl; do + if ! command -v "$tool" >/dev/null 2>&1; then + echo "::error::$tool is not on PATH" + echo "setup $tool is not on PATH" > "$OUT/failed.txt" + exit 2 + fi +done + +# One evaluation's scrapes, a file per service. +SNAPSHOT=$(mktemp -d) + +# Prints a service's metrics address as host:port. `kurtosis port print` answers +# `://:`, or `:` for a port with +# no application protocol; anything around the host and port is dropped, and +# curl's default (http) is what the metrics port speaks. +metrics_address() { + local addr + addr=$(kurtosis port print "$ENCLAVE" "$1" metrics) || return + addr="${addr#*://}" + printf '%s\n' "${addr%%/*}" +} + +# Prints a service's metrics, or nothing while it is unreachable. The address is +# looked up on every scrape, since a restarted service can come back on another +# host port. +scrape() { + local addr + addr=$(metrics_address "$1" 2>/dev/null) || return 0 + curl -sf --max-time 10 "$addr/metrics" 2>/dev/null || true +} + +# Scrapes every service once, so one evaluation reads each service as of one +# moment rather than once per condition. +take_snapshot() { + local service + for service in "${SERVICES[@]}"; do + scrape "$service" > "$SNAPSHOT/$service.prom" + done +} + +# Prints the sum of a series in the given exposition file, 0 if absent. A +# selector with labels matches that exact `name{labels}` token; a bare name +# matches the metric under any labels. Every series checked is a counter, so the +# sum is truncated to an integer for the shell's `-ge`: truncating rather than +# rounding keeps a value below 1 from meeting a `>= 1` condition, and `%.0f` +# prints a large or exponent-notation value (`1e+06`) whole, which `%d` need not. +value_of() { + awk -v sel="$2" ' + index(sel, "{") { if ($1 == sel) sum += $NF; next } + { split($1, p, "{"); if (p[1] == sel) sum += $NF } + END { printf "%.0f\n", int(sum) } + ' "$1" +} + +# Writes each service's builder-related series to /.prom. +dump_metrics() { + local service + for service in "${SERVICES[@]}"; do + scrape "$service" \ + | grep -E '^(lean_beacon_builder_|ethlambda_validator_.*(blinded|registration))' \ + > "$OUT/$service.prom" || true + done +} + +# Prints one line per condition and returns 1 if any is unmet. +evaluate() { + local entry svc series desc value unmet=0 + take_snapshot + for entry in "${CONDITIONS[@]}"; do + IFS='|' read -r svc series desc <<<"$entry" + if [ ! -s "$SNAPSHOT/$svc.prom" ]; then + echo "unmet $svc $series: no metrics, the service did not answer ($desc)" + unmet=1 + continue + fi + value=$(value_of "$SNAPSHOT/$svc.prom" "$series") + if [ "$value" -ge 1 ]; then + echo "ok $svc $series = $value ($desc)" + else + echo "unmet $svc $series = $value ($desc)" + unmet=1 + fi + done + return "$unmet" +} + +finish() { + dump_metrics + rm -rf "$SNAPSHOT" + echo "Builder metrics written to $OUT" +} +trap finish EXIT + +# `kurtosis run` has started every service by the time this runs, so each one +# must already have a metrics port. A service the package named differently +# would otherwise read as "not met yet" until the deadline, not as the setup +# problem it is. +missing=0 +for service in "${SERVICES[@]}"; do + if ! err=$(metrics_address "$service" 2>&1 >/dev/null); then + echo "::error::$service has no metrics port in enclave $ENCLAVE: ${err:-kurtosis port print failed}" + echo "setup $service has no metrics port in enclave $ENCLAVE" >> "$OUT/failed.txt" + missing=1 + fi +done +if [ "$missing" -ne 0 ]; then + echo "Check the service names above against \`kurtosis enclave inspect $ENCLAVE\`." + exit 2 +fi + +deadline=$((SECONDS + TIMEOUT_MIN * 60)) +status=1 +while :; do + summary=$(evaluate) && status=0 + grep '^unmet' <<<"$summary" > "$OUT/failed.txt" || true + if [ "$status" -eq 0 ]; then + break + fi + echo "[$(date -u +%H:%M:%S)] builder conditions not met yet:" + sed 's/^/ /' "$OUT/failed.txt" + if [ "$SECONDS" -ge "$deadline" ]; then + echo "::error::timed out after ${TIMEOUT_MIN} minutes waiting for builder blocks" + break + fi + sleep 30 +done + +echo +echo "$summary" +if [ "$status" -ne 0 ]; then + echo "Failed:" + cat "$OUT/failed.txt" + exit 1 +fi +echo "ethlambda built blocks through the builder"