From e345428433536f35b74f439095f7e282c320db56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:37:53 -0300 Subject: [PATCH 01/11] ci: add a Commit-Boost check to the kurtosis battery The builder PR stack (#694 to #705) lets ethlambda's beacon node and validator client build blocks through a MEV-Boost compatible builder, but nothing exercises it against a real relay and builder. Commit-Boost's PBS module is a drop-in MEV-Boost replacement, so a devnet with `mev_type: commit-boost` covers the whole path. The check reads the ethlambda services' builder metrics instead of running assertoor, because the base battery's assertoor tests currently fail on an unrelated spec-endpoint gap. It passes once registrations are forwarded, the ethlambda beacon node serves a builder payload, and the ethlambda validator client proposes blinded blocks both through that node and through lighthouse. The new `commit-boost` job runs on manual dispatch only: main's image has no `--builder` or `--builder-proposals` until the stack lands. It has its own ethereum-package pin, which needs the fork's builder-flags branch pushed. The relay, builder and PBS images are pinned by digest since the package defaults are floating tags. --- .github/workflows/kurtosis_assertoor.yml | 93 +++++++++++++ tooling/kurtosis-assertoor/README.md | 73 +++++++++++ .../commit_boost_params.yaml | 55 ++++++++ .../scripts/check-builder-blocks.sh | 123 ++++++++++++++++++ 4 files changed, 344 insertions(+) create mode 100644 tooling/kurtosis-assertoor/commit_boost_params.yaml create mode 100755 tooling/kurtosis-assertoor/scripts/check-builder-blocks.sh diff --git a/.github/workflows/kurtosis_assertoor.yml b/.github/workflows/kurtosis_assertoor.yml index 2e53fc8e4..8fec34608 100644 --- a/.github/workflows/kurtosis_assertoor.yml +++ b/.github/workflows/kurtosis_assertoor.yml @@ -17,6 +17,15 @@ on: description: "ethlambda image to test (default: ghcr.io/lambdaclass/ethlambda:unstable)" required: false type: string + battery: + description: "Which battery to run" + required: false + type: choice + options: + - assertoor + - commit-boost + - both + default: assertoor permissions: contents: read @@ -34,6 +43,12 @@ env: ETHEREUM_PACKAGE_REPO: https://github.com/lambdaclass/ethereum-package.git ETHEREUM_PACKAGE_REF: 67cb0e02dc0bd49ec9461cbf2ef7ac4a3232755f ENCLAVE: ethlambda-assertoor + # The Commit-Boost check needs the fork's `--builder` / `--builder-proposals` + # wiring for ethlambda, which the commit above lacks. It has its own pin so + # the base battery stays where it is. This commit is on the fork branch + # feat/ethlambda-builder-flags; see the README before changing it. + COMMIT_BOOST_PACKAGE_REF: 37fe57ea309b954361c0f33ac6efc570313ffaf6 + COMMIT_BOOST_ENCLAVE: ethlambda-commit-boost jobs: image: @@ -79,6 +94,7 @@ jobs: assertoor: name: Assertoor needs: image + if: github.event_name != 'workflow_dispatch' || inputs.battery != 'commit-boost' runs-on: ubuntu-latest timeout-minutes: 120 outputs: @@ -168,6 +184,83 @@ jobs: if: always() run: kurtosis enclave rm -f "$ENCLAVE" || true + # Dispatch-only for now: main's image does not have the builder flags until + # the builder PR stack (#694 to #705) lands. Once it is on main, drop the + # event condition so the check also runs on push and on pull requests. + commit-boost: + name: Commit-Boost + needs: image + if: github.event_name == 'workflow_dispatch' && inputs.battery != 'assertoor' + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - uses: actions/checkout@v6 + + - name: Free up disk space + uses: ./.github/actions/free-disk + + - name: Install kurtosis + run: | + echo "deb [trusted=yes] https://sdk.kurtosis.com/kurtosis-cli-release-artifacts/ /" \ + | sudo tee /etc/apt/sources.list.d/kurtosis.list + sudo apt-get update + sudo apt-get install -y "kurtosis-cli=$KURTOSIS_VERSION" + kurtosis analytics disable + kurtosis version + + # A clone for the same reason as in the assertoor job. + - name: Fetch ethereum-package at the pinned commit + run: | + git init "$RUNNER_TEMP/ethereum-package" + cd "$RUNNER_TEMP/ethereum-package" + git fetch --depth 1 "$ETHEREUM_PACKAGE_REPO" "$COMMIT_BOOST_PACKAGE_REF" + git checkout --detach FETCH_HEAD + test "$(git rev-parse HEAD)" = "$COMMIT_BOOST_PACKAGE_REF" + + - name: Point the args at the image under test + env: + IMAGE: ${{ needs.image.outputs.image }} + run: | + cp tooling/kurtosis-assertoor/commit_boost_params.yaml "$RUNNER_TEMP/args.yaml" + yq -i ' + (.participants[] | select(.cl_type == "ethlambda") | .cl_image) = strenv(IMAGE) | + (.participants[] | select(.vc_type == "ethlambda") | .vc_image) = strenv(IMAGE) + ' "$RUNNER_TEMP/args.yaml" + cat "$RUNNER_TEMP/args.yaml" + + - name: Start the devnet + run: | + kurtosis run "$RUNNER_TEMP/ethereum-package" \ + --enclave "$COMMIT_BOOST_ENCLAVE" --args-file "$RUNNER_TEMP/args.yaml" \ + | tee "$RUNNER_TEMP/kurtosis_output.txt" + + - name: Wait for builder blocks + run: | + tooling/kurtosis-assertoor/scripts/check-builder-blocks.sh \ + "$COMMIT_BOOST_ENCLAVE" 40 "$RUNNER_TEMP/builder-results" + + - name: Dump the enclave + if: always() + run: | + mkdir -p "$RUNNER_TEMP/dump" + cp "$RUNNER_TEMP/args.yaml" "$RUNNER_TEMP/dump/args.yaml" + cp "$RUNNER_TEMP/kurtosis_output.txt" "$RUNNER_TEMP/dump/" || true + cp -r "$RUNNER_TEMP/builder-results" "$RUNNER_TEMP/dump/" || true + cd "$RUNNER_TEMP/dump" + kurtosis enclave dump "$COMMIT_BOOST_ENCLAVE" || true + + - name: Upload the dump and results + if: always() + uses: actions/upload-artifact@v7 + with: + name: commit-boost-enclave-dump-${{ github.run_id }} + path: ${{ runner.temp }}/dump + if-no-files-found: warn + + - name: Remove the enclave + if: always() + run: kurtosis enclave rm -f "$COMMIT_BOOST_ENCLAVE" || true + notify: name: Post failure to Slack needs: [image, assertoor] diff --git a/tooling/kurtosis-assertoor/README.md b/tooling/kurtosis-assertoor/README.md index 84faa773a..153c2e4ec 100644 --- a/tooling/kurtosis-assertoor/README.md +++ b/tooling/kurtosis-assertoor/README.md @@ -68,3 +68,76 @@ field by field and marks a node whose spec differs offline (`invalid node specs: spec mismatch: GloasForkEpoch`); lighthouse reports the far-future value and ethlambda omits the key. The chain itself is healthy (all nodes share a head and finalize), so this is a spec-endpoint gap, not a harness problem. + +## Commit-Boost check + +Starts the same three participants with `mev_type: commit-boost` +(`commit_boost_params.yaml`): a flashbots mev-boost-relay, a reth-rbuilder +builder with its own lighthouse beacon node, and one +[Commit-Boost](https://github.com/Commit-Boost/commit-boost-client) PBS module +per participant, which stands in for MEV-Boost. The package gives the ethlambda +beacon node `--builder ` and the ethlambda validator client +`--builder-proposals`. There is no assertoor: the base battery's checks fail on +the spec-endpoint gap above, so `scripts/check-builder-blocks.sh` polls the +ethlambda services' Prometheus metrics instead, for up to the timeout it is given. + +| Service | Metric | Shows | +|---|---|---| +| `cl-3-ethlambda-geth` | `lean_beacon_builder_registrations_total{outcome="forwarded"}` >= 1 | Registrations reach the builder through the beacon node | +| `cl-3-ethlambda-geth` | `lean_beacon_builder_payload_source_total{source="builder"}` >= 1 | The beacon node's own `produceBlockV3` took a builder bid | +| `vc-3-geth-ethlambda` | `ethlambda_validator_builder_registrations_sent_total` >= 1 | The validator client registers its validators | +| `vc-3-geth-ethlambda` | `ethlambda_validator_blinded_blocks_proposed_total` >= 1 | It signs and publishes blinded blocks through ethlambda's beacon node | +| `vc-2-geth-lighthouse-ethlambda` | `ethlambda_validator_blinded_blocks_proposed_total` >= 1 | It does the same through a third-party beacon node (lighthouse) | + +Passing therefore covers the registration path, a builder block from ethlambda's +own beacon node, and ethlambda's validator client proposing blinded blocks +through lighthouse. The script writes every `lean_beacon_builder_*` and +`ethlambda_validator_*` blinded or registration series of the three services to +`/.prom`, pass or fail, and the unmet conditions to +`failed.txt`. + +The relay, builder and PBS images are pinned by digest in +`commit_boost_params.yaml` (the package's defaults are floating tags). + +### Run it locally + +The image needs the builder support, so build one from the builder PR stack +(#694 to #705) and set it as the `cl_image` and `vc_image` of the ethlambda +entries in `commit_boost_params.yaml`. From the repository root: + +```bash +git clone https://github.com/lambdaclass/ethereum-package.git /tmp/ethereum-package +git -C /tmp/ethereum-package checkout 37fe57ea309b954361c0f33ac6efc570313ffaf6 + +kurtosis run /tmp/ethereum-package --enclave ethlambda-commit-boost \ + --args-file tooling/kurtosis-assertoor/commit_boost_params.yaml + +tooling/kurtosis-assertoor/scripts/check-builder-blocks.sh \ + ethlambda-commit-boost 40 /tmp/builder-results + +kurtosis enclave rm -f ethlambda-commit-boost +``` + +### In CI + +The `commit-boost` job of `.github/workflows/kurtosis_assertoor.yml` runs on +manual dispatch only. Pick `battery: commit-boost` (or `both`) and give `image` +the tag of an image built from the builder PR stack; the default `assertoor` +runs only the base battery, as before. Its pin is `COMMIT_BOOST_PACKAGE_REF`, +separate from `ETHEREUM_PACKAGE_REF`, and it uploads `commit-boost-enclave-dump-`. + +It is dispatch-only because the image that pushes and pull requests test +(`sha-<7>` of `main`, or `unstable`) does not have `--builder` or +`--builder-proposals` until the builder PR stack is on `main`, so the check +would fail there. Once it lands, drop the event condition on the job to run it +on push and on pull requests. + +### The fork pin + +`COMMIT_BOOST_PACKAGE_REF` is `37fe57ea309b954361c0f33ac6efc570313ffaf6`, the +commit on the `lambdaclass/ethereum-package` branch `feat/ethlambda-builder-flags` +that adds ethlambda to `apply_external_builder_flags` (`--builder=` for the +beacon node, `--builder-proposals` for the validator client). That branch has +to be pushed before the job can fetch the commit. If the fork PR is +squash-merged, the commit is no longer reachable from the fork's default branch +and may be garbage collected, so re-pin to the squash commit. diff --git a/tooling/kurtosis-assertoor/commit_boost_params.yaml b/tooling/kurtosis-assertoor/commit_boost_params.yaml new file mode 100644 index 000000000..a46fa5ca5 --- /dev/null +++ b/tooling/kurtosis-assertoor/commit_boost_params.yaml @@ -0,0 +1,55 @@ +# The Commit-Boost check for ethlambda: the same three participants as +# network_params.yaml, plus `mev_type: commit-boost`, so ethlambda's beacon node +# and validator client build blocks through a Commit-Boost PBS module. There is +# no assertoor: the check is scripts/check-builder-blocks.sh, which reads the +# builder metrics of every ethlambda service. See README.md ("Commit-Boost +# check") for the steps. The package comes from a clone of +# lambdaclass/ethereum-package at the commit pinned in +# .github/workflows/kurtosis_assertoor.yml (COMMIT_BOOST_PACKAGE_REF). +# +# Participants, in the order ethereum-package launches them (the order fixes the +# service names check-builder-blocks.sh reads): +# 1. lighthouse + geth, lighthouse validator. It has to come first: the first +# participant is the network's bootnode, and ethlambda does not report an +# ENR over the beacon API. +# 2. lighthouse + geth, validated by the ethlambda validator client, so the +# validator client proposes blinded blocks through a beacon node that is +# not ethlambda. +# 3. ethlambda beacon node + geth, validated by the ethlambda validator client. +# +# `mev_type: commit-boost` adds a flashbots mev-boost-relay, a reth-rbuilder +# builder (with its own lighthouse beacon node) and one Commit-Boost PBS module +# per participant. The package passes each ethlambda beacon node `--builder` and +# each ethlambda validator client `--builder-proposals`, which only an image +# built with the builder support understands. The workflow rewrites the +# `cl_image` and `vc_image` of the ethlambda entries (selected by type, not +# position) to the image under test; the values below are what a local run uses. +participants: + - el_type: geth + cl_type: lighthouse + - el_type: geth + cl_type: lighthouse + vc_type: ethlambda + vc_image: ghcr.io/lambdaclass/ethlambda:unstable + - el_type: geth + cl_type: ethlambda + cl_image: ghcr.io/lambdaclass/ethlambda:unstable + vc_type: ethlambda + vc_image: ghcr.io/lambdaclass/ethlambda:unstable +network_params: + # ethlambda only supports the mainnet preset: 32-slot epochs of 12 s. + preset: mainnet + # Fulu is the last fork ethlambda processes; gloas stays unscheduled. + electra_fork_epoch: 0 + fulu_fork_epoch: 0 +mev_type: commit-boost +# The package defaults are floating tags (`main`, `develop`, `latest`), so a +# run would change under us. These pin what the tags pointed to on 2026-10-26; +# bump them deliberately. +mev_params: + # ethpandaops/mev-boost-relay:main + mev_relay_image: ethpandaops/mev-boost-relay@sha256:3600f04fcaf0bbb231f4737470f094fdf2314db54b4853197fa369aea90f8b19 + # ethpandaops/reth-rbuilder:develop + mev_builder_image: ethpandaops/reth-rbuilder@sha256:916ba84da2690e3e368f57da5b0c73bfd919b452de8b95f64895846e0f3a8308 + # ghcr.io/commit-boost/pbs:latest + mev_boost_image: ghcr.io/commit-boost/pbs@sha256:8612c03681d390ab7d0154f5db1581584cd265563b723947da90cd00ba7ca415 diff --git a/tooling/kurtosis-assertoor/scripts/check-builder-blocks.sh b/tooling/kurtosis-assertoor/scripts/check-builder-blocks.sh new file mode 100755 index 000000000..0cd9945a6 --- /dev/null +++ b/tooling/kurtosis-assertoor/scripts/check-builder-blocks.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# +# Waits until ethlambda has built blocks through the builder (Commit-Boost) and +# fails if that does not happen in time. +# +# Usage: check-builder-blocks.sh +# +# Reads Prometheus metrics through `kurtosis port print +# metrics`, and succeeds once every condition below holds: +# +# ethlambda beacon node (participant 3) +# lean_beacon_builder_registrations_total{outcome="forwarded"} >= 1 +# lean_beacon_builder_payload_source_total{source="builder"} >= 1 +# ethlambda validator client of participant 3 +# ethlambda_validator_builder_registrations_sent_total >= 1 +# ethlambda_validator_blinded_blocks_proposed_total >= 1 +# ethlambda validator client of participant 2 (on a lighthouse beacon node) +# ethlambda_validator_blinded_blocks_proposed_total >= 1 +# +# On every exit, writes each ethlambda service's builder metric lines to +# /.prom, and the conditions still unmet to +# /failed.txt. A condition unmet at the deadline fails the check. + +set -euo pipefail + +ENCLAVE="${1:?enclave name required}" +TIMEOUT_MIN="${2:?timeout in minutes required}" +OUT="${3:?output directory required}" + +# Service names follow ethereum-package: cl---, and vc--- +# with - appended when the validator client differs from the beacon node. +CL3="cl-3-ethlambda-geth" +VC2="vc-2-geth-lighthouse-ethlambda" +VC3="vc-3-geth-ethlambda" +SERVICES=("$CL3" "$VC2" "$VC3") + +# service|series (name, with its labels when the metric is labelled)|description +CONDITIONS=( + "$CL3|lean_beacon_builder_registrations_total{outcome=\"forwarded\"}|beacon node forwarded a registration" + "$CL3|lean_beacon_builder_payload_source_total{source=\"builder\"}|beacon node served a builder payload" + "$VC3|ethlambda_validator_builder_registrations_sent_total|validator client of participant 3 sent a registration" + "$VC3|ethlambda_validator_blinded_blocks_proposed_total|validator client of participant 3 proposed a blinded block" + "$VC2|ethlambda_validator_blinded_blocks_proposed_total|validator client of participant 2 proposed a blinded block" +) + +mkdir -p "$OUT" +deadline=$((SECONDS + TIMEOUT_MIN * 60)) +: > "$OUT/failed.txt" + +# Prints a service's metrics, or nothing while it is unreachable. +scrape() { + local addr + addr=$(kurtosis port print "$ENCLAVE" "$1" metrics 2>/dev/null) || return 0 + curl -sf --max-time 10 "${addr#*://}/metrics" 2>/dev/null || true +} + +# Prints the sum of a series over the given exposition text, 0 if absent. A +# selector with labels matches that exact `name{labels}` token; a bare name +# matches the metric under any labels. +value_of() { + awk -v sel="$2" ' + index(sel, "{") { if ($1 == sel) sum += $NF; next } + { split($1, p, "{"); if (p[1] == sel) sum += $NF } + END { printf "%d\n", sum } + ' <<<"$1" +} + +# Writes each service's builder-related series to /.prom. +dump_metrics() { + local service + for service in "${SERVICES[@]}"; do + scrape "$service" \ + | grep -E '^(lean_beacon_builder_|ethlambda_validator_.*(blinded|registration))' \ + > "$OUT/$service.prom" || true + done +} + +# Prints one line per condition and returns 1 if any is unmet. +evaluate() { + local entry svc series desc value unmet=0 + for entry in "${CONDITIONS[@]}"; do + IFS='|' read -r svc series desc <<<"$entry" + value=$(value_of "$(scrape "$svc")" "$series") + if [ "$value" -ge 1 ]; then + echo "ok $svc $series = $value ($desc)" + else + echo "unmet $svc $series = $value ($desc)" + unmet=1 + fi + done + return "$unmet" +} + +finish() { + dump_metrics + echo "Builder metrics written to $OUT" +} +trap finish EXIT + +status=1 +while :; do + if summary=$(evaluate); then + status=0 + break + fi + echo "[$(date -u +%H:%M:%S)] builder conditions not met yet:" + grep '^unmet' <<<"$summary" | sed 's/^/ /' + if [ "$SECONDS" -ge "$deadline" ]; then + echo "::error::timed out after ${TIMEOUT_MIN} minutes waiting for builder blocks" + break + fi + sleep 30 +done + +echo +echo "$summary" +grep '^unmet' <<<"$summary" > "$OUT/failed.txt" || true +if [ "$status" -ne 0 ]; then + echo "Failed:" + cat "$OUT/failed.txt" + exit 1 +fi +echo "ethlambda built blocks through the builder" From ad9ebeafb80835bb029b86960a296be5b323a941 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:47:28 -0300 Subject: [PATCH 02/11] docs: explain local images and the fork PR for the Commit-Boost check A contributor running the check locally needs to know how to get an image with the builder flags and that kurtosis will use it without pulling, or it looks like the run needs a published image. The fork pin also now has a PR to point at instead of an unpushed branch. --- tooling/kurtosis-assertoor/README.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tooling/kurtosis-assertoor/README.md b/tooling/kurtosis-assertoor/README.md index 153c2e4ec..f114d87c0 100644 --- a/tooling/kurtosis-assertoor/README.md +++ b/tooling/kurtosis-assertoor/README.md @@ -101,9 +101,13 @@ The relay, builder and PBS images are pinned by digest in ### Run it locally -The image needs the builder support, so build one from the builder PR stack -(#694 to #705) and set it as the `cl_image` and `vc_image` of the ethlambda -entries in `commit_boost_params.yaml`. From the repository root: +The image needs the builder support, so build one from a checkout of the +builder PR stack (#694 to #705). From that checkout's root, `make docker-build +DOCKER_TAG=` tags it `ghcr.io/lambdaclass/ethlambda:`; set that as the +`cl_image` and `vc_image` of the ethlambda entries in `commit_boost_params.yaml`. +Kurtosis uses an image that exists in the local Docker daemon and only pulls a +missing one (`kurtosis run --image-download`, default `missing`), so no setting +is needed; do not pass `--image-download always`. From the repository root: ```bash git clone https://github.com/lambdaclass/ethereum-package.git /tmp/ethereum-package @@ -137,7 +141,7 @@ on push and on pull requests. `COMMIT_BOOST_PACKAGE_REF` is `37fe57ea309b954361c0f33ac6efc570313ffaf6`, the commit on the `lambdaclass/ethereum-package` branch `feat/ethlambda-builder-flags` that adds ethlambda to `apply_external_builder_flags` (`--builder=` for the -beacon node, `--builder-proposals` for the validator client). That branch has -to be pushed before the job can fetch the commit. If the fork PR is +beacon node, `--builder-proposals` for the validator client). That branch is +lambdaclass/ethereum-package#22. If the fork PR is squash-merged, the commit is no longer reachable from the fork's default branch and may be garbage collected, so re-pin to the squash commit. From 3a7bf07fef33ac3370864308be95340f5ec36996 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 19:08:04 -0300 Subject: [PATCH 03/11] ci: work around two upstream breakages in the Commit-Boost check A local run did not start. The newest reth-rbuilder image parses only enode bootnodes while the package passes ENRs, and Commit-Boost's PBS requires SECONDS_PER_SLOT, which the genesis generator no longer writes. The first is fixed in the fork (PR #23), so the package pin moves to that commit. The second is worked around by giving Commit-Boost an explicit chain through `commit_boost_config`; its `chain_id` must be a hex string. The README lists both workarounds so they can be dropped when upstream is fixed, and explains the harmless "Builder is not answering" logs at startup. --- .github/workflows/kurtosis_assertoor.yml | 7 ++-- tooling/kurtosis-assertoor/README.md | 36 ++++++++++++++----- .../commit_boost_params.yaml | 27 ++++++++++++++ 3 files changed, 59 insertions(+), 11 deletions(-) diff --git a/.github/workflows/kurtosis_assertoor.yml b/.github/workflows/kurtosis_assertoor.yml index 8fec34608..082deb8e4 100644 --- a/.github/workflows/kurtosis_assertoor.yml +++ b/.github/workflows/kurtosis_assertoor.yml @@ -45,9 +45,10 @@ env: ENCLAVE: ethlambda-assertoor # The Commit-Boost check needs the fork's `--builder` / `--builder-proposals` # wiring for ethlambda, which the commit above lacks. It has its own pin so - # the base battery stays where it is. This commit is on the fork branch - # feat/ethlambda-builder-flags; see the README before changing it. - COMMIT_BOOST_PACKAGE_REF: 37fe57ea309b954361c0f33ac6efc570313ffaf6 + # the base battery stays where it is. Fork PR #22 (that wiring) is merged as + # a15e7df; this commit adds PR #23's fix for the builder's enode bootnodes on + # top. Re-pin to #23's merge commit once it lands; see the README. + COMMIT_BOOST_PACKAGE_REF: 7fa7d1225fe5d975300ecc5584bf28ca6d42d451 COMMIT_BOOST_ENCLAVE: ethlambda-commit-boost jobs: diff --git a/tooling/kurtosis-assertoor/README.md b/tooling/kurtosis-assertoor/README.md index f114d87c0..82496fa6d 100644 --- a/tooling/kurtosis-assertoor/README.md +++ b/tooling/kurtosis-assertoor/README.md @@ -111,7 +111,7 @@ is needed; do not pass `--image-download always`. From the repository root: ```bash git clone https://github.com/lambdaclass/ethereum-package.git /tmp/ethereum-package -git -C /tmp/ethereum-package checkout 37fe57ea309b954361c0f33ac6efc570313ffaf6 +git -C /tmp/ethereum-package checkout 7fa7d1225fe5d975300ecc5584bf28ca6d42d451 kurtosis run /tmp/ethereum-package --enclave ethlambda-commit-boost \ --args-file tooling/kurtosis-assertoor/commit_boost_params.yaml @@ -138,10 +138,30 @@ on push and on pull requests. ### The fork pin -`COMMIT_BOOST_PACKAGE_REF` is `37fe57ea309b954361c0f33ac6efc570313ffaf6`, the -commit on the `lambdaclass/ethereum-package` branch `feat/ethlambda-builder-flags` -that adds ethlambda to `apply_external_builder_flags` (`--builder=` for the -beacon node, `--builder-proposals` for the validator client). That branch is -lambdaclass/ethereum-package#22. If the fork PR is -squash-merged, the commit is no longer reachable from the fork's default branch -and may be garbage collected, so re-pin to the squash commit. +lambdaclass/ethereum-package#22 added ethlambda to `apply_external_builder_flags` +(`--builder=` for the beacon node, `--builder-proposals` for the validator +client) and is merged (`a15e7df`). `COMMIT_BOOST_PACKAGE_REF` is +`7fa7d1225fe5d975300ecc5584bf28ca6d42d451`, from +lambdaclass/ethereum-package#23, which adds the builder's enode bootnode fix on +top of it. Re-pin to #23's merge commit once it lands: a squash-merge leaves the +current commit unreachable from the fork's default branch, so it may be garbage +collected. + +### Upstream workarounds + +A local run found two breakages in images and packages we do not control: + +| Breakage | Worked around in | +|---|---| +| `ethpandaops/reth-rbuilder:develop` (newest image, 2026-07-17) parses only enode URLs as bootnodes, and the package has passed ENRs since its 08-05 refactor, so the builder's execution client refused them | fork PR #23, through the `COMMIT_BOOST_PACKAGE_REF` pin | +| Commit-Boost's PBS requires `SECONDS_PER_SLOT` in the chain config, and the genesis generator now writes only `SLOT_DURATION_MS` | `mev_params.commit_boost_config` in `commit_boost_params.yaml`, which gives the chain explicitly (`chain_id` has to be a hex string) | + +Drop each once the upstream is fixed. The explicit chain must change with the +network parameters. + +### Startup noise + +The Commit-Boost PBS modules start after the consensus clients, so at startup +ethlambda's beacon node logs "Builder is not answering" and its first +registration batch fails. The validator client re-sends registrations every +epoch, so this clears within an epoch and is not a failure. diff --git a/tooling/kurtosis-assertoor/commit_boost_params.yaml b/tooling/kurtosis-assertoor/commit_boost_params.yaml index a46fa5ca5..ca5d60353 100644 --- a/tooling/kurtosis-assertoor/commit_boost_params.yaml +++ b/tooling/kurtosis-assertoor/commit_boost_params.yaml @@ -24,6 +24,8 @@ # built with the builder support understands. The workflow rewrites the # `cl_image` and `vc_image` of the ethlambda entries (selected by type, not # position) to the image under test; the values below are what a local run uses. +# If the battery ever changes `network_id`, the preset or the fork epochs, the +# explicit Commit-Boost chain under `mev_params` must change with it. participants: - el_type: geth cl_type: lighthouse @@ -53,3 +55,28 @@ mev_params: mev_builder_image: ethpandaops/reth-rbuilder@sha256:916ba84da2690e3e368f57da5b0c73bfd919b452de8b95f64895846e0f3a8308 # ghcr.io/commit-boost/pbs:latest mev_boost_image: ghcr.io/commit-boost/pbs@sha256:8612c03681d390ab7d0154f5db1581584cd265563b723947da90cd00ba7ca415 + # Commit-Boost reads the chain from the package's config.yaml by default, + # and its parser requires SECONDS_PER_SLOT, which the genesis generator no + # longer writes (it writes SLOT_DURATION_MS). So the chain is given + # explicitly: kurtosis's defaults for this network (12 s slots, genesis fork + # version 0x10000038, fulu from genesis, chain id 3151908 = 0x301824). + # chain_id must be a hex string; a bare integer does not parse. The rest is + # the package's own template (static_files/mev/commit-boost/cb-config.toml.tmpl). + commit_boost_config: | + chain = { genesis_time_secs = {{ .Timestamp }}, slot_time_secs = 12, genesis_fork_version = "0x10000038", fulu_fork_slot = 0, chain_id = "0x301824" } + + [pbs] + host = "0.0.0.0" + port = {{ .Port }} + + {{ range $index, $relay := .Relays }} + [[relays]] + id = "mev_relay_{{$index}}" + url = "{{ $relay }}" + {{- end }} + + [logs.stdout] + level = "debug" + + [logs.file] + enabled=false From 6372671f19f429493c782a09da7493224c418618 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Tue, 27 Oct 2026 09:36:15 -0300 Subject: [PATCH 04/11] ci: start spamoor in the Commit-Boost check A local run registered every validator but never got a builder block: rbuilder submits no bid for a block without transactions, and the enclave had none, so the relay held no bids for any proposer and every getHeader answered 204. With transactions flowing, the same enclave served a builder payload within a few slots, and the ethlambda validator client signed the blinded block that the relay delivered. spamoor runs with the package's default spammers (transfers and blob transactions), pinned by digest like the other images. The README also explains the no_finality fallbacks a run counts before the chain first finalizes. --- tooling/kurtosis-assertoor/README.md | 10 ++++++++-- tooling/kurtosis-assertoor/commit_boost_params.yaml | 9 +++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/tooling/kurtosis-assertoor/README.md b/tooling/kurtosis-assertoor/README.md index 82496fa6d..1c5aef9ea 100644 --- a/tooling/kurtosis-assertoor/README.md +++ b/tooling/kurtosis-assertoor/README.md @@ -75,7 +75,8 @@ Starts the same three participants with `mev_type: commit-boost` (`commit_boost_params.yaml`): a flashbots mev-boost-relay, a reth-rbuilder builder with its own lighthouse beacon node, and one [Commit-Boost](https://github.com/Commit-Boost/commit-boost-client) PBS module -per participant, which stands in for MEV-Boost. The package gives the ethlambda +per participant, which stands in for MEV-Boost. It also starts spamoor, since +rbuilder submits no bid for a block without transactions. The package gives the ethlambda beacon node `--builder ` and the ethlambda validator client `--builder-proposals`. There is no assertoor: the base battery's checks fail on the spec-endpoint gap above, so `scripts/check-builder-blocks.sh` polls the @@ -96,7 +97,7 @@ through lighthouse. The script writes every `lean_beacon_builder_*` and `/.prom`, pass or fail, and the unmet conditions to `failed.txt`. -The relay, builder and PBS images are pinned by digest in +The relay, builder, PBS and spamoor images are pinned by digest in `commit_boost_params.yaml` (the package's defaults are floating tags). ### Run it locally @@ -165,3 +166,8 @@ The Commit-Boost PBS modules start after the consensus clients, so at startup ethlambda's beacon node logs "Builder is not answering" and its first registration batch fails. The validator client re-sends registrations every epoch, so this clears within an epoch and is not a failure. + +From epoch 4 until the chain first finalizes, the beacon node builds locally and +counts `lean_beacon_builder_fallback_total{reason="no_finality"}`: like +Lighthouse, it skips the builder when the proposal is more than 3 epochs past +the finalized checkpoint, and the genesis checkpoint counts as one. diff --git a/tooling/kurtosis-assertoor/commit_boost_params.yaml b/tooling/kurtosis-assertoor/commit_boost_params.yaml index ca5d60353..7f0929f23 100644 --- a/tooling/kurtosis-assertoor/commit_boost_params.yaml +++ b/tooling/kurtosis-assertoor/commit_boost_params.yaml @@ -45,6 +45,15 @@ network_params: electra_fork_epoch: 0 fulu_fork_epoch: 0 mev_type: commit-boost +# rbuilder submits nothing for a block with no transactions, so without a +# spammer the relay never holds a bid and every getHeader answers 204. spamoor's +# package defaults send type-2 transfers and blob transactions every slot, so +# bids also carry blob commitments. +additional_services: + - spamoor +spamoor_params: + # ethpandaops/spamoor:master on 2026-10-27 + image: ethpandaops/spamoor@sha256:e890785f5814218bbcb544c495820ff7b02dd554e6d714f82849a0ac8de680c5 # The package defaults are floating tags (`main`, `develop`, `latest`), so a # run would change under us. These pin what the tags pointed to on 2026-10-26; # bump them deliberately. From a211f4994b117f8abd86971cf49c462a2b787936 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:32:55 -0300 Subject: [PATCH 05/11] ci: run spamoor at a light load in the Commit-Boost check With spamoor's defaults (a chain-load group, 200 transfers and 6 blob transactions a slot) the relay's getHeader slowed to Commit-Boost's 950 ms timeout, so most proposals got no bid, and in 40 minutes the ethlambda validator client behind lighthouse never received one. 10 transfers and 1 blob transaction a slot keep bids coming without that slowdown: a local run met every condition about 12 minutes after the enclave started, and the relay delivered a builder block in 14 of the 32 slots of epoch 1. The package's own 100 Uniswap swaps a slot, which it adds whenever a builder runs, stay on; they are not configurable. --- tooling/kurtosis-assertoor/README.md | 6 ++-- .../commit_boost_params.yaml | 30 +++++++++++++++++-- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/tooling/kurtosis-assertoor/README.md b/tooling/kurtosis-assertoor/README.md index 1c5aef9ea..9570ef9ae 100644 --- a/tooling/kurtosis-assertoor/README.md +++ b/tooling/kurtosis-assertoor/README.md @@ -75,8 +75,10 @@ Starts the same three participants with `mev_type: commit-boost` (`commit_boost_params.yaml`): a flashbots mev-boost-relay, a reth-rbuilder builder with its own lighthouse beacon node, and one [Commit-Boost](https://github.com/Commit-Boost/commit-boost-client) PBS module -per participant, which stands in for MEV-Boost. It also starts spamoor, since -rbuilder submits no bid for a block without transactions. The package gives the ethlambda +per participant, which stands in for MEV-Boost. It also starts spamoor at a +light load, since rbuilder submits no bid for a block without transactions and +a heavy load slows the relay past Commit-Boost's getHeader timeout (see the +comment in the params file). The package gives the ethlambda beacon node `--builder ` and the ethlambda validator client `--builder-proposals`. There is no assertoor: the base battery's checks fail on the spec-endpoint gap above, so `scripts/check-builder-blocks.sh` polls the diff --git a/tooling/kurtosis-assertoor/commit_boost_params.yaml b/tooling/kurtosis-assertoor/commit_boost_params.yaml index 7f0929f23..ea232cdc6 100644 --- a/tooling/kurtosis-assertoor/commit_boost_params.yaml +++ b/tooling/kurtosis-assertoor/commit_boost_params.yaml @@ -46,14 +46,38 @@ network_params: fulu_fork_epoch: 0 mev_type: commit-boost # rbuilder submits nothing for a block with no transactions, so without a -# spammer the relay never holds a bid and every getHeader answers 204. spamoor's -# package defaults send type-2 transfers and blob transactions every slot, so -# bids also carry blob commitments. +# spammer the relay never holds a bid and every getHeader answers 204. The load +# is kept light on purpose: under spamoor's defaults (a chain-load group plus +# 200 transfers and 6 blob transactions a slot) the relay's getHeader slowed +# past Commit-Boost's 950 ms timeout on a local run, so most proposals got no +# bid. The package also always adds 100 Uniswap swaps a slot, sent to the +# builder's own execution client; that is not configurable. The blob spammer +# stays so bids carry blob commitments. additional_services: - spamoor spamoor_params: # ethpandaops/spamoor:master on 2026-10-27 image: ethpandaops/spamoor@sha256:e890785f5814218bbcb544c495820ff7b02dd554e6d714f82849a0ac8de680c5 + start_chainload: false + spammers: + - name: "EOA transfers" + description: "10 type-2 transfers a slot" + scenario: "eoatx" + config: + throughput: 10 + max_pending: 20 + max_wallets: 10 + base_fee: 20 + - name: "Blob transactions" + description: "1 type-4 transaction a slot with 1 sidecar" + scenario: "blob-combined" + config: + throughput: 1 + sidecars: 1 + max_pending: 2 + max_wallets: 2 + base_fee: 20 + blob_fee: 20 # The package defaults are floating tags (`main`, `develop`, `latest`), so a # run would change under us. These pin what the tags pointed to on 2026-10-26; # bump them deliberately. From 5831253c472051525b95070fc42c9efd351a7ba9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:39:03 -0300 Subject: [PATCH 06/11] ci: pin the Commit-Boost job's actions to the lockfile's versions #713 locked every workflow's actions in .github/workflows/actions.lock, after this job was written. Its checkout@v6 and upload-artifact@v7 no longer match the v6.1.0 and v7.0.2 the lockfile pins for this workflow, so verify-lockfile fails and the enforced lockfile refuses to start the workflow at all (startup_failure on the PR's Kurtosis Assertoor run). Use the same refs as the assertoor job beside it, including the `$/` same-repository form #713 gave the free-disk action. --- .github/workflows/kurtosis_assertoor.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/kurtosis_assertoor.yml b/.github/workflows/kurtosis_assertoor.yml index 56915fd50..376a4720a 100644 --- a/.github/workflows/kurtosis_assertoor.yml +++ b/.github/workflows/kurtosis_assertoor.yml @@ -195,10 +195,10 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 90 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.1.0 - name: Free up disk space - uses: ./.github/actions/free-disk + uses: $/.github/actions/free-disk - name: Install kurtosis run: | @@ -252,7 +252,7 @@ jobs: - name: Upload the dump and results if: always() - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@v7.0.2 with: name: commit-boost-enclave-dump-${{ github.run_id }} path: ${{ runner.temp }}/dump From 20cf458c72208798cdff9cb126250def2b6881e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:41:33 -0300 Subject: [PATCH 07/11] fix(ci): make the Commit-Boost check's metric reads and setup failures explicit From the automated review of this PR: - Counter values are summed and truncated in awk, then printed with %.0f, so a large or exponent-notation value (1e+06) reads whole and a value below 1 never meets a >= 1 condition. - Each service's metrics port must resolve before polling starts. A renamed service, a reordered participant list or a missing kurtosis/curl now fails at once with exit status 2 and kurtosis's own message, rather than reading as "not met yet" for 40 minutes. - One evaluation scrapes every service once, so the conditions on one service read the same snapshot. - A service that does not answer is reported as such, not as a 0. - failed.txt is rewritten after every evaluation, so a cancelled run still says what was unmet. - The address kurtosis prints is reduced to host:port, whatever scheme or path surrounds it. Checked against a stub kurtosis and local metrics servers, under macOS's bash 3.2 and under ubuntu:24.04's bash 5.2 with mawk; shellcheck is clean. --- tooling/kurtosis-assertoor/README.md | 7 +- .../scripts/check-builder-blocks.sh | 97 ++++++++++++++++--- 2 files changed, 86 insertions(+), 18 deletions(-) diff --git a/tooling/kurtosis-assertoor/README.md b/tooling/kurtosis-assertoor/README.md index 9570ef9ae..9144ec793 100644 --- a/tooling/kurtosis-assertoor/README.md +++ b/tooling/kurtosis-assertoor/README.md @@ -96,8 +96,11 @@ Passing therefore covers the registration path, a builder block from ethlambda's own beacon node, and ethlambda's validator client proposing blinded blocks through lighthouse. The script writes every `lean_beacon_builder_*` and `ethlambda_validator_*` blinded or registration series of the three services to -`/.prom`, pass or fail, and the unmet conditions to -`failed.txt`. +`/.prom`, pass or fail, and rewrites `failed.txt` with the +unmet conditions after every evaluation, so a cancelled run still says what was +missing. It stops at once, with exit status 2, if a service has no `metrics` +port: the service names depend on ethereum-package's naming and on the +participant order, and a mismatch is a setup problem, not a slow builder. The relay, builder, PBS and spamoor images are pinned by digest in `commit_boost_params.yaml` (the package's defaults are floating tags). diff --git a/tooling/kurtosis-assertoor/scripts/check-builder-blocks.sh b/tooling/kurtosis-assertoor/scripts/check-builder-blocks.sh index 0cd9945a6..14bd6dd35 100755 --- a/tooling/kurtosis-assertoor/scripts/check-builder-blocks.sh +++ b/tooling/kurtosis-assertoor/scripts/check-builder-blocks.sh @@ -17,9 +17,14 @@ # ethlambda validator client of participant 2 (on a lighthouse beacon node) # ethlambda_validator_blinded_blocks_proposed_total >= 1 # -# On every exit, writes each ethlambda service's builder metric lines to -# /.prom, and the conditions still unmet to -# /failed.txt. A condition unmet at the deadline fails the check. +# After every evaluation, writes the conditions still unmet to +# /failed.txt, so a run cut short still says what was missing. On +# every exit, writes each ethlambda service's builder metric lines to +# /.prom. +# +# Exits 1 if a condition is unmet at the deadline, and 2 at once if the check +# cannot run: kurtosis or curl is missing, or a service has no `metrics` port +# (ethereum-package renamed it, or the participants were reordered). set -euo pipefail @@ -44,25 +49,60 @@ CONDITIONS=( ) mkdir -p "$OUT" -deadline=$((SECONDS + TIMEOUT_MIN * 60)) : > "$OUT/failed.txt" -# Prints a service's metrics, or nothing while it is unreachable. +for tool in kurtosis curl; do + if ! command -v "$tool" >/dev/null 2>&1; then + echo "::error::$tool is not on PATH" + echo "setup $tool is not on PATH" > "$OUT/failed.txt" + exit 2 + fi +done + +# One evaluation's scrapes, a file per service. +SNAPSHOT=$(mktemp -d) + +# Prints a service's metrics address as host:port. `kurtosis port print` answers +# `://:`, or `:` for a port with +# no application protocol; anything around the host and port is dropped, and +# curl's default (http) is what the metrics port speaks. +metrics_address() { + local addr + addr=$(kurtosis port print "$ENCLAVE" "$1" metrics) || return + addr="${addr#*://}" + printf '%s\n' "${addr%%/*}" +} + +# Prints a service's metrics, or nothing while it is unreachable. The address is +# looked up on every scrape, since a restarted service can come back on another +# host port. scrape() { local addr - addr=$(kurtosis port print "$ENCLAVE" "$1" metrics 2>/dev/null) || return 0 - curl -sf --max-time 10 "${addr#*://}/metrics" 2>/dev/null || true + addr=$(metrics_address "$1" 2>/dev/null) || return 0 + curl -sf --max-time 10 "$addr/metrics" 2>/dev/null || true } -# Prints the sum of a series over the given exposition text, 0 if absent. A +# Scrapes every service once, so one evaluation reads each service as of one +# moment rather than once per condition. +take_snapshot() { + local service + for service in "${SERVICES[@]}"; do + scrape "$service" > "$SNAPSHOT/$service.prom" + done +} + +# Prints the sum of a series in the given exposition file, 0 if absent. A # selector with labels matches that exact `name{labels}` token; a bare name -# matches the metric under any labels. +# matches the metric under any labels. Every series checked is a counter, so the +# sum is truncated to an integer for the shell's `-ge`: truncating rather than +# rounding keeps a value below 1 from meeting a `>= 1` condition, and `%.0f` +# prints a large or exponent-notation value (`1e+06`) whole, which `%d` need not. value_of() { awk -v sel="$2" ' index(sel, "{") { if ($1 == sel) sum += $NF; next } { split($1, p, "{"); if (p[1] == sel) sum += $NF } - END { printf "%d\n", sum } - ' <<<"$1" + END { printf "%.0f\n", int(sum) } + ' "$1" } # Writes each service's builder-related series to /.prom. @@ -78,9 +118,15 @@ dump_metrics() { # Prints one line per condition and returns 1 if any is unmet. evaluate() { local entry svc series desc value unmet=0 + take_snapshot for entry in "${CONDITIONS[@]}"; do IFS='|' read -r svc series desc <<<"$entry" - value=$(value_of "$(scrape "$svc")" "$series") + if [ ! -s "$SNAPSHOT/$svc.prom" ]; then + echo "unmet $svc $series: no metrics, the service did not answer ($desc)" + unmet=1 + continue + fi + value=$(value_of "$SNAPSHOT/$svc.prom" "$series") if [ "$value" -ge 1 ]; then echo "ok $svc $series = $value ($desc)" else @@ -93,18 +139,38 @@ evaluate() { finish() { dump_metrics + rm -rf "$SNAPSHOT" echo "Builder metrics written to $OUT" } trap finish EXIT +# `kurtosis run` has started every service by the time this runs, so each one +# must already have a metrics port. A service the package named differently +# would otherwise read as "not met yet" until the deadline, not as the setup +# problem it is. +missing=0 +for service in "${SERVICES[@]}"; do + if ! err=$(metrics_address "$service" 2>&1 >/dev/null); then + echo "::error::$service has no metrics port in enclave $ENCLAVE: ${err:-kurtosis port print failed}" + echo "setup $service has no metrics port in enclave $ENCLAVE" >> "$OUT/failed.txt" + missing=1 + fi +done +if [ "$missing" -ne 0 ]; then + echo "Check the service names above against \`kurtosis enclave inspect $ENCLAVE\`." + exit 2 +fi + +deadline=$((SECONDS + TIMEOUT_MIN * 60)) status=1 while :; do - if summary=$(evaluate); then - status=0 + summary=$(evaluate) && status=0 + grep '^unmet' <<<"$summary" > "$OUT/failed.txt" || true + if [ "$status" -eq 0 ]; then break fi echo "[$(date -u +%H:%M:%S)] builder conditions not met yet:" - grep '^unmet' <<<"$summary" | sed 's/^/ /' + sed 's/^/ /' "$OUT/failed.txt" if [ "$SECONDS" -ge "$deadline" ]; then echo "::error::timed out after ${TIMEOUT_MIN} minutes waiting for builder blocks" break @@ -114,7 +180,6 @@ done echo echo "$summary" -grep '^unmet' <<<"$summary" > "$OUT/failed.txt" || true if [ "$status" -ne 0 ]; then echo "Failed:" cat "$OUT/failed.txt" From d3308afcb428bff7e4cfccf0d0759ac022e8e1b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:41:33 -0300 Subject: [PATCH 08/11] docs(ci): say what runs the base battery, and when notify must learn the Commit-Boost job notify reports pushes only and the commit-boost job runs on dispatch only, so adding the job to notify's needs today would wire up nothing. The note sits where the event condition is dropped, which is when it starts to matter. --- .github/workflows/kurtosis_assertoor.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/kurtosis_assertoor.yml b/.github/workflows/kurtosis_assertoor.yml index 376a4720a..c112f39ec 100644 --- a/.github/workflows/kurtosis_assertoor.yml +++ b/.github/workflows/kurtosis_assertoor.yml @@ -95,6 +95,8 @@ jobs: assertoor: name: Assertoor needs: image + # Every push and pull request, and every dispatch except one that asked for + # the commit-boost battery alone: `assertoor` and `both` both run this job. if: github.event_name != 'workflow_dispatch' || inputs.battery != 'commit-boost' runs-on: ubuntu-latest timeout-minutes: 120 @@ -187,7 +189,10 @@ jobs: # Dispatch-only for now: main's image does not have the builder flags until # the builder PR stack (#694 to #705) lands. Once it is on main, drop the - # event condition so the check also runs on push and on pull requests. + # event condition so the check also runs on push and on pull requests, and at + # the same time add `commit-boost` to `notify`'s `needs` and its failed.txt to + # the Slack text: `notify` reports pushes only, so until then it can never + # see this job. commit-boost: name: Commit-Boost needs: image From db90e13d245c97095444c81cb28a8efbff20096c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:41:33 -0300 Subject: [PATCH 09/11] docs(ci): say how to bump the Commit-Boost image digests --- tooling/kurtosis-assertoor/commit_boost_params.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tooling/kurtosis-assertoor/commit_boost_params.yaml b/tooling/kurtosis-assertoor/commit_boost_params.yaml index ea232cdc6..d2bdea324 100644 --- a/tooling/kurtosis-assertoor/commit_boost_params.yaml +++ b/tooling/kurtosis-assertoor/commit_boost_params.yaml @@ -80,7 +80,9 @@ spamoor_params: blob_fee: 20 # The package defaults are floating tags (`main`, `develop`, `latest`), so a # run would change under us. These pin what the tags pointed to on 2026-10-26; -# bump them deliberately. +# bump them deliberately. To bump one, read the tag's current digest from the +# `Digest:` line of `docker buildx imagetools inspect :`, and update +# the date here with it. mev_params: # ethpandaops/mev-boost-relay:main mev_relay_image: ethpandaops/mev-boost-relay@sha256:3600f04fcaf0bbb231f4737470f094fdf2314db54b4853197fa369aea90f8b19 From a3a3588be43c39bd9bb5f8c78d28070c16dab827 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:57:56 -0300 Subject: [PATCH 10/11] fix(ci): start the Commit-Boost check's validator clients with --init-slashing-protection Since #734 the ethlambda validator client refuses keys with no slashing-protection record, so both ethlambda VCs exited at startup and kurtosis rolled them back. network_params.yaml has passed the flag since then; this file now does the same, for the same reason: genesis keys in a fresh directory have never signed. --- tooling/kurtosis-assertoor/commit_boost_params.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tooling/kurtosis-assertoor/commit_boost_params.yaml b/tooling/kurtosis-assertoor/commit_boost_params.yaml index d2bdea324..5022ec4ae 100644 --- a/tooling/kurtosis-assertoor/commit_boost_params.yaml +++ b/tooling/kurtosis-assertoor/commit_boost_params.yaml @@ -33,11 +33,17 @@ participants: cl_type: lighthouse vc_type: ethlambda vc_image: ghcr.io/lambdaclass/ethlambda:unstable + # Genesis keys in a fresh directory: nothing has ever signed with them, so + # registering them without history is safe. Without it the validator + # client refuses to start (no slashing-protection record), as in + # network_params.yaml. + vc_extra_params: ["--init-slashing-protection"] - el_type: geth cl_type: ethlambda cl_image: ghcr.io/lambdaclass/ethlambda:unstable vc_type: ethlambda vc_image: ghcr.io/lambdaclass/ethlambda:unstable + vc_extra_params: ["--init-slashing-protection"] network_params: # ethlambda only supports the mainnet preset: 32-slot epochs of 12 s. preset: mainnet From 396bb609e2d8f2c90447d7d8b43c47a5999a47d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:57:56 -0300 Subject: [PATCH 11/11] fix(ci): fail the Commit-Boost job's devnet step when kurtosis run fails The step piped kurtosis run into tee under bash -e without pipefail, so it took tee's status: a Starlark error passed the step and surfaced only as missing services in the next one. --- .github/workflows/kurtosis_assertoor.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/kurtosis_assertoor.yml b/.github/workflows/kurtosis_assertoor.yml index c112f39ec..796091b64 100644 --- a/.github/workflows/kurtosis_assertoor.yml +++ b/.github/workflows/kurtosis_assertoor.yml @@ -234,8 +234,11 @@ jobs: ' "$RUNNER_TEMP/args.yaml" cat "$RUNNER_TEMP/args.yaml" + # pipefail, or the step takes tee's status: a Starlark error that rolled + # back a service would pass here and surface only in the next step. - name: Start the devnet run: | + set -o pipefail kurtosis run "$RUNNER_TEMP/ethereum-package" \ --enclave "$COMMIT_BOOST_ENCLAVE" --args-file "$RUNNER_TEMP/args.yaml" \ | tee "$RUNNER_TEMP/kurtosis_output.txt"