From d47abbab8cfaf82abec5785b88f36891fd3c7a4c Mon Sep 17 00:00:00 2001 From: syntrust Date: Wed, 7 Jan 2026 12:00:38 +0800 Subject: [PATCH 01/24] healthcheck --- .../workflows/archive_service_healthcheck.yml | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 .github/workflows/archive_service_healthcheck.yml diff --git a/.github/workflows/archive_service_healthcheck.yml b/.github/workflows/archive_service_healthcheck.yml new file mode 100644 index 00000000..c1b4d5c6 --- /dev/null +++ b/.github/workflows/archive_service_healthcheck.yml @@ -0,0 +1,93 @@ +name: Beacon blob_sidecars healthcheck + +on: + schedule: + # Every 8 hours (UTC) + - cron: '0 */8 * * *' + workflow_dispatch: {} + +jobs: + check: + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - name: Check blob_sidecars endpoint + shell: bash + run: | + set -euo pipefail + + url='https://archive.mainnet.ethstorage.io:9645/eth/v1/beacon/blob_sidecars/13164810?indices=3' + + # Capture status code; use retries/timeouts to reduce flakiness. + code="$( + curl --silent --show-error --location \ + --retry 3 --retry-delay 5 --retry-all-errors \ + --connect-timeout 10 --max-time 30 \ + --output /dev/null --write-out '%{http_code}' \ + "$url" \ + || echo '000' + )" + + echo "HTTP status: $code" + if [[ "$code" != "200" ]]; then + echo "Expected HTTP 200 but got $code for: $url" >&2 + exit 1 + fi + + - name: Decide whether to send daily success email + id: daily_success + if: ${{ success() && github.event_name == 'schedule' }} + shell: bash + run: | + set -euo pipefail + + # This workflow runs every 8 hours (00/08/16 UTC). Send success email only once per day. + hour_utc="$(date -u +%H)" + if [[ "$hour_utc" == "00" ]]; then + echo "send=true" >> "$GITHUB_OUTPUT" + else + echo "send=false" >> "$GITHUB_OUTPUT" + fi + + - name: Send daily success email + if: ${{ success() && steps.daily_success.outputs.send == 'true' }} + uses: dawidd6/action-send-mail@v6 + with: + server_address: smtp.gmail.com + server_port: 465 + username: ${{ secrets.SMTP_USERNAME }} + password: ${{ secrets.SMTP_PASSWORD }} + from: ES Archive Service + to: ${{ secrets.ARCHIVE_SERVICE_EMAIL_TO }} + subject: "✅ Archive Service OK (daily)" + body: | + Daily success report: archive service health check passed. + + URL: + https://archive.mainnet.ethstorage.io:9645/eth/v1/beacon/blob_sidecars/13164810?indices=3 + + Repo: ${{ github.repository }} + Workflow: ${{ github.workflow }} + Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + - name: Send email on failure + if: failure() + uses: dawidd6/action-send-mail@v6 + with: + server_address: smtp.gmail.com + server_port: 465 + username: ${{ secrets.SMTP_USERNAME }} + password: ${{ secrets.SMTP_PASSWORD }} + from: ES Archive Service + to: ${{ secrets.ARCHIVE_SERVICE_EMAIL_TO }} + subject: "❌ Archive Service FAILED" + body: | + The scheduled archive service health check failed. + + URL: + https://archive.mainnet.ethstorage.io:9645/eth/v1/beacon/blob_sidecars/13164810?indices=3 + + Repo: ${{ github.repository }} + Workflow: ${{ github.workflow }} + Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} From 474badcc8f62f7f3be7ad6c83322fa63bc5e2427 Mon Sep 17 00:00:00 2001 From: syntrust Date: Wed, 7 Jan 2026 14:26:40 +0800 Subject: [PATCH 02/24] debug --- .github/workflows/archive_service_healthcheck.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/archive_service_healthcheck.yml b/.github/workflows/archive_service_healthcheck.yml index c1b4d5c6..7958201c 100644 --- a/.github/workflows/archive_service_healthcheck.yml +++ b/.github/workflows/archive_service_healthcheck.yml @@ -1,10 +1,17 @@ -name: Beacon blob_sidecars healthcheck +name: Archive Service Healthcheck + +env: + ACTIONS_RUNNER_DEBUG: true + ACTIONS_STEP_DEBUG: true on: + push: + branches: + - archive_alert/* schedule: # Every 8 hours (UTC) - cron: '0 */8 * * *' - workflow_dispatch: {} + workflow_dispatch: jobs: check: From c77aed18c91eebb83bd50d1f0926d9e2804a87b4 Mon Sep 17 00:00:00 2001 From: syntrust Date: Wed, 7 Jan 2026 15:19:17 +0800 Subject: [PATCH 03/24] send email on manual trigger --- .github/workflows/archive_service_healthcheck.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/archive_service_healthcheck.yml b/.github/workflows/archive_service_healthcheck.yml index 7958201c..a6c3285b 100644 --- a/.github/workflows/archive_service_healthcheck.yml +++ b/.github/workflows/archive_service_healthcheck.yml @@ -44,12 +44,18 @@ jobs: - name: Decide whether to send daily success email id: daily_success - if: ${{ success() && github.event_name == 'schedule' }} + if: ${{ success() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} shell: bash run: | set -euo pipefail - # This workflow runs every 8 hours (00/08/16 UTC). Send success email only once per day. + # Manual runs: always email on success. + if [[ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]]; then + echo "send=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # Scheduled runs: every 8 hours (00/08/16 UTC). Send success email only once per day. hour_utc="$(date -u +%H)" if [[ "$hour_utc" == "00" ]]; then echo "send=true" >> "$GITHUB_OUTPUT" From 39756ec789b7a6506dfc7801cd5c3623930cdd1c Mon Sep 17 00:00:00 2001 From: syntrust Date: Wed, 7 Jan 2026 15:25:29 +0800 Subject: [PATCH 04/24] debug --- .github/workflows/archive_service_healthcheck.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/archive_service_healthcheck.yml b/.github/workflows/archive_service_healthcheck.yml index a6c3285b..3a8d0623 100644 --- a/.github/workflows/archive_service_healthcheck.yml +++ b/.github/workflows/archive_service_healthcheck.yml @@ -12,6 +12,9 @@ on: # Every 8 hours (UTC) - cron: '0 */8 * * *' workflow_dispatch: + pull_request: + branches: + - main jobs: check: From 287b1dc0ccd0e115e92e56eb7f08f1f70b76765c Mon Sep 17 00:00:00 2001 From: syntrust Date: Wed, 7 Jan 2026 15:42:21 +0800 Subject: [PATCH 05/24] debug --- .../workflows/archive_service_healthcheck.yml | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/archive_service_healthcheck.yml b/.github/workflows/archive_service_healthcheck.yml index 3a8d0623..ca10424b 100644 --- a/.github/workflows/archive_service_healthcheck.yml +++ b/.github/workflows/archive_service_healthcheck.yml @@ -47,24 +47,25 @@ jobs: - name: Decide whether to send daily success email id: daily_success - if: ${{ success() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} shell: bash run: | set -euo pipefail + send=false + # Manual runs: always email on success. if [[ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]]; then - echo "send=true" >> "$GITHUB_OUTPUT" - exit 0 + send=true + elif [[ "${GITHUB_EVENT_NAME}" == "schedule" ]]; then + # Scheduled runs: every 8 hours (00/08/16 UTC). Send success email only once per day. + hour_utc="$(date -u +%H)" + if [[ "$hour_utc" == "00" ]]; then + send=true + fi fi - # Scheduled runs: every 8 hours (00/08/16 UTC). Send success email only once per day. - hour_utc="$(date -u +%H)" - if [[ "$hour_utc" == "00" ]]; then - echo "send=true" >> "$GITHUB_OUTPUT" - else - echo "send=false" >> "$GITHUB_OUTPUT" - fi + echo "Event: ${GITHUB_EVENT_NAME}" + echo "send=$send" >> "$GITHUB_OUTPUT" - name: Send daily success email if: ${{ success() && steps.daily_success.outputs.send == 'true' }} From b4aebdb8c8f7755d3e558f01f4e098eab9c006a5 Mon Sep 17 00:00:00 2001 From: syntrust Date: Wed, 7 Jan 2026 16:21:25 +0800 Subject: [PATCH 06/24] check content --- .../workflows/archive_service_healthcheck.yml | 55 ++++++++++++++++--- 1 file changed, 48 insertions(+), 7 deletions(-) diff --git a/.github/workflows/archive_service_healthcheck.yml b/.github/workflows/archive_service_healthcheck.yml index ca10424b..9dd4b8f9 100644 --- a/.github/workflows/archive_service_healthcheck.yml +++ b/.github/workflows/archive_service_healthcheck.yml @@ -23,28 +23,62 @@ jobs: steps: - name: Check blob_sidecars endpoint + id: check_endpoint shell: bash run: | set -euo pipefail url='https://archive.mainnet.ethstorage.io:9645/eth/v1/beacon/blob_sidecars/13164810?indices=3' + expected_blob_prefix='0x31000000f000ac44aa0a1981e2' + + resp_file="${RUNNER_TEMP}/blob_sidecars.json" + + write_failure_detail() { + local detail="$1" + local delim="__FAILURE_DETAIL__" + { + echo "failure_detail<<${delim}" + echo "$detail" + echo "${delim}" + } >> "$GITHUB_OUTPUT" + } + + # Ensure file exists so it can be attached in the failure email. + : > "$resp_file" # Capture status code; use retries/timeouts to reduce flakiness. code="$( curl --silent --show-error --location \ --retry 3 --retry-delay 5 --retry-all-errors \ --connect-timeout 10 --max-time 30 \ - --output /dev/null --write-out '%{http_code}' \ + --output "$resp_file" --write-out '%{http_code}' \ "$url" \ || echo '000' )" echo "HTTP status: $code" if [[ "$code" != "200" ]]; then - echo "Expected HTTP 200 but got $code for: $url" >&2 + detail="HTTP 200 expected but got ${code}\n\nURL:\n${url}" + write_failure_detail "$detail" + exit 1 + fi + + blob="$(jq -r '.data[0].blob // empty' "$resp_file")" + if [[ -z "$blob" ]]; then + detail="Missing .data[0].blob in response\n\nURL:\n${url}" + write_failure_detail "$detail" + exit 1 + fi + + if [[ "$blob" != "${expected_blob_prefix}"* ]]; then + detail="Blob prefix mismatch\n\nURL:\n${url}\n\nExpected prefix:\n${expected_blob_prefix}\n\nActual blob starts with:\n${blob:0:${#expected_blob_prefix}}" + write_failure_detail "$detail" exit 1 fi + echo "Blob prefix OK" + write_failure_detail "" + - name: Decide whether to send daily success email id: daily_success shell: bash @@ -64,6 +98,9 @@ jobs: fi fi + # DEBUG + send=true + echo "Event: ${GITHUB_EVENT_NAME}" echo "send=$send" >> "$GITHUB_OUTPUT" @@ -99,12 +136,16 @@ jobs: from: ES Archive Service to: ${{ secrets.ARCHIVE_SERVICE_EMAIL_TO }} subject: "❌ Archive Service FAILED" + attachments: ${{ runner.temp }}/blob_sidecars.json body: | - The scheduled archive service health check failed. + The archive service health check failed. - URL: - https://archive.mainnet.ethstorage.io:9645/eth/v1/beacon/blob_sidecars/13164810?indices=3 + Event: ${{ github.event_name }} + URL: https://archive.mainnet.ethstorage.io:9645/eth/v1/beacon/blob_sidecars/13164810?indices=3 + + Summary: + ${{ steps.check_endpoint.outputs.failure_detail }} + + Full response is attached: blob_sidecars.json - Repo: ${{ github.repository }} - Workflow: ${{ github.workflow }} Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} From 453c2775b99a2d5a58b2591f2392f4184652b6e8 Mon Sep 17 00:00:00 2001 From: syntrust Date: Wed, 7 Jan 2026 16:26:46 +0800 Subject: [PATCH 07/24] Trigger workflow From 64083a85f9aec464abc985f3d772511d0e7363a0 Mon Sep 17 00:00:00 2001 From: syntrust Date: Wed, 7 Jan 2026 16:33:57 +0800 Subject: [PATCH 08/24] debug --- .../workflows/archive_service_healthcheck.yml | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/.github/workflows/archive_service_healthcheck.yml b/.github/workflows/archive_service_healthcheck.yml index 9dd4b8f9..82b89e24 100644 --- a/.github/workflows/archive_service_healthcheck.yml +++ b/.github/workflows/archive_service_healthcheck.yml @@ -3,11 +3,10 @@ name: Archive Service Healthcheck env: ACTIONS_RUNNER_DEBUG: true ACTIONS_STEP_DEBUG: true + ARCHIVE_BLOB_SIDECARS_URL: https://archive.mainnet.ethstorage.io:9645/eth/v1/beacon/blob_sidecars/13164810?indices=3 + ARCHIVE_BLOB_PREFIX: 0x31000000f000ac44aa0a1981e2x on: - push: - branches: - - archive_alert/* schedule: # Every 8 hours (UTC) - cron: '0 */8 * * *' @@ -28,8 +27,8 @@ jobs: run: | set -euo pipefail - url='https://archive.mainnet.ethstorage.io:9645/eth/v1/beacon/blob_sidecars/13164810?indices=3' - expected_blob_prefix='0x31000000f000ac44aa0a1981e2' + url='${{ env.ARCHIVE_BLOB_SIDECARS_URL }}' + expected_blob_prefix='${{ env.ARCHIVE_BLOB_PREFIX }}' resp_file="${RUNNER_TEMP}/blob_sidecars.json" @@ -98,9 +97,6 @@ jobs: fi fi - # DEBUG - send=true - echo "Event: ${GITHUB_EVENT_NAME}" echo "send=$send" >> "$GITHUB_OUTPUT" @@ -119,7 +115,7 @@ jobs: Daily success report: archive service health check passed. URL: - https://archive.mainnet.ethstorage.io:9645/eth/v1/beacon/blob_sidecars/13164810?indices=3 + ${{ env.ARCHIVE_BLOB_SIDECARS_URL }} Repo: ${{ github.repository }} Workflow: ${{ github.workflow }} @@ -134,14 +130,14 @@ jobs: username: ${{ secrets.SMTP_USERNAME }} password: ${{ secrets.SMTP_PASSWORD }} from: ES Archive Service - to: ${{ secrets.ARCHIVE_SERVICE_EMAIL_TO }} + to: lundeng@quarkchain.org subject: "❌ Archive Service FAILED" attachments: ${{ runner.temp }}/blob_sidecars.json body: | The archive service health check failed. Event: ${{ github.event_name }} - URL: https://archive.mainnet.ethstorage.io:9645/eth/v1/beacon/blob_sidecars/13164810?indices=3 + URL: ${{ env.ARCHIVE_BLOB_SIDECARS_URL }} Summary: ${{ steps.check_endpoint.outputs.failure_detail }} From d740c076918f10475f0308313ce4873e4fb66b53 Mon Sep 17 00:00:00 2001 From: syntrust Date: Wed, 7 Jan 2026 16:42:55 +0800 Subject: [PATCH 09/24] debug --- .github/workflows/archive_service_healthcheck.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/archive_service_healthcheck.yml b/.github/workflows/archive_service_healthcheck.yml index 82b89e24..cddd7fa8 100644 --- a/.github/workflows/archive_service_healthcheck.yml +++ b/.github/workflows/archive_service_healthcheck.yml @@ -106,8 +106,8 @@ jobs: with: server_address: smtp.gmail.com server_port: 465 - username: ${{ secrets.SMTP_USERNAME }} - password: ${{ secrets.SMTP_PASSWORD }} + username: ${{ secrets.ARCHIVE_SERVICE_SMTP_USERNAME }} + password: ${{ secrets.ARCHIVE_SERVICE_SMTP_PASSWORD }} from: ES Archive Service to: ${{ secrets.ARCHIVE_SERVICE_EMAIL_TO }} subject: "✅ Archive Service OK (daily)" @@ -127,10 +127,10 @@ jobs: with: server_address: smtp.gmail.com server_port: 465 - username: ${{ secrets.SMTP_USERNAME }} - password: ${{ secrets.SMTP_PASSWORD }} + username: ${{ secrets.ARCHIVE_SERVICE_SMTP_USERNAME }} + password: ${{ secrets.ARCHIVE_SERVICE_SMTP_PASSWORD }} from: ES Archive Service - to: lundeng@quarkchain.org + to: ${{ secrets.ARCHIVE_SERVICE_EMAIL_TO }} subject: "❌ Archive Service FAILED" attachments: ${{ runner.temp }}/blob_sidecars.json body: | From e06397e816bef6004ef12dc470958c9dcf90c222 Mon Sep 17 00:00:00 2001 From: syntrust Date: Wed, 7 Jan 2026 17:18:23 +0800 Subject: [PATCH 10/24] debug --- .../workflows/archive_service_healthcheck.yml | 23 ++++--------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/.github/workflows/archive_service_healthcheck.yml b/.github/workflows/archive_service_healthcheck.yml index cddd7fa8..d4c0d663 100644 --- a/.github/workflows/archive_service_healthcheck.yml +++ b/.github/workflows/archive_service_healthcheck.yml @@ -32,16 +32,6 @@ jobs: resp_file="${RUNNER_TEMP}/blob_sidecars.json" - write_failure_detail() { - local detail="$1" - local delim="__FAILURE_DETAIL__" - { - echo "failure_detail<<${delim}" - echo "$detail" - echo "${delim}" - } >> "$GITHUB_OUTPUT" - } - # Ensure file exists so it can be attached in the failure email. : > "$resp_file" @@ -57,26 +47,22 @@ jobs: echo "HTTP status: $code" if [[ "$code" != "200" ]]; then - detail="HTTP 200 expected but got ${code}\n\nURL:\n${url}" - write_failure_detail "$detail" + echo "HTTP 200 expected but got ${code}" >> $GITHUB_OUTPUT exit 1 fi blob="$(jq -r '.data[0].blob // empty' "$resp_file")" if [[ -z "$blob" ]]; then - detail="Missing .data[0].blob in response\n\nURL:\n${url}" - write_failure_detail "$detail" + echo "Missing .data[0].blob in response" >> $GITHUB_OUTPUT exit 1 fi if [[ "$blob" != "${expected_blob_prefix}"* ]]; then - detail="Blob prefix mismatch\n\nURL:\n${url}\n\nExpected prefix:\n${expected_blob_prefix}\n\nActual blob starts with:\n${blob:0:${#expected_blob_prefix}}" - write_failure_detail "$detail" + echo "Blob data mismatch" >> $GITHUB_OUTPUT exit 1 fi - echo "Blob prefix OK" - write_failure_detail "" + echo "Blob data OK" - name: Decide whether to send daily success email id: daily_success @@ -139,7 +125,6 @@ jobs: Event: ${{ github.event_name }} URL: ${{ env.ARCHIVE_BLOB_SIDECARS_URL }} - Summary: ${{ steps.check_endpoint.outputs.failure_detail }} Full response is attached: blob_sidecars.json From ca4c13c8e788c5be4fe9f6d45b75ba6fc74f2b7a Mon Sep 17 00:00:00 2001 From: syntrust Date: Wed, 7 Jan 2026 17:26:24 +0800 Subject: [PATCH 11/24] debug --- .github/workflows/archive_service_healthcheck.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/archive_service_healthcheck.yml b/.github/workflows/archive_service_healthcheck.yml index d4c0d663..7323e45f 100644 --- a/.github/workflows/archive_service_healthcheck.yml +++ b/.github/workflows/archive_service_healthcheck.yml @@ -47,18 +47,19 @@ jobs: echo "HTTP status: $code" if [[ "$code" != "200" ]]; then - echo "HTTP 200 expected but got ${code}" >> $GITHUB_OUTPUT + failure_detail="Unexpected HTTP status code: expected 200, got ${code}" + echo "failure_detail=$failure_detail" >> $GITHUB_OUTPUT exit 1 fi blob="$(jq -r '.data[0].blob // empty' "$resp_file")" if [[ -z "$blob" ]]; then - echo "Missing .data[0].blob in response" >> $GITHUB_OUTPUT + echo "failure_detail=Missing .data[0].blob in response" >> $GITHUB_OUTPUT exit 1 fi if [[ "$blob" != "${expected_blob_prefix}"* ]]; then - echo "Blob data mismatch" >> $GITHUB_OUTPUT + echo "failure_detail=Blob data mismatch" >> $GITHUB_OUTPUT exit 1 fi From 5d32dae1be41b890d94d3afbdc655a570e53f26f Mon Sep 17 00:00:00 2001 From: syntrust Date: Wed, 7 Jan 2026 17:33:59 +0800 Subject: [PATCH 12/24] debug --- .github/workflows/archive_service_healthcheck.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/archive_service_healthcheck.yml b/.github/workflows/archive_service_healthcheck.yml index 7323e45f..bc4da075 100644 --- a/.github/workflows/archive_service_healthcheck.yml +++ b/.github/workflows/archive_service_healthcheck.yml @@ -4,7 +4,7 @@ env: ACTIONS_RUNNER_DEBUG: true ACTIONS_STEP_DEBUG: true ARCHIVE_BLOB_SIDECARS_URL: https://archive.mainnet.ethstorage.io:9645/eth/v1/beacon/blob_sidecars/13164810?indices=3 - ARCHIVE_BLOB_PREFIX: 0x31000000f000ac44aa0a1981e2x + ARCHIVE_BLOB_PREFIX: 0x31000000f000ac44aa0a1981e2 on: schedule: @@ -84,6 +84,9 @@ jobs: fi fi + # Debug + send=true + echo "Event: ${GITHUB_EVENT_NAME}" echo "send=$send" >> "$GITHUB_OUTPUT" @@ -97,17 +100,13 @@ jobs: password: ${{ secrets.ARCHIVE_SERVICE_SMTP_PASSWORD }} from: ES Archive Service to: ${{ secrets.ARCHIVE_SERVICE_EMAIL_TO }} - subject: "✅ Archive Service OK (daily)" + subject: "✅ Archive Service OK" body: | Daily success report: archive service health check passed. URL: ${{ env.ARCHIVE_BLOB_SIDECARS_URL }} - Repo: ${{ github.repository }} - Workflow: ${{ github.workflow }} - Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - - name: Send email on failure if: failure() uses: dawidd6/action-send-mail@v6 From e0e939c9fdc0a4fec73ec2f891be429c940ecf93 Mon Sep 17 00:00:00 2001 From: syntrust Date: Wed, 7 Jan 2026 17:39:49 +0800 Subject: [PATCH 13/24] debug --- .github/workflows/archive_service_healthcheck.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/archive_service_healthcheck.yml b/.github/workflows/archive_service_healthcheck.yml index bc4da075..27715d44 100644 --- a/.github/workflows/archive_service_healthcheck.yml +++ b/.github/workflows/archive_service_healthcheck.yml @@ -4,7 +4,7 @@ env: ACTIONS_RUNNER_DEBUG: true ACTIONS_STEP_DEBUG: true ARCHIVE_BLOB_SIDECARS_URL: https://archive.mainnet.ethstorage.io:9645/eth/v1/beacon/blob_sidecars/13164810?indices=3 - ARCHIVE_BLOB_PREFIX: 0x31000000f000ac44aa0a1981e2 + ARCHIVE_BLOB_PREFIX: "0x31000000f000ac44aa0a1981e2" on: schedule: From 9e08ad09635e14c03dee3b31d81f30cfff623641 Mon Sep 17 00:00:00 2001 From: syntrust Date: Wed, 7 Jan 2026 18:18:28 +0800 Subject: [PATCH 14/24] fix permission --- .github/workflows/archive_service_healthcheck.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/archive_service_healthcheck.yml b/.github/workflows/archive_service_healthcheck.yml index 27715d44..27abc35b 100644 --- a/.github/workflows/archive_service_healthcheck.yml +++ b/.github/workflows/archive_service_healthcheck.yml @@ -1,8 +1,8 @@ name: Archive Service Healthcheck - +permissions: + contents: read + pull-requests: write env: - ACTIONS_RUNNER_DEBUG: true - ACTIONS_STEP_DEBUG: true ARCHIVE_BLOB_SIDECARS_URL: https://archive.mainnet.ethstorage.io:9645/eth/v1/beacon/blob_sidecars/13164810?indices=3 ARCHIVE_BLOB_PREFIX: "0x31000000f000ac44aa0a1981e2" From 662fc0fbcbf4e97a56a71bbf55538ed1e1d7849e Mon Sep 17 00:00:00 2001 From: syntrust Date: Thu, 8 Jan 2026 16:30:56 +0800 Subject: [PATCH 15/24] Check latest blob --- .../workflows/archive_service_healthcheck.yml | 155 +++++++++---- .../scripts/check_latest_blob.sh | 203 ++++++++++++++++++ 2 files changed, 322 insertions(+), 36 deletions(-) create mode 100644 integration_tests/scripts/check_latest_blob.sh diff --git a/.github/workflows/archive_service_healthcheck.yml b/.github/workflows/archive_service_healthcheck.yml index 27abc35b..0233c6e7 100644 --- a/.github/workflows/archive_service_healthcheck.yml +++ b/.github/workflows/archive_service_healthcheck.yml @@ -2,9 +2,6 @@ name: Archive Service Healthcheck permissions: contents: read pull-requests: write -env: - ARCHIVE_BLOB_SIDECARS_URL: https://archive.mainnet.ethstorage.io:9645/eth/v1/beacon/blob_sidecars/13164810?indices=3 - ARCHIVE_BLOB_PREFIX: "0x31000000f000ac44aa0a1981e2" on: schedule: @@ -16,9 +13,17 @@ on: - main jobs: - check: + check_blob_sidecars_API_for_critical_blob: + name: Check blob_sidecars API for critical blob runs-on: ubuntu-latest timeout-minutes: 5 + env: + ARCHIVE_BLOB_SIDECARS_URL: https://archive.mainnet.ethstorage.io:9645/eth/v1/beacon/blob_sidecars/13164810?indices=3 + ARCHIVE_BLOB_PREFIX: "0x31000000f000ac44aa0a1981e2" + + outputs: + failure_detail: ${{ steps.check_endpoint.outputs.failure_detail }} + daily_success_send: ${{ steps.daily_success.outputs.send }} steps: - name: Check blob_sidecars endpoint @@ -84,31 +89,120 @@ jobs: fi fi - # Debug - send=true - echo "Event: ${GITHUB_EVENT_NAME}" echo "send=$send" >> "$GITHUB_OUTPUT" - - name: Send daily success email - if: ${{ success() && steps.daily_success.outputs.send == 'true' }} - uses: dawidd6/action-send-mail@v6 + check_latest_uploaded_blob: + name: Check latest blob with blobs API from Beacon + runs-on: ubuntu-latest + timeout-minutes: 10 + + outputs: + failure_detail: ${{ steps.putblob.outputs.failure_detail }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install foundry (cast) + uses: foundry-rs/foundry-toolchain@v1 with: - server_address: smtp.gmail.com - server_port: 465 - username: ${{ secrets.ARCHIVE_SERVICE_SMTP_USERNAME }} - password: ${{ secrets.ARCHIVE_SERVICE_SMTP_PASSWORD }} - from: ES Archive Service - to: ${{ secrets.ARCHIVE_SERVICE_EMAIL_TO }} - subject: "✅ Archive Service OK" - body: | - Daily success report: archive service health check passed. + version: stable + + - name: Find latest PutBlob and fetch beacon blobs + id: putblob + shell: bash + env: + EL_RPC_URL: ${{ secrets.ARCHIVE_SERVICE_EL_RPC_URL }} + BEACON_API: ${{ secrets.ARCHIVE_SERVICE_BEACON_API }} + run: | + set -euo pipefail + out_file="$(mktemp)" + if bash ./integration_tests/scripts/check_latest_blob.sh >"$out_file" 2>&1; then + exit 0 + fi + + echo "failure_detail<> "$GITHUB_OUTPUT" + tail -n 80 "$out_file" >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + cat "$out_file" >&2 + exit 1 - URL: - ${{ env.ARCHIVE_BLOB_SIDECARS_URL }} + notify: + name: Notify (combined) + runs-on: ubuntu-latest + timeout-minutes: 5 + needs: + - check_blob_sidecars_API_for_critical_blob + - check_latest_uploaded_blob + if: ${{ always() }} - - name: Send email on failure - if: failure() + steps: + - name: Compose email + id: compose + shell: bash + env: + EVENT_NAME: ${{ github.event_name }} + BLOB_RESULT: ${{ needs.check_blob_sidecars_API_for_critical_blob.result }} + PUTBLOB_RESULT: ${{ needs.check_latest_uploaded_blob.result }} + BLOB_FAILURE_DETAIL: ${{ needs.check_blob_sidecars_API_for_critical_blob.outputs.failure_detail }} + PUTBLOB_FAILURE_DETAIL: ${{ needs.check_latest_uploaded_blob.outputs.failure_detail }} + ARCHIVE_BLOB_SIDECARS_URL: ${{ env.ARCHIVE_BLOB_SIDECARS_URL }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + + send=false + overall="OK" + + if [[ "$BLOB_RESULT" != "success" || "$PUTBLOB_RESULT" != "success" ]]; then + overall="FAILED" + send=true + else + # Success: send only on workflow_dispatch, or once per day at 00 UTC for scheduled runs. + if [[ "$EVENT_NAME" == "workflow_dispatch" ]]; then + send=true + elif [[ "$EVENT_NAME" == "schedule" ]]; then + hour_utc="$(date -u +%H)" + if [[ "$hour_utc" == "00" ]]; then + send=true + fi + fi + fi + + if [[ "$overall" == "OK" ]]; then + subject="✅ Archive Service OK" + else + subject="❌ Archive Service FAILED" + fi + + # Debug + send=true + + echo "send=$send" >> "$GITHUB_OUTPUT" + echo "subject=$subject" >> "$GITHUB_OUTPUT" + + echo "body<> "$GITHUB_OUTPUT" + echo "Archive Service Healthcheck: $overall" >> "$GITHUB_OUTPUT" + echo >> "$GITHUB_OUTPUT" + echo "Event: $EVENT_NAME" >> "$GITHUB_OUTPUT" + echo "Run: $RUN_URL" >> "$GITHUB_OUTPUT" + echo >> "$GITHUB_OUTPUT" + echo "check_blob_sidecars_API_for_critical_blob: $BLOB_RESULT" >> "$GITHUB_OUTPUT" + echo " URL: $ARCHIVE_BLOB_SIDECARS_URL" >> "$GITHUB_OUTPUT" + if [[ -n "${BLOB_FAILURE_DETAIL:-}" ]]; then + echo " Failure detail: $BLOB_FAILURE_DETAIL" >> "$GITHUB_OUTPUT" + fi + echo >> "$GITHUB_OUTPUT" + echo "check_latest_uploaded_blob: $PUTBLOB_RESULT" >> "$GITHUB_OUTPUT" + if [[ -n "${PUTBLOB_FAILURE_DETAIL:-}" ]]; then + echo " Failure detail (tail):" >> "$GITHUB_OUTPUT" + echo "$PUTBLOB_FAILURE_DETAIL" >> "$GITHUB_OUTPUT" + fi + echo "EOF" >> "$GITHUB_OUTPUT" + + - name: Send combined email + if: ${{ steps.compose.outputs.send == 'true' }} uses: dawidd6/action-send-mail@v6 with: server_address: smtp.gmail.com @@ -117,16 +211,5 @@ jobs: password: ${{ secrets.ARCHIVE_SERVICE_SMTP_PASSWORD }} from: ES Archive Service to: ${{ secrets.ARCHIVE_SERVICE_EMAIL_TO }} - subject: "❌ Archive Service FAILED" - attachments: ${{ runner.temp }}/blob_sidecars.json - body: | - The archive service health check failed. - - Event: ${{ github.event_name }} - URL: ${{ env.ARCHIVE_BLOB_SIDECARS_URL }} - - ${{ steps.check_endpoint.outputs.failure_detail }} - - Full response is attached: blob_sidecars.json - - Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + subject: ${{ steps.compose.outputs.subject }} + body: ${{ steps.compose.outputs.body }} diff --git a/integration_tests/scripts/check_latest_blob.sh b/integration_tests/scripts/check_latest_blob.sh new file mode 100644 index 00000000..602f2829 --- /dev/null +++ b/integration_tests/scripts/check_latest_blob.sh @@ -0,0 +1,203 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Hardcoded mainnet constants +genesis_time=1606824023 +seconds_per_slot=12 + +# Search back this many blocks for PutBlob events +lookback=2400 + +ES_CONTRACT_ADDRESS="${ES_CONTRACT_ADDRESS:-0xf0193d6E8fc186e77b6E63af4151db07524f6a7A}" +ARCHIVE_RPC_URL="${ARCHIVE_RPC_URL:-https://archive.mainnet.ethstorage.io:9645}" + +EL_RPC_URL="${EL_RPC_URL:-}" +BEACON_API="${BEACON_API:-}" + +if [[ -z "${EL_RPC_URL:-}" ]]; then + echo "Missing EL_RPC_URL" >&2 + exit 2 +fi + +if [[ -z "${BEACON_API:-}" ]]; then + echo "Missing BEACON_API" >&2 + exit 2 +fi + +for bin in cast jq curl; do + command -v "$bin" >/dev/null 2>&1 || { echo "Missing dependency: $bin" >&2; exit 2; } +done + +topic0="$(cast sig-event "PutBlob(uint256,uint256,bytes32)")" + +latest_dec="$(cast block-number -r "$EL_RPC_URL")" +if [[ -z "$latest_dec" ]]; then +echo "Failed to get latest block" >&2 +exit 1 +fi + +from_dec=$(( latest_dec > lookback ? latest_dec - lookback : 0 )) +from_hex="$(cast to-hex "$from_dec")" +to_hex="$(cast to-hex "$latest_dec")" + +echo "Searching PutBlob logs" +echo " contract: $ES_CONTRACT_ADDRESS" +echo " blocks: $from_dec..$latest_dec" + +params="$(jq -nc --arg from "$from_hex" --arg to "$to_hex" --arg addr "$ES_CONTRACT_ADDRESS" --arg t0 "$topic0" '[{fromBlock:$from,toBlock:$to,address:$addr,topics:[$t0]}]')" +logs_json="$(cast rpc --raw -r "$EL_RPC_URL" eth_getLogs "$params")" + +# Handle both possible shapes: +# - raw JSON-RPC: {"jsonrpc":"2.0","id":...,"result":[...]} +# - cast-decoded result: [...] +if echo "$logs_json" | jq -e 'type=="object" and has("error")' >/dev/null 2>&1; then + echo "EL RPC returned error:" >&2 + echo "$logs_json" | jq -c '.error' >&2 + exit 1 +fi + +logs_arr="$(echo "$logs_json" | jq -c 'if type=="object" and has("result") then .result elif type=="array" then . else [] end')" + +# eth_getLogs result order is ascending by blockNumber/logIndex; take the last entry. +last_log="$(echo "$logs_arr" | jq -c 'last')" +if [[ "$last_log" == "null" || -z "$last_log" ]]; then + echo "No PutBlob logs found in lookback window" >&2 + exit 1 +fi + +# PutBlob(uint256 indexed kvIdx, uint256 indexed kvSize, bytes32 indexed dataHash) +dataHash="$(echo "$last_log" | jq -r '.topics[3] // empty')" + +if [[ -z "$dataHash" ]]; then + echo "Missing expected PutBlob topic (need topics[3])" >&2 + echo "$last_log" | jq -c . >&2 + exit 1 +fi + +echo "PutBlob params" +echo " dataHash: $dataHash" + +block_hex="$(echo "$last_log" | jq -r '.blockNumber')" +tx_hash="$(echo "$last_log" | jq -r '.transactionHash')" + +# Get block timestamp +block_json="$(cast rpc --raw -r "$EL_RPC_URL" eth_getBlockByNumber "[\"$block_hex\", false]")" +if echo "$block_json" | jq -e 'type=="object" and has("error")' >/dev/null 2>&1; then + echo "EL RPC returned error (eth_getBlockByNumber):" >&2 + echo "$block_json" | jq -c '.error' >&2 + exit 1 +fi + +block_obj="$(echo "$block_json" | jq -c 'if type=="object" and has("result") then .result elif type=="object" then . else empty end')" +if [[ -z "$block_obj" || "$block_obj" == "null" ]]; then + echo "eth_getBlockByNumber returned empty block for $block_hex" >&2 + exit 1 +fi + +ts_hex="$(echo "$block_obj" | jq -r '.timestamp // empty')" +if [[ -z "$ts_hex" || "$ts_hex" == "null" ]]; then + echo "Missing block timestamp in eth_getBlockByNumber response" >&2 + echo "$block_obj" | jq -c . >&2 + exit 1 +fi + +if [[ "$ts_hex" == 0x* || "$ts_hex" == 0X* ]]; then + ts_dec=$((16#${ts_hex#0x})) +else + # Some providers may return decimal timestamps. + ts_dec=$((ts_hex)) +fi + +if (( ts_dec < genesis_time )); then + echo "Block timestamp is before genesis_time" >&2 + exit 1 +fi + +slot=$(((ts_dec - genesis_time) / seconds_per_slot)) + +echo "Latest PutBlob" +echo " tx: $tx_hash" +echo " block: $(cast to-dec "$block_hex")" +echo " slot: $slot" + +# Compare `.data` between beacon blobs API and archive service blobs API. +beacon_url="$BEACON_API/eth/v1/beacon/blobs/$slot?versioned_hashes=${dataHash}" +archive_url="$ARCHIVE_RPC_URL/eth/v1/beacon/blobs/$slot?versioned_hashes=${dataHash}" + +echo "Fetching blobs (filtered by versioned_hashes)" +echo " beacon: $beacon_url" +echo " archiveService: $archive_url" + +beacon_resp="$( + curl --silent --show-error --location \ + --retry 3 --retry-delay 2 --retry-all-errors \ + --connect-timeout 10 --max-time 60 \ + --write-out '\n%{http_code}' \ + "$beacon_url" \ + || printf '\n000' +)" +beacon_code="${beacon_resp##*$'\n'}" +beacon_body="${beacon_resp%$'\n'*}" +echo "Beacon /blobs status: $beacon_code" +if [[ "$beacon_code" != "200" ]]; then + echo "Beacon blobs request failed (HTTP $beacon_code)" >&2 + echo "URL: $beacon_url" >&2 + exit 1 +fi + +archive_resp="$( + curl --silent --show-error --location \ + --retry 3 --retry-delay 2 --retry-all-errors \ + --connect-timeout 10 --max-time 60 \ + --write-out '\n%{http_code}' \ + "$archive_url" \ + || printf '\n000' +)" +archive_code="${archive_resp##*$'\n'}" +archive_body="${archive_resp%$'\n'*}" +echo "ArchiveService /blobs status: $archive_code" +if [[ "$archive_code" != "200" ]]; then + echo "Archive service blobs request failed (HTTP $archive_code)" >&2 + echo "URL: $archive_url" >&2 + exit 1 +fi + +# Basic sanity: both responses have `.data` as array +if [[ "$(jq -r '.data | type' <<<"$beacon_body")" != "array" ]]; then + echo "Unexpected beacon /blobs response shape" >&2 + jq -c . <<<"$beacon_body" >&2 || true + exit 1 +fi +if [[ "$(jq -r '.data | type' <<<"$archive_body")" != "array" ]]; then + echo "Unexpected archive service /blobs response shape" >&2 + jq -c . <<<"$archive_body" >&2 || true + exit 1 +fi + +beacon_data_norm="$(jq -cS '.data' <<<"$beacon_body")" +archive_data_norm="$(jq -cS '.data' <<<"$archive_body")" + +if [[ "$beacon_data_norm" == "$archive_data_norm" ]]; then + echo "OK: beacon .data matches archive service .data" + jq -r '.data | length' <<<"$beacon_body" | awk '{print " blobCount: " $1}' + exit 0 +fi + +echo "Mismatch: beacon .data differs from archive service .data" >&2 +echo "Beacon URL: $beacon_url" >&2 +echo "ArchiveService URL: $archive_url" >&2 + +beacon_len="$(jq -r '.data | length' <<<"$beacon_body" 2>/dev/null || echo '?')" +archive_len="$(jq -r '.data | length' <<<"$archive_body" 2>/dev/null || echo '?')" +echo "beacon data length: $beacon_len" >&2 +echo "archive data length: $archive_len" >&2 + +# Compare `.data` contents +if command -v diff >/dev/null 2>&1; then + diff -u \ + <(jq -S '.data' <<<"$beacon_body") \ + <(jq -S '.data' <<<"$archive_body") \ + >&2 || true +fi + +exit 1 \ No newline at end of file From d33876d506ed760a1756619683b07d9775950a24 Mon Sep 17 00:00:00 2001 From: syntrust Date: Thu, 8 Jan 2026 16:46:43 +0800 Subject: [PATCH 16/24] debug --- .../workflows/archive_service_healthcheck.yml | 17 +++++++++++------ integration_tests/scripts/check_latest_blob.sh | 5 +++++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.github/workflows/archive_service_healthcheck.yml b/.github/workflows/archive_service_healthcheck.yml index 0233c6e7..261cb46a 100644 --- a/.github/workflows/archive_service_healthcheck.yml +++ b/.github/workflows/archive_service_healthcheck.yml @@ -23,6 +23,7 @@ jobs: outputs: failure_detail: ${{ steps.check_endpoint.outputs.failure_detail }} + archive_blob_sidecars_url: ${{ steps.check_endpoint.outputs.archive_blob_sidecars_url }} daily_success_send: ${{ steps.daily_success.outputs.send }} steps: @@ -35,6 +36,8 @@ jobs: url='${{ env.ARCHIVE_BLOB_SIDECARS_URL }}' expected_blob_prefix='${{ env.ARCHIVE_BLOB_PREFIX }}' + echo "archive_blob_sidecars_url=$url" >> "$GITHUB_OUTPUT" + resp_file="${RUNNER_TEMP}/blob_sidecars.json" # Ensure file exists so it can be attached in the failure email. @@ -99,6 +102,7 @@ jobs: outputs: failure_detail: ${{ steps.putblob.outputs.failure_detail }} + archive_url: ${{ steps.putblob.outputs.archive_url }} steps: - name: Checkout @@ -147,7 +151,8 @@ jobs: PUTBLOB_RESULT: ${{ needs.check_latest_uploaded_blob.result }} BLOB_FAILURE_DETAIL: ${{ needs.check_blob_sidecars_API_for_critical_blob.outputs.failure_detail }} PUTBLOB_FAILURE_DETAIL: ${{ needs.check_latest_uploaded_blob.outputs.failure_detail }} - ARCHIVE_BLOB_SIDECARS_URL: ${{ env.ARCHIVE_BLOB_SIDECARS_URL }} + PUTBLOB_ARCHIVE_URL: ${{ needs.check_latest_uploaded_blob.outputs.archive_url }} + ARCHIVE_BLOB_SIDECARS_URL: ${{ needs.check_blob_sidecars_API_for_critical_blob.outputs.archive_blob_sidecars_url }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | set -euo pipefail @@ -171,14 +176,11 @@ jobs: fi if [[ "$overall" == "OK" ]]; then - subject="✅ Archive Service OK" + subject="✅ Archive Service Healthcheck OK" else - subject="❌ Archive Service FAILED" + subject="❌ Archive Service Healthcheck FAILED" fi - # Debug - send=true - echo "send=$send" >> "$GITHUB_OUTPUT" echo "subject=$subject" >> "$GITHUB_OUTPUT" @@ -195,6 +197,9 @@ jobs: fi echo >> "$GITHUB_OUTPUT" echo "check_latest_uploaded_blob: $PUTBLOB_RESULT" >> "$GITHUB_OUTPUT" + if [[ -n "${PUTBLOB_ARCHIVE_URL:-}" ]]; then + echo " Archive URL: $PUTBLOB_ARCHIVE_URL" >> "$GITHUB_OUTPUT" + fi if [[ -n "${PUTBLOB_FAILURE_DETAIL:-}" ]]; then echo " Failure detail (tail):" >> "$GITHUB_OUTPUT" echo "$PUTBLOB_FAILURE_DETAIL" >> "$GITHUB_OUTPUT" diff --git a/integration_tests/scripts/check_latest_blob.sh b/integration_tests/scripts/check_latest_blob.sh index 602f2829..bc9f2dbb 100644 --- a/integration_tests/scripts/check_latest_blob.sh +++ b/integration_tests/scripts/check_latest_blob.sh @@ -124,6 +124,11 @@ echo " slot: $slot" beacon_url="$BEACON_API/eth/v1/beacon/blobs/$slot?versioned_hashes=${dataHash}" archive_url="$ARCHIVE_RPC_URL/eth/v1/beacon/blobs/$slot?versioned_hashes=${dataHash}" +if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + echo "beacon_url=$beacon_url" >> "$GITHUB_OUTPUT" + echo "archive_url=$archive_url" >> "$GITHUB_OUTPUT" +fi + echo "Fetching blobs (filtered by versioned_hashes)" echo " beacon: $beacon_url" echo " archiveService: $archive_url" From 15393a80afb4ca6d0e65f0adac787c81c99a07b8 Mon Sep 17 00:00:00 2001 From: syntrust Date: Thu, 8 Jan 2026 16:50:38 +0800 Subject: [PATCH 17/24] debug --- .github/workflows/archive_service_healthcheck.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/archive_service_healthcheck.yml b/.github/workflows/archive_service_healthcheck.yml index 261cb46a..8e3298d7 100644 --- a/.github/workflows/archive_service_healthcheck.yml +++ b/.github/workflows/archive_service_healthcheck.yml @@ -122,14 +122,17 @@ jobs: run: | set -euo pipefail out_file="$(mktemp)" - if bash ./integration_tests/scripts/check_latest_blob.sh >"$out_file" 2>&1; then + set +e + bash ./integration_tests/scripts/check_latest_blob.sh 2>&1 | tee "$out_file" + status=${PIPESTATUS[0]} + set -e + if [[ "$status" == "0" ]]; then exit 0 fi echo "failure_detail<> "$GITHUB_OUTPUT" tail -n 80 "$out_file" >> "$GITHUB_OUTPUT" echo "EOF" >> "$GITHUB_OUTPUT" - cat "$out_file" >&2 exit 1 notify: @@ -181,6 +184,9 @@ jobs: subject="❌ Archive Service Healthcheck FAILED" fi + # Debug + send=true + echo "send=$send" >> "$GITHUB_OUTPUT" echo "subject=$subject" >> "$GITHUB_OUTPUT" From b93e560d54d25ad5e03a26026c8a78bda784e086 Mon Sep 17 00:00:00 2001 From: syntrust Date: Thu, 8 Jan 2026 17:15:46 +0800 Subject: [PATCH 18/24] use finalized --- integration_tests/scripts/check_latest_blob.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) mode change 100644 => 100755 integration_tests/scripts/check_latest_blob.sh diff --git a/integration_tests/scripts/check_latest_blob.sh b/integration_tests/scripts/check_latest_blob.sh old mode 100644 new mode 100755 index bc9f2dbb..16fa888b --- a/integration_tests/scripts/check_latest_blob.sh +++ b/integration_tests/scripts/check_latest_blob.sh @@ -36,7 +36,8 @@ echo "Failed to get latest block" >&2 exit 1 fi -from_dec=$(( latest_dec > lookback ? latest_dec - lookback : 0 )) +latest_dec=$((latest_dec - 64)) # adjust to finalized +from_dec=$((latest_dec - lookback)) from_hex="$(cast to-hex "$from_dec")" to_hex="$(cast to-hex "$latest_dec")" From 37a89e9ad0995724bbd2abe4c494c4c4eb5a672d Mon Sep 17 00:00:00 2001 From: syntrust Date: Thu, 8 Jan 2026 17:19:15 +0800 Subject: [PATCH 19/24] rm debug --- .github/workflows/archive_service_healthcheck.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/archive_service_healthcheck.yml b/.github/workflows/archive_service_healthcheck.yml index 8e3298d7..d9e90fc4 100644 --- a/.github/workflows/archive_service_healthcheck.yml +++ b/.github/workflows/archive_service_healthcheck.yml @@ -184,9 +184,6 @@ jobs: subject="❌ Archive Service Healthcheck FAILED" fi - # Debug - send=true - echo "send=$send" >> "$GITHUB_OUTPUT" echo "subject=$subject" >> "$GITHUB_OUTPUT" From 3f7141c7d26bbfd59d343af768d3bb2592c5bc6e Mon Sep 17 00:00:00 2001 From: syntrust Date: Thu, 8 Jan 2026 17:19:38 +0800 Subject: [PATCH 20/24] rm debug --- .github/workflows/archive_service_healthcheck.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/archive_service_healthcheck.yml b/.github/workflows/archive_service_healthcheck.yml index d9e90fc4..77a1dd04 100644 --- a/.github/workflows/archive_service_healthcheck.yml +++ b/.github/workflows/archive_service_healthcheck.yml @@ -8,9 +8,6 @@ on: # Every 8 hours (UTC) - cron: '0 */8 * * *' workflow_dispatch: - pull_request: - branches: - - main jobs: check_blob_sidecars_API_for_critical_blob: From c81bbf4549e211c6803bf85efb8b41964561afdb Mon Sep 17 00:00:00 2001 From: syntrust Date: Thu, 8 Jan 2026 18:09:42 +0800 Subject: [PATCH 21/24] compare hash --- .../workflows/archive_service_healthcheck.yml | 15 +++++++--- .../scripts/check_latest_blob.sh | 30 ++++--------------- 2 files changed, 17 insertions(+), 28 deletions(-) diff --git a/.github/workflows/archive_service_healthcheck.yml b/.github/workflows/archive_service_healthcheck.yml index 77a1dd04..d343bab8 100644 --- a/.github/workflows/archive_service_healthcheck.yml +++ b/.github/workflows/archive_service_healthcheck.yml @@ -16,7 +16,7 @@ jobs: timeout-minutes: 5 env: ARCHIVE_BLOB_SIDECARS_URL: https://archive.mainnet.ethstorage.io:9645/eth/v1/beacon/blob_sidecars/13164810?indices=3 - ARCHIVE_BLOB_PREFIX: "0x31000000f000ac44aa0a1981e2" + ARCHIVE_BLOB_HASH: "5475d05275aaae328b99a4f4058ac1e121eaa4e4d4d378d292d6130f32d6ede0" outputs: failure_detail: ${{ steps.check_endpoint.outputs.failure_detail }} @@ -31,7 +31,7 @@ jobs: set -euo pipefail url='${{ env.ARCHIVE_BLOB_SIDECARS_URL }}' - expected_blob_prefix='${{ env.ARCHIVE_BLOB_PREFIX }}' + expected_blob_hash='${{ env.ARCHIVE_BLOB_HASH }}' echo "archive_blob_sidecars_url=$url" >> "$GITHUB_OUTPUT" @@ -63,8 +63,15 @@ jobs: exit 1 fi - if [[ "$blob" != "${expected_blob_prefix}"* ]]; then - echo "failure_detail=Blob data mismatch" >> $GITHUB_OUTPUT + # Compare by SHA-256 hash. + actual_blob_hash="$(printf '%s' "$blob" | sha256sum | awk '{print $1}')" + if [[ -z "$actual_blob_hash" ]]; then + echo "failure_detail=Failed to compute blob sha256" >> $GITHUB_OUTPUT + exit 1 + fi + + if [[ "$actual_blob_hash" != "$expected_blob_hash" ]]; then + echo "failure_detail=Blob hash mismatch (expected ${expected_blob_hash}, got ${actual_blob_hash})" >> $GITHUB_OUTPUT exit 1 fi diff --git a/integration_tests/scripts/check_latest_blob.sh b/integration_tests/scripts/check_latest_blob.sh index 16fa888b..73b44534 100755 --- a/integration_tests/scripts/check_latest_blob.sh +++ b/integration_tests/scripts/check_latest_blob.sh @@ -171,39 +171,21 @@ fi # Basic sanity: both responses have `.data` as array if [[ "$(jq -r '.data | type' <<<"$beacon_body")" != "array" ]]; then echo "Unexpected beacon /blobs response shape" >&2 - jq -c . <<<"$beacon_body" >&2 || true exit 1 fi if [[ "$(jq -r '.data | type' <<<"$archive_body")" != "array" ]]; then echo "Unexpected archive service /blobs response shape" >&2 - jq -c . <<<"$archive_body" >&2 || true exit 1 fi -beacon_data_norm="$(jq -cS '.data' <<<"$beacon_body")" -archive_data_norm="$(jq -cS '.data' <<<"$archive_body")" +# Minimal comparison: hash the full `.data` array JSON and compare. +beacon_data_hash="$(jq -c '.data' <<<"$beacon_body" | sha256sum)" +archive_data_hash="$(jq -c '.data' <<<"$archive_body" | sha256sum)" -if [[ "$beacon_data_norm" == "$archive_data_norm" ]]; then - echo "OK: beacon .data matches archive service .data" - jq -r '.data | length' <<<"$beacon_body" | awk '{print " blobCount: " $1}' +if [[ "$beacon_data_hash" == "$archive_data_hash" ]]; then + echo "✅ Match" exit 0 fi -echo "Mismatch: beacon .data differs from archive service .data" >&2 -echo "Beacon URL: $beacon_url" >&2 -echo "ArchiveService URL: $archive_url" >&2 - -beacon_len="$(jq -r '.data | length' <<<"$beacon_body" 2>/dev/null || echo '?')" -archive_len="$(jq -r '.data | length' <<<"$archive_body" 2>/dev/null || echo '?')" -echo "beacon data length: $beacon_len" >&2 -echo "archive data length: $archive_len" >&2 - -# Compare `.data` contents -if command -v diff >/dev/null 2>&1; then - diff -u \ - <(jq -S '.data' <<<"$beacon_body") \ - <(jq -S '.data' <<<"$archive_body") \ - >&2 || true -fi - +echo "❌ Mismatch" >&2 exit 1 \ No newline at end of file From 913e0e2a3851b7717a115b9036ba62de8e52d263 Mon Sep 17 00:00:00 2001 From: syntrust Date: Thu, 8 Jan 2026 18:11:22 +0800 Subject: [PATCH 22/24] debug --- .github/workflows/archive_service_healthcheck.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/archive_service_healthcheck.yml b/.github/workflows/archive_service_healthcheck.yml index d343bab8..700b2cb0 100644 --- a/.github/workflows/archive_service_healthcheck.yml +++ b/.github/workflows/archive_service_healthcheck.yml @@ -8,6 +8,9 @@ on: # Every 8 hours (UTC) - cron: '0 */8 * * *' workflow_dispatch: + pull_request: + branches: + - main jobs: check_blob_sidecars_API_for_critical_blob: From fec59da8a86e055df243307c92510302cbc15a08 Mon Sep 17 00:00:00 2001 From: syntrust Date: Thu, 8 Jan 2026 18:23:31 +0800 Subject: [PATCH 23/24] debug --- .github/workflows/archive_service_healthcheck.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/archive_service_healthcheck.yml b/.github/workflows/archive_service_healthcheck.yml index 700b2cb0..aca14a35 100644 --- a/.github/workflows/archive_service_healthcheck.yml +++ b/.github/workflows/archive_service_healthcheck.yml @@ -8,9 +8,9 @@ on: # Every 8 hours (UTC) - cron: '0 */8 * * *' workflow_dispatch: - pull_request: + push: branches: - - main + - archive_alert jobs: check_blob_sidecars_API_for_critical_blob: From b09fd11a2083e954551ed88b18da16200ee838fc Mon Sep 17 00:00:00 2001 From: syntrust Date: Thu, 8 Jan 2026 18:31:28 +0800 Subject: [PATCH 24/24] Trigger workflow