From 0b954d0aa420779105bbacd4459e14406dc9274b Mon Sep 17 00:00:00 2001 From: Louis Parkin Date: Fri, 10 Jul 2026 17:20:47 +0200 Subject: [PATCH 01/19] STAC-25142 Add GitHub Actions lint + unit-test workflow (agent CI migration phase 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ports filename_linting + unbranded/branded unit_tests from .gitlab-ci-agent.yml to GitHub Actions, tracking the active stackstate-7.78.2 pipeline. - filename-linting on `docker` (cheap gate; only needs `git ls-files`). - unbranded/branded unit tests on `xlarge` inside the datadog_build image (tag 7af9194f), pulled via the read-only quay proxy. - 7.78.2 deltas vs 7.71.2: `--major-version` dropped from agent.build/inv test (removed upstream in DD 7.78), and branded runs scripts/verify_branding_literals.sh after fix_branding.sh. Test-execution policy (architect directive — minimize redundant runs): each suite runs once. filename-lint on every PR; the two heavy ~2h suites on pull_request, deduped via concurrency cancel-in-progress. They move to merge_group (run once at land) at phase 6; master/version-branch pushes then build+publish without re-running tests. Security: minimal contents: read, SHA-pinned checkout, persist-credentials false, no PATs, registry-proxy secret only via container.credentials, fork PRs skipped. Zizmor clean; actionlint clean apart from self-hosted labels. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/lint-and-unit-tests.yml | 156 ++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 .github/workflows/lint-and-unit-tests.yml diff --git a/.github/workflows/lint-and-unit-tests.yml b/.github/workflows/lint-and-unit-tests.yml new file mode 100644 index 000000000000..e889595f3979 --- /dev/null +++ b/.github/workflows/lint-and-unit-tests.yml @@ -0,0 +1,156 @@ +name: Lint and unit tests + +# Ported from .gitlab-ci-agent.yml (filename_linting, unbranded_unit_tests, +# branded_unit_tests) as part of the GitLab -> GitHub migration (STAC-25142), +# phase 1: lint + unit tests. Tracks the stackstate-7.78.2 pipeline. +# +# Test-execution policy (architect directive — minimize redundant runs): +# * filename-linting is a cheap gate and runs on every PR. +# * The two unit suites are heavy (each does a full `inv agent.build` + full +# `inv test`, ~2h on an XL runner). They run ONCE per change here, deduped +# via `concurrency: cancel-in-progress`. When the merge queue is enabled +# (phase 6) they move to `merge_group` (run once at land) and drop from +# `pull_request`; master/version-branch pushes then build+publish WITHOUT +# re-running tests. Follow-ups tracked in the migration scope doc: +# - path-filter the heavy suites (skip when no Go/build inputs changed) +# - open question for architects: do we need BOTH unbranded and branded +# on every path, or branded-as-gate + unbranded on the queue only? +# +# Prerequisites for a GREEN run (see scope doc): +# * The GitLab push mirror to this repo must be disabled (done 2026-07-10), +# otherwise a mirror sync overwrites anything pushed to GitHub. +# * This is a PUBLIC repo. The read-only registry-proxy credentials must be +# available to it (org-inherited or provisioned via pulumi-infra): +# vars REGISTRY_HOST, REGISTRY_USER + secret REGISTRY_PASSWORD. Org secrets +# are never exposed to fork PRs, so external-fork PRs skip these jobs. + +on: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + # conda env baked into the datadog_build image. + CONDA_ENV: ddpy3 + +jobs: + filename-linting: + name: Filename linting + # Same-repo PRs only: the registry-proxy secret is not exposed to fork PRs. + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: docker + timeout-minutes: 20 + container: + # datadog_build image (built by datadog-agent-buildimages), pulled via the + # read-only quay proxy — mirrors BUILD_IMAGE in .gitlab-ci.yml (7.78.2 tag). + image: ${{ vars.REGISTRY_HOST }}/quay/stackstate/datadog_build_linux_x64:7af9194f + credentials: + username: ${{ vars.REGISTRY_USER }} + password: ${{ secrets.REGISTRY_PASSWORD }} + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: inv linter.filenames + run: | + set -eo pipefail + # linter.filenames only shells out to `git ls-files` + path checks, so + # it needs no Go deps/vendoring (the GitLab job carried that as boilerplate). + . /root/miniforge3/etc/profile.d/conda.sh + conda activate "${CONDA_ENV}" + inv -e linter.filenames + + unbranded-unit-tests: + name: Unit tests (unbranded / DataDog) + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: xlarge + timeout-minutes: 150 + container: + image: ${{ vars.REGISTRY_HOST }}/quay/stackstate/datadog_build_linux_x64:7af9194f + credentials: + username: ${{ vars.REGISTRY_USER }} + password: ${{ secrets.REGISTRY_PASSWORD }} + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Build agent and run unit tests + run: | + set -eo pipefail + export PATH="$PATH:/usr/local/go/bin" + . /root/miniforge3/etc/profile.d/conda.sh + conda activate "${CONDA_ENV}" + go clean -modcache + go work sync + go work vendor + inv -e deps --verbose + # /go/bin is not cached on ephemeral runners; re-run deps to get tool + # binaries (golint, misspell, ...). + inv deps + export AGENT_GITHUB_ORG=DataDog + export GITHUB_ORG=DataDog + export BRANDED=false + export AGENT_REPO_NAME=datadog-agent + inv -e agent.build --race + gofmt -l -w -s ./pkg ./cmd + inv -e rtloader.test + invoke install-tools + # Go 1.25+ no longer ships prebuilt tool binaries; build covdata into + # GOTOOLDIR so `go test -cover` resolves it (golang/go#75031). + go build -o "$(go env GOTOOLDIR)/covdata" cmd/covdata + # Drop build tags for features StackState does not ship in its images. + export STS_UT_BUILD_EXCLUDE="oracle,trivy,trivy_no_javadb,nvml,jetson,bundle_installer,systemd" + inv -e test --coverage --race --profile --cpus 4 --build-exclude="${STS_UT_BUILD_EXCLUDE}" --timeout=600 + + branded-unit-tests: + name: Unit tests (branded / StackState) + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: xlarge + timeout-minutes: 150 + container: + image: ${{ vars.REGISTRY_HOST }}/quay/stackstate/datadog_build_linux_x64:7af9194f + credentials: + username: ${{ vars.REGISTRY_USER }} + password: ${{ secrets.REGISTRY_PASSWORD }} + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Build agent and run unit tests + run: | + set -eo pipefail + export PATH="$PATH:/usr/local/go/bin" + . /root/miniforge3/etc/profile.d/conda.sh + conda activate "${CONDA_ENV}" + go clean -modcache + go work sync + go work vendor + inv -e deps --verbose + inv deps + export AGENT_GITHUB_ORG=DataDog + export GITHUB_ORG=DataDog + export BRANDED=true + export AGENT_REPO_NAME=datadog-agent + export OMNIBUS_FORCE_PACKAGES=true + # Rebrand DataDog -> StackState, then verify no unbranded literals remain. + ./fix_branding.sh + ./scripts/verify_branding_literals.sh + inv -e agent.build --race + gofmt -l -w -s ./pkg ./cmd + inv -e rtloader.test + invoke install-tools + go build -o "$(go env GOTOOLDIR)/covdata" cmd/covdata + export STS_UT_BUILD_EXCLUDE="oracle,trivy,trivy_no_javadb,nvml,jetson,bundle_installer,systemd" + inv -e test --coverage --race --profile --cpus 4 --build-exclude="${STS_UT_BUILD_EXCLUDE}" --timeout=600 From d9ca26904a33d530403e4623e84a9116a18f7dc7 Mon Sep 17 00:00:00 2001 From: Louis Parkin Date: Wed, 22 Jul 2026 09:13:58 +0200 Subject: [PATCH 02/19] STAC-25142 CI: run lint+unit on dedicated public runners (docker-public/xlarge-public) Public repos can't use the shared docker/xlarge runners; use the isolated agent-public pool (pulumi-infra #227 + argocd-apps #115). --- .github/workflows/lint-and-unit-tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/lint-and-unit-tests.yml b/.github/workflows/lint-and-unit-tests.yml index e889595f3979..73b728bfaf77 100644 --- a/.github/workflows/lint-and-unit-tests.yml +++ b/.github/workflows/lint-and-unit-tests.yml @@ -44,7 +44,7 @@ jobs: name: Filename linting # Same-repo PRs only: the registry-proxy secret is not exposed to fork PRs. if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository - runs-on: docker + runs-on: docker-public timeout-minutes: 20 container: # datadog_build image (built by datadog-agent-buildimages), pulled via the @@ -71,7 +71,7 @@ jobs: unbranded-unit-tests: name: Unit tests (unbranded / DataDog) if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository - runs-on: xlarge + runs-on: xlarge-public timeout-minutes: 150 container: image: ${{ vars.REGISTRY_HOST }}/quay/stackstate/datadog_build_linux_x64:7af9194f @@ -115,7 +115,7 @@ jobs: branded-unit-tests: name: Unit tests (branded / StackState) if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository - runs-on: xlarge + runs-on: xlarge-public timeout-minutes: 150 container: image: ${{ vars.REGISTRY_HOST }}/quay/stackstate/datadog_build_linux_x64:7af9194f From 51c513e596ad4815dd035ed07096eb84aac38fdf Mon Sep 17 00:00:00 2001 From: Louis Parkin Date: Fri, 24 Jul 2026 11:07:35 +0200 Subject: [PATCH 03/19] STAC-25142 CI: run workflow steps in bash (dash lacks set -o pipefail; conda needs bash) --- .github/workflows/lint-and-unit-tests.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/lint-and-unit-tests.yml b/.github/workflows/lint-and-unit-tests.yml index 73b728bfaf77..fd582da0db00 100644 --- a/.github/workflows/lint-and-unit-tests.yml +++ b/.github/workflows/lint-and-unit-tests.yml @@ -35,6 +35,12 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +# The datadog_build container's default shell is dash, which lacks `set -o pipefail` +# and mis-handles the conda activation the steps rely on; force bash. +defaults: + run: + shell: bash + env: # conda env baked into the datadog_build image. CONDA_ENV: ddpy3 From 936010c7365acc9f8fae7003c8a2309f753708bb Mon Sep 17 00:00:00 2001 From: Louis Parkin Date: Fri, 24 Jul 2026 11:18:35 +0200 Subject: [PATCH 04/19] STAC-25142 CI: mark workspace safe.directory (git dubious ownership in container) --- .github/workflows/lint-and-unit-tests.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/lint-and-unit-tests.yml b/.github/workflows/lint-and-unit-tests.yml index fd582da0db00..ff8420e6cac4 100644 --- a/.github/workflows/lint-and-unit-tests.yml +++ b/.github/workflows/lint-and-unit-tests.yml @@ -72,6 +72,9 @@ jobs: # it needs no Go deps/vendoring (the GitLab job carried that as boilerplate). . /root/miniforge3/etc/profile.d/conda.sh conda activate "${CONDA_ENV}" + # Work volume is owned by the ARC runner uid but the job container runs as + # root, so git rejects the repo as "dubious ownership"; mark it safe. + git config --global --add safe.directory '*' inv -e linter.filenames unbranded-unit-tests: @@ -96,6 +99,9 @@ jobs: export PATH="$PATH:/usr/local/go/bin" . /root/miniforge3/etc/profile.d/conda.sh conda activate "${CONDA_ENV}" + # Work volume is owned by the ARC runner uid but the job container runs as + # root, so git rejects the repo as "dubious ownership"; mark it safe. + git config --global --add safe.directory '*' go clean -modcache go work sync go work vendor @@ -140,6 +146,9 @@ jobs: export PATH="$PATH:/usr/local/go/bin" . /root/miniforge3/etc/profile.d/conda.sh conda activate "${CONDA_ENV}" + # Work volume is owned by the ARC runner uid but the job container runs as + # root, so git rejects the repo as "dubious ownership"; mark it safe. + git config --global --add safe.directory '*' go clean -modcache go work sync go work vendor From adc6b70b82e5516c68ca58fc2c1ae70908c07bc9 Mon Sep 17 00:00:00 2001 From: Louis Parkin Date: Fri, 24 Jul 2026 13:39:14 +0200 Subject: [PATCH 05/19] STAC-25142 CI: full-depth checkout for unit tests so git describe finds the 3.x version tag --- .github/workflows/lint-and-unit-tests.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/lint-and-unit-tests.yml b/.github/workflows/lint-and-unit-tests.yml index ff8420e6cac4..e23bbfdcafa6 100644 --- a/.github/workflows/lint-and-unit-tests.yml +++ b/.github/workflows/lint-and-unit-tests.yml @@ -92,6 +92,9 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false + # Full history + tags so `inv agent.build` resolves the version via + # `git describe --tags --match "3.*"` (needs the 3.x tag reachable from HEAD). + fetch-depth: 0 - name: Build agent and run unit tests run: | @@ -139,6 +142,9 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false + # Full history + tags so `inv agent.build` resolves the version via + # `git describe --tags --match "3.*"` (needs the 3.x tag reachable from HEAD). + fetch-depth: 0 - name: Build agent and run unit tests run: | From c7d7a421acdb1c6c23048527c538eb2107bec5b5 Mon Sep 17 00:00:00 2001 From: Louis Parkin Date: Fri, 24 Jul 2026 14:42:00 +0200 Subject: [PATCH 06/19] STAC-25142 Pass workspace dir to fix_branding.sh (CI_PROJECT_DIR is GitLab-only) --- .github/workflows/lint-and-unit-tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint-and-unit-tests.yml b/.github/workflows/lint-and-unit-tests.yml index e23bbfdcafa6..0847a13a2f54 100644 --- a/.github/workflows/lint-and-unit-tests.yml +++ b/.github/workflows/lint-and-unit-tests.yml @@ -166,7 +166,9 @@ jobs: export AGENT_REPO_NAME=datadog-agent export OMNIBUS_FORCE_PACKAGES=true # Rebrand DataDog -> StackState, then verify no unbranded literals remain. - ./fix_branding.sh + # fix_branding.sh defaults its target dir to $CI_PROJECT_DIR (a GitLab + # built-in that is unset here); pass the checkout root explicitly. + ./fix_branding.sh "${GITHUB_WORKSPACE}" ./scripts/verify_branding_literals.sh inv -e agent.build --race gofmt -l -w -s ./pkg ./cmd From c07d0c871e320457cdb7d0273d92b17cfb6e7090 Mon Sep 17 00:00:00 2001 From: Louis Parkin Date: Mon, 27 Jul 2026 10:39:23 +0200 Subject: [PATCH 07/19] STAC-25142 Add GitHub Actions binary-build workflow (agent CI migration phase 2) Port build_binaries and build_cluster_agent from .gitlab-ci-agent.yml to .github/workflows/build-binaries.yml: build the branded (StackState) production agent (+ rtloader) and cluster-agent in the datadog_build container on public xlarge runners, once per change (pull_request), deduped via concurrency. Carry over the GitLab pipeline speedups: self-contained jobs with no shared deps prerequisite (the Go module cache is untransportable between jobs and shared vendoring breaks the go.mod<->vendor reconcile), version.txt generated in-job from git, and a runner-local persistent-cache follow-up noted for the real warm-cache win. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/build-binaries.yml | 171 +++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 .github/workflows/build-binaries.yml diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml new file mode 100644 index 000000000000..1a013eb36d8a --- /dev/null +++ b/.github/workflows/build-binaries.yml @@ -0,0 +1,171 @@ +name: Binary builds + +# Ported from .gitlab-ci-agent.yml (build_binaries, build_cluster_agent) as part +# of the GitLab -> GitHub migration (STAC-25142), phase 2: binary builds. Tracks +# the stackstate-7.78.2 pipeline. Builds the branded (StackState) production agent +# and cluster-agent from the DataDog fork, validating that the shippable binaries +# compile on GitHub runners before the packaging/image phases build on top of them. +# +# Speedup concepts carried over from the GitLab pipeline work (STAC-25396): +# * Self-contained jobs, NO shared "prepare deps once" job. On GitLab, deps_deb +# sat on the critical path as a serial prefix that every build job depended on, +# but it never actually shared anything reusable: the Go module cache is too +# big to transport between jobs (~540k files, 2-3 GB) and a shared vendor/ dir +# breaks vendoring (`go work sync` mutates go.mod, so each job must re-vendor +# against its own checkout). So every job here reconciles its own deps +# (go work sync + go work vendor + inv deps) instead of depending on a deps job. +# * deps_deb's only genuinely consumed output was version.txt, which is git-based +# (inv agent.version), not module-cache-based -- so build-cluster-agent just +# generates it in-job rather than pulling it from an upstream job. +# * Each suite runs once on the path to master, deduped via +# `concurrency: cancel-in-progress` (a newer push cancels the in-flight build). +# +# Known follow-ups (same as the GitLab analysis): +# * The real build-time win is a WARM module cache, which needs persistent +# runner-local storage (PVC-backed GOMODCACHE/GOCACHE on the ARC pool) -- an +# SRE / pulumi-infra change, not workflow YAML. actions/cache does not help: +# it hits the same file-count/size tax that made the GitLab cache a net loss. +# * Phase 6 (merge queue): move the heavy builds to `merge_group` so they run +# once at land time and drop from `pull_request`; master/version-branch pushes +# then build + publish without re-running these. +# +# Prerequisites for a GREEN run are the same as lint-and-unit-tests.yml: the GitLab +# push mirror must stay disabled, and this PUBLIC repo needs the read-only +# registry-proxy credentials (vars REGISTRY_HOST/REGISTRY_USER + secret +# REGISTRY_PASSWORD) for the container: image pull. Org secrets are never exposed +# to fork PRs, so external-fork PRs skip these jobs by design. + +on: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +# The datadog_build container's default shell is dash, which lacks `set -o pipefail` +# and mis-handles the conda activation the steps rely on; force bash. +defaults: + run: + shell: bash + +env: + # conda env baked into the datadog_build image. + CONDA_ENV: ddpy3 + +jobs: + build-agent: + name: Build agent binary (branded / StackState) + # Same-repo PRs only: the registry-proxy secret is not exposed to fork PRs. + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: xlarge-public + timeout-minutes: 120 + container: + # datadog_build image (built by datadog-agent-buildimages), pulled via the + # read-only quay proxy -- mirrors BUILD_IMAGE in .gitlab-ci.yml (7.78.2 tag). + image: ${{ vars.REGISTRY_HOST }}/quay/stackstate/datadog_build_linux_x64:7af9194f + credentials: + username: ${{ vars.REGISTRY_USER }} + password: ${{ secrets.REGISTRY_PASSWORD }} + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + # Full history + tags so `inv agent.build` resolves the version via + # `git describe --tags --match "3.*"` (needs the 3.x tag reachable from HEAD). + fetch-depth: 0 + + - name: Build branded agent (production, with rtloader) + run: | + set -eo pipefail + export PATH="$PATH:/usr/local/go/bin" + . /root/miniforge3/etc/profile.d/conda.sh + conda activate "${CONDA_ENV}" + # Work volume is owned by the ARC runner uid but the job container runs as + # root, so git rejects the repo as "dubious ownership"; mark it safe. + git config --global --add safe.directory '*' + # Self-contained deps: reconcile go.mod <-> vendor for this checkout + # (cross-job cache/vendor sharing is not viable here -- see header). + go clean -modcache + go work sync + go work vendor + inv -e deps --verbose + # rtloader is the C++/Python bridge the full agent links against; the + # cluster-agent doesn't need it, but the production agent does. + inv -e rtloader.make + inv -e rtloader.install + export AGENT_GITHUB_ORG=DataDog + export GITHUB_ORG=DataDog + export BRANDED=true + export AGENT_REPO_NAME=datadog-agent + # Rebrand DataDog -> StackState. fix_branding.sh defaults its target dir to + # $CI_PROJECT_DIR (a GitLab built-in that is unset here); pass the checkout + # root explicitly. + ./fix_branding.sh "${GITHUB_WORKSPACE}" + # Production (non-race) build -- the shippable binary, distinct from the + # race-instrumented build in the unit-test workflow. + inv -e agent.build + ls -la bin/agent + + build-cluster-agent: + name: Build cluster-agent binary (branded / StackState) + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: xlarge-public + timeout-minutes: 120 + container: + image: ${{ vars.REGISTRY_HOST }}/quay/stackstate/datadog_build_linux_x64:7af9194f + credentials: + username: ${{ vars.REGISTRY_USER }} + password: ${{ secrets.REGISTRY_PASSWORD }} + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + # Full history + tags so the version resolves via `git describe`. + fetch-depth: 0 + + - name: Build branded cluster-agent + run: | + set -eo pipefail + export PATH="$PATH:/usr/local/go/bin" + . /root/miniforge3/etc/profile.d/conda.sh + conda activate "${CONDA_ENV}" + # Work volume is owned by the ARC runner uid but the job container runs as + # root, so git rejects the repo as "dubious ownership"; mark it safe. + git config --global --add safe.directory '*' + # Self-contained deps (see header): reconcile go.mod <-> vendor per job. + go clean -modcache + go work sync + go work vendor + inv -e deps --verbose + export AGENT_GITHUB_ORG=DataDog + export GITHUB_ORG=DataDog + export BRANDED=true + export AGENT_REPO_NAME=datadog-agent + # Rebrand DataDog -> StackState; pass the checkout root (CI_PROJECT_DIR is GitLab-only). + ./fix_branding.sh "${GITHUB_WORKSPACE}" + inv -e cluster-agent.build + # version.txt is a build artifact for the downstream packaging/image phases. + # deps_deb produced it on GitLab; there is no shared deps job here, so + # generate it in-job. inv agent.version is git-based (not module-cache-based) + # and cheap. No `-e`: its stdout must be only the version string. + inv agent.version -u > version.txt + ls -la bin/stackstate-cluster-agent + + - name: Upload cluster-agent build artifacts + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: cluster-agent-binary + # Paths match the GitLab build_cluster_agent artifacts; the cluster-agent + # image phase (phase 5) consumes the binary + Dockerfile manifest. + path: | + bin/stackstate-cluster-agent/ + Dockerfiles/cluster-agent/stackstate-cluster.yaml + version.txt + retention-days: 5 + if-no-files-found: error From ae94fa845e09256af2199f2c336d90697846dc60 Mon Sep 17 00:00:00 2001 From: Louis Parkin Date: Tue, 28 Jul 2026 09:46:25 +0200 Subject: [PATCH 08/19] STAC-25429-godeps-cache-image: warm Go module cache via prebuilt cache image Speeds up the migrated stackstate-agent GitHub Actions build/test jobs by replacing each job's cold `go clean -modcache` + `inv deps` (go mod download + tidy) reconcile with a warm GOMODCACHE delivered as a prebuilt CI image, keyed by a content hash of the module graph. Mirrors the StackGraph ci-metadata / build-ci-image pattern. - .github/scripts/agent-godeps-cache-metadata.sh: content-addressed image tag from every go.mod/go.sum + go.work + modules.yml plus the Dockerfile/script themselves, so any graph or mechanism change rotates the tag. - .github/docker/godeps-cache/Dockerfile: FROM datadog_build, bakes external modules into GOMODCACHE via a per-module `go mod download` loop. - .github/workflows/godeps-cache.yml: reusable workflow (tag lookup + build only when missing); pushes to quay.io/stackstate (private), pulled by consumers through the registry.tooling quay proxy with the existing REGISTRY_* creds. - build-binaries.yml / lint-and-unit-tests.yml: add the godeps-cache job, repoint the 4 heavy jobs to the cache image, drop the per-job modcache reset + inv deps (keep `go work sync` / `go work vendor` against the warm cache). Zizmor clean (narrow unpinned-images ignore on the content-addressed container refs, which are effectively digest-pinned but computed per run). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/docker/godeps-cache/Dockerfile | 31 ++++ .../scripts/agent-godeps-cache-metadata.sh | 39 +++++ .github/workflows/build-binaries.yml | 81 ++++++---- .github/workflows/godeps-cache.yml | 142 ++++++++++++++++++ .github/workflows/lint-and-unit-tests.yml | 53 +++++-- 5 files changed, 304 insertions(+), 42 deletions(-) create mode 100644 .github/docker/godeps-cache/Dockerfile create mode 100755 .github/scripts/agent-godeps-cache-metadata.sh create mode 100644 .github/workflows/godeps-cache.yml diff --git a/.github/docker/godeps-cache/Dockerfile b/.github/docker/godeps-cache/Dockerfile new file mode 100644 index 000000000000..e1badfa38a82 --- /dev/null +++ b/.github/docker/godeps-cache/Dockerfile @@ -0,0 +1,31 @@ +# Go dependency cache image for stackstate-agent CI (STAC-25429). Derives FROM the +# private datadog_build CI image and bakes the workspace's external Go module +# downloads into GOMODCACHE, so consumer jobs get a warm module cache with no +# per-job `go mod download`/network round-trips. Rebuilt only when the module graph +# hash changes -- the image tag is content-addressed +# (see .github/scripts/agent-godeps-cache-metadata.sh). +# +# No `# syntax=` frontend directive and no BuildKit secret mounts: the base image is +# pulled by the daemon after `docker login` (creds stay in the daemon, not the +# build), so the build itself needs no secrets and no Docker Hub frontend pull. +ARG BASE_IMAGE +FROM ${BASE_IMAGE} + +# Pre-download external modules for every module in the workspace. manifests/ +# preserves the repo's directory layout so nested go.mod files (and their local +# `replace ../` targets) resolve. GOFLAGS=-mod=mod lets `go mod download` reconcile +# go.sum in-container without a committed change; only the populated module cache is +# kept -- the manifests themselves are removed from the final image. +COPY manifests/ /workspace/ +RUN set -eux; \ + git config --global --add safe.directory '*'; \ + export PATH="${PATH}:/usr/local/go/bin"; \ + export GOFLAGS=-mod=mod; \ + cd /workspace; \ + find . -type f -name go.mod -print0 \ + | while IFS= read -r -d '' modfile; do \ + moddir="$(dirname "${modfile}")"; \ + echo "go mod download :: ${moddir}"; \ + ( cd "${moddir}" && go mod download ); \ + done; \ + rm -rf /workspace diff --git a/.github/scripts/agent-godeps-cache-metadata.sh b/.github/scripts/agent-godeps-cache-metadata.sh new file mode 100755 index 000000000000..af6692f5612f --- /dev/null +++ b/.github/scripts/agent-godeps-cache-metadata.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Computes the content-addressed tag for the stackstate-agent Go dependency cache +# image (STAC-25429) and the two image refs it is referenced by: +# * push ref -> quay.io/stackstate/sts-ci-images (authoritative; build + exists check) +# * pull ref -> ${REGISTRY_HOST}/quay/... (registry.tooling proxy; job container) +# +# Mirrors StackVista/StackGraph .github/scripts/stackgraph-ci-metadata.sh: the tag is +# a hash of the dependency-defining inputs, so an unchanged module graph reuses the +# same image across every branch/PR/default build (the tag *is* the cache key), while +# any change to the graph rotates the tag and a stale cache is never reused. + +agent_godeps_compute_metadata() { + : "${QUAY_REGISTRY:?QUAY_REGISTRY is required}" # quay.io + : "${REGISTRY_HOST:?REGISTRY_HOST is required}" # registry.tooling.stackstate.io (quay proxy) + : "${BASE_IMAGE_TAG:?BASE_IMAGE_TAG is required}" # datadog_build tag the cache derives FROM + + # Hash inputs: every module manifest (go.work + go.work.sum + modules.yml + all + # nested go.mod/go.sum) plus the base image tag and the two files that define the + # cache mechanism itself (Dockerfile + this script), so changing how the cache is + # built also rotates the tag. + local godeps_hash + godeps_hash="$( + { + git ls-files -z -- \ + '*go.mod' '*go.sum' 'go.work' 'go.work.sum' 'modules.yml' \ + '.github/docker/godeps-cache/Dockerfile' \ + '.github/scripts/agent-godeps-cache-metadata.sh' \ + | LC_ALL=C sort -z \ + | xargs -0 sha256sum + printf 'base:%s\n' "${BASE_IMAGE_TAG}" + } | sha256sum | cut -c1-16 + )" + + local image_tag="stackstate-agent-godeps-${godeps_hash}" + # shellcheck disable=SC2034 # consumed by the sourcing workflow step + ci_image_push="${QUAY_REGISTRY}/stackstate/sts-ci-images:${image_tag}" + # shellcheck disable=SC2034 # consumed by the sourcing workflow step + ci_image="${REGISTRY_HOST}/quay/stackstate/sts-ci-images:${image_tag}" +} diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index 1a013eb36d8a..f7363a6605e4 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -6,34 +6,38 @@ name: Binary builds # and cluster-agent from the DataDog fork, validating that the shippable binaries # compile on GitHub runners before the packaging/image phases build on top of them. # -# Speedup concepts carried over from the GitLab pipeline work (STAC-25396): -# * Self-contained jobs, NO shared "prepare deps once" job. On GitLab, deps_deb -# sat on the critical path as a serial prefix that every build job depended on, -# but it never actually shared anything reusable: the Go module cache is too -# big to transport between jobs (~540k files, 2-3 GB) and a shared vendor/ dir -# breaks vendoring (`go work sync` mutates go.mod, so each job must re-vendor -# against its own checkout). So every job here reconciles its own deps -# (go work sync + go work vendor + inv deps) instead of depending on a deps job. +# Speedup concepts carried over from the GitLab pipeline work (STAC-25396) and the +# warm-cache follow-up (STAC-25429): +# * WARM Go module cache via a prebuilt cache image (STAC-25429), keyed by a hash +# of the module graph -- the StackGraph ci-metadata/build-ci-image pattern. The +# godeps-cache reusable workflow (.github/workflows/godeps-cache.yml) publishes +# an image FROM datadog_build with all external modules baked into GOMODCACHE; +# the build jobs use it as their `container:` so the cache is already warm. This +# replaces the earlier "each job cold-reconciles its own deps" approach: there +# is no `go clean -modcache` and no per-job `inv deps` (go mod download + tidy). +# The module cache is too big to transport via actions/cache (~540k files, +# 2-3 GB), which is why it ships as an image layer instead. Jobs still run +# `go work sync` + `go work vendor` per checkout (vendoring mutates go.mod, so +# vendor/ is materialised per job) -- but now against the warm cache, offline. # * deps_deb's only genuinely consumed output was version.txt, which is git-based # (inv agent.version), not module-cache-based -- so build-cluster-agent just # generates it in-job rather than pulling it from an upstream job. # * Each suite runs once on the path to master, deduped via # `concurrency: cancel-in-progress` (a newer push cancels the in-flight build). # -# Known follow-ups (same as the GitLab analysis): -# * The real build-time win is a WARM module cache, which needs persistent -# runner-local storage (PVC-backed GOMODCACHE/GOCACHE on the ARC pool) -- an -# SRE / pulumi-infra change, not workflow YAML. actions/cache does not help: -# it hits the same file-count/size tax that made the GitLab cache a net loss. +# Known follow-ups: +# * go.mod tidiness is no longer verified in these jobs (the `inv deps` reconcile +# is gone). Add a single check-mod-tidy gate rather than paying tidy in every job. # * Phase 6 (merge queue): move the heavy builds to `merge_group` so they run # once at land time and drop from `pull_request`; master/version-branch pushes # then build + publish without re-running these. # -# Prerequisites for a GREEN run are the same as lint-and-unit-tests.yml: the GitLab -# push mirror must stay disabled, and this PUBLIC repo needs the read-only -# registry-proxy credentials (vars REGISTRY_HOST/REGISTRY_USER + secret -# REGISTRY_PASSWORD) for the container: image pull. Org secrets are never exposed -# to fork PRs, so external-fork PRs skip these jobs by design. +# Prerequisites for a GREEN run: the GitLab push mirror must stay disabled, and this +# PUBLIC repo needs the read-only registry-proxy credentials (vars REGISTRY_HOST/ +# REGISTRY_USER + secret REGISTRY_PASSWORD) to pull the cache image, plus quay push +# credentials (var QUAY_USER + secret QUAY_PASSWORD) for the godeps-cache build job. +# The cache image is PRIVATE (its base datadog_build is private). Org secrets are +# never exposed to fork PRs, so external-fork PRs skip these jobs by design. on: pull_request: @@ -57,16 +61,29 @@ env: CONDA_ENV: ddpy3 jobs: + godeps-cache: + name: Go dependency cache image + # Same-repo PRs only: the registry/quay secrets are not exposed to fork PRs. + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + uses: ./.github/workflows/godeps-cache.yml + secrets: + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }} + build-agent: name: Build agent binary (branded / StackState) # Same-repo PRs only: the registry-proxy secret is not exposed to fork PRs. if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + needs: godeps-cache runs-on: xlarge-public timeout-minutes: 120 container: - # datadog_build image (built by datadog-agent-buildimages), pulled via the - # read-only quay proxy -- mirrors BUILD_IMAGE in .gitlab-ci.yml (7.78.2 tag). - image: ${{ vars.REGISTRY_HOST }}/quay/stackstate/datadog_build_linux_x64:7af9194f + # Warm Go module cache image (godeps-cache job); pulled via the same + # read-only registry proxy + REGISTRY_* credentials as datadog_build. The tag is + # a sha256 content hash of the module graph (see godeps-cache.yml), so this ref + # is effectively digest-pinned; it cannot be a static digest because it is + # computed per run -- hence the narrow unpinned-images ignore below. + image: ${{ needs.godeps-cache.outputs.ci_image }} # zizmor: ignore[unpinned-images] credentials: username: ${{ vars.REGISTRY_USER }} password: ${{ secrets.REGISTRY_PASSWORD }} @@ -88,12 +105,12 @@ jobs: # Work volume is owned by the ARC runner uid but the job container runs as # root, so git rejects the repo as "dubious ownership"; mark it safe. git config --global --add safe.directory '*' - # Self-contained deps: reconcile go.mod <-> vendor for this checkout - # (cross-job cache/vendor sharing is not viable here -- see header). - go clean -modcache + # GOMODCACHE is pre-warmed by the godeps-cache image (STAC-25429), keyed + # to the module-graph hash, so there is no `go clean -modcache` and no + # per-job `inv deps` (go mod download + tidy) reconcile. Materialise + # vendor/ for this checkout against the warm cache (offline; no download). go work sync go work vendor - inv -e deps --verbose # rtloader is the C++/Python bridge the full agent links against; the # cluster-agent doesn't need it, but the production agent does. inv -e rtloader.make @@ -114,10 +131,16 @@ jobs: build-cluster-agent: name: Build cluster-agent binary (branded / StackState) if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + needs: godeps-cache runs-on: xlarge-public timeout-minutes: 120 container: - image: ${{ vars.REGISTRY_HOST }}/quay/stackstate/datadog_build_linux_x64:7af9194f + # Warm Go module cache image (godeps-cache job); pulled via the same + # read-only registry proxy + REGISTRY_* credentials as datadog_build. The tag is + # a sha256 content hash of the module graph (see godeps-cache.yml), so this ref + # is effectively digest-pinned; it cannot be a static digest because it is + # computed per run -- hence the narrow unpinned-images ignore below. + image: ${{ needs.godeps-cache.outputs.ci_image }} # zizmor: ignore[unpinned-images] credentials: username: ${{ vars.REGISTRY_USER }} password: ${{ secrets.REGISTRY_PASSWORD }} @@ -138,11 +161,11 @@ jobs: # Work volume is owned by the ARC runner uid but the job container runs as # root, so git rejects the repo as "dubious ownership"; mark it safe. git config --global --add safe.directory '*' - # Self-contained deps (see header): reconcile go.mod <-> vendor per job. - go clean -modcache + # GOMODCACHE is pre-warmed by the godeps-cache image (STAC-25429); no + # `go clean -modcache` and no per-job `inv deps` reconcile. Materialise + # vendor/ for this checkout against the warm cache (offline; no download). go work sync go work vendor - inv -e deps --verbose export AGENT_GITHUB_ORG=DataDog export GITHUB_ORG=DataDog export BRANDED=true diff --git a/.github/workflows/godeps-cache.yml b/.github/workflows/godeps-cache.yml new file mode 100644 index 000000000000..e5fcf8ba2ff3 --- /dev/null +++ b/.github/workflows/godeps-cache.yml @@ -0,0 +1,142 @@ +name: Go dependency cache image + +# Reusable workflow (STAC-25429): builds and publishes a Go module cache image for +# stackstate-agent CI, keyed by a hash of the module graph. Mirrors the StackGraph +# ci-metadata / build-ci-image pattern (its .github/workflows/ci.yml): the image tag +# is content-addressed, so an unchanged dependency graph reuses an already-published +# image across every branch/PR/default build (the tag *is* the cache key), and the +# heavy Go module download only runs when the graph actually changes. +# +# The image is FROM the private datadog_build CI image with the workspace's external +# modules baked into GOMODCACHE. It is pushed to quay.io/stackstate/sts-ci-images +# (private) and pulled by consumer jobs through the registry.tooling quay proxy with +# the same read-only REGISTRY_* credentials they already use for datadog_build. +# Because the base image is private, the cache image MUST stay private -- it is never +# pushed to a public registry (that would leak base-image layers). + +on: + workflow_call: + inputs: + base_image_tag: + description: Tag of the datadog_build_linux_x64 base image the cache derives FROM. + type: string + default: "7af9194f" + outputs: + ci_image: + description: Proxy pull ref of the cache image, for use as a consumer job `container.image`. + value: ${{ jobs.metadata.outputs.ci_image }} + image_exists: + description: Whether the computed cache image tag already exists in the registry. + value: ${{ jobs.metadata.outputs.image_exists }} + secrets: + REGISTRY_PASSWORD: + description: Password for the read-only registry.tooling quay proxy (pull the base image). + required: true + QUAY_PASSWORD: + description: Password for quay.io/stackstate (inspect + push the cache image). + required: true + +permissions: + contents: read + +jobs: + metadata: + name: Compute cache image tag and look up existing image + runs-on: docker-public + timeout-minutes: 15 + outputs: + ci_image: ${{ steps.metadata.outputs.ci_image }} + image_exists: ${{ steps.metadata.outputs.image_exists }} + env: + QUAY_REGISTRY: quay.io + REGISTRY_HOST: ${{ vars.REGISTRY_HOST }} + QUAY_USER: ${{ vars.QUAY_USER }} + QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }} + BASE_IMAGE_TAG: ${{ inputs.base_image_tag }} + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Compute cache image metadata and check existence + id: metadata + run: | + set -euo pipefail + source .github/scripts/agent-godeps-cache-metadata.sh + agent_godeps_compute_metadata + + # Presence-gated login: same-repo runs have QUAY_* wired; a run without + # them still computes the tag and reports the image as missing. + if [[ -n "$QUAY_USER" && -n "$QUAY_PASSWORD" ]]; then + echo "$QUAY_PASSWORD" | docker login --username "$QUAY_USER" --password-stdin "$QUAY_REGISTRY" + fi + + if docker manifest inspect "$ci_image_push" >/dev/null 2>&1; then + image_exists=true + else + image_exists=false + fi + + { + echo "ci_image=${ci_image}" + echo "image_exists=${image_exists}" + } >> "$GITHUB_OUTPUT" + + echo "Cache push ref: ${ci_image_push}" + echo "Cache pull ref: ${ci_image}" + echo "Cache image exists: ${image_exists}" + + build: + name: Build and publish missing cache image + if: ${{ needs.metadata.outputs.image_exists != 'true' }} + needs: metadata + runs-on: docker-public + timeout-minutes: 60 + env: + QUAY_REGISTRY: quay.io + REGISTRY_HOST: ${{ vars.REGISTRY_HOST }} + REGISTRY_USER: ${{ vars.REGISTRY_USER }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + QUAY_USER: ${{ vars.QUAY_USER }} + QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }} + BASE_IMAGE_TAG: ${{ inputs.base_image_tag }} + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Build and publish Go dependency cache image + run: | + set -euo pipefail + source .github/scripts/agent-godeps-cache-metadata.sh + agent_godeps_compute_metadata + + base_image="${REGISTRY_HOST}/quay/stackstate/datadog_build_linux_x64:${BASE_IMAGE_TAG}" + + # Minimal build context: the module manifests (repo paths preserved so + # nested modules and their local `replace ../` targets resolve) plus the + # cache Dockerfile. Assembled outside the checkout so `docker build` + # streams only what the image needs. + context="$(mktemp -d)" + mkdir -p "${context}/manifests" + git ls-files -z -- \ + '*go.mod' '*go.sum' 'go.work' 'go.work.sum' 'modules.yml' \ + | while IFS= read -r -d '' f; do + mkdir -p "${context}/manifests/$(dirname "$f")" + cp "$f" "${context}/manifests/${f}" + done + cp .github/docker/godeps-cache/Dockerfile "${context}/Dockerfile" + + # Log into both registries: pull the private base via the registry.tooling + # proxy, push the result to quay.io/stackstate. + echo "$REGISTRY_PASSWORD" | docker login --username "$REGISTRY_USER" --password-stdin "$REGISTRY_HOST" + echo "$QUAY_PASSWORD" | docker login --username "$QUAY_USER" --password-stdin "$QUAY_REGISTRY" + + docker build \ + --pull \ + --build-arg "BASE_IMAGE=${base_image}" \ + --tag "$ci_image_push" \ + "$context" + docker push "$ci_image_push" diff --git a/.github/workflows/lint-and-unit-tests.yml b/.github/workflows/lint-and-unit-tests.yml index 0847a13a2f54..246e80e10f6c 100644 --- a/.github/workflows/lint-and-unit-tests.yml +++ b/.github/workflows/lint-and-unit-tests.yml @@ -20,9 +20,13 @@ name: Lint and unit tests # * The GitLab push mirror to this repo must be disabled (done 2026-07-10), # otherwise a mirror sync overwrites anything pushed to GitHub. # * This is a PUBLIC repo. The read-only registry-proxy credentials must be -# available to it (org-inherited or provisioned via pulumi-infra): -# vars REGISTRY_HOST, REGISTRY_USER + secret REGISTRY_PASSWORD. Org secrets -# are never exposed to fork PRs, so external-fork PRs skip these jobs. +# available to it (org-inherited or provisioned via pulumi-infra) to pull the +# cache image: vars REGISTRY_HOST, REGISTRY_USER + secret REGISTRY_PASSWORD. +# The godeps-cache build job additionally needs quay push credentials (var +# QUAY_USER + secret QUAY_PASSWORD). The warm Go module cache is delivered by +# the godeps-cache reusable workflow (STAC-25429; see build-binaries.yml header +# and .github/workflows/godeps-cache.yml). Org secrets are never exposed to fork +# PRs, so external-fork PRs skip these jobs. on: pull_request: @@ -46,6 +50,15 @@ env: CONDA_ENV: ddpy3 jobs: + godeps-cache: + name: Go dependency cache image + # Same-repo PRs only: the registry/quay secrets are not exposed to fork PRs. + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + uses: ./.github/workflows/godeps-cache.yml + secrets: + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }} + filename-linting: name: Filename linting # Same-repo PRs only: the registry-proxy secret is not exposed to fork PRs. @@ -80,10 +93,16 @@ jobs: unbranded-unit-tests: name: Unit tests (unbranded / DataDog) if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + needs: godeps-cache runs-on: xlarge-public timeout-minutes: 150 container: - image: ${{ vars.REGISTRY_HOST }}/quay/stackstate/datadog_build_linux_x64:7af9194f + # Warm Go module cache image (godeps-cache job); pulled via the same + # read-only registry proxy + REGISTRY_* credentials as datadog_build. The tag is + # a sha256 content hash of the module graph (see godeps-cache.yml), so this ref + # is effectively digest-pinned; it cannot be a static digest because it is + # computed per run -- hence the narrow unpinned-images ignore below. + image: ${{ needs.godeps-cache.outputs.ci_image }} # zizmor: ignore[unpinned-images] credentials: username: ${{ vars.REGISTRY_USER }} password: ${{ secrets.REGISTRY_PASSWORD }} @@ -105,13 +124,14 @@ jobs: # Work volume is owned by the ARC runner uid but the job container runs as # root, so git rejects the repo as "dubious ownership"; mark it safe. git config --global --add safe.directory '*' - go clean -modcache + # GOMODCACHE is pre-warmed by the godeps-cache image (STAC-25429), keyed + # to the module-graph hash: no `go clean -modcache` and no per-job + # `inv deps` reconcile. Materialise vendor/ against the warm cache + # (offline). Go tool binaries come from `invoke install-tools` below -- + # the removed second `inv deps` never installed them (it is go mod + # download + tidy, not a tool install). go work sync go work vendor - inv -e deps --verbose - # /go/bin is not cached on ephemeral runners; re-run deps to get tool - # binaries (golint, misspell, ...). - inv deps export AGENT_GITHUB_ORG=DataDog export GITHUB_ORG=DataDog export BRANDED=false @@ -130,10 +150,16 @@ jobs: branded-unit-tests: name: Unit tests (branded / StackState) if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + needs: godeps-cache runs-on: xlarge-public timeout-minutes: 150 container: - image: ${{ vars.REGISTRY_HOST }}/quay/stackstate/datadog_build_linux_x64:7af9194f + # Warm Go module cache image (godeps-cache job); pulled via the same + # read-only registry proxy + REGISTRY_* credentials as datadog_build. The tag is + # a sha256 content hash of the module graph (see godeps-cache.yml), so this ref + # is effectively digest-pinned; it cannot be a static digest because it is + # computed per run -- hence the narrow unpinned-images ignore below. + image: ${{ needs.godeps-cache.outputs.ci_image }} # zizmor: ignore[unpinned-images] credentials: username: ${{ vars.REGISTRY_USER }} password: ${{ secrets.REGISTRY_PASSWORD }} @@ -155,11 +181,12 @@ jobs: # Work volume is owned by the ARC runner uid but the job container runs as # root, so git rejects the repo as "dubious ownership"; mark it safe. git config --global --add safe.directory '*' - go clean -modcache + # GOMODCACHE is pre-warmed by the godeps-cache image (STAC-25429): no + # `go clean -modcache` and no per-job `inv deps` reconcile. Materialise + # vendor/ against the warm cache (offline). Go tool binaries come from + # `invoke install-tools` below. go work sync go work vendor - inv -e deps --verbose - inv deps export AGENT_GITHUB_ORG=DataDog export GITHUB_ORG=DataDog export BRANDED=true From 7bedf74380494c6d8e9c8f3bf9755af37d264f50 Mon Sep 17 00:00:00 2001 From: Louis Parkin Date: Tue, 28 Jul 2026 12:57:55 +0200 Subject: [PATCH 09/19] STAC-25142 Add GitHub Actions omnibus DEB build workflow --- .github/workflows/build-deb.yml | 240 ++++++++++++++++++++++++++++++++ 1 file changed, 240 insertions(+) create mode 100644 .github/workflows/build-deb.yml diff --git a/.github/workflows/build-deb.yml b/.github/workflows/build-deb.yml new file mode 100644 index 000000000000..63c84fba8ca8 --- /dev/null +++ b/.github/workflows/build-deb.yml @@ -0,0 +1,240 @@ +name: DEB package build + +# Ported from .gitlab-ci-agent.yml (build_deb, test_deb_renaming) as part of the +# GitLab -> GitHub migration (STAC-25142), phase 3: omnibus DEB packaging. Tracks +# the stackstate-7.78.2 pipeline. +# +# Scope: the PR lane only -- packaging plus the branding gate. `sign_deb` and the +# S3 publish stay out by design and land in phase 4: signing is a release action, +# it needs the GPG key material, and org secrets are never exposed to fork PRs on +# what is a PUBLIC repo. Validating a change and releasing it are different trust +# boundaries and belong in different workflows. +# +# Deliberate differences from the GitLab job: +# * `deps_deb` is NOT ported. Its only downstream-consumed artifact was +# version.txt, produced by `inv agent.version -u` -- a pure git-tag derivation +# that needs no upstream job, just `fetch-depth: 0`. Its other output +# (vendor.tar.gz) is consumed by nobody (see the STAC-25396 note at +# .gitlab-ci-agent.yml:68-75). Dropping it removes a job, an artifact +# round-trip and a cache for zero loss. +# * No build caches yet -- this is a deliberate cold port. The GitLab job carried +# three (omnibus base_dir + .gems, bazel install, bazel repository_cache), none +# of which transfer directly: the GitHub-hosted Actions cache is capped at 10 GB +# per repo and the omnibus cache alone is expected to exceed it, and STAC-25396 +# already established that caches of this shape hit the file-count tax and +# artifact 413s on this repo. The cache backend is its own ticket; the cold +# wall-clock this job produces is the number that tells us what it is worth. +# Bazel state is still pinned under XDG_CACHE_HOME below so a cache layer can +# adopt those paths unchanged. +# * The GitLab job's `.omnibus` <-> `/omnibus` move dance is dropped along with +# the cache. Note it was already inert there: it staged the cache at /omnibus +# while the build ran with --base-dir /.omnibus. +# +# Prerequisites for a GREEN run (beyond the phase 1/2 registry-proxy credentials): +# * GITLAB_PACKAGE_REGISTRY_PYPI_SIMPLE_URL / _USER / _READONLY_PASSWORD must +# reach this repo as REPO-level secrets, provisioned through pulumi-infra +# (github/repoVariables/resources.yaml, `- repository: stackstate-agent`). +# Repo-level, not org-level: org secrets default to visibility=private, which +# excludes this PUBLIC repo, and widening them to visibility=all -- the +# STAC-25350 workaround used for the read-only REGISTRY_* pull credentials -- +# would expose a package-registry credential to every public repo in the org. +# `.gitlab-scripts/setup_artifact_registry.sh` hard-exits when they are unset, +# so build-deb stays red until they land. The agent genuinely needs a private +# Python index for non-upstream packages, so these must be provisioned rather +# than dropped. Use the READONLY password, never the write one. Credentials go +# to pip.conf + ~/.netrc, never into a URL -- see wiki/concepts/ci-credentials.md. + +on: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +# The datadog_build container's default shell is dash, which lacks `set -o pipefail` +# and mis-handles the conda activation the steps rely on; force bash. +defaults: + run: + shell: bash + +env: + # conda env baked into the datadog_build image. + CONDA_ENV: ddpy3 + # MAJOR_VERSION in .gitlab-ci.yml; selects the built .deb for the branding gate. + MAJOR_VERSION: '3' + +jobs: + build-deb: + name: Build DEB package (omnibus, branded / StackState) + # Same-repo PRs only: the registry-proxy secret is not exposed to fork PRs. + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: xlarge-public + # Cold omnibus build: no git cache, so every embedded software (openssl, cpython, + # openscap, sqlite, ...) compiles from source. Generous until we have a real number. + timeout-minutes: 300 + container: + # datadog_build image (built by datadog-agent-buildimages), pulled via the + # read-only quay proxy -- mirrors BUILD_IMAGE in .gitlab-ci.yml (7.78.2 tag). + image: ${{ vars.REGISTRY_HOST }}/quay/stackstate/datadog_build_linux_x64:7af9194f + credentials: + username: ${{ vars.REGISTRY_USER }} + password: ${{ secrets.REGISTRY_PASSWORD }} + env: + # Must live OUTSIDE the checkout: omnibus git-clones into base_dir and that + # conflicts with the agent working tree when nested under it. + OMNIBUS_BASE_DIR: /omnibus + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + # Full history + tags so `inv agent.version` resolves via `git describe`. + fetch-depth: 0 + + - name: Build DEB package with omnibus + env: + # Repo-level secrets (see header). Read via `secrets.` for all three + # because pulumi-infra's `value: "secret:..."` form provisions a secret + # unless the entry is marked `type: variable` -- the mismatch flagged in + # the stackpacks block of github/repoVariables/resources.yaml. + GITLAB_PACKAGE_REGISTRY_PYPI_SIMPLE_URL: ${{ secrets.GITLAB_PACKAGE_REGISTRY_PYPI_SIMPLE_URL }} + GITLAB_PACKAGE_REGISTRY_USER: ${{ secrets.GITLAB_PACKAGE_REGISTRY_USER }} + GITLAB_PACKAGE_REGISTRY_READONLY_PASSWORD: ${{ secrets.GITLAB_PACKAGE_REGISTRY_READONLY_PASSWORD }} + run: | + set -eo pipefail + export PATH="$PATH:/usr/local/go/bin" + . /root/miniforge3/etc/profile.d/conda.sh + conda activate "${CONDA_ENV}" + # omnibus is Ruby; the build image ships it under rvm. + . /usr/local/rvm/scripts/rvm + # Work volume is owned by the ARC runner uid but the job container runs as + # root, so git rejects the repo as "dubious ownership"; mark it safe. + git config --global --add safe.directory '*' + + # omnibus resolves the agent source through GOPATH rather than the + # workspace path; GitLab's before_script created this symlink. + mkdir -p /go/src/github.com/StackVista + rm -rf /go/src/github.com/StackVista/stackstate-agent + ln -s "${GITHUB_WORKSPACE}" /go/src/github.com/StackVista/stackstate-agent + + # DD 7.78 builds some dependencies through bazelisk from + # omnibus/config/software/datadog-agent-dependencies.rb. bazelisk aborts + # with "XDG_CACHE_HOME () must denote a directory in CI!" unless these + # point at real directories. + export XDG_CACHE_HOME="${GITHUB_WORKSPACE}/.cache" + export BAZELISK_HOME="${XDG_CACHE_HOME}/bazelisk" + mkdir -p "${XDG_CACHE_HOME}/bazel" "${XDG_CACHE_HOME}/bazel-repo" "${BAZELISK_HOME}" + # .bazelrc:75 `try-import %workspace%/user.bazelrc` is the upstream-blessed + # extension point; it applies to every bazelisk shell-out omnibus makes. + # user.bazelrc is gitignored. + cat > "${GITHUB_WORKSPACE}/user.bazelrc" < StackState BEFORE vendoring: branding rewrites import + # paths, so vendor/ must be materialised against the rewritten tree. + # fix_branding.sh defaults its target dir to $CI_PROJECT_DIR (a GitLab + # built-in that is unset here); pass the checkout root explicitly. + ./fix_branding.sh "${GITHUB_WORKSPACE}" + + go clean -modcache + go work sync + go work vendor + inv -e deps --verbose + # version.txt is consumed by the packaging/image phases. deps_deb produced + # it on GitLab; generate it in-job instead. No `-e`: stdout must be only + # the version string. + inv agent.version -u > version.txt + cat version.txt + + # Writes pip.conf + ~/.netrc for the private Python index. Hard-exits if + # the GITLAB_PACKAGE_REGISTRY_* variables above are unset. + source ./.gitlab-scripts/setup_artifact_registry.sh + + export LD_LIBRARY_PATH="/opt/stackstate-agent/embedded/lib/python3.12/site-packages/psycopg2_binary.libs:/opt/stackstate-agent/embedded/lib" + + # --install-directory pins the omnibus paths to the BRANDED install dir. + # Without it tasks/omnibus.py derives them from the unbranded + # /opt/datadog-agent while the branded build uses /opt/stackstate-agent, + # and the post-build step then targets a nonexistent directory. + inv -e omnibus.build \ + --gem-path "${GITHUB_WORKSPACE}/.gems" \ + --base-dir "${OMNIBUS_BASE_DIR}" \ + --go-mod-cache "${GITHUB_WORKSPACE}/vendor" \ + --skip-deps \ + --skip-sign \ + --install-directory /opt/stackstate-agent + + - name: Collect package outputs + run: | + set -eo pipefail + mkdir -p outcomes + cp -r "${OMNIBUS_BASE_DIR}/pkg" outcomes/ + cp -r Dockerfiles outcomes/ + ls -la outcomes/pkg outcomes/Dockerfiles + + - name: Upload DEB package and image manifests + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: deb-package + # Paths match the GitLab build_deb artifacts; the branding gate below and + # the image phase (phase 5) consume them. + path: | + outcomes/pkg/*.deb + outcomes/pkg/*.json + outcomes/Dockerfiles/agent + outcomes/Dockerfiles/cluster-agent + outcomes/Dockerfiles/dogstatsd + outcomes/Dockerfiles/manifests + version.txt + retention-days: 5 + if-no-files-found: error + + test-deb-renaming: + name: DEB branding verification (no DataDog references) + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + needs: build-deb + # Cheap gate: unpacks the .deb and greps it. Does not need the XL class. + runs-on: docker-public + timeout-minutes: 30 + container: + # Same build image: test_deb.sh needs `ar` from binutils to unpack the .deb. + image: ${{ vars.REGISTRY_HOST }}/quay/stackstate/datadog_build_linux_x64:7af9194f + credentials: + username: ${{ vars.REGISTRY_USER }} + password: ${{ secrets.REGISTRY_PASSWORD }} + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Download DEB package + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: deb-package + + - name: Verify package carries no DataDog branding + run: | + set -eo pipefail + git config --global --add safe.directory '*' + # test_deb.sh takes exactly one package and rejects anything else, so + # resolve the glob here and fail loudly rather than passing it through + # (the GitLab job relied on the glob expanding to exactly one file). + shopt -s nullglob + debs=(outcomes/pkg/stackstate-agent_"${MAJOR_VERSION}"*.deb) + if [ "${#debs[@]}" -ne 1 ]; then + echo "Expected exactly one stackstate-agent_${MAJOR_VERSION}*.deb, found ${#debs[@]}: ${debs[*]}" >&2 + exit 1 + fi + ./test/renaming/test_deb.sh "${debs[0]}" From 229df15c89115aeb137b16de649b99a960479592 Mon Sep 17 00:00:00 2001 From: Louis Parkin Date: Tue, 28 Jul 2026 13:07:52 +0200 Subject: [PATCH 10/19] STAC-25142 Add agent and cluster-agent container image builds --- .github/workflows/build-binaries.yml | 57 +++++++++++++++++++++ .github/workflows/build-deb.yml | 76 +++++++++++++++++++++++++--- 2 files changed, 125 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index 1a013eb36d8a..c98c8ec5e886 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -169,3 +169,60 @@ jobs: version.txt retention-days: 5 if-no-files-found: error + + build-cluster-agent-image: + name: Build cluster-agent container image (docker build, no push on PR) + # Ported from pre_release_cluster_agent_image. Lives in this workflow rather + # than build-deb.yml so it can `needs:` the job that produces its input -- + # cross-workflow artifact hand-off would need run-id plumbing for no gain. + # Stops at `docker build` + a runtime check: the GitLab job also pushed to + # quay.io/stackstate, which the PR lane of a PUBLIC repo must not do. + # Publishing lands in phase 4, gated on the branch. + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + needs: build-cluster-agent + # DinD class: docker CLI in the runner image, dockerd in a native sidecar. + runs-on: docker-public + timeout-minutes: 45 + env: + LOCAL_IMAGE: stackstate-cluster-agent:ci + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Download cluster-agent binary + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: cluster-agent-binary + + - name: Log in to the registry proxy + env: + REGISTRY_HOST: ${{ vars.REGISTRY_HOST }} + REGISTRY_USER: ${{ vars.REGISTRY_USER }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + run: | + set -eo pipefail + # Dockerfiles/cluster-agent pulls its ubuntu builder stage through the + # proxy; the BCI stages come from registry.suse.com and need no auth. + printf '%s' "${REGISTRY_PASSWORD}" | docker login -u "${REGISTRY_USER}" --password-stdin "${REGISTRY_HOST}" + + - name: Build cluster-agent image + run: | + set -eo pipefail + # bin/stackstate-cluster-agent is a DIRECTORY (binary + dist/), and it + # must stay one: the Dockerfile COPYs it to + # /opt/stackstate-agent/bin/stackstate-cluster-agent/ and entrypoint.sh + # puts that directory on PATH so CMD can exec `stackstate-cluster-agent` + # by name. Flattening it to a bare binary breaks the image. + cp -r bin/stackstate-cluster-agent Dockerfiles/cluster-agent/ + docker build -t "${LOCAL_IMAGE}" Dockerfiles/cluster-agent + + - name: Smoke test cluster-agent image + run: | + set -eo pipefail + # Bypass entrypoint.sh: it hard-exits without STS_API_KEY, which a + # version check has no business needing. + docker run --rm \ + --entrypoint /opt/stackstate-agent/bin/stackstate-cluster-agent/stackstate-cluster-agent \ + "${LOCAL_IMAGE}" version diff --git a/.github/workflows/build-deb.yml b/.github/workflows/build-deb.yml index 63c84fba8ca8..bcd229db52a5 100644 --- a/.github/workflows/build-deb.yml +++ b/.github/workflows/build-deb.yml @@ -1,14 +1,22 @@ name: DEB package build -# Ported from .gitlab-ci-agent.yml (build_deb, test_deb_renaming) as part of the -# GitLab -> GitHub migration (STAC-25142), phase 3: omnibus DEB packaging. Tracks -# the stackstate-7.78.2 pipeline. +# Ported from .gitlab-ci-agent.yml (build_deb, test_deb_renaming, +# pre_release_main_agent_image) as part of the GitLab -> GitHub migration +# (STAC-25142). Tracks the stackstate-7.78.2 pipeline. # -# Scope: the PR lane only -- packaging plus the branding gate. `sign_deb` and the -# S3 publish stay out by design and land in phase 4: signing is a release action, -# it needs the GPG key material, and org secrets are never exposed to fork PRs on -# what is a PUBLIC repo. Validating a change and releasing it are different trust -# boundaries and belong in different workflows. +# Scope: the PR lane only -- package, verify branding, build the agent image and +# smoke-test it. Nothing is signed, pushed or published here: +# * `sign_deb` and the S3 publish are release actions needing GPG key material. +# * `pre_release_main_agent_image` also PUSHES to quay.io/stackstate; this job +# stops at `docker build` + a runtime check, so the PR lane needs no write +# credential at all -- only the read-only proxy creds already in use. +# On a PUBLIC repo that separation is the point: validating a change and releasing +# it are different trust boundaries and belong in different workflows. Publishing +# lands in phase 4, gated on the branch, not on the PR. +# +# Only the amd64 pipeline is ported. GitLab fans .gitlab-ci-agent.yml out twice +# from .gitlab-ci.yml (agent-x86 / agent-arm); the arm64 half needs an arm64 XL +# runner class, which does not exist yet (only `arm64-docker`). Tracked separately. # # Deliberate differences from the GitLab job: # * `deps_deb` is NOT ported. Its only downstream-consumed artifact was @@ -238,3 +246,55 @@ jobs: exit 1 fi ./test/renaming/test_deb.sh "${debs[0]}" + + build-agent-image: + name: Build agent container image (docker build, no push on PR) + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + needs: build-deb + # DinD class: docker CLI in the runner image, dockerd in a native sidecar. + runs-on: docker-public + timeout-minutes: 60 + env: + # ARCH in the agent-x86 trigger block of .gitlab-ci.yml; also the .deb suffix. + ARCH: amd64 + LOCAL_IMAGE: stackstate-agent:ci + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Download DEB package + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: deb-package + + - name: Log in to the registry proxy + env: + REGISTRY_HOST: ${{ vars.REGISTRY_HOST }} + REGISTRY_USER: ${{ vars.REGISTRY_USER }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + run: | + set -eo pipefail + # Dockerfiles/agent pulls its ubuntu extract stage through the proxy; + # the BCI stages come from registry.suse.com and need no auth. + printf '%s' "${REGISTRY_PASSWORD}" | docker login -u "${REGISTRY_USER}" --password-stdin "${REGISTRY_HOST}" + + - name: Build agent image + run: | + set -eo pipefail + shopt -s nullglob + debs=(outcomes/pkg/stackstate-agent_*_"${ARCH}".deb) + if [ "${#debs[@]}" -ne 1 ]; then + echo "Expected exactly one stackstate-agent_*_${ARCH}.deb, found ${#debs[@]}: ${debs[*]}" >&2 + exit 1 + fi + cp "${debs[0]}" Dockerfiles/agent/ + # publish_image.sh also passes --build-arg S6_ARCH; the agent Dockerfile + # declares no such ARG, so it is dropped here rather than kept as a warning. + docker build --build-arg ARCH="${ARCH}" -t "${LOCAL_IMAGE}" Dockerfiles/agent + + - name: Smoke test agent image + run: | + set -eo pipefail + docker run --rm --entrypoint /opt/stackstate-agent/bin/agent/agent "${LOCAL_IMAGE}" version From c45407dea6c65e1b96f23bc4ebf5437aa0d77ce5 Mon Sep 17 00:00:00 2001 From: Louis Parkin Date: Tue, 28 Jul 2026 13:24:04 +0200 Subject: [PATCH 11/19] STAC-25142 Read the PyPI index URL as a variable, not a secret --- .github/workflows/build-deb.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-deb.yml b/.github/workflows/build-deb.yml index bcd229db52a5..839bb6c5793a 100644 --- a/.github/workflows/build-deb.yml +++ b/.github/workflows/build-deb.yml @@ -40,7 +40,7 @@ name: DEB package build # # Prerequisites for a GREEN run (beyond the phase 1/2 registry-proxy credentials): # * GITLAB_PACKAGE_REGISTRY_PYPI_SIMPLE_URL / _USER / _READONLY_PASSWORD must -# reach this repo as REPO-level secrets, provisioned through pulumi-infra +# reach this repo at REPO level, provisioned through pulumi-infra # (github/repoVariables/resources.yaml, `- repository: stackstate-agent`). # Repo-level, not org-level: org secrets default to visibility=private, which # excludes this PUBLIC repo, and widening them to visibility=all -- the @@ -105,11 +105,11 @@ jobs: - name: Build DEB package with omnibus env: - # Repo-level secrets (see header). Read via `secrets.` for all three - # because pulumi-infra's `value: "secret:..."` form provisions a secret - # unless the entry is marked `type: variable` -- the mismatch flagged in - # the stackpacks block of github/repoVariables/resources.yaml. - GITLAB_PACKAGE_REGISTRY_PYPI_SIMPLE_URL: ${{ secrets.GITLAB_PACKAGE_REGISTRY_PYPI_SIMPLE_URL }} + # Repo-level, provisioned by pulumi-infra (see header). The index URL is a + # plain variable -- it is not secret, the same URL is hardcoded in this repo + # at omnibus/package-scripts/publish_image.sh -- while the user and password + # are secrets. Scheme-less: setup_artifact_registry.sh prepends https://. + GITLAB_PACKAGE_REGISTRY_PYPI_SIMPLE_URL: ${{ vars.GITLAB_PACKAGE_REGISTRY_PYPI_SIMPLE_URL }} GITLAB_PACKAGE_REGISTRY_USER: ${{ secrets.GITLAB_PACKAGE_REGISTRY_USER }} GITLAB_PACKAGE_REGISTRY_READONLY_PASSWORD: ${{ secrets.GITLAB_PACKAGE_REGISTRY_READONLY_PASSWORD }} run: | From ecb9c60a3b565266a4362a0c557d06242a3b8cd3 Mon Sep 17 00:00:00 2001 From: Louis Parkin Date: Tue, 28 Jul 2026 13:43:38 +0200 Subject: [PATCH 12/19] STAC-25429 Push the Go dep cache to its own quay repo, not sts-ci-images --- .github/scripts/agent-godeps-cache-metadata.sh | 16 +++++++++++----- .github/workflows/godeps-cache.yml | 16 +++++++++++----- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/.github/scripts/agent-godeps-cache-metadata.sh b/.github/scripts/agent-godeps-cache-metadata.sh index af6692f5612f..00eedbf97c02 100755 --- a/.github/scripts/agent-godeps-cache-metadata.sh +++ b/.github/scripts/agent-godeps-cache-metadata.sh @@ -1,8 +1,13 @@ #!/usr/bin/env bash # Computes the content-addressed tag for the stackstate-agent Go dependency cache # image (STAC-25429) and the two image refs it is referenced by: -# * push ref -> quay.io/stackstate/sts-ci-images (authoritative; build + exists check) -# * pull ref -> ${REGISTRY_HOST}/quay/... (registry.tooling proxy; job container) +# * push ref -> quay.io/stackstate/stackstate-agent-godeps-cache (authoritative) +# * pull ref -> ${REGISTRY_HOST}/quay/... (registry.tooling proxy) +# +# The cache lives in its own quay repo rather than the shared sts-ci-images, which also +# holds the ARC runner images: stackstate-agent is PUBLIC, so whatever credential this +# workflow carries is reachable from any org member's same-repo branch. A dedicated repo +# keeps that reach down to a rebuildable cache. # # Mirrors StackVista/StackGraph .github/scripts/stackgraph-ci-metadata.sh: the tag is # a hash of the dependency-defining inputs, so an unchanged module graph reuses the @@ -31,9 +36,10 @@ agent_godeps_compute_metadata() { } | sha256sum | cut -c1-16 )" - local image_tag="stackstate-agent-godeps-${godeps_hash}" + local image_repo="stackstate/stackstate-agent-godeps-cache" + local image_tag="godeps-${godeps_hash}" # shellcheck disable=SC2034 # consumed by the sourcing workflow step - ci_image_push="${QUAY_REGISTRY}/stackstate/sts-ci-images:${image_tag}" + ci_image_push="${QUAY_REGISTRY}/${image_repo}:${image_tag}" # shellcheck disable=SC2034 # consumed by the sourcing workflow step - ci_image="${REGISTRY_HOST}/quay/stackstate/sts-ci-images:${image_tag}" + ci_image="${REGISTRY_HOST}/quay/${image_repo}:${image_tag}" } diff --git a/.github/workflows/godeps-cache.yml b/.github/workflows/godeps-cache.yml index e5fcf8ba2ff3..5b671a4742ce 100644 --- a/.github/workflows/godeps-cache.yml +++ b/.github/workflows/godeps-cache.yml @@ -8,11 +8,17 @@ name: Go dependency cache image # heavy Go module download only runs when the graph actually changes. # # The image is FROM the private datadog_build CI image with the workspace's external -# modules baked into GOMODCACHE. It is pushed to quay.io/stackstate/sts-ci-images -# (private) and pulled by consumer jobs through the registry.tooling quay proxy with -# the same read-only REGISTRY_* credentials they already use for datadog_build. -# Because the base image is private, the cache image MUST stay private -- it is never -# pushed to a public registry (that would leak base-image layers). +# modules baked into GOMODCACHE. It is pushed to the dedicated private quay repo +# quay.io/stackstate/stackstate-agent-godeps-cache and pulled by consumer jobs through +# the registry.tooling quay proxy with the same read-only REGISTRY_* credentials they +# already use for datadog_build. Because the base image is private, the cache image +# MUST stay private -- it is never pushed to a public registry (that would leak +# base-image layers). +# +# Dedicated repo, not the shared sts-ci-images: that one also holds the ARC runner +# images, and stackstate-agent is PUBLIC, so any org member's same-repo branch can use +# whatever push credential this workflow carries. Scoping it to a rebuildable cache +# keeps a compromised branch away from the images the CI fleet boots from. on: workflow_call: From ffabcac22c074bfb8f1420e0e7d33d186b4b11ea Mon Sep 17 00:00:00 2001 From: Louis Parkin Date: Tue, 28 Jul 2026 14:13:41 +0200 Subject: [PATCH 13/19] STAC-25429 Correct visibility comments: datadog_build base and cache repo are public --- .github/docker/godeps-cache/Dockerfile | 2 +- .github/workflows/godeps-cache.yml | 17 ++++++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/docker/godeps-cache/Dockerfile b/.github/docker/godeps-cache/Dockerfile index e1badfa38a82..776191265398 100644 --- a/.github/docker/godeps-cache/Dockerfile +++ b/.github/docker/godeps-cache/Dockerfile @@ -1,5 +1,5 @@ # Go dependency cache image for stackstate-agent CI (STAC-25429). Derives FROM the -# private datadog_build CI image and bakes the workspace's external Go module +# datadog_build CI image and bakes the workspace's external Go module # downloads into GOMODCACHE, so consumer jobs get a warm module cache with no # per-job `go mod download`/network round-trips. Rebuilt only when the module graph # hash changes -- the image tag is content-addressed diff --git a/.github/workflows/godeps-cache.yml b/.github/workflows/godeps-cache.yml index 5b671a4742ce..c079a3966aec 100644 --- a/.github/workflows/godeps-cache.yml +++ b/.github/workflows/godeps-cache.yml @@ -7,13 +7,16 @@ name: Go dependency cache image # image across every branch/PR/default build (the tag *is* the cache key), and the # heavy Go module download only runs when the graph actually changes. # -# The image is FROM the private datadog_build CI image with the workspace's external -# modules baked into GOMODCACHE. It is pushed to the dedicated private quay repo +# The image is FROM the datadog_build CI image with the workspace's external modules +# baked into GOMODCACHE. It is pushed to the dedicated quay repo # quay.io/stackstate/stackstate-agent-godeps-cache and pulled by consumer jobs through # the registry.tooling quay proxy with the same read-only REGISTRY_* credentials they -# already use for datadog_build. Because the base image is private, the cache image -# MUST stay private -- it is never pushed to a public registry (that would leak -# base-image layers). +# already use for datadog_build. +# +# Both that repo and the datadog_build base are public (terraform-infra quay/locals.tf). +# Nothing private is baked in: this repo is open source and the module set is already +# public in the committed go.mod files. Public read also keeps the pull path off the +# registry.tooling proxy robot's private-repo grants, which it does not hold. # # Dedicated repo, not the shared sts-ci-images: that one also holds the ARC runner # images, and stackstate-agent is PUBLIC, so any org member's same-repo branch can use @@ -135,8 +138,8 @@ jobs: done cp .github/docker/godeps-cache/Dockerfile "${context}/Dockerfile" - # Log into both registries: pull the private base via the registry.tooling - # proxy, push the result to quay.io/stackstate. + # Log into both registries: pull the base via the registry.tooling proxy, + # push the result to quay.io/stackstate. echo "$REGISTRY_PASSWORD" | docker login --username "$REGISTRY_USER" --password-stdin "$REGISTRY_HOST" echo "$QUAY_PASSWORD" | docker login --username "$QUAY_USER" --password-stdin "$QUAY_REGISTRY" From e2e6e1c0ebd6b02e67648669131deb9a70dda3d0 Mon Sep 17 00:00:00 2001 From: Louis Parkin Date: Tue, 28 Jul 2026 15:45:09 +0200 Subject: [PATCH 14/19] STAC-25142 Restore the cluster-agent binary's exec bit after artifact download --- .github/workflows/build-binaries.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index c98c8ec5e886..17c29336c992 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -210,6 +210,12 @@ jobs: - name: Build cluster-agent image run: | set -eo pipefail + # actions/download-artifact does not preserve the executable bit (GitLab + # artifacts did). The Dockerfile's `chmod +x` targets + # opt/stackstate-agent/bin/stackstate-cluster-agent, which is the enclosing + # DIRECTORY, not the binary inside it -- so nothing in the build restores + # it and the image ships a non-executable agent. + chmod +x bin/stackstate-cluster-agent/stackstate-cluster-agent # bin/stackstate-cluster-agent is a DIRECTORY (binary + dist/), and it # must stay one: the Dockerfile COPYs it to # /opt/stackstate-agent/bin/stackstate-cluster-agent/ and entrypoint.sh From 7049264e457a28ff51d73cac46f5a9f314cd2429 Mon Sep 17 00:00:00 2001 From: Louis Parkin Date: Tue, 28 Jul 2026 15:46:15 +0200 Subject: [PATCH 15/19] STAC-25429 fix godeps cache image baking an empty module cache RUN ran under dash so the read -d '' loop never iterated; -mod=mod is rejected in workspace mode. --- .github/docker/godeps-cache/Dockerfile | 29 ++++++++++++++++---------- .github/workflows/godeps-cache.yml | 22 ++++++++++++++----- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/.github/docker/godeps-cache/Dockerfile b/.github/docker/godeps-cache/Dockerfile index 776191265398..6137b966249f 100644 --- a/.github/docker/godeps-cache/Dockerfile +++ b/.github/docker/godeps-cache/Dockerfile @@ -11,21 +11,28 @@ ARG BASE_IMAGE FROM ${BASE_IMAGE} -# Pre-download external modules for every module in the workspace. manifests/ -# preserves the repo's directory layout so nested go.mod files (and their local -# `replace ../` targets) resolve. GOFLAGS=-mod=mod lets `go mod download` reconcile -# go.sum in-container without a committed change; only the populated module cache is -# kept -- the manifests themselves are removed from the final image. +# `read -d ''` and `pipefail` are bashisms and RUN defaults to /bin/sh (dash), where +# the download loop silently iterates zero times and publishes an empty cache. +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# Pre-download external modules for every module in the workspace, mirroring what +# `inv deps` does per module (tasks/libs/common/go.py) at the same parallelism. +# manifests/ preserves the repo's directory layout so nested go.mod files (and their +# local `replace ../` targets) resolve. No GOFLAGS override: go.work puts this in +# workspace mode, where anything other than -mod=readonly is rejected outright. Only +# the populated module cache is kept -- the manifests are removed from the final image. COPY manifests/ /workspace/ RUN set -eux; \ git config --global --add safe.directory '*'; \ export PATH="${PATH}:/usr/local/go/bin"; \ - export GOFLAGS=-mod=mod; \ cd /workspace; \ + modules="$(find . -type f -name go.mod | wc -l)"; \ + echo "workspace modules: ${modules}"; \ + test "${modules}" -gt 0; \ find . -type f -name go.mod -print0 \ - | while IFS= read -r -d '' modfile; do \ - moddir="$(dirname "${modfile}")"; \ - echo "go mod download :: ${moddir}"; \ - ( cd "${moddir}" && go mod download ); \ - done; \ + | xargs -0 -n1 -P8 bash -c \ + 'moddir="$(dirname "$1")"; echo "go mod download :: ${moddir}"; cd "${moddir}" && go mod download' _; \ + cache="$(go env GOMODCACHE)"; \ + du -sh "${cache}"; \ + test "$(du -sm "${cache}" | cut -f1)" -gt 100; \ rm -rf /workspace diff --git a/.github/workflows/godeps-cache.yml b/.github/workflows/godeps-cache.yml index c079a3966aec..b29452c7376e 100644 --- a/.github/workflows/godeps-cache.yml +++ b/.github/workflows/godeps-cache.yml @@ -102,6 +102,11 @@ jobs: needs: metadata runs-on: docker-public timeout-minutes: 60 + # Consumer workflows call this in parallel on the same commit; without a shared gate + # they all see image_exists=false and build the same content-addressed tag at once. + concurrency: + group: godeps-cache-${{ needs.metadata.outputs.ci_image }} + cancel-in-progress: false env: QUAY_REGISTRY: quay.io REGISTRY_HOST: ${{ vars.REGISTRY_HOST }} @@ -124,6 +129,18 @@ jobs: base_image="${REGISTRY_HOST}/quay/stackstate/datadog_build_linux_x64:${BASE_IMAGE_TAG}" + # Log into both registries: pull the base via the registry.tooling proxy, + # push the result to quay.io/stackstate. + echo "$REGISTRY_PASSWORD" | docker login --username "$REGISTRY_USER" --password-stdin "$REGISTRY_HOST" + echo "$QUAY_PASSWORD" | docker login --username "$QUAY_USER" --password-stdin "$QUAY_REGISTRY" + + # The metadata lookup can be stale by the time this job holds the concurrency + # slot -- a sibling workflow may have published the same tag while it queued. + if docker manifest inspect "$ci_image_push" >/dev/null 2>&1; then + echo "Already published by a concurrent run: ${ci_image_push}" + exit 0 + fi + # Minimal build context: the module manifests (repo paths preserved so # nested modules and their local `replace ../` targets resolve) plus the # cache Dockerfile. Assembled outside the checkout so `docker build` @@ -138,11 +155,6 @@ jobs: done cp .github/docker/godeps-cache/Dockerfile "${context}/Dockerfile" - # Log into both registries: pull the base via the registry.tooling proxy, - # push the result to quay.io/stackstate. - echo "$REGISTRY_PASSWORD" | docker login --username "$REGISTRY_USER" --password-stdin "$REGISTRY_HOST" - echo "$QUAY_PASSWORD" | docker login --username "$QUAY_USER" --password-stdin "$QUAY_REGISTRY" - docker build \ --pull \ --build-arg "BASE_IMAGE=${base_image}" \ From 4e986befe4e8aa5dd99ee330c9eaa80301dd9431 Mon Sep 17 00:00:00 2001 From: Louis Parkin Date: Wed, 29 Jul 2026 10:52:00 +0200 Subject: [PATCH 16/19] STAC-25429 keep only module zips in the godeps cache image --- .github/docker/godeps-cache/Dockerfile | 29 ++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/.github/docker/godeps-cache/Dockerfile b/.github/docker/godeps-cache/Dockerfile index 6137b966249f..c0166cefcda4 100644 --- a/.github/docker/godeps-cache/Dockerfile +++ b/.github/docker/godeps-cache/Dockerfile @@ -5,6 +5,14 @@ # hash changes -- the image tag is content-addressed # (see .github/scripts/agent-godeps-cache-metadata.sh). # +# Only the module ZIPS ($GOMODCACHE/cache/download) are kept; the extracted module +# trees are deleted before the layer closes and Go re-extracts on demand. Measured on +# a full cache, the extracted trees are ~79% of the bytes but ~97% of the *files*, and +# image pull cost here is bound by filesystem metadata, not transfer: warm-proxy pulls +# of the full-cache image spent 0.7m downloading and 4.9m extracting. Shipping ~400k +# tiny files through overlayfs cost more than it saved (+3.0m pull against ~1.45m of +# avoided in-build downloading, so ~1.5m/job net loss). +# # No `# syntax=` frontend directive and no BuildKit secret mounts: the base image is # pulled by the daemon after `docker login` (creds stay in the daemon, not the # build), so the build itself needs no secrets and no Docker Hub frontend pull. @@ -20,7 +28,13 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] # manifests/ preserves the repo's directory layout so nested go.mod files (and their # local `replace ../` targets) resolve. No GOFLAGS override: go.work puts this in # workspace mode, where anything other than -mod=readonly is rejected outright. Only -# the populated module cache is kept -- the manifests are removed from the final image. +# the zip cache is kept -- the manifests and extracted trees are removed in the same +# RUN, so the layer diff only ever contains the final state. +# +# The prune needs `chmod -R u+w` first: Go writes the module cache read-only (dirs +# 0555, files 0444) and `rm -rf` cannot descend into it as a non-root user. +# cache/vcs holds full git clones for any module fetched outside the proxy; it is not +# needed to rebuild a module from its zip, so it goes too. COPY manifests/ /workspace/ RUN set -eux; \ git config --global --add safe.directory '*'; \ @@ -33,6 +47,17 @@ RUN set -eux; \ | xargs -0 -n1 -P8 bash -c \ 'moddir="$(dirname "$1")"; echo "go mod download :: ${moddir}"; cd "${moddir}" && go mod download' _; \ cache="$(go env GOMODCACHE)"; \ + echo "== populated module cache =="; \ + du -sh "${cache}"; \ + echo "files: $(find "${cache}" -type f | wc -l)"; \ + echo "== zip cache =="; \ + du -sh "${cache}/cache/download"; \ + echo "files: $(find "${cache}/cache/download" -type f | wc -l)"; \ + test "$(du -sm "${cache}/cache/download" | cut -f1)" -gt 100; \ + find "${cache}" -mindepth 1 -maxdepth 1 ! -name cache -exec chmod -R u+w {} +; \ + find "${cache}" -mindepth 1 -maxdepth 1 ! -name cache -exec rm -rf {} +; \ + if [ -d "${cache}/cache/vcs" ]; then chmod -R u+w "${cache}/cache/vcs"; rm -rf "${cache}/cache/vcs"; fi; \ + echo "== retained in image =="; \ du -sh "${cache}"; \ - test "$(du -sm "${cache}" | cut -f1)" -gt 100; \ + echo "files: $(find "${cache}" -type f | wc -l)"; \ rm -rf /workspace From 0ebc9f1331da8650864bc0bca513401550a9a188 Mon Sep 17 00:00:00 2001 From: Louis Parkin Date: Wed, 29 Jul 2026 13:32:06 +0200 Subject: [PATCH 17/19] STAC-25459 wire inv check-mod-tidy into CI, gated on the godeps cache hash --- .github/workflows/lint-and-unit-tests.yml | 58 +++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/.github/workflows/lint-and-unit-tests.yml b/.github/workflows/lint-and-unit-tests.yml index 246e80e10f6c..a9c9a58a7c82 100644 --- a/.github/workflows/lint-and-unit-tests.yml +++ b/.github/workflows/lint-and-unit-tests.yml @@ -31,6 +31,11 @@ name: Lint and unit tests on: pull_request: workflow_dispatch: + inputs: + force_mod_tidy: + description: Run the go.mod tidiness check even when the module graph is unchanged. + type: boolean + default: false permissions: contents: read @@ -90,6 +95,59 @@ jobs: git config --global --add safe.directory '*' inv -e linter.filenames + mod-tidy: + name: Go module tidiness (go.mod / go.sum) + # Nothing in this repo's CI has ever verified module tidiness (STAC-25459): + # `inv deps` only downloads, and the `go mod tidy` that used to sit in the GitLab + # reconcile mutated the tree rather than failing on a diff. `inv check-mod-tidy` + # has always existed (tasks/go.py) and does fail on a diff -- it was just never wired up. + # + # Gated on the godeps-cache tag, which is a sha256 over exactly **/go.mod, + # **/go.sum, go.work, go.work.sum and modules.yml. `image_exists == true` therefore + # means those files are byte-identical to a tree this check already passed on: the + # cache tag doubles as the cached verdict, so the check costs nothing on the PRs + # that do not touch the module graph. Dispatch with force_mod_tidy to re-run it + # against an unchanged graph. + # + # Blind spot: a source-only change that drops the last import of a dependency + # leaves a stale `require` without touching any hashed file, so this stays skipped. + # That does not break builds; a scheduled force_mod_tidy run would cover it. + if: >- + ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) + && (needs.godeps-cache.outputs.image_exists != 'true' || inputs.force_mod_tidy) }} + needs: godeps-cache + # XL for headroom: check-mod-tidy walks all ~186 workspace modules serially and + # holds the full module graph. Revisit once there is a wall-clock measurement. + runs-on: xlarge-public + timeout-minutes: 60 + container: + # Warm Go module cache image (godeps-cache job); see the unit-test jobs below + # for why this ref cannot be a static digest. + image: ${{ needs.godeps-cache.outputs.ci_image }} # zizmor: ignore[unpinned-images] + credentials: + username: ${{ vars.REGISTRY_USER }} + password: ${{ secrets.REGISTRY_PASSWORD }} + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + # No fetch-depth: check-mod-tidy diffs the working tree, not history. + persist-credentials: false + + - name: inv check-mod-tidy + run: | + set -eo pipefail + export PATH="$PATH:/usr/local/go/bin" + . /root/miniforge3/etc/profile.d/conda.sh + conda activate "${CONDA_ENV}" + # check-mod-tidy reports by mutating the tree (`go work sync`, then `go mod + # tidy` per module) and diffing it, so git has to accept the checkout. + git config --global --add safe.directory '*' + # `go mod tidy` resolves the full module graph including test dependencies of + # dependencies, which is wider than the `go mod download` set baked into the + # cache image, so expect some network fetches here even on a warm cache. + inv -e check-mod-tidy + unbranded-unit-tests: name: Unit tests (unbranded / DataDog) if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository From 9a2658bb100ee9441f1e25ebb79ec1f7d5baa3e5 Mon Sep 17 00:00:00 2001 From: Louis Parkin Date: Wed, 29 Jul 2026 16:48:34 +0200 Subject: [PATCH 18/19] STAC-25429 Build the DEB package in the warm Go module cache image --- .github/workflows/build-deb.yml | 60 ++++++++++++++++++++++++--------- 1 file changed, 45 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-deb.yml b/.github/workflows/build-deb.yml index 839bb6c5793a..72daaf7183bc 100644 --- a/.github/workflows/build-deb.yml +++ b/.github/workflows/build-deb.yml @@ -25,20 +25,33 @@ name: DEB package build # (vendor.tar.gz) is consumed by nobody (see the STAC-25396 note at # .gitlab-ci-agent.yml:68-75). Dropping it removes a job, an artifact # round-trip and a cache for zero loss. -# * No build caches yet -- this is a deliberate cold port. The GitLab job carried -# three (omnibus base_dir + .gems, bazel install, bazel repository_cache), none -# of which transfer directly: the GitHub-hosted Actions cache is capped at 10 GB -# per repo and the omnibus cache alone is expected to exceed it, and STAC-25396 -# already established that caches of this shape hit the file-count tax and -# artifact 413s on this repo. The cache backend is its own ticket; the cold -# wall-clock this job produces is the number that tells us what it is worth. -# Bazel state is still pinned under XDG_CACHE_HOME below so a cache layer can -# adopt those paths unchanged. +# * WARM Go module cache via the prebuilt cache image (STAC-25429), the same way +# the binary builds and unit tests get theirs: this workflow calls the +# godeps-cache reusable workflow and runs build-deb inside the image it +# publishes, so GOMODCACHE is already populated. That replaces the cold +# `go clean -modcache` + `inv deps` reconcile the port started with. The tag is +# a content hash of the module graph, so this workflow shares the image with +# the other two rather than building a third one -- and on a PR that leaves +# go.mod alone, nothing is built at all. Branding is safe against a cache baked +# from the unbranded tree: fix_branding.sh rewrites .go sources and testdata +# only, never go.mod/go.sum/go.work, so the module graph is identical either +# side of it. +# * The OMNIBUS and BAZEL caches are still absent -- this stays a cold port for +# those. The GitLab job carried three (omnibus base_dir + .gems, bazel install, +# bazel repository_cache), none of which transfer directly: the GitHub-hosted +# Actions cache is capped at 10 GB per repo and the omnibus cache alone is +# expected to exceed it, and STAC-25396 already established that caches of this +# shape hit the file-count tax and artifact 413s on this repo. That backend is +# its own ticket; the wall-clock this job produces is the number that tells us +# what it is worth. Bazel state is still pinned under XDG_CACHE_HOME below so a +# cache layer can adopt those paths unchanged. # * The GitLab job's `.omnibus` <-> `/omnibus` move dance is dropped along with # the cache. Note it was already inert there: it staged the cache at /omnibus # while the build ran with --base-dir /.omnibus. # -# Prerequisites for a GREEN run (beyond the phase 1/2 registry-proxy credentials): +# Prerequisites for a GREEN run (beyond the phase 1/2 registry-proxy credentials +# and the quay push credentials the godeps-cache build job needs -- var QUAY_USER +# + secret QUAY_PASSWORD, see godeps-cache.yml): # * GITLAB_PACKAGE_REGISTRY_PYPI_SIMPLE_URL / _USER / _READONLY_PASSWORD must # reach this repo at REPO level, provisioned through pulumi-infra # (github/repoVariables/resources.yaml, `- repository: stackstate-agent`). @@ -76,18 +89,32 @@ env: MAJOR_VERSION: '3' jobs: + godeps-cache: + name: Go dependency cache image + # Same-repo PRs only: the registry/quay secrets are not exposed to fork PRs. + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + uses: ./.github/workflows/godeps-cache.yml + secrets: + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }} + build-deb: name: Build DEB package (omnibus, branded / StackState) # Same-repo PRs only: the registry-proxy secret is not exposed to fork PRs. if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + needs: godeps-cache runs-on: xlarge-public # Cold omnibus build: no git cache, so every embedded software (openssl, cpython, # openscap, sqlite, ...) compiles from source. Generous until we have a real number. timeout-minutes: 300 container: - # datadog_build image (built by datadog-agent-buildimages), pulled via the - # read-only quay proxy -- mirrors BUILD_IMAGE in .gitlab-ci.yml (7.78.2 tag). - image: ${{ vars.REGISTRY_HOST }}/quay/stackstate/datadog_build_linux_x64:7af9194f + # Warm Go module cache image (godeps-cache job), FROM the same datadog_build + # image this job used to run directly and pulled through the same read-only + # registry proxy + REGISTRY_* credentials. The tag is a sha256 content hash of + # the module graph (see godeps-cache.yml), so this ref is effectively + # digest-pinned; it cannot be a static digest because it is computed per run -- + # hence the narrow unpinned-images ignore below. + image: ${{ needs.godeps-cache.outputs.ci_image }} # zizmor: ignore[unpinned-images] credentials: username: ${{ vars.REGISTRY_USER }} password: ${{ secrets.REGISTRY_PASSWORD }} @@ -155,10 +182,13 @@ jobs: # built-in that is unset here); pass the checkout root explicitly. ./fix_branding.sh "${GITHUB_WORKSPACE}" - go clean -modcache + # GOMODCACHE is pre-warmed by the godeps-cache image (STAC-25429), keyed + # to the module-graph hash, so there is no `go clean -modcache` and no + # per-job `inv deps` (go mod download + tidy) reconcile. Materialise + # vendor/ for this checkout against the warm cache (offline; no download). + # omnibus.build is handed this vendor/ as its --go-mod-cache below. go work sync go work vendor - inv -e deps --verbose # version.txt is consumed by the packaging/image phases. deps_deb produced # it on GitLab; generate it in-job instead. No `-e`: stdout must be only # the version string. From b048ba9528b41f34432ef58ef6c667de19e8fa6e Mon Sep 17 00:00:00 2001 From: Louis Parkin Date: Thu, 30 Jul 2026 15:27:58 +0200 Subject: [PATCH 19/19] STAC-25494 stagger godeps-cache builds instead of gating on concurrency --- .github/workflows/build-binaries.yml | 5 +++ .github/workflows/build-deb.yml | 5 +++ .github/workflows/godeps-cache.yml | 37 ++++++++++++++++++++--- .github/workflows/lint-and-unit-tests.yml | 4 +++ 4 files changed, 46 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index d6d0dca547ae..4534dff0d2da 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -66,6 +66,11 @@ jobs: # Same-repo PRs only: the registry/quay secrets are not exposed to fork PRs. if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository uses: ./.github/workflows/godeps-cache.yml + # Wait for Lint and unit tests to publish the image before building it here + # (STAC-25494). Covers a ~7m build plus runner scheduling; on timeout this builds + # it itself, so a failed sibling costs latency, not correctness. + with: + build_delay_seconds: 900 secrets: REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }} diff --git a/.github/workflows/build-deb.yml b/.github/workflows/build-deb.yml index 72daaf7183bc..1893497c583f 100644 --- a/.github/workflows/build-deb.yml +++ b/.github/workflows/build-deb.yml @@ -94,6 +94,11 @@ jobs: # Same-repo PRs only: the registry/quay secrets are not exposed to fork PRs. if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository uses: ./.github/workflows/godeps-cache.yml + # Wait for Lint and unit tests to publish the image before building it here + # (STAC-25494). Covers a ~7m build plus runner scheduling; on timeout this builds + # it itself, so a failed sibling costs latency, not correctness. + with: + build_delay_seconds: 900 secrets: REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }} diff --git a/.github/workflows/godeps-cache.yml b/.github/workflows/godeps-cache.yml index b29452c7376e..175e5d01cda6 100644 --- a/.github/workflows/godeps-cache.yml +++ b/.github/workflows/godeps-cache.yml @@ -30,6 +30,14 @@ on: description: Tag of the datadog_build_linux_x64 base image the cache derives FROM. type: string default: "7af9194f" + build_delay_seconds: + description: >- + How long to wait for a sibling workflow to publish the cache image before + building it here. Consumers stagger this so one builds immediately and the + rest wait out a normal build first. See the build job for why this is not a + concurrency group. + type: number + default: 0 outputs: ci_image: description: Proxy pull ref of the cache image, for use as a consumer job `container.image`. @@ -104,9 +112,13 @@ jobs: timeout-minutes: 60 # Consumer workflows call this in parallel on the same commit; without a shared gate # they all see image_exists=false and build the same content-addressed tag at once. - concurrency: - group: godeps-cache-${{ needs.metadata.outputs.ci_image }} - cancel-in-progress: false + # + # That gate used to be a `concurrency` group. It cannot be: GitHub allows one + # running and one *pending* entry per group, and cancels the pending one when a + # third arrives -- even with cancel-in-progress: false. With three consumers the + # loser's build was cancelled, and cancellation propagates to the caller, so a whole + # workflow (unit tests included) ended as "cancelled" on exactly the PRs that change + # the module graph. STAC-25494. Callers stagger `build_delay_seconds` instead. env: QUAY_REGISTRY: quay.io REGISTRY_HOST: ${{ vars.REGISTRY_HOST }} @@ -115,6 +127,7 @@ jobs: QUAY_USER: ${{ vars.QUAY_USER }} QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }} BASE_IMAGE_TAG: ${{ inputs.base_image_tag }} + BUILD_DELAY_SECONDS: ${{ inputs.build_delay_seconds }} steps: - name: Check out repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -134,8 +147,22 @@ jobs: echo "$REGISTRY_PASSWORD" | docker login --username "$REGISTRY_USER" --password-stdin "$REGISTRY_HOST" echo "$QUAY_PASSWORD" | docker login --username "$QUAY_USER" --password-stdin "$QUAY_REGISTRY" - # The metadata lookup can be stale by the time this job holds the concurrency - # slot -- a sibling workflow may have published the same tag while it queued. + # Staggered wait: give the caller designated to build first (delay 0) a full + # build's worth of time before duplicating its work. Polling before the first + # build would only waste the delay, which is why the builder gets 0. + waited=0 + while [ "$waited" -lt "${BUILD_DELAY_SECONDS}" ]; do + if docker manifest inspect "$ci_image_push" >/dev/null 2>&1; then + echo "Published by a sibling workflow after ${waited}s: ${ci_image_push}" + exit 0 + fi + sleep 20 + waited=$((waited + 20)) + done + + # Backstop: the metadata lookup, and any wait above, can be stale by the time + # we get here. Pushing the same content-addressed tag twice is harmless, but + # rebuilding it is not free. if docker manifest inspect "$ci_image_push" >/dev/null 2>&1; then echo "Already published by a concurrent run: ${ci_image_push}" exit 0 diff --git a/.github/workflows/lint-and-unit-tests.yml b/.github/workflows/lint-and-unit-tests.yml index a9c9a58a7c82..ef033ec8b934 100644 --- a/.github/workflows/lint-and-unit-tests.yml +++ b/.github/workflows/lint-and-unit-tests.yml @@ -60,6 +60,10 @@ jobs: # Same-repo PRs only: the registry/quay secrets are not exposed to fork PRs. if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository uses: ./.github/workflows/godeps-cache.yml + # Designated builder of the cache image (STAC-25494): no wait, so this workflow -- + # the longest of the three -- is never held up. The other two wait it out. + with: + build_delay_seconds: 0 secrets: REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }}