STAC-25429 Warm the Go module cache with a prebuilt, content-addressed CI image

.github/docker/godeps-cache/Dockerfile

@@ -0,0 +1,63 @@
+# Go dependency cache image for stackstate-agent CI (STAC-25429). Derives FROM the
+# datadog_build CI image and bakes the workspace's external Go module
+# downloads into GOMODCACHE, so consumer jobs get a warm module cache with no
+# per-job `go mod download`/network round-trips. Rebuilt only when the module graph
+# hash changes -- the image tag is content-addressed
+# (see .github/scripts/agent-godeps-cache-metadata.sh).
+#
+# Only the module ZIPS ($GOMODCACHE/cache/download) are kept; the extracted module
+# trees are deleted before the layer closes and Go re-extracts on demand. Measured on
+# a full cache, the extracted trees are ~79% of the bytes but ~97% of the *files*, and
+# image pull cost here is bound by filesystem metadata, not transfer: warm-proxy pulls
+# of the full-cache image spent 0.7m downloading and 4.9m extracting. Shipping ~400k
+# tiny files through overlayfs cost more than it saved (+3.0m pull against ~1.45m of
+# avoided in-build downloading, so ~1.5m/job net loss).
+#
+# No `# syntax=` frontend directive and no BuildKit secret mounts: the base image is
+# pulled by the daemon after `docker login` (creds stay in the daemon, not the
+# build), so the build itself needs no secrets and no Docker Hub frontend pull.
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}
+
+# `read -d ''` and `pipefail` are bashisms and RUN defaults to /bin/sh (dash), where
+# the download loop silently iterates zero times and publishes an empty cache.
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+# Pre-download external modules for every module in the workspace, mirroring what
+# `inv deps` does per module (tasks/libs/common/go.py) at the same parallelism.
+# manifests/ preserves the repo's directory layout so nested go.mod files (and their
+# local `replace ../` targets) resolve. No GOFLAGS override: go.work puts this in
+# workspace mode, where anything other than -mod=readonly is rejected outright. Only
+# the zip cache is kept -- the manifests and extracted trees are removed in the same
+# RUN, so the layer diff only ever contains the final state.
+#
+# The prune needs `chmod -R u+w` first: Go writes the module cache read-only (dirs
+# 0555, files 0444) and `rm -rf` cannot descend into it as a non-root user.
+# cache/vcs holds full git clones for any module fetched outside the proxy; it is not
+# needed to rebuild a module from its zip, so it goes too.
+COPY manifests/ /workspace/
+RUN set -eux; \
+    git config --global --add safe.directory '*'; \
+    export PATH="${PATH}:/usr/local/go/bin"; \
+    cd /workspace; \
+    modules="$(find . -type f -name go.mod | wc -l)"; \
+    echo "workspace modules: ${modules}"; \
+    test "${modules}" -gt 0; \
+    find . -type f -name go.mod -print0 \
+      | xargs -0 -n1 -P8 bash -c \
+          'moddir="$(dirname "$1")"; echo "go mod download :: ${moddir}"; cd "${moddir}" && go mod download' _; \
+    cache="$(go env GOMODCACHE)"; \
+    echo "== populated module cache =="; \
+    du -sh "${cache}"; \
+    echo "files: $(find "${cache}" -type f | wc -l)"; \
+    echo "== zip cache =="; \
+    du -sh "${cache}/cache/download"; \
+    echo "files: $(find "${cache}/cache/download" -type f | wc -l)"; \
+    test "$(du -sm "${cache}/cache/download" | cut -f1)" -gt 100; \
+    find "${cache}" -mindepth 1 -maxdepth 1 ! -name cache -exec chmod -R u+w {} +; \
+    find "${cache}" -mindepth 1 -maxdepth 1 ! -name cache -exec rm -rf {} +; \
+    if [ -d "${cache}/cache/vcs" ]; then chmod -R u+w "${cache}/cache/vcs"; rm -rf "${cache}/cache/vcs"; fi; \
+    echo "== retained in image =="; \
+    du -sh "${cache}"; \
+    echo "files: $(find "${cache}" -type f | wc -l)"; \
+    rm -rf /workspace

.github/scripts/agent-godeps-cache-metadata.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+# Computes the content-addressed tag for the stackstate-agent Go dependency cache
+# image (STAC-25429) and the two image refs it is referenced by:
+#   * push  ref -> quay.io/stackstate/stackstate-agent-godeps-cache  (authoritative)
+#   * pull  ref -> ${REGISTRY_HOST}/quay/...                         (registry.tooling proxy)
+#
+# The cache lives in its own quay repo rather than the shared sts-ci-images, which also
+# holds the ARC runner images: stackstate-agent is PUBLIC, so whatever credential this
+# workflow carries is reachable from any org member's same-repo branch. A dedicated repo
+# keeps that reach down to a rebuildable cache.
+#
+# Mirrors StackVista/StackGraph .github/scripts/stackgraph-ci-metadata.sh: the tag is
+# a hash of the dependency-defining inputs, so an unchanged module graph reuses the
+# same image across every branch/PR/default build (the tag *is* the cache key), while
+# any change to the graph rotates the tag and a stale cache is never reused.
+
+agent_godeps_compute_metadata() {
+  : "${QUAY_REGISTRY:?QUAY_REGISTRY is required}"     # quay.io
+  : "${REGISTRY_HOST:?REGISTRY_HOST is required}"     # registry.tooling.stackstate.io (quay proxy)
+  : "${BASE_IMAGE_TAG:?BASE_IMAGE_TAG is required}"   # datadog_build tag the cache derives FROM
+
+  # Hash inputs: every module manifest (go.work + go.work.sum + modules.yml + all
+  # nested go.mod/go.sum) plus the base image tag and the two files that define the
+  # cache mechanism itself (Dockerfile + this script), so changing how the cache is
+  # built also rotates the tag.
+  local godeps_hash
+  godeps_hash="$(
+    {
+      git ls-files -z -- \
+        '*go.mod' '*go.sum' 'go.work' 'go.work.sum' 'modules.yml' \
+        '.github/docker/godeps-cache/Dockerfile' \
+        '.github/scripts/agent-godeps-cache-metadata.sh' \
+        | LC_ALL=C sort -z \
+        | xargs -0 sha256sum
+      printf 'base:%s\n' "${BASE_IMAGE_TAG}"
+    } | sha256sum | cut -c1-16
+  )"
+
+  local image_repo="stackstate/stackstate-agent-godeps-cache"
+  local image_tag="godeps-${godeps_hash}"
+  # shellcheck disable=SC2034  # consumed by the sourcing workflow step
+  ci_image_push="${QUAY_REGISTRY}/${image_repo}:${image_tag}"
+  # shellcheck disable=SC2034  # consumed by the sourcing workflow step
+  ci_image="${REGISTRY_HOST}/quay/${image_repo}:${image_tag}"
+}

.github/workflows/build-binaries.yml

@@ -6,34 +6,38 @@ name: Binary builds
 # and cluster-agent from the DataDog fork, validating that the shippable binaries
 # compile on GitHub runners before the packaging/image phases build on top of them.
 #
-# Speedup concepts carried over from the GitLab pipeline work (STAC-25396):
-#   * Self-contained jobs, NO shared "prepare deps once" job. On GitLab, deps_deb
-#     sat on the critical path as a serial prefix that every build job depended on,
-#     but it never actually shared anything reusable: the Go module cache is too
-#     big to transport between jobs (~540k files, 2-3 GB) and a shared vendor/ dir
-#     breaks vendoring (`go work sync` mutates go.mod, so each job must re-vendor
-#     against its own checkout). So every job here reconciles its own deps
-#     (go work sync + go work vendor + inv deps) instead of depending on a deps job.
+# Speedup concepts carried over from the GitLab pipeline work (STAC-25396) and the
+# warm-cache follow-up (STAC-25429):
+#   * WARM Go module cache via a prebuilt cache image (STAC-25429), keyed by a hash
+#     of the module graph -- the StackGraph ci-metadata/build-ci-image pattern. The
+#     godeps-cache reusable workflow (.github/workflows/godeps-cache.yml) publishes
+#     an image FROM datadog_build with all external modules baked into GOMODCACHE;
+#     the build jobs use it as their `container:` so the cache is already warm. This
+#     replaces the earlier "each job cold-reconciles its own deps" approach: there
+#     is no `go clean -modcache` and no per-job `inv deps` (go mod download + tidy).
+#     The module cache is too big to transport via actions/cache (~540k files,
+#     2-3 GB), which is why it ships as an image layer instead. Jobs still run
+#     `go work sync` + `go work vendor` per checkout (vendoring mutates go.mod, so
+#     vendor/ is materialised per job) -- but now against the warm cache, offline.
 #   * deps_deb's only genuinely consumed output was version.txt, which is git-based
 #     (inv agent.version), not module-cache-based -- so build-cluster-agent just
 #     generates it in-job rather than pulling it from an upstream job.
 #   * Each suite runs once on the path to master, deduped via
 #     `concurrency: cancel-in-progress` (a newer push cancels the in-flight build).
 #
-# Known follow-ups (same as the GitLab analysis):
-#   * The real build-time win is a WARM module cache, which needs persistent
-#     runner-local storage (PVC-backed GOMODCACHE/GOCACHE on the ARC pool) -- an
-#     SRE / pulumi-infra change, not workflow YAML. actions/cache does not help:
-#     it hits the same file-count/size tax that made the GitLab cache a net loss.
+# Known follow-ups:
+#   * go.mod tidiness is no longer verified in these jobs (the `inv deps` reconcile
+#     is gone). Add a single check-mod-tidy gate rather than paying tidy in every job.
 #   * Phase 6 (merge queue): move the heavy builds to `merge_group` so they run
 #     once at land time and drop from `pull_request`; master/version-branch pushes
 #     then build + publish without re-running these.
 #
-# Prerequisites for a GREEN run are the same as lint-and-unit-tests.yml: the GitLab
-# push mirror must stay disabled, and this PUBLIC repo needs the read-only
-# registry-proxy credentials (vars REGISTRY_HOST/REGISTRY_USER + secret
-# REGISTRY_PASSWORD) for the container: image pull. Org secrets are never exposed
-# to fork PRs, so external-fork PRs skip these jobs by design.
+# Prerequisites for a GREEN run: the GitLab push mirror must stay disabled, and this
+# PUBLIC repo needs the read-only registry-proxy credentials (vars REGISTRY_HOST/
+# REGISTRY_USER + secret REGISTRY_PASSWORD) to pull the cache image, plus quay push
+# credentials (var QUAY_USER + secret QUAY_PASSWORD) for the godeps-cache build job.
+# The cache image is PRIVATE (its base datadog_build is private). Org secrets are
+# never exposed to fork PRs, so external-fork PRs skip these jobs by design.
 
 on:
   pull_request:
@@ -57,16 +61,29 @@ env:
   CONDA_ENV: ddpy3
 
 jobs:
+  godeps-cache:
+    name: Go dependency cache image
+    # Same-repo PRs only: the registry/quay secrets are not exposed to fork PRs.
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+    uses: ./.github/workflows/godeps-cache.yml
+    secrets:
+      REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
+      QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }}
+
   build-agent:
     name: Build agent binary (branded / StackState)
     # Same-repo PRs only: the registry-proxy secret is not exposed to fork PRs.
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+    needs: godeps-cache
     runs-on: xlarge-public
     timeout-minutes: 120
     container:
-      # datadog_build image (built by datadog-agent-buildimages), pulled via the
-      # read-only quay proxy -- mirrors BUILD_IMAGE in .gitlab-ci.yml (7.78.2 tag).
-      image: ${{ vars.REGISTRY_HOST }}/quay/stackstate/datadog_build_linux_x64:7af9194f
+      # Warm Go module cache image (godeps-cache job); pulled via the same
+      # read-only registry proxy + REGISTRY_* credentials as datadog_build. The tag is
+      # a sha256 content hash of the module graph (see godeps-cache.yml), so this ref
+      # is effectively digest-pinned; it cannot be a static digest because it is
+      # computed per run -- hence the narrow unpinned-images ignore below.
+      image: ${{ needs.godeps-cache.outputs.ci_image }} # zizmor: ignore[unpinned-images]
       credentials:
         username: ${{ vars.REGISTRY_USER }}
         password: ${{ secrets.REGISTRY_PASSWORD }}
@@ -88,12 +105,12 @@ jobs:
           # Work volume is owned by the ARC runner uid but the job container runs as
           # root, so git rejects the repo as "dubious ownership"; mark it safe.
           git config --global --add safe.directory '*'
-          # Self-contained deps: reconcile go.mod <-> vendor for this checkout
-          # (cross-job cache/vendor sharing is not viable here -- see header).
-          go clean -modcache
+          # GOMODCACHE is pre-warmed by the godeps-cache image (STAC-25429), keyed
+          # to the module-graph hash, so there is no `go clean -modcache` and no
+          # per-job `inv deps` (go mod download + tidy) reconcile. Materialise
+          # vendor/ for this checkout against the warm cache (offline; no download).
           go work sync
           go work vendor
-          inv -e deps --verbose
           # rtloader is the C++/Python bridge the full agent links against; the
           # cluster-agent doesn't need it, but the production agent does.
           inv -e rtloader.make
@@ -114,10 +131,16 @@ jobs:
   build-cluster-agent:
     name: Build cluster-agent binary (branded / StackState)
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+    needs: godeps-cache
     runs-on: xlarge-public
     timeout-minutes: 120
     container:
-      image: ${{ vars.REGISTRY_HOST }}/quay/stackstate/datadog_build_linux_x64:7af9194f
+      # Warm Go module cache image (godeps-cache job); pulled via the same
+      # read-only registry proxy + REGISTRY_* credentials as datadog_build. The tag is
+      # a sha256 content hash of the module graph (see godeps-cache.yml), so this ref
+      # is effectively digest-pinned; it cannot be a static digest because it is
+      # computed per run -- hence the narrow unpinned-images ignore below.
+      image: ${{ needs.godeps-cache.outputs.ci_image }} # zizmor: ignore[unpinned-images]
       credentials:
         username: ${{ vars.REGISTRY_USER }}
         password: ${{ secrets.REGISTRY_PASSWORD }}
@@ -138,11 +161,11 @@ jobs:
           # Work volume is owned by the ARC runner uid but the job container runs as
           # root, so git rejects the repo as "dubious ownership"; mark it safe.
           git config --global --add safe.directory '*'
-          # Self-contained deps (see header): reconcile go.mod <-> vendor per job.
-          go clean -modcache
+          # GOMODCACHE is pre-warmed by the godeps-cache image (STAC-25429); no
+          # `go clean -modcache` and no per-job `inv deps` reconcile. Materialise
+          # vendor/ for this checkout against the warm cache (offline; no download).
           go work sync
           go work vendor
-          inv -e deps --verbose
           export AGENT_GITHUB_ORG=DataDog
           export GITHUB_ORG=DataDog
           export BRANDED=true