diff --git a/.github/docker/godeps-cache/Dockerfile b/.github/docker/godeps-cache/Dockerfile new file mode 100644 index 000000000000..c0166cefcda4 --- /dev/null +++ b/.github/docker/godeps-cache/Dockerfile @@ -0,0 +1,63 @@ +# Go dependency cache image for stackstate-agent CI (STAC-25429). Derives FROM the +# datadog_build CI image and bakes the workspace's external Go module +# downloads into GOMODCACHE, so consumer jobs get a warm module cache with no +# per-job `go mod download`/network round-trips. Rebuilt only when the module graph +# hash changes -- the image tag is content-addressed +# (see .github/scripts/agent-godeps-cache-metadata.sh). +# +# Only the module ZIPS ($GOMODCACHE/cache/download) are kept; the extracted module +# trees are deleted before the layer closes and Go re-extracts on demand. Measured on +# a full cache, the extracted trees are ~79% of the bytes but ~97% of the *files*, and +# image pull cost here is bound by filesystem metadata, not transfer: warm-proxy pulls +# of the full-cache image spent 0.7m downloading and 4.9m extracting. Shipping ~400k +# tiny files through overlayfs cost more than it saved (+3.0m pull against ~1.45m of +# avoided in-build downloading, so ~1.5m/job net loss). +# +# No `# syntax=` frontend directive and no BuildKit secret mounts: the base image is +# pulled by the daemon after `docker login` (creds stay in the daemon, not the +# build), so the build itself needs no secrets and no Docker Hub frontend pull. +ARG BASE_IMAGE +FROM ${BASE_IMAGE} + +# `read -d ''` and `pipefail` are bashisms and RUN defaults to /bin/sh (dash), where +# the download loop silently iterates zero times and publishes an empty cache. +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# Pre-download external modules for every module in the workspace, mirroring what +# `inv deps` does per module (tasks/libs/common/go.py) at the same parallelism. +# manifests/ preserves the repo's directory layout so nested go.mod files (and their +# local `replace ../` targets) resolve. No GOFLAGS override: go.work puts this in +# workspace mode, where anything other than -mod=readonly is rejected outright. Only +# the zip cache is kept -- the manifests and extracted trees are removed in the same +# RUN, so the layer diff only ever contains the final state. +# +# The prune needs `chmod -R u+w` first: Go writes the module cache read-only (dirs +# 0555, files 0444) and `rm -rf` cannot descend into it as a non-root user. +# cache/vcs holds full git clones for any module fetched outside the proxy; it is not +# needed to rebuild a module from its zip, so it goes too. +COPY manifests/ /workspace/ +RUN set -eux; \ + git config --global --add safe.directory '*'; \ + export PATH="${PATH}:/usr/local/go/bin"; \ + cd /workspace; \ + modules="$(find . -type f -name go.mod | wc -l)"; \ + echo "workspace modules: ${modules}"; \ + test "${modules}" -gt 0; \ + find . -type f -name go.mod -print0 \ + | xargs -0 -n1 -P8 bash -c \ + 'moddir="$(dirname "$1")"; echo "go mod download :: ${moddir}"; cd "${moddir}" && go mod download' _; \ + cache="$(go env GOMODCACHE)"; \ + echo "== populated module cache =="; \ + du -sh "${cache}"; \ + echo "files: $(find "${cache}" -type f | wc -l)"; \ + echo "== zip cache =="; \ + du -sh "${cache}/cache/download"; \ + echo "files: $(find "${cache}/cache/download" -type f | wc -l)"; \ + test "$(du -sm "${cache}/cache/download" | cut -f1)" -gt 100; \ + find "${cache}" -mindepth 1 -maxdepth 1 ! -name cache -exec chmod -R u+w {} +; \ + find "${cache}" -mindepth 1 -maxdepth 1 ! -name cache -exec rm -rf {} +; \ + if [ -d "${cache}/cache/vcs" ]; then chmod -R u+w "${cache}/cache/vcs"; rm -rf "${cache}/cache/vcs"; fi; \ + echo "== retained in image =="; \ + du -sh "${cache}"; \ + echo "files: $(find "${cache}" -type f | wc -l)"; \ + rm -rf /workspace diff --git a/.github/scripts/agent-godeps-cache-metadata.sh b/.github/scripts/agent-godeps-cache-metadata.sh new file mode 100755 index 000000000000..00eedbf97c02 --- /dev/null +++ b/.github/scripts/agent-godeps-cache-metadata.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Computes the content-addressed tag for the stackstate-agent Go dependency cache +# image (STAC-25429) and the two image refs it is referenced by: +# * push ref -> quay.io/stackstate/stackstate-agent-godeps-cache (authoritative) +# * pull ref -> ${REGISTRY_HOST}/quay/... (registry.tooling proxy) +# +# The cache lives in its own quay repo rather than the shared sts-ci-images, which also +# holds the ARC runner images: stackstate-agent is PUBLIC, so whatever credential this +# workflow carries is reachable from any org member's same-repo branch. A dedicated repo +# keeps that reach down to a rebuildable cache. +# +# Mirrors StackVista/StackGraph .github/scripts/stackgraph-ci-metadata.sh: the tag is +# a hash of the dependency-defining inputs, so an unchanged module graph reuses the +# same image across every branch/PR/default build (the tag *is* the cache key), while +# any change to the graph rotates the tag and a stale cache is never reused. + +agent_godeps_compute_metadata() { + : "${QUAY_REGISTRY:?QUAY_REGISTRY is required}" # quay.io + : "${REGISTRY_HOST:?REGISTRY_HOST is required}" # registry.tooling.stackstate.io (quay proxy) + : "${BASE_IMAGE_TAG:?BASE_IMAGE_TAG is required}" # datadog_build tag the cache derives FROM + + # Hash inputs: every module manifest (go.work + go.work.sum + modules.yml + all + # nested go.mod/go.sum) plus the base image tag and the two files that define the + # cache mechanism itself (Dockerfile + this script), so changing how the cache is + # built also rotates the tag. + local godeps_hash + godeps_hash="$( + { + git ls-files -z -- \ + '*go.mod' '*go.sum' 'go.work' 'go.work.sum' 'modules.yml' \ + '.github/docker/godeps-cache/Dockerfile' \ + '.github/scripts/agent-godeps-cache-metadata.sh' \ + | LC_ALL=C sort -z \ + | xargs -0 sha256sum + printf 'base:%s\n' "${BASE_IMAGE_TAG}" + } | sha256sum | cut -c1-16 + )" + + local image_repo="stackstate/stackstate-agent-godeps-cache" + local image_tag="godeps-${godeps_hash}" + # shellcheck disable=SC2034 # consumed by the sourcing workflow step + ci_image_push="${QUAY_REGISTRY}/${image_repo}:${image_tag}" + # shellcheck disable=SC2034 # consumed by the sourcing workflow step + ci_image="${REGISTRY_HOST}/quay/${image_repo}:${image_tag}" +} diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml new file mode 100644 index 000000000000..4534dff0d2da --- /dev/null +++ b/.github/workflows/build-binaries.yml @@ -0,0 +1,262 @@ +name: Binary builds + +# Ported from .gitlab-ci-agent.yml (build_binaries, build_cluster_agent) as part +# of the GitLab -> GitHub migration (STAC-25142), phase 2: binary builds. Tracks +# the stackstate-7.78.2 pipeline. Builds the branded (StackState) production agent +# and cluster-agent from the DataDog fork, validating that the shippable binaries +# compile on GitHub runners before the packaging/image phases build on top of them. +# +# Speedup concepts carried over from the GitLab pipeline work (STAC-25396) and the +# warm-cache follow-up (STAC-25429): +# * WARM Go module cache via a prebuilt cache image (STAC-25429), keyed by a hash +# of the module graph -- the StackGraph ci-metadata/build-ci-image pattern. The +# godeps-cache reusable workflow (.github/workflows/godeps-cache.yml) publishes +# an image FROM datadog_build with all external modules baked into GOMODCACHE; +# the build jobs use it as their `container:` so the cache is already warm. This +# replaces the earlier "each job cold-reconciles its own deps" approach: there +# is no `go clean -modcache` and no per-job `inv deps` (go mod download + tidy). +# The module cache is too big to transport via actions/cache (~540k files, +# 2-3 GB), which is why it ships as an image layer instead. Jobs still run +# `go work sync` + `go work vendor` per checkout (vendoring mutates go.mod, so +# vendor/ is materialised per job) -- but now against the warm cache, offline. +# * deps_deb's only genuinely consumed output was version.txt, which is git-based +# (inv agent.version), not module-cache-based -- so build-cluster-agent just +# generates it in-job rather than pulling it from an upstream job. +# * Each suite runs once on the path to master, deduped via +# `concurrency: cancel-in-progress` (a newer push cancels the in-flight build). +# +# Known follow-ups: +# * go.mod tidiness is no longer verified in these jobs (the `inv deps` reconcile +# is gone). Add a single check-mod-tidy gate rather than paying tidy in every job. +# * Phase 6 (merge queue): move the heavy builds to `merge_group` so they run +# once at land time and drop from `pull_request`; master/version-branch pushes +# then build + publish without re-running these. +# +# Prerequisites for a GREEN run: the GitLab push mirror must stay disabled, and this +# PUBLIC repo needs the read-only registry-proxy credentials (vars REGISTRY_HOST/ +# REGISTRY_USER + secret REGISTRY_PASSWORD) to pull the cache image, plus quay push +# credentials (var QUAY_USER + secret QUAY_PASSWORD) for the godeps-cache build job. +# The cache image is PRIVATE (its base datadog_build is private). Org secrets are +# never exposed to fork PRs, so external-fork PRs skip these jobs by design. + +on: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +# The datadog_build container's default shell is dash, which lacks `set -o pipefail` +# and mis-handles the conda activation the steps rely on; force bash. +defaults: + run: + shell: bash + +env: + # conda env baked into the datadog_build image. + CONDA_ENV: ddpy3 + +jobs: + godeps-cache: + name: Go dependency cache image + # Same-repo PRs only: the registry/quay secrets are not exposed to fork PRs. + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + uses: ./.github/workflows/godeps-cache.yml + # Wait for Lint and unit tests to publish the image before building it here + # (STAC-25494). Covers a ~7m build plus runner scheduling; on timeout this builds + # it itself, so a failed sibling costs latency, not correctness. + with: + build_delay_seconds: 900 + secrets: + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }} + + build-agent: + name: Build agent binary (branded / StackState) + # Same-repo PRs only: the registry-proxy secret is not exposed to fork PRs. + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + needs: godeps-cache + runs-on: xlarge-public + timeout-minutes: 120 + container: + # Warm Go module cache image (godeps-cache job); pulled via the same + # read-only registry proxy + REGISTRY_* credentials as datadog_build. The tag is + # a sha256 content hash of the module graph (see godeps-cache.yml), so this ref + # is effectively digest-pinned; it cannot be a static digest because it is + # computed per run -- hence the narrow unpinned-images ignore below. + image: ${{ needs.godeps-cache.outputs.ci_image }} # zizmor: ignore[unpinned-images] + credentials: + username: ${{ vars.REGISTRY_USER }} + password: ${{ secrets.REGISTRY_PASSWORD }} + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + # Full history + tags so `inv agent.build` resolves the version via + # `git describe --tags --match "3.*"` (needs the 3.x tag reachable from HEAD). + fetch-depth: 0 + + - name: Build branded agent (production, with rtloader) + run: | + set -eo pipefail + export PATH="$PATH:/usr/local/go/bin" + . /root/miniforge3/etc/profile.d/conda.sh + conda activate "${CONDA_ENV}" + # Work volume is owned by the ARC runner uid but the job container runs as + # root, so git rejects the repo as "dubious ownership"; mark it safe. + git config --global --add safe.directory '*' + # GOMODCACHE is pre-warmed by the godeps-cache image (STAC-25429), keyed + # to the module-graph hash, so there is no `go clean -modcache` and no + # per-job `inv deps` (go mod download + tidy) reconcile. Materialise + # vendor/ for this checkout against the warm cache (offline; no download). + go work sync + go work vendor + # rtloader is the C++/Python bridge the full agent links against; the + # cluster-agent doesn't need it, but the production agent does. + inv -e rtloader.make + inv -e rtloader.install + export AGENT_GITHUB_ORG=DataDog + export GITHUB_ORG=DataDog + export BRANDED=true + export AGENT_REPO_NAME=datadog-agent + # Rebrand DataDog -> StackState. fix_branding.sh defaults its target dir to + # $CI_PROJECT_DIR (a GitLab built-in that is unset here); pass the checkout + # root explicitly. + ./fix_branding.sh "${GITHUB_WORKSPACE}" + # Production (non-race) build -- the shippable binary, distinct from the + # race-instrumented build in the unit-test workflow. + inv -e agent.build + ls -la bin/agent + + build-cluster-agent: + name: Build cluster-agent binary (branded / StackState) + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + needs: godeps-cache + runs-on: xlarge-public + timeout-minutes: 120 + container: + # Warm Go module cache image (godeps-cache job); pulled via the same + # read-only registry proxy + REGISTRY_* credentials as datadog_build. The tag is + # a sha256 content hash of the module graph (see godeps-cache.yml), so this ref + # is effectively digest-pinned; it cannot be a static digest because it is + # computed per run -- hence the narrow unpinned-images ignore below. + image: ${{ needs.godeps-cache.outputs.ci_image }} # zizmor: ignore[unpinned-images] + credentials: + username: ${{ vars.REGISTRY_USER }} + password: ${{ secrets.REGISTRY_PASSWORD }} + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + # Full history + tags so the version resolves via `git describe`. + fetch-depth: 0 + + - name: Build branded cluster-agent + run: | + set -eo pipefail + export PATH="$PATH:/usr/local/go/bin" + . /root/miniforge3/etc/profile.d/conda.sh + conda activate "${CONDA_ENV}" + # Work volume is owned by the ARC runner uid but the job container runs as + # root, so git rejects the repo as "dubious ownership"; mark it safe. + git config --global --add safe.directory '*' + # GOMODCACHE is pre-warmed by the godeps-cache image (STAC-25429); no + # `go clean -modcache` and no per-job `inv deps` reconcile. Materialise + # vendor/ for this checkout against the warm cache (offline; no download). + go work sync + go work vendor + export AGENT_GITHUB_ORG=DataDog + export GITHUB_ORG=DataDog + export BRANDED=true + export AGENT_REPO_NAME=datadog-agent + # Rebrand DataDog -> StackState; pass the checkout root (CI_PROJECT_DIR is GitLab-only). + ./fix_branding.sh "${GITHUB_WORKSPACE}" + inv -e cluster-agent.build + # version.txt is a build artifact for the downstream packaging/image phases. + # deps_deb produced it on GitLab; there is no shared deps job here, so + # generate it in-job. inv agent.version is git-based (not module-cache-based) + # and cheap. No `-e`: its stdout must be only the version string. + inv agent.version -u > version.txt + ls -la bin/stackstate-cluster-agent + + - name: Upload cluster-agent build artifacts + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: cluster-agent-binary + # Paths match the GitLab build_cluster_agent artifacts; the cluster-agent + # image phase (phase 5) consumes the binary + Dockerfile manifest. + path: | + bin/stackstate-cluster-agent/ + Dockerfiles/cluster-agent/stackstate-cluster.yaml + version.txt + retention-days: 5 + if-no-files-found: error + + build-cluster-agent-image: + name: Build cluster-agent container image (docker build, no push on PR) + # Ported from pre_release_cluster_agent_image. Lives in this workflow rather + # than build-deb.yml so it can `needs:` the job that produces its input -- + # cross-workflow artifact hand-off would need run-id plumbing for no gain. + # Stops at `docker build` + a runtime check: the GitLab job also pushed to + # quay.io/stackstate, which the PR lane of a PUBLIC repo must not do. + # Publishing lands in phase 4, gated on the branch. + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + needs: build-cluster-agent + # DinD class: docker CLI in the runner image, dockerd in a native sidecar. + runs-on: docker-public + timeout-minutes: 45 + env: + LOCAL_IMAGE: stackstate-cluster-agent:ci + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Download cluster-agent binary + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: cluster-agent-binary + + - name: Log in to the registry proxy + env: + REGISTRY_HOST: ${{ vars.REGISTRY_HOST }} + REGISTRY_USER: ${{ vars.REGISTRY_USER }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + run: | + set -eo pipefail + # Dockerfiles/cluster-agent pulls its ubuntu builder stage through the + # proxy; the BCI stages come from registry.suse.com and need no auth. + printf '%s' "${REGISTRY_PASSWORD}" | docker login -u "${REGISTRY_USER}" --password-stdin "${REGISTRY_HOST}" + + - name: Build cluster-agent image + run: | + set -eo pipefail + # actions/download-artifact does not preserve the executable bit (GitLab + # artifacts did). The Dockerfile's `chmod +x` targets + # opt/stackstate-agent/bin/stackstate-cluster-agent, which is the enclosing + # DIRECTORY, not the binary inside it -- so nothing in the build restores + # it and the image ships a non-executable agent. + chmod +x bin/stackstate-cluster-agent/stackstate-cluster-agent + # bin/stackstate-cluster-agent is a DIRECTORY (binary + dist/), and it + # must stay one: the Dockerfile COPYs it to + # /opt/stackstate-agent/bin/stackstate-cluster-agent/ and entrypoint.sh + # puts that directory on PATH so CMD can exec `stackstate-cluster-agent` + # by name. Flattening it to a bare binary breaks the image. + cp -r bin/stackstate-cluster-agent Dockerfiles/cluster-agent/ + docker build -t "${LOCAL_IMAGE}" Dockerfiles/cluster-agent + + - name: Smoke test cluster-agent image + run: | + set -eo pipefail + # Bypass entrypoint.sh: it hard-exits without STS_API_KEY, which a + # version check has no business needing. + docker run --rm \ + --entrypoint /opt/stackstate-agent/bin/stackstate-cluster-agent/stackstate-cluster-agent \ + "${LOCAL_IMAGE}" version diff --git a/.github/workflows/build-deb.yml b/.github/workflows/build-deb.yml new file mode 100644 index 000000000000..1893497c583f --- /dev/null +++ b/.github/workflows/build-deb.yml @@ -0,0 +1,335 @@ +name: DEB package build + +# Ported from .gitlab-ci-agent.yml (build_deb, test_deb_renaming, +# pre_release_main_agent_image) as part of the GitLab -> GitHub migration +# (STAC-25142). Tracks the stackstate-7.78.2 pipeline. +# +# Scope: the PR lane only -- package, verify branding, build the agent image and +# smoke-test it. Nothing is signed, pushed or published here: +# * `sign_deb` and the S3 publish are release actions needing GPG key material. +# * `pre_release_main_agent_image` also PUSHES to quay.io/stackstate; this job +# stops at `docker build` + a runtime check, so the PR lane needs no write +# credential at all -- only the read-only proxy creds already in use. +# On a PUBLIC repo that separation is the point: validating a change and releasing +# it are different trust boundaries and belong in different workflows. Publishing +# lands in phase 4, gated on the branch, not on the PR. +# +# Only the amd64 pipeline is ported. GitLab fans .gitlab-ci-agent.yml out twice +# from .gitlab-ci.yml (agent-x86 / agent-arm); the arm64 half needs an arm64 XL +# runner class, which does not exist yet (only `arm64-docker`). Tracked separately. +# +# Deliberate differences from the GitLab job: +# * `deps_deb` is NOT ported. Its only downstream-consumed artifact was +# version.txt, produced by `inv agent.version -u` -- a pure git-tag derivation +# that needs no upstream job, just `fetch-depth: 0`. Its other output +# (vendor.tar.gz) is consumed by nobody (see the STAC-25396 note at +# .gitlab-ci-agent.yml:68-75). Dropping it removes a job, an artifact +# round-trip and a cache for zero loss. +# * WARM Go module cache via the prebuilt cache image (STAC-25429), the same way +# the binary builds and unit tests get theirs: this workflow calls the +# godeps-cache reusable workflow and runs build-deb inside the image it +# publishes, so GOMODCACHE is already populated. That replaces the cold +# `go clean -modcache` + `inv deps` reconcile the port started with. The tag is +# a content hash of the module graph, so this workflow shares the image with +# the other two rather than building a third one -- and on a PR that leaves +# go.mod alone, nothing is built at all. Branding is safe against a cache baked +# from the unbranded tree: fix_branding.sh rewrites .go sources and testdata +# only, never go.mod/go.sum/go.work, so the module graph is identical either +# side of it. +# * The OMNIBUS and BAZEL caches are still absent -- this stays a cold port for +# those. The GitLab job carried three (omnibus base_dir + .gems, bazel install, +# bazel repository_cache), none of which transfer directly: the GitHub-hosted +# Actions cache is capped at 10 GB per repo and the omnibus cache alone is +# expected to exceed it, and STAC-25396 already established that caches of this +# shape hit the file-count tax and artifact 413s on this repo. That backend is +# its own ticket; the wall-clock this job produces is the number that tells us +# what it is worth. Bazel state is still pinned under XDG_CACHE_HOME below so a +# cache layer can adopt those paths unchanged. +# * The GitLab job's `.omnibus` <-> `/omnibus` move dance is dropped along with +# the cache. Note it was already inert there: it staged the cache at /omnibus +# while the build ran with --base-dir /.omnibus. +# +# Prerequisites for a GREEN run (beyond the phase 1/2 registry-proxy credentials +# and the quay push credentials the godeps-cache build job needs -- var QUAY_USER +# + secret QUAY_PASSWORD, see godeps-cache.yml): +# * GITLAB_PACKAGE_REGISTRY_PYPI_SIMPLE_URL / _USER / _READONLY_PASSWORD must +# reach this repo at REPO level, provisioned through pulumi-infra +# (github/repoVariables/resources.yaml, `- repository: stackstate-agent`). +# Repo-level, not org-level: org secrets default to visibility=private, which +# excludes this PUBLIC repo, and widening them to visibility=all -- the +# STAC-25350 workaround used for the read-only REGISTRY_* pull credentials -- +# would expose a package-registry credential to every public repo in the org. +# `.gitlab-scripts/setup_artifact_registry.sh` hard-exits when they are unset, +# so build-deb stays red until they land. The agent genuinely needs a private +# Python index for non-upstream packages, so these must be provisioned rather +# than dropped. Use the READONLY password, never the write one. Credentials go +# to pip.conf + ~/.netrc, never into a URL -- see wiki/concepts/ci-credentials.md. + +on: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +# The datadog_build container's default shell is dash, which lacks `set -o pipefail` +# and mis-handles the conda activation the steps rely on; force bash. +defaults: + run: + shell: bash + +env: + # conda env baked into the datadog_build image. + CONDA_ENV: ddpy3 + # MAJOR_VERSION in .gitlab-ci.yml; selects the built .deb for the branding gate. + MAJOR_VERSION: '3' + +jobs: + godeps-cache: + name: Go dependency cache image + # Same-repo PRs only: the registry/quay secrets are not exposed to fork PRs. + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + uses: ./.github/workflows/godeps-cache.yml + # Wait for Lint and unit tests to publish the image before building it here + # (STAC-25494). Covers a ~7m build plus runner scheduling; on timeout this builds + # it itself, so a failed sibling costs latency, not correctness. + with: + build_delay_seconds: 900 + secrets: + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }} + + build-deb: + name: Build DEB package (omnibus, branded / StackState) + # Same-repo PRs only: the registry-proxy secret is not exposed to fork PRs. + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + needs: godeps-cache + runs-on: xlarge-public + # Cold omnibus build: no git cache, so every embedded software (openssl, cpython, + # openscap, sqlite, ...) compiles from source. Generous until we have a real number. + timeout-minutes: 300 + container: + # Warm Go module cache image (godeps-cache job), FROM the same datadog_build + # image this job used to run directly and pulled through the same read-only + # registry proxy + REGISTRY_* credentials. The tag is a sha256 content hash of + # the module graph (see godeps-cache.yml), so this ref is effectively + # digest-pinned; it cannot be a static digest because it is computed per run -- + # hence the narrow unpinned-images ignore below. + image: ${{ needs.godeps-cache.outputs.ci_image }} # zizmor: ignore[unpinned-images] + credentials: + username: ${{ vars.REGISTRY_USER }} + password: ${{ secrets.REGISTRY_PASSWORD }} + env: + # Must live OUTSIDE the checkout: omnibus git-clones into base_dir and that + # conflicts with the agent working tree when nested under it. + OMNIBUS_BASE_DIR: /omnibus + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + # Full history + tags so `inv agent.version` resolves via `git describe`. + fetch-depth: 0 + + - name: Build DEB package with omnibus + env: + # Repo-level, provisioned by pulumi-infra (see header). The index URL is a + # plain variable -- it is not secret, the same URL is hardcoded in this repo + # at omnibus/package-scripts/publish_image.sh -- while the user and password + # are secrets. Scheme-less: setup_artifact_registry.sh prepends https://. + GITLAB_PACKAGE_REGISTRY_PYPI_SIMPLE_URL: ${{ vars.GITLAB_PACKAGE_REGISTRY_PYPI_SIMPLE_URL }} + GITLAB_PACKAGE_REGISTRY_USER: ${{ secrets.GITLAB_PACKAGE_REGISTRY_USER }} + GITLAB_PACKAGE_REGISTRY_READONLY_PASSWORD: ${{ secrets.GITLAB_PACKAGE_REGISTRY_READONLY_PASSWORD }} + run: | + set -eo pipefail + export PATH="$PATH:/usr/local/go/bin" + . /root/miniforge3/etc/profile.d/conda.sh + conda activate "${CONDA_ENV}" + # omnibus is Ruby; the build image ships it under rvm. + . /usr/local/rvm/scripts/rvm + # Work volume is owned by the ARC runner uid but the job container runs as + # root, so git rejects the repo as "dubious ownership"; mark it safe. + git config --global --add safe.directory '*' + + # omnibus resolves the agent source through GOPATH rather than the + # workspace path; GitLab's before_script created this symlink. + mkdir -p /go/src/github.com/StackVista + rm -rf /go/src/github.com/StackVista/stackstate-agent + ln -s "${GITHUB_WORKSPACE}" /go/src/github.com/StackVista/stackstate-agent + + # DD 7.78 builds some dependencies through bazelisk from + # omnibus/config/software/datadog-agent-dependencies.rb. bazelisk aborts + # with "XDG_CACHE_HOME () must denote a directory in CI!" unless these + # point at real directories. + export XDG_CACHE_HOME="${GITHUB_WORKSPACE}/.cache" + export BAZELISK_HOME="${XDG_CACHE_HOME}/bazelisk" + mkdir -p "${XDG_CACHE_HOME}/bazel" "${XDG_CACHE_HOME}/bazel-repo" "${BAZELISK_HOME}" + # .bazelrc:75 `try-import %workspace%/user.bazelrc` is the upstream-blessed + # extension point; it applies to every bazelisk shell-out omnibus makes. + # user.bazelrc is gitignored. + cat > "${GITHUB_WORKSPACE}/user.bazelrc" < StackState BEFORE vendoring: branding rewrites import + # paths, so vendor/ must be materialised against the rewritten tree. + # fix_branding.sh defaults its target dir to $CI_PROJECT_DIR (a GitLab + # built-in that is unset here); pass the checkout root explicitly. + ./fix_branding.sh "${GITHUB_WORKSPACE}" + + # GOMODCACHE is pre-warmed by the godeps-cache image (STAC-25429), keyed + # to the module-graph hash, so there is no `go clean -modcache` and no + # per-job `inv deps` (go mod download + tidy) reconcile. Materialise + # vendor/ for this checkout against the warm cache (offline; no download). + # omnibus.build is handed this vendor/ as its --go-mod-cache below. + go work sync + go work vendor + # version.txt is consumed by the packaging/image phases. deps_deb produced + # it on GitLab; generate it in-job instead. No `-e`: stdout must be only + # the version string. + inv agent.version -u > version.txt + cat version.txt + + # Writes pip.conf + ~/.netrc for the private Python index. Hard-exits if + # the GITLAB_PACKAGE_REGISTRY_* variables above are unset. + source ./.gitlab-scripts/setup_artifact_registry.sh + + export LD_LIBRARY_PATH="/opt/stackstate-agent/embedded/lib/python3.12/site-packages/psycopg2_binary.libs:/opt/stackstate-agent/embedded/lib" + + # --install-directory pins the omnibus paths to the BRANDED install dir. + # Without it tasks/omnibus.py derives them from the unbranded + # /opt/datadog-agent while the branded build uses /opt/stackstate-agent, + # and the post-build step then targets a nonexistent directory. + inv -e omnibus.build \ + --gem-path "${GITHUB_WORKSPACE}/.gems" \ + --base-dir "${OMNIBUS_BASE_DIR}" \ + --go-mod-cache "${GITHUB_WORKSPACE}/vendor" \ + --skip-deps \ + --skip-sign \ + --install-directory /opt/stackstate-agent + + - name: Collect package outputs + run: | + set -eo pipefail + mkdir -p outcomes + cp -r "${OMNIBUS_BASE_DIR}/pkg" outcomes/ + cp -r Dockerfiles outcomes/ + ls -la outcomes/pkg outcomes/Dockerfiles + + - name: Upload DEB package and image manifests + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: deb-package + # Paths match the GitLab build_deb artifacts; the branding gate below and + # the image phase (phase 5) consume them. + path: | + outcomes/pkg/*.deb + outcomes/pkg/*.json + outcomes/Dockerfiles/agent + outcomes/Dockerfiles/cluster-agent + outcomes/Dockerfiles/dogstatsd + outcomes/Dockerfiles/manifests + version.txt + retention-days: 5 + if-no-files-found: error + + test-deb-renaming: + name: DEB branding verification (no DataDog references) + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + needs: build-deb + # Cheap gate: unpacks the .deb and greps it. Does not need the XL class. + runs-on: docker-public + timeout-minutes: 30 + container: + # Same build image: test_deb.sh needs `ar` from binutils to unpack the .deb. + image: ${{ vars.REGISTRY_HOST }}/quay/stackstate/datadog_build_linux_x64:7af9194f + credentials: + username: ${{ vars.REGISTRY_USER }} + password: ${{ secrets.REGISTRY_PASSWORD }} + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Download DEB package + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: deb-package + + - name: Verify package carries no DataDog branding + run: | + set -eo pipefail + git config --global --add safe.directory '*' + # test_deb.sh takes exactly one package and rejects anything else, so + # resolve the glob here and fail loudly rather than passing it through + # (the GitLab job relied on the glob expanding to exactly one file). + shopt -s nullglob + debs=(outcomes/pkg/stackstate-agent_"${MAJOR_VERSION}"*.deb) + if [ "${#debs[@]}" -ne 1 ]; then + echo "Expected exactly one stackstate-agent_${MAJOR_VERSION}*.deb, found ${#debs[@]}: ${debs[*]}" >&2 + exit 1 + fi + ./test/renaming/test_deb.sh "${debs[0]}" + + build-agent-image: + name: Build agent container image (docker build, no push on PR) + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + needs: build-deb + # DinD class: docker CLI in the runner image, dockerd in a native sidecar. + runs-on: docker-public + timeout-minutes: 60 + env: + # ARCH in the agent-x86 trigger block of .gitlab-ci.yml; also the .deb suffix. + ARCH: amd64 + LOCAL_IMAGE: stackstate-agent:ci + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Download DEB package + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: deb-package + + - name: Log in to the registry proxy + env: + REGISTRY_HOST: ${{ vars.REGISTRY_HOST }} + REGISTRY_USER: ${{ vars.REGISTRY_USER }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + run: | + set -eo pipefail + # Dockerfiles/agent pulls its ubuntu extract stage through the proxy; + # the BCI stages come from registry.suse.com and need no auth. + printf '%s' "${REGISTRY_PASSWORD}" | docker login -u "${REGISTRY_USER}" --password-stdin "${REGISTRY_HOST}" + + - name: Build agent image + run: | + set -eo pipefail + shopt -s nullglob + debs=(outcomes/pkg/stackstate-agent_*_"${ARCH}".deb) + if [ "${#debs[@]}" -ne 1 ]; then + echo "Expected exactly one stackstate-agent_*_${ARCH}.deb, found ${#debs[@]}: ${debs[*]}" >&2 + exit 1 + fi + cp "${debs[0]}" Dockerfiles/agent/ + # publish_image.sh also passes --build-arg S6_ARCH; the agent Dockerfile + # declares no such ARG, so it is dropped here rather than kept as a warning. + docker build --build-arg ARCH="${ARCH}" -t "${LOCAL_IMAGE}" Dockerfiles/agent + + - name: Smoke test agent image + run: | + set -eo pipefail + docker run --rm --entrypoint /opt/stackstate-agent/bin/agent/agent "${LOCAL_IMAGE}" version diff --git a/.github/workflows/godeps-cache.yml b/.github/workflows/godeps-cache.yml new file mode 100644 index 000000000000..175e5d01cda6 --- /dev/null +++ b/.github/workflows/godeps-cache.yml @@ -0,0 +1,190 @@ +name: Go dependency cache image + +# Reusable workflow (STAC-25429): builds and publishes a Go module cache image for +# stackstate-agent CI, keyed by a hash of the module graph. Mirrors the StackGraph +# ci-metadata / build-ci-image pattern (its .github/workflows/ci.yml): the image tag +# is content-addressed, so an unchanged dependency graph reuses an already-published +# image across every branch/PR/default build (the tag *is* the cache key), and the +# heavy Go module download only runs when the graph actually changes. +# +# The image is FROM the datadog_build CI image with the workspace's external modules +# baked into GOMODCACHE. It is pushed to the dedicated quay repo +# quay.io/stackstate/stackstate-agent-godeps-cache and pulled by consumer jobs through +# the registry.tooling quay proxy with the same read-only REGISTRY_* credentials they +# already use for datadog_build. +# +# Both that repo and the datadog_build base are public (terraform-infra quay/locals.tf). +# Nothing private is baked in: this repo is open source and the module set is already +# public in the committed go.mod files. Public read also keeps the pull path off the +# registry.tooling proxy robot's private-repo grants, which it does not hold. +# +# Dedicated repo, not the shared sts-ci-images: that one also holds the ARC runner +# images, and stackstate-agent is PUBLIC, so any org member's same-repo branch can use +# whatever push credential this workflow carries. Scoping it to a rebuildable cache +# keeps a compromised branch away from the images the CI fleet boots from. + +on: + workflow_call: + inputs: + base_image_tag: + description: Tag of the datadog_build_linux_x64 base image the cache derives FROM. + type: string + default: "7af9194f" + build_delay_seconds: + description: >- + How long to wait for a sibling workflow to publish the cache image before + building it here. Consumers stagger this so one builds immediately and the + rest wait out a normal build first. See the build job for why this is not a + concurrency group. + type: number + default: 0 + outputs: + ci_image: + description: Proxy pull ref of the cache image, for use as a consumer job `container.image`. + value: ${{ jobs.metadata.outputs.ci_image }} + image_exists: + description: Whether the computed cache image tag already exists in the registry. + value: ${{ jobs.metadata.outputs.image_exists }} + secrets: + REGISTRY_PASSWORD: + description: Password for the read-only registry.tooling quay proxy (pull the base image). + required: true + QUAY_PASSWORD: + description: Password for quay.io/stackstate (inspect + push the cache image). + required: true + +permissions: + contents: read + +jobs: + metadata: + name: Compute cache image tag and look up existing image + runs-on: docker-public + timeout-minutes: 15 + outputs: + ci_image: ${{ steps.metadata.outputs.ci_image }} + image_exists: ${{ steps.metadata.outputs.image_exists }} + env: + QUAY_REGISTRY: quay.io + REGISTRY_HOST: ${{ vars.REGISTRY_HOST }} + QUAY_USER: ${{ vars.QUAY_USER }} + QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }} + BASE_IMAGE_TAG: ${{ inputs.base_image_tag }} + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Compute cache image metadata and check existence + id: metadata + run: | + set -euo pipefail + source .github/scripts/agent-godeps-cache-metadata.sh + agent_godeps_compute_metadata + + # Presence-gated login: same-repo runs have QUAY_* wired; a run without + # them still computes the tag and reports the image as missing. + if [[ -n "$QUAY_USER" && -n "$QUAY_PASSWORD" ]]; then + echo "$QUAY_PASSWORD" | docker login --username "$QUAY_USER" --password-stdin "$QUAY_REGISTRY" + fi + + if docker manifest inspect "$ci_image_push" >/dev/null 2>&1; then + image_exists=true + else + image_exists=false + fi + + { + echo "ci_image=${ci_image}" + echo "image_exists=${image_exists}" + } >> "$GITHUB_OUTPUT" + + echo "Cache push ref: ${ci_image_push}" + echo "Cache pull ref: ${ci_image}" + echo "Cache image exists: ${image_exists}" + + build: + name: Build and publish missing cache image + if: ${{ needs.metadata.outputs.image_exists != 'true' }} + needs: metadata + runs-on: docker-public + timeout-minutes: 60 + # Consumer workflows call this in parallel on the same commit; without a shared gate + # they all see image_exists=false and build the same content-addressed tag at once. + # + # That gate used to be a `concurrency` group. It cannot be: GitHub allows one + # running and one *pending* entry per group, and cancels the pending one when a + # third arrives -- even with cancel-in-progress: false. With three consumers the + # loser's build was cancelled, and cancellation propagates to the caller, so a whole + # workflow (unit tests included) ended as "cancelled" on exactly the PRs that change + # the module graph. STAC-25494. Callers stagger `build_delay_seconds` instead. + env: + QUAY_REGISTRY: quay.io + REGISTRY_HOST: ${{ vars.REGISTRY_HOST }} + REGISTRY_USER: ${{ vars.REGISTRY_USER }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + QUAY_USER: ${{ vars.QUAY_USER }} + QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }} + BASE_IMAGE_TAG: ${{ inputs.base_image_tag }} + BUILD_DELAY_SECONDS: ${{ inputs.build_delay_seconds }} + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Build and publish Go dependency cache image + run: | + set -euo pipefail + source .github/scripts/agent-godeps-cache-metadata.sh + agent_godeps_compute_metadata + + base_image="${REGISTRY_HOST}/quay/stackstate/datadog_build_linux_x64:${BASE_IMAGE_TAG}" + + # Log into both registries: pull the base via the registry.tooling proxy, + # push the result to quay.io/stackstate. + echo "$REGISTRY_PASSWORD" | docker login --username "$REGISTRY_USER" --password-stdin "$REGISTRY_HOST" + echo "$QUAY_PASSWORD" | docker login --username "$QUAY_USER" --password-stdin "$QUAY_REGISTRY" + + # Staggered wait: give the caller designated to build first (delay 0) a full + # build's worth of time before duplicating its work. Polling before the first + # build would only waste the delay, which is why the builder gets 0. + waited=0 + while [ "$waited" -lt "${BUILD_DELAY_SECONDS}" ]; do + if docker manifest inspect "$ci_image_push" >/dev/null 2>&1; then + echo "Published by a sibling workflow after ${waited}s: ${ci_image_push}" + exit 0 + fi + sleep 20 + waited=$((waited + 20)) + done + + # Backstop: the metadata lookup, and any wait above, can be stale by the time + # we get here. Pushing the same content-addressed tag twice is harmless, but + # rebuilding it is not free. + if docker manifest inspect "$ci_image_push" >/dev/null 2>&1; then + echo "Already published by a concurrent run: ${ci_image_push}" + exit 0 + fi + + # Minimal build context: the module manifests (repo paths preserved so + # nested modules and their local `replace ../` targets resolve) plus the + # cache Dockerfile. Assembled outside the checkout so `docker build` + # streams only what the image needs. + context="$(mktemp -d)" + mkdir -p "${context}/manifests" + git ls-files -z -- \ + '*go.mod' '*go.sum' 'go.work' 'go.work.sum' 'modules.yml' \ + | while IFS= read -r -d '' f; do + mkdir -p "${context}/manifests/$(dirname "$f")" + cp "$f" "${context}/manifests/${f}" + done + cp .github/docker/godeps-cache/Dockerfile "${context}/Dockerfile" + + docker build \ + --pull \ + --build-arg "BASE_IMAGE=${base_image}" \ + --tag "$ci_image_push" \ + "$context" + docker push "$ci_image_push" diff --git a/.github/workflows/lint-and-unit-tests.yml b/.github/workflows/lint-and-unit-tests.yml new file mode 100644 index 000000000000..ef033ec8b934 --- /dev/null +++ b/.github/workflows/lint-and-unit-tests.yml @@ -0,0 +1,268 @@ +name: Lint and unit tests + +# Ported from .gitlab-ci-agent.yml (filename_linting, unbranded_unit_tests, +# branded_unit_tests) as part of the GitLab -> GitHub migration (STAC-25142), +# phase 1: lint + unit tests. Tracks the stackstate-7.78.2 pipeline. +# +# Test-execution policy (architect directive — minimize redundant runs): +# * filename-linting is a cheap gate and runs on every PR. +# * The two unit suites are heavy (each does a full `inv agent.build` + full +# `inv test`, ~2h on an XL runner). They run ONCE per change here, deduped +# via `concurrency: cancel-in-progress`. When the merge queue is enabled +# (phase 6) they move to `merge_group` (run once at land) and drop from +# `pull_request`; master/version-branch pushes then build+publish WITHOUT +# re-running tests. Follow-ups tracked in the migration scope doc: +# - path-filter the heavy suites (skip when no Go/build inputs changed) +# - open question for architects: do we need BOTH unbranded and branded +# on every path, or branded-as-gate + unbranded on the queue only? +# +# Prerequisites for a GREEN run (see scope doc): +# * The GitLab push mirror to this repo must be disabled (done 2026-07-10), +# otherwise a mirror sync overwrites anything pushed to GitHub. +# * This is a PUBLIC repo. The read-only registry-proxy credentials must be +# available to it (org-inherited or provisioned via pulumi-infra) to pull the +# cache image: vars REGISTRY_HOST, REGISTRY_USER + secret REGISTRY_PASSWORD. +# The godeps-cache build job additionally needs quay push credentials (var +# QUAY_USER + secret QUAY_PASSWORD). The warm Go module cache is delivered by +# the godeps-cache reusable workflow (STAC-25429; see build-binaries.yml header +# and .github/workflows/godeps-cache.yml). Org secrets are never exposed to fork +# PRs, so external-fork PRs skip these jobs. + +on: + pull_request: + workflow_dispatch: + inputs: + force_mod_tidy: + description: Run the go.mod tidiness check even when the module graph is unchanged. + type: boolean + default: false + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +# The datadog_build container's default shell is dash, which lacks `set -o pipefail` +# and mis-handles the conda activation the steps rely on; force bash. +defaults: + run: + shell: bash + +env: + # conda env baked into the datadog_build image. + CONDA_ENV: ddpy3 + +jobs: + godeps-cache: + name: Go dependency cache image + # Same-repo PRs only: the registry/quay secrets are not exposed to fork PRs. + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + uses: ./.github/workflows/godeps-cache.yml + # Designated builder of the cache image (STAC-25494): no wait, so this workflow -- + # the longest of the three -- is never held up. The other two wait it out. + with: + build_delay_seconds: 0 + secrets: + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }} + + filename-linting: + name: Filename linting + # Same-repo PRs only: the registry-proxy secret is not exposed to fork PRs. + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: docker-public + timeout-minutes: 20 + container: + # datadog_build image (built by datadog-agent-buildimages), pulled via the + # read-only quay proxy — mirrors BUILD_IMAGE in .gitlab-ci.yml (7.78.2 tag). + image: ${{ vars.REGISTRY_HOST }}/quay/stackstate/datadog_build_linux_x64:7af9194f + credentials: + username: ${{ vars.REGISTRY_USER }} + password: ${{ secrets.REGISTRY_PASSWORD }} + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: inv linter.filenames + run: | + set -eo pipefail + # linter.filenames only shells out to `git ls-files` + path checks, so + # it needs no Go deps/vendoring (the GitLab job carried that as boilerplate). + . /root/miniforge3/etc/profile.d/conda.sh + conda activate "${CONDA_ENV}" + # Work volume is owned by the ARC runner uid but the job container runs as + # root, so git rejects the repo as "dubious ownership"; mark it safe. + git config --global --add safe.directory '*' + inv -e linter.filenames + + mod-tidy: + name: Go module tidiness (go.mod / go.sum) + # Nothing in this repo's CI has ever verified module tidiness (STAC-25459): + # `inv deps` only downloads, and the `go mod tidy` that used to sit in the GitLab + # reconcile mutated the tree rather than failing on a diff. `inv check-mod-tidy` + # has always existed (tasks/go.py) and does fail on a diff -- it was just never wired up. + # + # Gated on the godeps-cache tag, which is a sha256 over exactly **/go.mod, + # **/go.sum, go.work, go.work.sum and modules.yml. `image_exists == true` therefore + # means those files are byte-identical to a tree this check already passed on: the + # cache tag doubles as the cached verdict, so the check costs nothing on the PRs + # that do not touch the module graph. Dispatch with force_mod_tidy to re-run it + # against an unchanged graph. + # + # Blind spot: a source-only change that drops the last import of a dependency + # leaves a stale `require` without touching any hashed file, so this stays skipped. + # That does not break builds; a scheduled force_mod_tidy run would cover it. + if: >- + ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) + && (needs.godeps-cache.outputs.image_exists != 'true' || inputs.force_mod_tidy) }} + needs: godeps-cache + # XL for headroom: check-mod-tidy walks all ~186 workspace modules serially and + # holds the full module graph. Revisit once there is a wall-clock measurement. + runs-on: xlarge-public + timeout-minutes: 60 + container: + # Warm Go module cache image (godeps-cache job); see the unit-test jobs below + # for why this ref cannot be a static digest. + image: ${{ needs.godeps-cache.outputs.ci_image }} # zizmor: ignore[unpinned-images] + credentials: + username: ${{ vars.REGISTRY_USER }} + password: ${{ secrets.REGISTRY_PASSWORD }} + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + # No fetch-depth: check-mod-tidy diffs the working tree, not history. + persist-credentials: false + + - name: inv check-mod-tidy + run: | + set -eo pipefail + export PATH="$PATH:/usr/local/go/bin" + . /root/miniforge3/etc/profile.d/conda.sh + conda activate "${CONDA_ENV}" + # check-mod-tidy reports by mutating the tree (`go work sync`, then `go mod + # tidy` per module) and diffing it, so git has to accept the checkout. + git config --global --add safe.directory '*' + # `go mod tidy` resolves the full module graph including test dependencies of + # dependencies, which is wider than the `go mod download` set baked into the + # cache image, so expect some network fetches here even on a warm cache. + inv -e check-mod-tidy + + unbranded-unit-tests: + name: Unit tests (unbranded / DataDog) + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + needs: godeps-cache + runs-on: xlarge-public + timeout-minutes: 150 + container: + # Warm Go module cache image (godeps-cache job); pulled via the same + # read-only registry proxy + REGISTRY_* credentials as datadog_build. The tag is + # a sha256 content hash of the module graph (see godeps-cache.yml), so this ref + # is effectively digest-pinned; it cannot be a static digest because it is + # computed per run -- hence the narrow unpinned-images ignore below. + image: ${{ needs.godeps-cache.outputs.ci_image }} # zizmor: ignore[unpinned-images] + credentials: + username: ${{ vars.REGISTRY_USER }} + password: ${{ secrets.REGISTRY_PASSWORD }} + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + # Full history + tags so `inv agent.build` resolves the version via + # `git describe --tags --match "3.*"` (needs the 3.x tag reachable from HEAD). + fetch-depth: 0 + + - name: Build agent and run unit tests + run: | + set -eo pipefail + export PATH="$PATH:/usr/local/go/bin" + . /root/miniforge3/etc/profile.d/conda.sh + conda activate "${CONDA_ENV}" + # Work volume is owned by the ARC runner uid but the job container runs as + # root, so git rejects the repo as "dubious ownership"; mark it safe. + git config --global --add safe.directory '*' + # GOMODCACHE is pre-warmed by the godeps-cache image (STAC-25429), keyed + # to the module-graph hash: no `go clean -modcache` and no per-job + # `inv deps` reconcile. Materialise vendor/ against the warm cache + # (offline). Go tool binaries come from `invoke install-tools` below -- + # the removed second `inv deps` never installed them (it is go mod + # download + tidy, not a tool install). + go work sync + go work vendor + export AGENT_GITHUB_ORG=DataDog + export GITHUB_ORG=DataDog + export BRANDED=false + export AGENT_REPO_NAME=datadog-agent + inv -e agent.build --race + gofmt -l -w -s ./pkg ./cmd + inv -e rtloader.test + invoke install-tools + # Go 1.25+ no longer ships prebuilt tool binaries; build covdata into + # GOTOOLDIR so `go test -cover` resolves it (golang/go#75031). + go build -o "$(go env GOTOOLDIR)/covdata" cmd/covdata + # Drop build tags for features StackState does not ship in its images. + export STS_UT_BUILD_EXCLUDE="oracle,trivy,trivy_no_javadb,nvml,jetson,bundle_installer,systemd" + inv -e test --coverage --race --profile --cpus 4 --build-exclude="${STS_UT_BUILD_EXCLUDE}" --timeout=600 + + branded-unit-tests: + name: Unit tests (branded / StackState) + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + needs: godeps-cache + runs-on: xlarge-public + timeout-minutes: 150 + container: + # Warm Go module cache image (godeps-cache job); pulled via the same + # read-only registry proxy + REGISTRY_* credentials as datadog_build. The tag is + # a sha256 content hash of the module graph (see godeps-cache.yml), so this ref + # is effectively digest-pinned; it cannot be a static digest because it is + # computed per run -- hence the narrow unpinned-images ignore below. + image: ${{ needs.godeps-cache.outputs.ci_image }} # zizmor: ignore[unpinned-images] + credentials: + username: ${{ vars.REGISTRY_USER }} + password: ${{ secrets.REGISTRY_PASSWORD }} + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + # Full history + tags so `inv agent.build` resolves the version via + # `git describe --tags --match "3.*"` (needs the 3.x tag reachable from HEAD). + fetch-depth: 0 + + - name: Build agent and run unit tests + run: | + set -eo pipefail + export PATH="$PATH:/usr/local/go/bin" + . /root/miniforge3/etc/profile.d/conda.sh + conda activate "${CONDA_ENV}" + # Work volume is owned by the ARC runner uid but the job container runs as + # root, so git rejects the repo as "dubious ownership"; mark it safe. + git config --global --add safe.directory '*' + # GOMODCACHE is pre-warmed by the godeps-cache image (STAC-25429): no + # `go clean -modcache` and no per-job `inv deps` reconcile. Materialise + # vendor/ against the warm cache (offline). Go tool binaries come from + # `invoke install-tools` below. + go work sync + go work vendor + export AGENT_GITHUB_ORG=DataDog + export GITHUB_ORG=DataDog + export BRANDED=true + export AGENT_REPO_NAME=datadog-agent + export OMNIBUS_FORCE_PACKAGES=true + # Rebrand DataDog -> StackState, then verify no unbranded literals remain. + # fix_branding.sh defaults its target dir to $CI_PROJECT_DIR (a GitLab + # built-in that is unset here); pass the checkout root explicitly. + ./fix_branding.sh "${GITHUB_WORKSPACE}" + ./scripts/verify_branding_literals.sh + inv -e agent.build --race + gofmt -l -w -s ./pkg ./cmd + inv -e rtloader.test + invoke install-tools + go build -o "$(go env GOTOOLDIR)/covdata" cmd/covdata + export STS_UT_BUILD_EXCLUDE="oracle,trivy,trivy_no_javadb,nvml,jetson,bundle_installer,systemd" + inv -e test --coverage --race --profile --cpus 4 --build-exclude="${STS_UT_BUILD_EXCLUDE}" --timeout=600