From 60ce0b132a7d23bae6dcd7ad2a0ca71ebe9c4580 Mon Sep 17 00:00:00 2001 From: Cheena Malhotra Date: Tue, 21 Jul 2026 17:17:18 -0700 Subject: [PATCH 01/22] Add SqlClient performance test pipeline for internal Perf Test Lab Adds a new Azure DevOps pipeline that runs the Microsoft.Data.SqlClient BenchmarkDotNet performance tests on the internal "Perf Test Lab" by consuming the reusable extends template v1/Perf.Test.Job.yml from the InternalDriverTools/PerfTest repository (per wiki page 284). Files: - eng/pipelines/perf/sqlclient-perf-pipeline.yml: extends the perf template. - eng/pipelines/perf/scripts/run-perf-tests.sh: Linux on-VM entry point. - eng/pipelines/perf/scripts/run-perf-tests.ps1: Windows on-VM entry point. The on-VM scripts install the pinned .NET SDK (global.json), create the perf database, inject the VM SQL Server connection string into the benchmark runner config, pin the client to the reserved CPU set, run the benchmarks, and collect BenchmarkDotNet artifacts for the template to publish. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4a21e2a3-6efa-45c7-9bfd-81e7f3d24a89 --- eng/pipelines/perf/scripts/run-perf-tests.ps1 | 231 +++++++++++++++ eng/pipelines/perf/scripts/run-perf-tests.sh | 279 ++++++++++++++++++ .../perf/sqlclient-perf-pipeline.yml | 156 ++++++++++ 3 files changed, 666 insertions(+) create mode 100644 eng/pipelines/perf/scripts/run-perf-tests.ps1 create mode 100755 eng/pipelines/perf/scripts/run-perf-tests.sh create mode 100644 eng/pipelines/perf/sqlclient-perf-pipeline.yml diff --git a/eng/pipelines/perf/scripts/run-perf-tests.ps1 b/eng/pipelines/perf/scripts/run-perf-tests.ps1 new file mode 100644 index 0000000000..22a490c13c --- /dev/null +++ b/eng/pipelines/perf/scripts/run-perf-tests.ps1 @@ -0,0 +1,231 @@ +#################################################################################################### +# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this +# file to you under the MIT license. See the LICENSE file in the project root for more information. +#################################################################################################### +# +# run-perf-tests.ps1 +# +# Entry point executed ON the Perf Test Lab Windows VM by the InternalDriverTools/PerfTest extends +# template (v1/Perf.Test.Job.yml). The template SCPs the driver source tree to the VM, runs this +# script over SSH, then SCPs the results sub-directory back and publishes it as a pipeline artifact. +# +# This is the Windows counterpart of run-perf-tests.sh. See that file for the full description of +# responsibilities. On Windows the benchmark client is pinned to the reserved CPU set via the +# process ProcessorAffinity mask (derived from PERF_CLIENT_CPUS) instead of taskset. +# +# Environment variables injected by the template (see wiki "Performance Test Automation"): +# SQL_SERVER Host/IP of the SQL Server on the perf VM (e.g. localhost). +# SQL_PASSWORD SQL Server 'sa' password. +# PERF_CLIENT_CPUS Core range reserved for the test client, e.g. "16-31". +# PERF_SQL_CPUS Core range SQL Server is pinned to, e.g. "0-15" (informational). +# +[CmdletBinding()] +param( + [string]$Configuration = "Release", + [string]$Framework = "net9.0", + [string]$ResultsSubdir = "perf-results" +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version Latest + +#################################################################################################### +# Resolve paths +#################################################################################################### + +# This script lives at /eng/pipelines/perf/scripts/run-perf-tests.ps1, so the repo root is +# four levels up. +$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$RepoRoot = (Resolve-Path (Join-Path $ScriptDir "..\..\..\..")).Path +$PerfProject = Join-Path $RepoRoot "src\Microsoft.Data.SqlClient\tests\PerformanceTests\Microsoft.Data.SqlClient.PerformanceTests.csproj" +$PerfDir = Split-Path -Parent $PerfProject +$ResultsDir = Join-Path $RepoRoot $ResultsSubdir + +$SqlServer = if ($env:SQL_SERVER) { $env:SQL_SERVER } else { "localhost" } +$SqlPassword = $env:SQL_PASSWORD +$DbName = "sqlclient-perf-db" + +Write-Host "==================================================================" +Write-Host " SqlClient Performance Tests" +Write-Host "==================================================================" +Write-Host " Repo root : $RepoRoot" +Write-Host " Perf project : $PerfProject" +Write-Host " Configuration : $Configuration" +Write-Host " Framework : $Framework" +Write-Host " Results dir : $ResultsDir" +Write-Host " SQL_SERVER : $SqlServer" +Write-Host " PERF_CLIENT_CPUS: $($env:PERF_CLIENT_CPUS)" +Write-Host " PERF_SQL_CPUS : $($env:PERF_SQL_CPUS)" +Write-Host "==================================================================" + +if (-not (Test-Path $PerfProject)) { + throw "Performance test project not found at $PerfProject" +} +if ([string]::IsNullOrEmpty($SqlPassword)) { + throw "SQL_PASSWORD environment variable is not set (expected from the perf template)." +} + +New-Item -ItemType Directory -Force -Path $ResultsDir | Out-Null + +$env:DOTNET_NOLOGO = "1" +$env:DOTNET_CLI_TELEMETRY_OPTOUT = "1" +$env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE = "1" + +#################################################################################################### +# 1. Install the .NET SDK (pinned by global.json) and the runtimes for the target frameworks. +#################################################################################################### + +function Install-DotNet { + $globalJson = Get-Content (Join-Path $RepoRoot "global.json") -Raw + # Strip // comments so ConvertFrom-Json accepts the file. + $globalJson = ($globalJson -split "`n" | ForEach-Object { $_ -replace '//.*$', '' }) -join "`n" + $sdkVersion = (ConvertFrom-Json $globalJson).sdk.version + if ([string]::IsNullOrEmpty($sdkVersion)) { + throw "Could not determine SDK version from global.json" + } + + $dotnetRoot = Join-Path $env:USERPROFILE ".dotnet" + $env:DOTNET_ROOT = $dotnetRoot + $env:PATH = "$dotnetRoot;$dotnetRoot\tools;$env:PATH" + + Write-Host "Installing .NET SDK $sdkVersion into $dotnetRoot ..." + $installScript = Join-Path $env:TEMP "dotnet-install.ps1" + Invoke-WebRequest -UseBasicParsing "https://dot.net/v1/dotnet-install.ps1" -OutFile $installScript + + & $installScript -Version $sdkVersion -InstallDir $dotnetRoot + foreach ($channel in @("8.0", "9.0", "10.0")) { + & $installScript -Channel $channel -Runtime dotnet -InstallDir $dotnetRoot + } +} + +$hasNet10Sdk = $false +if (Get-Command dotnet -ErrorAction SilentlyContinue) { + if ((dotnet --list-sdks) -match '^10\.0\.') { $hasNet10Sdk = $true } +} +if ($hasNet10Sdk) { + Write-Host "Using pre-installed dotnet: $((Get-Command dotnet).Source)" +} else { + Install-DotNet +} + +dotnet --info + +#################################################################################################### +# 2. Create the perf database on the VM's SQL Server. +# +# The benchmark runners create their own tables but not the database, so create it here +# (idempotently) using the Microsoft.Data.SqlClient assembly that ships with the SDK-less runtime, +# invoked through a tiny inline program. sqlcmd is used when available; otherwise we fall back to a +# .NET one-liner via the perf project's own driver reference. +#################################################################################################### + +Write-Host "Ensuring database [$DbName] exists on $SqlServer ..." + +$sqlcmd = Get-Command sqlcmd -ErrorAction SilentlyContinue +if ($sqlcmd) { + & $sqlcmd.Source -S $SqlServer -U sa -P $SqlPassword -C -b -l 30 ` + -Q "IF DB_ID('$DbName') IS NULL CREATE DATABASE [$DbName];" + if ($LASTEXITCODE -ne 0) { throw "sqlcmd failed to create database [$DbName] (exit $LASTEXITCODE)." } + Write-Host "Database [$DbName] is ready." +} else { + throw "sqlcmd was not found on the VM; cannot create the perf database [$DbName]." +} + +#################################################################################################### +# 3. Inject the VM's SQL Server connection string into the benchmark runner config. +#################################################################################################### + +$RunnerConfig = Join-Path $RepoRoot "perf-runnerconfig.json" +$env:RUNNER_CONFIG = $RunnerConfig + +$srcConfig = Join-Path $PerfDir "runnerconfig.jsonc" +$rawConfig = Get-Content $srcConfig -Raw +# Strip // line comments so ConvertFrom-Json accepts the .jsonc content. +$rawConfig = ($rawConfig -split "`n" | ForEach-Object { $_ -replace '(?m)^\s*//.*$', '' }) -join "`n" +$cfg = ConvertFrom-Json $rawConfig + +$cfg.ConnectionString = "Server=tcp:$SqlServer,1433;User ID=sa;Password=$SqlPassword;Initial Catalog=$DbName;TrustServerCertificate=True;Encrypt=False;" +$cfg | ConvertTo-Json -Depth 10 | Set-Content -Path $RunnerConfig -Encoding UTF8 +Write-Host "Wrote runner config to $RunnerConfig (Server=tcp:$SqlServer,1433; Initial Catalog=$DbName)" + +#################################################################################################### +# 4 & 5. Run the benchmarks, pinned to the reserved client CPU set. +#################################################################################################### + +$RunDir = Join-Path $RepoRoot "perf-run" +if (Test-Path $RunDir) { Remove-Item -Recurse -Force $RunDir } +New-Item -ItemType Directory -Force -Path $RunDir | Out-Null + +Write-Host "Building performance tests ($Configuration, $Framework) ..." +dotnet build $PerfProject -c $Configuration -f $Framework --nologo -v minimal +if ($LASTEXITCODE -ne 0) { throw "Build failed (exit $LASTEXITCODE)." } + +# Convert a CPU range like "16-31" (or a comma list "16,17,18") into an affinity bitmask. +function Get-AffinityMask([string]$cpuSpec) { + if ([string]::IsNullOrEmpty($cpuSpec)) { return $null } + [long]$mask = 0 + foreach ($part in $cpuSpec.Split(",")) { + if ($part -match '^\s*(\d+)\s*-\s*(\d+)\s*$') { + for ($c = [int]$Matches[1]; $c -le [int]$Matches[2]; $c++) { $mask = $mask -bor ([long]1 -shl $c) } + } elseif ($part -match '^\s*(\d+)\s*$') { + $mask = $mask -bor ([long]1 -shl [int]$Matches[1]) + } + } + return $mask +} + +Push-Location $RunDir +try { + Write-Host "Starting benchmarks: dotnet run --project $PerfProject -c $Configuration -f $Framework --no-build" + $psi = New-Object System.Diagnostics.ProcessStartInfo + $psi.FileName = "dotnet" + $psi.Arguments = "run --project `"$PerfProject`" -c $Configuration -f $Framework --no-build" + $psi.UseShellExecute = $false + $psi.WorkingDirectory = $RunDir + + $proc = [System.Diagnostics.Process]::Start($psi) + + $mask = Get-AffinityMask $env:PERF_CLIENT_CPUS + if ($null -ne $mask -and $mask -gt 0) { + try { + $proc.ProcessorAffinity = [System.IntPtr]$mask + Write-Host "Pinned benchmark client (PID $($proc.Id)) to CPUs $($env:PERF_CLIENT_CPUS) (mask 0x$($mask.ToString('X')))." + } catch { + Write-Warning "Failed to set ProcessorAffinity: $_" + } + } else { + Write-Warning "PERF_CLIENT_CPUS unset; running without CPU pinning." + } + + $proc.WaitForExit() + if ($proc.ExitCode -ne 0) { throw "Benchmark run failed (exit $($proc.ExitCode))." } +} finally { + Pop-Location +} + +#################################################################################################### +# 6. Collect BenchmarkDotNet artifacts into the results sub-directory. +#################################################################################################### + +$ArtifactsDir = Join-Path $RunDir "BenchmarkDotNet.Artifacts" +if (Test-Path $ArtifactsDir) { + Write-Host "Collecting BenchmarkDotNet artifacts into $ResultsDir ..." + # Keep the full artifact tree (logs + per-run report folder) for detailed inspection. + Copy-Item -Recurse -Force $ArtifactsDir (Join-Path $ResultsDir "BenchmarkDotNet.Artifacts") + # Also flatten the report files (github markdown, csv, html) to the TOP of the results folder. + # The collect-results template auto-attaches top-level results/*.md files as run summaries, so + # placing the *-report-github.md reports here makes them show up on the pipeline run. + $ReportsDir = Join-Path $ArtifactsDir "results" + if (Test-Path $ReportsDir) { + Copy-Item -Recurse -Force (Join-Path $ReportsDir "*") $ResultsDir + } +} else { + Write-Warning "No BenchmarkDotNet.Artifacts directory was produced at $ArtifactsDir." +} + +Write-Host "Collected results:" +Get-ChildItem -Recurse -File $ResultsDir | ForEach-Object { $_.FullName } | Sort-Object + +Write-Host "==================================================================" +Write-Host " Performance run complete." +Write-Host "==================================================================" diff --git a/eng/pipelines/perf/scripts/run-perf-tests.sh b/eng/pipelines/perf/scripts/run-perf-tests.sh new file mode 100755 index 0000000000..ff7829a68b --- /dev/null +++ b/eng/pipelines/perf/scripts/run-perf-tests.sh @@ -0,0 +1,279 @@ +#!/usr/bin/env bash +#################################################################################################### +# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this +# file to you under the MIT license. See the LICENSE file in the project root for more information. +#################################################################################################### +# +# run-perf-tests.sh +# +# Entry point executed ON the Perf Test Lab Linux VM by the InternalDriverTools/PerfTest extends +# template (v1/Perf.Test.Job.yml). The template SCPs the driver source tree to the VM, runs this +# script over SSH, then SCPs the results sub-directory back and publishes it as a pipeline artifact. +# +# Responsibilities: +# 1. Install the .NET SDK pinned by the repo's global.json (plus the runtime for the target TFM). +# 2. Create the perf database on the VM's SQL Server (the benchmark runners create tables but not +# the database). +# 3. Inject the VM's SQL Server connection string into the benchmark runner config. +# 4. Pin the benchmark client to the reserved CPU set (PERF_CLIENT_CPUS) so it does not contend +# with SQL Server (which is pinned to the disjoint PERF_SQL_CPUS set). +# 5. Run the BenchmarkDotNet performance tests. +# 6. Collect the BenchmarkDotNet artifacts into the results sub-directory. +# +# Environment variables injected by the template (see wiki "Performance Test Automation"): +# SQL_SERVER Host/IP of the SQL Server on the perf VM (e.g. localhost). +# SQL_PASSWORD SQL Server 'sa' password. +# PERF_CLIENT_CPUS Core range reserved for the test client, e.g. "16-31". +# PERF_SQL_CPUS Core range SQL Server is pinned to, e.g. "0-15" (informational). +# +set -euo pipefail + +#################################################################################################### +# Argument parsing +#################################################################################################### + +configuration="Release" +framework="net9.0" +resultsSubDir="perf-results" + +usage() { + echo "Usage: $0 [--configuration ] [--framework ] [--results-subdir ]" >&2 +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --configuration) configuration="$2"; shift 2 ;; + --framework) framework="$2"; shift 2 ;; + --results-subdir) resultsSubDir="$2"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown argument: $1" >&2; usage; exit 2 ;; + esac +done + +#################################################################################################### +# Resolve paths +#################################################################################################### + +# This script lives at /eng/pipelines/perf/scripts/run-perf-tests.sh, so the repo root is four +# levels up. Deriving it from the script location keeps us independent of the working directory the +# template runs us from. +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../../../.." >/dev/null 2>&1 && pwd)" + +PERF_PROJECT="${REPO_ROOT}/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Microsoft.Data.SqlClient.PerformanceTests.csproj" +PERF_DIR="$(dirname -- "${PERF_PROJECT}")" +RESULTS_DIR="${REPO_ROOT}/${resultsSubDir}" + +echo "==================================================================" +echo " SqlClient Performance Tests" +echo "==================================================================" +echo " Repo root : ${REPO_ROOT}" +echo " Perf project : ${PERF_PROJECT}" +echo " Configuration : ${configuration}" +echo " Framework : ${framework}" +echo " Results dir : ${RESULTS_DIR}" +echo " SQL_SERVER : ${SQL_SERVER:-}" +echo " PERF_CLIENT_CPUS: ${PERF_CLIENT_CPUS:-}" +echo " PERF_SQL_CPUS : ${PERF_SQL_CPUS:-}" +echo "==================================================================" + +if [[ ! -f "${PERF_PROJECT}" ]]; then + echo "ERROR: Performance test project not found at ${PERF_PROJECT}" >&2 + exit 1 +fi + +: "${SQL_SERVER:=localhost}" +if [[ -z "${SQL_PASSWORD:-}" ]]; then + echo "ERROR: SQL_PASSWORD environment variable is not set (expected from the perf template)." >&2 + exit 1 +fi + +mkdir -p "${RESULTS_DIR}" + +export DOTNET_NOLOGO=1 +export DOTNET_CLI_TELEMETRY_OPTOUT=1 +export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 + +#################################################################################################### +# 1. Install the .NET SDK (pinned by global.json) and the runtime for the target framework. +#################################################################################################### + +install_dotnet() { + local sdkVersion + # Extract the SDK version from global.json (strip // comments, then read "version"). + sdkVersion="$(sed 's://.*::' "${REPO_ROOT}/global.json" \ + | tr -d '\r' \ + | grep -oE '"version"[[:space:]]*:[[:space:]]*"[^"]+"' \ + | head -n1 \ + | sed -E 's/.*"version"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/')" + + if [[ -z "${sdkVersion}" ]]; then + echo "ERROR: Could not determine SDK version from ${REPO_ROOT}/global.json" >&2 + exit 1 + fi + + export DOTNET_ROOT="${HOME}/.dotnet" + export PATH="${DOTNET_ROOT}:${DOTNET_ROOT}/tools:${PATH}" + + echo "Installing .NET SDK ${sdkVersion} into ${DOTNET_ROOT} ..." + local installScript + installScript="$(mktemp)" + curl -fsSL https://dot.net/v1/dotnet-install.sh -o "${installScript}" + chmod +x "${installScript}" + + # SDK pinned by global.json (used to build MDS + the perf project). + "${installScript}" --version "${sdkVersion}" --install-dir "${DOTNET_ROOT}" --no-path + + # Shared runtimes for the frameworks the benchmarks may run against. Installing all three keeps + # the script robust regardless of the --framework selected by the pipeline. + local channel + for channel in 8.0 9.0 10.0; do + "${installScript}" --channel "${channel}" --runtime dotnet --install-dir "${DOTNET_ROOT}" --no-path + done + + rm -f "${installScript}" +} + +# Reuse a pre-installed SDK only if it already satisfies global.json; otherwise install locally. +if command -v dotnet >/dev/null 2>&1 && dotnet --list-sdks 2>/dev/null | grep -q '10\.0\.'; then + echo "Using pre-installed dotnet: $(command -v dotnet)" + export DOTNET_ROOT="$(dirname -- "$(command -v dotnet)")" +else + install_dotnet +fi + +dotnet --info + +#################################################################################################### +# 2. Create the perf database on the VM's SQL Server. +# +# The benchmark runners connect to Initial Catalog=sqlclient-perf-db and create their own tables, +# but they do NOT create the database itself, so we create it here (idempotently). +#################################################################################################### + +DB_NAME="sqlclient-perf-db" + +find_sqlcmd() { + if command -v sqlcmd >/dev/null 2>&1; then + command -v sqlcmd + return 0 + fi + # mssql-tools default install locations. + local candidate + for candidate in /opt/mssql-tools18/bin/sqlcmd /opt/mssql-tools/bin/sqlcmd; do + if [[ -x "${candidate}" ]]; then + echo "${candidate}" + return 0 + fi + done + return 1 +} + +echo "Ensuring database [${DB_NAME}] exists on ${SQL_SERVER} ..." +if SQLCMD="$(find_sqlcmd)"; then + # -C trusts the server certificate (mssql-tools18 requires encryption by default). + "${SQLCMD}" -S "${SQL_SERVER}" -U sa -P "${SQL_PASSWORD}" -C -b -l 30 \ + -Q "IF DB_ID('${DB_NAME}') IS NULL CREATE DATABASE [${DB_NAME}];" + echo "Database [${DB_NAME}] is ready." +else + echo "ERROR: sqlcmd was not found on the VM; cannot create the perf database [${DB_NAME}]." >&2 + echo " Looked for 'sqlcmd' on PATH and in /opt/mssql-tools18/bin and /opt/mssql-tools/bin." >&2 + exit 1 +fi + +#################################################################################################### +# 3. Inject the VM's SQL Server connection string into the benchmark runner config. +# +# The perf app reads its config from the file named by RUNNER_CONFIG (falling back to +# runnerconfig.jsonc in the working directory). We copy the checked-in config and replace only the +# ConnectionString value so all benchmark tuning (iterations, row counts, enabled flags) is +# preserved. python3 is used to JSON-escape the (potentially special-character) password safely. +#################################################################################################### + +RUNNER_CONFIG="${REPO_ROOT}/perf-runnerconfig.json" +export RUNNER_CONFIG + +python3 - "$PERF_DIR/runnerconfig.jsonc" "$RUNNER_CONFIG" <<'PY' +import json, os, re, sys + +src, dst = sys.argv[1], sys.argv[2] + +with open(src, "r", encoding="utf-8-sig") as fh: + text = fh.read() + +# Strip // line comments so the .jsonc content parses as JSON. The checked-in config has no "//" +# inside string values, so a simple line-comment strip is safe here. +text = re.sub(r'(?m)^\s*//.*$', '', text) +cfg = json.loads(text) + +server = os.environ["SQL_SERVER"] +password = os.environ["SQL_PASSWORD"] +db = os.environ.get("DB_NAME", "sqlclient-perf-db") + +cfg["ConnectionString"] = ( + f"Server=tcp:{server},1433;User ID=sa;Password={password};" + f"Initial Catalog={db};TrustServerCertificate=True;Encrypt=False;" +) + +with open(dst, "w", encoding="utf-8") as fh: + json.dump(cfg, fh, indent=2) + +print(f"Wrote runner config to {dst} (Server=tcp:{server},1433; Initial Catalog={db})") +PY + +#################################################################################################### +# 4 & 5. Run the benchmarks, pinned to the reserved client CPU set. +# +# BenchmarkDotNet writes its artifacts to ./BenchmarkDotNet.Artifacts relative to the working +# directory, so we run from a dedicated run directory and collect from there afterwards. +#################################################################################################### + +RUN_DIR="${REPO_ROOT}/perf-run" +rm -rf "${RUN_DIR}" +mkdir -p "${RUN_DIR}" + +# Build once up-front so the timed run uses --no-build (no build noise inside the measured process). +echo "Building performance tests (${configuration}, ${framework}) ..." +dotnet build "${PERF_PROJECT}" -c "${configuration}" -f "${framework}" --nologo -v minimal + +# Assemble the run command, pinning to PERF_CLIENT_CPUS when available. +run_cmd=(dotnet run --project "${PERF_PROJECT}" -c "${configuration}" -f "${framework}" --no-build) + +if [[ -n "${PERF_CLIENT_CPUS:-}" ]] && command -v taskset >/dev/null 2>&1; then + echo "Pinning benchmark client to CPUs ${PERF_CLIENT_CPUS} via taskset." + run_cmd=(taskset -c "${PERF_CLIENT_CPUS}" "${run_cmd[@]}") +else + echo "WARNING: PERF_CLIENT_CPUS unset or taskset unavailable; running without CPU pinning." >&2 +fi + +echo "Running: ${run_cmd[*]}" +( + cd "${RUN_DIR}" + "${run_cmd[@]}" +) + +#################################################################################################### +# 6. Collect BenchmarkDotNet artifacts into the results sub-directory. +#################################################################################################### + +ARTIFACTS_DIR="${RUN_DIR}/BenchmarkDotNet.Artifacts" +if [[ -d "${ARTIFACTS_DIR}" ]]; then + echo "Collecting BenchmarkDotNet artifacts into ${RESULTS_DIR} ..." + # Keep the full artifact tree (logs + per-run report folder) for detailed inspection. + cp -R "${ARTIFACTS_DIR}" "${RESULTS_DIR}/BenchmarkDotNet.Artifacts" + # Also flatten the report files (github markdown, csv, html) to the TOP of the results folder. + # The collect-results template auto-attaches top-level results/*.md files as run summaries, so + # placing the *-report-github.md reports here makes them show up on the pipeline run. + if [[ -d "${ARTIFACTS_DIR}/results" ]]; then + cp -R "${ARTIFACTS_DIR}/results/." "${RESULTS_DIR}/" + fi +else + echo "WARNING: No BenchmarkDotNet.Artifacts directory was produced at ${ARTIFACTS_DIR}." >&2 +fi + +echo "Collected results:" +find "${RESULTS_DIR}" -type f | sort + +echo "==================================================================" +echo " Performance run complete." +echo "==================================================================" diff --git a/eng/pipelines/perf/sqlclient-perf-pipeline.yml b/eng/pipelines/perf/sqlclient-perf-pipeline.yml new file mode 100644 index 0000000000..e816c44d4e --- /dev/null +++ b/eng/pipelines/perf/sqlclient-perf-pipeline.yml @@ -0,0 +1,156 @@ +#################################################################################################### +# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this +# file to you under the MIT license. See the LICENSE file in the project root for more information. +#################################################################################################### + +# SqlClient Performance Test pipeline. +# +# This pipeline runs the Microsoft.Data.SqlClient BenchmarkDotNet performance tests on the internal +# "Perf Test Lab" (Azure Dedicated Hosts). It does so by consuming the reusable extends template +# published by the InternalDriverTools/PerfTest repository: +# +# v1/Perf.Test.Job.yml@PerfTemplates +# +# See the design/usage guidance on the wiki: +# InternalDriverTools.wiki - "Performance Test Automation" (page 284) +# +# The template owns the full run lifecycle (SSH key generation, dedicated-host VM provisioning, test +# execution over SSH, result collection, and teardown). All this pipeline supplies is: +# +# * the source directory to copy to the VM (testRootDir), +# * the entry-point script to execute on the VM (testScript + testScriptArgs), +# * the results subfolder to copy back (testResultsSubDir), and +# * optional post-test steps to surface the results. +# +# The on-VM script (eng/pipelines/perf/scripts/run-perf-tests.*) installs the .NET SDK, creates the +# perf database, injects the VM's SQL Server connection string into the benchmark runner config, +# pins the client to the reserved CPU set, runs the benchmarks, and collects the BenchmarkDotNet +# artifacts into the results subfolder. + +# Set the pipeline run name to the day-of-year and the daily run counter. +name: $(DayOfYear)$(Rev:rr) + +# This pipeline is manual/scheduled only; never trigger on PRs or commits. +pr: none +trigger: none + +# Uncomment to run nightly. Times are UTC. +# schedules: +# - cron: "0 6 * * *" +# displayName: Nightly performance run +# branches: +# include: +# - main +# always: true + +parameters: + + # Target OS for the perf VM and the benchmark client. + - name: platform + displayName: Platform + type: string + default: linux + values: + - linux + - windows + + # The build configuration used to build and run the benchmarks on the VM. Performance numbers are + # only meaningful in Release, so that is the default (and only) supported value. + - name: buildConfiguration + displayName: Build Configuration + type: string + default: Release + values: + - Release + + # The .NET runtime the benchmarks are executed against. Must be one of the target frameworks of + # the PerformanceTests project (net8.0/net9.0/net10.0). + - name: dotnetFramework + displayName: .NET Framework (TFM) + type: string + default: net9.0 + values: + - net8.0 + - net9.0 + - net10.0 + + # Maximum time (minutes) the template waits for the benchmark run on the VM before timing out. + - name: testTimeoutMinutes + displayName: Test Timeout (minutes) + type: number + default: 180 + + # The folder name this repository ('self') is checked out into. The Perf.Test.Job template + # performs a MULTI-REPO checkout ('checkout: self' + the template repo), so Azure DevOps places + # 'self' under $(Build.SourcesDirectory)/ rather than at $(Build.SourcesDirectory) + # directly (mirroring how the template repo lands at $(Build.SourcesDirectory)/PerfTest). This + # must match the ADO repository name for the SqlClient repo hosting this pipeline. + - name: sourcesSubDir + displayName: Self checkout folder name + type: string + default: dotnet-sqlclient + +# Reference the PerfTest repository that hosts the reusable extends template. Driver-team projects +# have been onboarded with the Agent Pools and Service Connections required to consume it. +resources: + repositories: + - repository: PerfTemplates + type: git + name: InternalDriverTools/PerfTest + ref: refs/heads/main + +# Consume the Perf Test Lab template. The template defines the whole job/stage structure; we only +# pass parameters into it. +extends: + template: v1/Perf.Test.Job.yml@PerfTemplates + parameters: + platform: ${{ parameters.platform }} + + # The entire driver source tree is copied to the VM so the benchmarks (which reference + # Microsoft.Data.SqlClient as a project) can be built from source against the current commit. + # Under the template's multi-repo checkout, 'self' lands in a repo-named subfolder. + testRootDir: $(Build.SourcesDirectory)/${{ parameters.sourcesSubDir }} + + # Entry-point script (relative to testRootDir), selected by platform. Linux uses bash, Windows + # uses PowerShell, per the template's .sh/.ps1 convention. + ${{ if eq(parameters.platform, 'windows') }}: + testScript: eng/pipelines/perf/scripts/run-perf-tests.ps1 + ${{ else }}: + testScript: eng/pipelines/perf/scripts/run-perf-tests.sh + + # Arguments forwarded to the script. The script also reads SQL_SERVER, SQL_PASSWORD and + # PERF_CLIENT_CPUS directly from the VM session environment (injected by the template). + testScriptArgs: '--configuration ${{ parameters.buildConfiguration }} --framework ${{ parameters.dotnetFramework }} --results-subdir perf-results' + + # Subfolder (relative to testRootDir on the VM) the script writes results into. The template + # copies this VM folder back to the agent, but always lands it at a fixed location: + # $(Build.ArtifactStagingDirectory)/results (and publishes it as the 'perf-results' artifact). + testResultsSubDir: perf-results + + testTimeoutMinutes: ${{ parameters.testTimeoutMinutes }} + jobName: SqlClientPerf + + # Post-test steps run on the agent after results have been copied back. The template already + # publishes the results artifact and attaches any top-level results/*.md as run summaries; this + # step additionally surfaces the BenchmarkDotNet markdown reports in the build log. + steps: + - bash: | + set -euo pipefail + resultsDir="$(Build.ArtifactStagingDirectory)/results" + echo "=== Performance results ($resultsDir) ===" + if [ ! -d "$resultsDir" ]; then + echo "##vso[task.logissue type=warning]No results directory was collected from the VM." + exit 0 + fi + find "$resultsDir" -type f | sort + echo + # Print every BenchmarkDotNet GitHub-markdown report that was produced. + while IFS= read -r report; do + echo "----------------------------------------------------------------" + echo "### $report" + echo "----------------------------------------------------------------" + cat "$report" + echo + done < <(find "$resultsDir" -type f -name '*-report-github.md' | sort) + displayName: 'Show performance results' + condition: succeededOrFailed() From 41e5449d0f71b50a3f3917227551cac3744a22fe Mon Sep 17 00:00:00 2001 From: Cheena Malhotra Date: Tue, 21 Jul 2026 18:57:48 -0700 Subject: [PATCH 02/22] Add baseline-vs-current perf comparison and Kusto ingestion Extend the SqlClient performance pipeline to: - Run benchmarks against a released NuGet baseline (default 7.0.2, overridable at queue time) and against the branch under test built from source, both CPU-pinned, in two passes on the VM. - Compare the two passes and emit a per-benchmark delta (markdown + json) surfaced as the run summary. - Translate BenchmarkDotNet "full" JSON into the perf-results Kusto schema (PerfRun + PerfBenchmarkResult, wiki 270) and optionally ingest via an ADO service connection (AzureCLI@2 + queued ingestion). Enables JsonExporter.Full and adds a CPM VersionOverride switch (MdsPackageVersion) so the baseline pass can pin a released MDS version restored from NuGet.org through a dedicated single-source config. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4a21e2a3-6efa-45c7-9bfd-81e7f3d24a89 --- eng/pipelines/perf/README.md | 170 +++++++++++++ .../perf/kusto/PerfBenchmarkResult.kql | 54 ++++ eng/pipelines/perf/kusto/PerfRun.kql | 44 ++++ eng/pipelines/perf/scripts/compare_perf.py | 231 +++++++++++++++++ eng/pipelines/perf/scripts/ingest_kusto.py | 93 +++++++ eng/pipelines/perf/scripts/perf_to_kusto.py | 233 ++++++++++++++++++ eng/pipelines/perf/scripts/run-perf-tests.ps1 | 162 ++++++++---- eng/pipelines/perf/scripts/run-perf-tests.sh | 139 ++++++++--- .../perf/sqlclient-perf-pipeline.yml | 159 +++++++++++- .../Config/BenchmarkConfig.cs | 4 + ...oft.Data.SqlClient.PerformanceTests.csproj | 10 +- 11 files changed, 1218 insertions(+), 81 deletions(-) create mode 100644 eng/pipelines/perf/README.md create mode 100644 eng/pipelines/perf/kusto/PerfBenchmarkResult.kql create mode 100644 eng/pipelines/perf/kusto/PerfRun.kql create mode 100644 eng/pipelines/perf/scripts/compare_perf.py create mode 100644 eng/pipelines/perf/scripts/ingest_kusto.py create mode 100644 eng/pipelines/perf/scripts/perf_to_kusto.py diff --git a/eng/pipelines/perf/README.md b/eng/pipelines/perf/README.md new file mode 100644 index 0000000000..0c16f1e126 --- /dev/null +++ b/eng/pipelines/perf/README.md @@ -0,0 +1,170 @@ +# SqlClient Performance Test Pipeline + +This directory contains the Azure DevOps pipeline and supporting scripts that run the +Microsoft.Data.SqlClient [BenchmarkDotNet](https://benchmarkdotnet.org/) performance tests on the +internal **Perf Test Lab** (Azure Dedicated Hosts), compare the branch under test against a released +NuGet baseline, and ingest the results into the InternalDriverTools **perf-results** Kusto database. + +> Internal references: +> - Performance Test Automation — InternalDriverTools wiki page **284** +> - Performance Results Database Specification — InternalDriverTools wiki page **270** + +## Contents + +| Path | Purpose | +| ---- | ------- | +| `sqlclient-perf-pipeline.yml` | The pipeline. Extends `v1/Perf.Test.Job.yml@PerfTemplates`. | +| `scripts/run-perf-tests.sh` | Linux on-VM entry point: install SDK, create DB, two benchmark passes, compare. | +| `scripts/run-perf-tests.ps1` | Windows equivalent (ProcessorAffinity instead of `taskset`). | +| `scripts/compare_perf.py` | Compares baseline vs current BenchmarkDotNet JSON → delta (md + json). | +| `scripts/perf_to_kusto.py` | Translates BenchmarkDotNet "full" JSON → Kusto `PerfRun` + `PerfBenchmarkResult` NDJSON. | +| `scripts/ingest_kusto.py` | Queued Kusto ingestion (az CLI auth, runs on the agent). | +| `kusto/PerfRun.kql` | Create-table + JSON ingestion mapping for `PerfRun`. | +| `kusto/PerfBenchmarkResult.kql` | Create-table + JSON ingestion mapping for `PerfBenchmarkResult`. | + +## Architecture + +``` +Queue pipeline (ADO) + │ + ▼ +extends: v1/Perf.Test.Job.yml@PerfTemplates + │ (provisions a dedicated-host VM, SCPs the repo, runs the script over SSH, + │ SCPs back, publishes it, tears the VM down) + ▼ +ON THE VM ── run-perf-tests.{sh,ps1} + 1. Install the .NET SDK pinned by global.json (+ runtimes). + 2. Create the perf database on the VM's SQL Server. + 3. Inject the VM SQL connection string into runnerconfig. + 4. Baseline pass → MDS from NuGet.org (Package mode) → results/baseline/ + 5. Current pass → MDS built from source (ProjectReference) → results/current/ + (both pinned to PERF_CLIENT_CPUS) + 6. compare_perf.py → results/comparison/ + results/summary.md + ▼ +ON THE AGENT ── pipeline post-test steps + • Show the BenchmarkDotNet markdown reports in the log. + • perf_to_kusto.py → NDJSON for both passes (published as 'perf-kusto-payloads'). + • (optional) AzureCLI@2 + ingest_kusto.py → Kusto perf-results database. +``` + +The extends template only exposes **post-test** steps to consumers (no pre-build hook), so **both +benchmark passes run inside the on-VM script**. Translation and ingestion run on the **agent** +because that is where the pipeline's AAD identity / service connection and the native pipeline +context variables are available (the VM is behind NAT and lacks the pipeline identity). + +## Parameters + +| Parameter | Default | Description | +| --------- | ------- | ----------- | +| `platform` | `linux` | `linux` or `windows` VM + client. | +| `buildConfiguration` | `Release` | Only `Release` produces meaningful numbers. | +| `dotnetFramework` | `net9.0` | TFM the benchmarks run against (`net8.0`/`net9.0`/`net10.0`). | +| `testTimeoutMinutes` | `180` | Template timeout waiting for the VM run. | +| `sourcesSubDir` | `dotnet-sqlclient` | Folder the repo (`self`) is checked out into under the template's multi-repo checkout. Must match the ADO repo name. | +| `baselineVersion` | `7.0.2` | **Baseline Version** — released MDS the branch is compared against. Empty = current-only (no baseline pass / comparison). | +| `regressionThreshold` | `10` | Percent slowdown (current vs baseline mean) flagged as a regression. | +| `kustoClusterUri` | *(empty)* | Kusto cluster URI. Blank ⇒ ingestion skipped. | +| `kustoDatabase` | `PerfResults` | Target Kusto database. | +| `kustoServiceConnection` | *(empty)* | ADO ARM service connection whose SP has ingest rights. Blank ⇒ ingestion skipped. | +| `driverName` | `Microsoft.Data.SqlClient` | Recorded on every row (`DriverName` / `DerivedRunId`). | + +### Managing the baseline version + +`baselineVersion` is **manually managed**. After each stable release is published to NuGet.org, +bump the `default` in `sqlclient-perf-pipeline.yml` (e.g. `7.0.2` → the next stable). It can also be +overridden at queue time without editing the pipeline. + +## Two-pass build model + +The `PerformanceTests` project references Microsoft.Data.SqlClient two ways, selected by MSBuild: + +- **Current** (default): `ProjectReference` to the in-repo source — the branch under test. +- **Baseline**: `ReferenceType=Package` turns the reference into a `PackageReference`. Because the + repo uses **Central Package Management (CPM)**, the version is pinned with `VersionOverride` via + `-p:MdsPackageVersion=` (a plain `Version` is ignored under CPM). + +The VM's `NuGet.config` exposes only the governed feed, and CPM rejects multiple unmapped sources +(`NU1507`). The baseline pass therefore restores through a **dedicated single-source config** +(`perf-baseline-nuget.config`, generated at runtime) pointing only at `https://api.nuget.org/v3/index.json`. + +## Comparison output + +`compare_perf.py` matches benchmarks by `(Type, Method, Parameters)` and reports, per benchmark, the +baseline/current mean (ms), mean %Δ, allocation %Δ, and a status (`regression` / `improvement` / +`unchanged` / `new` / `removed`). Outputs: + +- `results/comparison/comparison.md` (also copied to `results/summary.md`, which the template + attaches as the run summary), +- `results/comparison/comparison.json` (structured, for tooling). + +## Kusto schema & ingestion + +Schema follows wiki 270. Two tables: + +- **`PerfRun`** — one row per run (baseline OR current): `DerivedRunId` (PK = + `DriverName|CommitHash|PipelineRunId`), driver/machine/agent, pipeline id + build URL, branch + + `BranchCategory`, `VersionString`, commit hash/date, `IsComparableBase`, `IngestedAt`. +- **`PerfBenchmarkResult`** — one row per benchmark: `BenchmarkId` (PK = + `DerivedRunId|BenchmarkName|MethodName|ParameterSignature`), timings in **milliseconds** + (BenchmarkDotNet reports nanoseconds; values are divided by 1,000,000), percentiles, throughput, + allocation, runtime/platform, and `DriverSpecificMetrics` (GC collections, lock contentions, …). + +The baseline and current passes share the pipeline run id and (for the current pass) the commit, so +to keep their `DerivedRunId`s distinct the baseline row uses `CommitHash = v` and +`IsComparableBase = true`; the current row uses the real commit and `IsComparableBase = false` with +the triggering branch name. + +Source = BenchmarkDotNet **JSON "full"** exporter files (`*-report-full.json`). The exporter is +enabled in `Config/BenchmarkConfig.cs` (`JsonExporter.Full`). + +### One-time database setup + +Run the KQL scripts once against the target database (Kusto query editor, or +`.execute database script`): + +``` +kusto/PerfRun.kql +kusto/PerfBenchmarkResult.kql +``` + +They create the tables (`.create-merge`, idempotent) and the JSON ingestion mappings +(`PerfRun_json_mapping`, `PerfBenchmarkResult_json_mapping`) referenced by `ingest_kusto.py`. + +### Authentication + +Ingestion runs in an `AzureCLI@2` task using the ADO **ARM service connection** +(`kustoServiceConnection`). That connection's **service principal** must be granted, on the target +database: + +- **Database Ingestor** (to ingest), and +- **Database Viewer** (recommended, for verification queries). + +`ingest_kusto.py` authenticates to Kusto with `with_az_cli_authentication` (the service connection +is already `az login`'d inside the task) and performs a **queued** ingestion against the +data-management (`ingest-`) endpoint. + +### Running before the cluster exists + +Ingestion is **conditional**: it only runs when both `kustoClusterUri` and `kustoServiceConnection` +are supplied. Until the cluster and service connection are provisioned, the pipeline still runs both +passes, produces the comparison, and publishes the translated NDJSON as the `perf-kusto-payloads` +artifact for manual/backfill ingestion. + +## Running the pipeline + +1. In ADO, open **SqlClient-Performance-Tests** and select **Run pipeline**. +2. Choose the branch to benchmark; optionally override `baselineVersion` and (once available) the + Kusto parameters. +3. After the run, review the **run summary** (comparison) and the `perf-results` / + `perf-kusto-payloads` artifacts. + +## Troubleshooting + +| Symptom | Likely cause / fix | +| ------- | ------------------ | +| `NU1507` during the baseline pass | Multiple NuGet sources under CPM. The baseline uses a single-source config; ensure `perf-baseline-nuget.config` is being passed via `-p:RestoreConfigFile`. | +| Baseline restore fails to find MDS | `baselineVersion` isn't a published NuGet.org version, or the VM has no outbound access to `api.nuget.org`. | +| No comparison / summary | The baseline pass was skipped (empty `baselineVersion`) or one pass produced no `*-report-full.json`. | +| Ingestion step skipped | `kustoClusterUri` or `kustoServiceConnection` is blank (expected until the cluster is provisioned). | +| Ingestion auth error | The service connection's SP lacks **Database Ingestor** on the target database. | +| Benchmarks not CPU-pinned | `PERF_CLIENT_CPUS` was not injected, or `taskset` is unavailable on the VM. | diff --git a/eng/pipelines/perf/kusto/PerfBenchmarkResult.kql b/eng/pipelines/perf/kusto/PerfBenchmarkResult.kql new file mode 100644 index 0000000000..68c6343328 --- /dev/null +++ b/eng/pipelines/perf/kusto/PerfBenchmarkResult.kql @@ -0,0 +1,54 @@ +// PerfBenchmarkResult table + JSON ingestion mapping for the SqlClient +// performance-results database. Schema follows InternalDriverTools wiki page +// 270 ("Performance Results Database Specification") and the SqlClient +// BenchmarkDotNet conversion subpage. +// +// Ingestion (perf_to_kusto.py output) references the mapping named +// 'PerfBenchmarkResult_json_mapping'. + +.create-merge table PerfBenchmarkResult ( + BenchmarkId: string, + DerivedRunId: string, + BenchmarkName: string, + MethodName: string, + ParameterSignature: string, + ParameterBag: dynamic, + JobName: string, + MeanMs: real, + P50Ms: real, + P95Ms: real, + P99Ms: real, + ThroughputOpsPerSec: real, + ErrorMs: real, + StdDevMs: real, + MemoryAllocatedBytes: real, + Runtime: string, + Platform: string, + DriverSpecificMetrics: dynamic, + IngestedAt: datetime +) + +.create-or-alter table PerfBenchmarkResult ingestion json mapping "PerfBenchmarkResult_json_mapping" +``` +[ + { "column": "BenchmarkId", "path": "$.BenchmarkId" }, + { "column": "DerivedRunId", "path": "$.DerivedRunId" }, + { "column": "BenchmarkName", "path": "$.BenchmarkName" }, + { "column": "MethodName", "path": "$.MethodName" }, + { "column": "ParameterSignature", "path": "$.ParameterSignature" }, + { "column": "ParameterBag", "path": "$.ParameterBag" }, + { "column": "JobName", "path": "$.JobName" }, + { "column": "MeanMs", "path": "$.MeanMs" }, + { "column": "P50Ms", "path": "$.P50Ms" }, + { "column": "P95Ms", "path": "$.P95Ms" }, + { "column": "P99Ms", "path": "$.P99Ms" }, + { "column": "ThroughputOpsPerSec", "path": "$.ThroughputOpsPerSec" }, + { "column": "ErrorMs", "path": "$.ErrorMs" }, + { "column": "StdDevMs", "path": "$.StdDevMs" }, + { "column": "MemoryAllocatedBytes", "path": "$.MemoryAllocatedBytes" }, + { "column": "Runtime", "path": "$.Runtime" }, + { "column": "Platform", "path": "$.Platform" }, + { "column": "DriverSpecificMetrics","path": "$.DriverSpecificMetrics" }, + { "column": "IngestedAt", "path": "$.IngestedAt" } +] +``` diff --git a/eng/pipelines/perf/kusto/PerfRun.kql b/eng/pipelines/perf/kusto/PerfRun.kql new file mode 100644 index 0000000000..9be9aebe99 --- /dev/null +++ b/eng/pipelines/perf/kusto/PerfRun.kql @@ -0,0 +1,44 @@ +// PerfRun table + JSON ingestion mapping for the SqlClient performance-results +// database. Schema follows InternalDriverTools wiki page 270 ("Performance +// Results Database Specification"). +// +// Run once against the target database (e.g. in the Kusto query editor or via +// `az kusto` / `.execute database script`). Ingestion (perf_to_kusto.py output) +// references the mapping named 'PerfRun_json_mapping'. + +.create-merge table PerfRun ( + DerivedRunId: string, + DriverName: string, + MachineName: string, + AgentName: string, + PipelineRunId: string, + BuildUrl: string, + RunDate: datetime, + BranchName: string, + BranchCategory: string, + VersionString: string, + CommitHash: string, + CommitDate: datetime, + IsComparableBase: bool, + IngestedAt: datetime +) + +.create-or-alter table PerfRun ingestion json mapping "PerfRun_json_mapping" +``` +[ + { "column": "DerivedRunId", "path": "$.DerivedRunId" }, + { "column": "DriverName", "path": "$.DriverName" }, + { "column": "MachineName", "path": "$.MachineName" }, + { "column": "AgentName", "path": "$.AgentName" }, + { "column": "PipelineRunId", "path": "$.PipelineRunId" }, + { "column": "BuildUrl", "path": "$.BuildUrl" }, + { "column": "RunDate", "path": "$.RunDate" }, + { "column": "BranchName", "path": "$.BranchName" }, + { "column": "BranchCategory", "path": "$.BranchCategory" }, + { "column": "VersionString", "path": "$.VersionString" }, + { "column": "CommitHash", "path": "$.CommitHash" }, + { "column": "CommitDate", "path": "$.CommitDate" }, + { "column": "IsComparableBase", "path": "$.IsComparableBase" }, + { "column": "IngestedAt", "path": "$.IngestedAt" } +] +``` diff --git a/eng/pipelines/perf/scripts/compare_perf.py b/eng/pipelines/perf/scripts/compare_perf.py new file mode 100644 index 0000000000..e5e8d253b4 --- /dev/null +++ b/eng/pipelines/perf/scripts/compare_perf.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +"""Compare two sets of BenchmarkDotNet "full" JSON reports and emit a delta. + +Reads every ``*-report-full.json`` under a baseline directory and a current +directory, matches benchmarks by (Type, Method, Parameters), and computes the +per-benchmark delta in mean execution time and allocated memory. + +Outputs: + * a GitHub-flavoured markdown table (``--out-md``), and + * a structured JSON document (``--out-json``) + +The script only uses the Python standard library so it can run on the perf VM +without installing any packages. +""" + +import argparse +import glob +import json +import os +import sys + + +NS_PER_MS = 1_000_000.0 + + +def _load_benchmarks(directory): + """Return {key: record} for every benchmark found under *directory*. + + key = "Type.Method(Parameters)". record carries the mean (ns), the + allocated bytes/op, and the display fields used for reporting. + """ + records = {} + pattern = os.path.join(directory, "**", "*-report-full.json") + for path in sorted(glob.glob(pattern, recursive=True)): + try: + with open(path, "r", encoding="utf-8-sig") as handle: + data = json.load(handle) + except (OSError, ValueError) as exc: + print(f"WARNING: could not parse {path}: {exc}", file=sys.stderr) + continue + + for bench in data.get("Benchmarks", []): + btype = bench.get("Type", "") + method = bench.get("Method", "") + params = bench.get("Parameters", "") or "" + stats = bench.get("Statistics") or {} + mean_ns = stats.get("Mean") + if mean_ns is None: + continue + memory = bench.get("Memory") or {} + alloc = memory.get("BytesAllocatedPerOperation") + + key = f"{btype}.{method}({params})" + records[key] = { + "benchmarkName": btype, + "methodName": method, + "parameterSignature": params, + "meanNs": float(mean_ns), + "allocatedBytes": float(alloc) if alloc is not None else None, + } + return records + + +def _pct(baseline, current): + if baseline in (None, 0): + return None + return (current - baseline) / baseline * 100.0 + + +def build_comparison(baseline_dir, current_dir, threshold_pct): + baseline = _load_benchmarks(baseline_dir) + current = _load_benchmarks(current_dir) + + entries = [] + for key in sorted(set(baseline) | set(current)): + b = baseline.get(key) + c = current.get(key) + ref = c or b + entry = { + "key": key, + "benchmarkName": ref["benchmarkName"], + "methodName": ref["methodName"], + "parameterSignature": ref["parameterSignature"], + "baselineMeanMs": (b["meanNs"] / NS_PER_MS) if b else None, + "currentMeanMs": (c["meanNs"] / NS_PER_MS) if c else None, + "baselineAllocBytes": b["allocatedBytes"] if b else None, + "currentAllocBytes": c["allocatedBytes"] if c else None, + } + + if b and c: + entry["meanDeltaPct"] = _pct(b["meanNs"], c["meanNs"]) + entry["meanRatio"] = (c["meanNs"] / b["meanNs"]) if b["meanNs"] else None + if b["allocatedBytes"] and c["allocatedBytes"] is not None: + entry["allocDeltaPct"] = _pct(b["allocatedBytes"], c["allocatedBytes"]) + else: + entry["allocDeltaPct"] = None + delta = entry["meanDeltaPct"] + if delta is None: + entry["status"] = "unknown" + elif delta > threshold_pct: + entry["status"] = "regression" + elif delta < -threshold_pct: + entry["status"] = "improvement" + else: + entry["status"] = "unchanged" + elif c and not b: + entry["meanDeltaPct"] = None + entry["meanRatio"] = None + entry["allocDeltaPct"] = None + entry["status"] = "current-only" + else: + entry["meanDeltaPct"] = None + entry["meanRatio"] = None + entry["allocDeltaPct"] = None + entry["status"] = "baseline-only" + + entries.append(entry) + + # Sort worst-regression first, then by name for stability. + def _sort_key(e): + d = e["meanDeltaPct"] + return (-(d if d is not None else -1e18), e["key"]) + + entries.sort(key=_sort_key) + return entries + + +def _fmt_ms(value): + return f"{value:.4f}" if value is not None else "-" + + +def _fmt_pct(value): + if value is None: + return "-" + return f"{value:+.2f}%" + + +def _fmt_bytes(value): + return f"{int(value)}" if value is not None else "-" + + +def render_markdown(entries, baseline_version, threshold_pct): + regressions = [e for e in entries if e["status"] == "regression"] + improvements = [e for e in entries if e["status"] == "improvement"] + + lines = [] + lines.append("# SqlClient Performance Comparison") + lines.append("") + lines.append(f"Baseline: **{baseline_version}**  |  " + f"Regression threshold: **{threshold_pct:.0f}%**") + lines.append("") + lines.append(f"- Benchmarks compared: **{len(entries)}**") + lines.append(f"- Regressions (slower > {threshold_pct:.0f}%): **{len(regressions)}**") + lines.append(f"- Improvements (faster > {threshold_pct:.0f}%): **{len(improvements)}**") + lines.append("") + lines.append("| Status | Benchmark | Method | Params | Baseline (ms) | " + "Current (ms) | Mean Δ | Alloc Δ |") + lines.append("| ------ | --------- | ------ | ------ | ------------- | " + "------------ | ------ | ------- |") + + icon = { + "regression": "🔴 regression", + "improvement": "🟢 improvement", + "unchanged": "⚪ unchanged", + "current-only": "🆕 new", + "baseline-only": "➖ removed", + "unknown": "❔", + } + for e in entries: + lines.append( + "| {status} | {name} | {method} | {params} | {base} | {cur} | " + "{delta} | {alloc} |".format( + status=icon.get(e["status"], e["status"]), + name=e["benchmarkName"], + method=e["methodName"], + params=e["parameterSignature"] or "-", + base=_fmt_ms(e["baselineMeanMs"]), + cur=_fmt_ms(e["currentMeanMs"]), + delta=_fmt_pct(e["meanDeltaPct"]), + alloc=_fmt_pct(e["allocDeltaPct"]), + ) + ) + lines.append("") + return "\n".join(lines) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--baseline-dir", required=True) + parser.add_argument("--current-dir", required=True) + parser.add_argument("--out-md", required=True) + parser.add_argument("--out-json", required=True) + parser.add_argument("--baseline-version", default="baseline") + parser.add_argument("--threshold", type=float, default=10.0, + help="Percent slowdown that counts as a regression.") + parser.add_argument("--fail-on-regression", action="store_true", + help="Exit non-zero if any regression is detected.") + args = parser.parse_args(argv) + + entries = build_comparison(args.baseline_dir, args.current_dir, args.threshold) + + os.makedirs(os.path.dirname(os.path.abspath(args.out_md)), exist_ok=True) + os.makedirs(os.path.dirname(os.path.abspath(args.out_json)), exist_ok=True) + + markdown = render_markdown(entries, args.baseline_version, args.threshold) + with open(args.out_md, "w", encoding="utf-8") as handle: + handle.write(markdown + "\n") + + with open(args.out_json, "w", encoding="utf-8") as handle: + json.dump( + { + "baselineVersion": args.baseline_version, + "thresholdPct": args.threshold, + "entries": entries, + }, + handle, + indent=2, + ) + + regressions = [e for e in entries if e["status"] == "regression"] + print(f"Compared {len(entries)} benchmarks; {len(regressions)} regression(s).") + print(f"Wrote {args.out_md} and {args.out_json}.") + + if args.fail_on_regression and regressions: + print("Regressions detected; failing as requested.", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eng/pipelines/perf/scripts/ingest_kusto.py b/eng/pipelines/perf/scripts/ingest_kusto.py new file mode 100644 index 0000000000..5235744ebf --- /dev/null +++ b/eng/pipelines/perf/scripts/ingest_kusto.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Ingest translated perf-results NDJSON files into Azure Data Explorer (Kusto). + +Runs on the pipeline agent inside an ``AzureCLI@2`` task so that the Azure DevOps +service connection's service principal is already logged in via ``az``. The +script therefore authenticates to Kusto with *az CLI* credentials and performs a +queued ingestion of the PerfRun and PerfBenchmarkResult NDJSON files produced by +``perf_to_kusto.py``. + +Requires the ``azure-kusto-ingest`` package (``pip install azure-kusto-data +azure-kusto-ingest``); the pipeline step installs it before invoking this script. +""" + +import argparse +import os +import sys + + +def _ingest(cluster, database, table, mapping, data_file): + from azure.kusto.data import KustoConnectionStringBuilder + from azure.kusto.ingest import ( + QueuedIngestClient, + IngestionProperties, + FileDescriptor, + ) + from azure.kusto.data.data_format import DataFormat + from azure.kusto.ingest import ReportLevel + + if not os.path.exists(data_file) or os.path.getsize(data_file) == 0: + print(f"Skipping {table}: '{data_file}' is missing or empty.") + return 0 + + # Queued ingestion targets the data-management ("ingest-") endpoint. + ingest_uri = cluster + if "://" in cluster: + scheme, rest = cluster.split("://", 1) + if not rest.startswith("ingest-"): + ingest_uri = f"{scheme}://ingest-{rest}" + + kcsb = KustoConnectionStringBuilder.with_az_cli_authentication(ingest_uri) + client = QueuedIngestClient(kcsb) + + props = IngestionProperties( + database=database, + table=table, + data_format=DataFormat.MULTIJSON, + ingestion_mapping_reference=mapping, + report_level=ReportLevel.FailuresAndSuccesses, + ) + + descriptor = FileDescriptor(data_file, os.path.getsize(data_file)) + client.ingest_from_file(descriptor, ingestion_properties=props) + print(f"Queued ingestion of '{data_file}' -> {database}.{table} " + f"(mapping '{mapping}').") + return 0 + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cluster", required=True, + help="Kusto cluster URI, e.g. https://..kusto.windows.net") + parser.add_argument("--database", required=True) + parser.add_argument("--run-file", required=True, + help="PerfRun NDJSON file(s), comma-separated.") + parser.add_argument("--results-file", required=True, + help="PerfBenchmarkResult NDJSON file(s), comma-separated.") + parser.add_argument("--run-table", default="PerfRun") + parser.add_argument("--results-table", default="PerfBenchmarkResult") + parser.add_argument("--run-mapping", default="PerfRun_json_mapping") + parser.add_argument("--results-mapping", default="PerfBenchmarkResult_json_mapping") + args = parser.parse_args(argv) + + try: + import azure.kusto.ingest # noqa: F401 + except ImportError: + print("ERROR: azure-kusto-ingest is not installed. " + "Run 'pip install azure-kusto-data azure-kusto-ingest'.", + file=sys.stderr) + return 2 + + for data_file in [f.strip() for f in args.run_file.split(",") if f.strip()]: + _ingest(args.cluster, args.database, args.run_table, + args.run_mapping, data_file) + for data_file in [f.strip() for f in args.results_file.split(",") if f.strip()]: + _ingest(args.cluster, args.database, args.results_table, + args.results_mapping, data_file) + + print("Ingestion requests submitted (queued).") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eng/pipelines/perf/scripts/perf_to_kusto.py b/eng/pipelines/perf/scripts/perf_to_kusto.py new file mode 100644 index 0000000000..53fde0c680 --- /dev/null +++ b/eng/pipelines/perf/scripts/perf_to_kusto.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +"""Translate BenchmarkDotNet "full" JSON reports into Kusto perf-results rows. + +Implements the mapping documented on the InternalDriverTools wiki, page 270 +("Performance Results Database Specification") and the SqlClient conversion +subpage. Produces two newline-delimited JSON (NDJSON) files ready for Kusto +ingestion: + + * PerfRun - one row describing this run (baseline OR current). + * PerfBenchmarkResult - one row per benchmark method/parameter combination. + +All benchmark timings in BenchmarkDotNet are expressed in NANOSECONDS; the +schema stores milliseconds, so every time value is divided by 1,000,000. + +Only the Python standard library is required. +""" + +import argparse +import datetime +import glob +import json +import os +import sys + + +NS_PER_MS = 1_000_000.0 + + +def _utcnow_iso(): + return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +def _branch_category(branch_name): + """Map a git ref to one of the schema's BranchCategory buckets.""" + if not branch_name: + return "other" + name = branch_name + for prefix in ("refs/heads/", "refs/"): + if name.startswith(prefix): + name = name[len(prefix):] + break + if name.startswith("pull/") or branch_name.startswith("refs/pull/"): + return "pull_request" + if name == "main" or name == "master": + return "main" + if name.startswith("release/"): + return "release" + if name.startswith("dev/"): + return "dev" + if name.startswith("feat/") or name.startswith("feature/"): + return "feature" + return "other" + + +def _parse_parameters(parameters): + """Parse BenchmarkDotNet's 'Parameters' string into a structured bag. + + Example input: "RowCount=1000, UseAsync=True" + """ + bag = {} + if not parameters: + return bag + for part in parameters.split(","): + part = part.strip() + if not part or "=" not in part: + continue + key, _, value = part.partition("=") + bag[key.strip()] = value.strip() + return bag + + +def _driver_specific_metrics(bench): + metrics = {} + memory = bench.get("Memory") or {} + for field in ("Gen0Collections", "Gen1Collections", "Gen2Collections", + "TotalOperations", "BytesAllocatedPerOperation"): + if field in memory and memory[field] is not None: + metrics[field] = memory[field] + for metric in bench.get("Metrics", []) or []: + descriptor = metric.get("Descriptor") or {} + legend = descriptor.get("Legend") or descriptor.get("Id") + if legend is not None and metric.get("Value") is not None: + metrics[legend] = metric["Value"] + return metrics + + +def _percentile(stats, name): + pct = stats.get("Percentiles") or {} + value = pct.get(name) + return (value / NS_PER_MS) if value is not None else None + + +def translate(input_dir, ctx): + """Return (run_row, [result_rows]) for every full-report JSON in input_dir.""" + derived_run_id = "{driver}|{commit}|{pipeline}".format( + driver=ctx["driver_name"], + commit=ctx["commit_hash"], + pipeline=ctx["pipeline_run_id"], + ) + now = _utcnow_iso() + + run_row = { + "DerivedRunId": derived_run_id, + "DriverName": ctx["driver_name"], + "MachineName": ctx["machine_name"], + "AgentName": ctx["agent_name"], + "PipelineRunId": ctx["pipeline_run_id"], + "BuildUrl": ctx["build_url"], + "RunDate": ctx["run_date"] or now, + "BranchName": ctx["branch_name"], + "BranchCategory": _branch_category(ctx["branch_name"]), + "VersionString": ctx["version_string"], + "CommitHash": ctx["commit_hash"], + "CommitDate": ctx["commit_date"] or None, + "IsComparableBase": bool(ctx["is_comparable_base"]), + "IngestedAt": now, + } + + result_rows = [] + runtime_seen = None + platform_seen = None + + pattern = os.path.join(input_dir, "**", "*-report-full.json") + for path in sorted(glob.glob(pattern, recursive=True)): + try: + with open(path, "r", encoding="utf-8-sig") as handle: + data = json.load(handle) + except (OSError, ValueError) as exc: + print(f"WARNING: could not parse {path}: {exc}", file=sys.stderr) + continue + + host = data.get("HostEnvironmentInfo") or {} + runtime = host.get("RuntimeVersion") + architecture = host.get("Architecture") + runtime_seen = runtime_seen or runtime + platform_seen = platform_seen or architecture + + for bench in data.get("Benchmarks", []): + stats = bench.get("Statistics") or {} + mean_ns = stats.get("Mean") + if mean_ns is None: + continue + mean_ms = mean_ns / NS_PER_MS + + btype = bench.get("Type", "") + method = bench.get("Method", "") + params = bench.get("Parameters", "") or "" + memory = bench.get("Memory") or {} + + benchmark_id = "{run}|{name}|{method}|{sig}".format( + run=derived_run_id, name=btype, method=method, sig=params) + + result_rows.append({ + "BenchmarkId": benchmark_id, + "DerivedRunId": derived_run_id, + "BenchmarkName": btype, + "MethodName": method, + "ParameterSignature": params, + "ParameterBag": _parse_parameters(params), + "JobName": (bench.get("Job") or bench.get("JobConfig") or ""), + "MeanMs": mean_ms, + "P50Ms": _percentile(stats, "P50"), + "P95Ms": _percentile(stats, "P95"), + "P99Ms": _percentile(stats, "P99"), + "ThroughputOpsPerSec": (1000.0 / mean_ms) if mean_ms else None, + "ErrorMs": (stats.get("StandardError") / NS_PER_MS) + if stats.get("StandardError") is not None else None, + "StdDevMs": (stats.get("StandardDeviation") / NS_PER_MS) + if stats.get("StandardDeviation") is not None else None, + "MemoryAllocatedBytes": memory.get("BytesAllocatedPerOperation"), + "Runtime": runtime, + "Platform": architecture, + "DriverSpecificMetrics": _driver_specific_metrics(bench), + "IngestedAt": now, + }) + + return run_row, result_rows + + +def _write_ndjson(path, rows): + os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True) + with open(path, "w", encoding="utf-8") as handle: + for row in rows: + handle.write(json.dumps(row, separators=(",", ":")) + "\n") + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input-dir", required=True, + help="Directory containing *-report-full.json for one pass.") + parser.add_argument("--out-run", required=True, help="PerfRun NDJSON output path.") + parser.add_argument("--out-results", required=True, + help="PerfBenchmarkResult NDJSON output path.") + parser.add_argument("--driver-name", default="Microsoft.Data.SqlClient") + parser.add_argument("--machine-name", default="") + parser.add_argument("--agent-name", default="") + parser.add_argument("--pipeline-run-id", required=True) + parser.add_argument("--build-url", default="") + parser.add_argument("--run-date", default="") + parser.add_argument("--branch-name", default="") + parser.add_argument("--version-string", default="") + parser.add_argument("--commit-hash", required=True) + parser.add_argument("--commit-date", default="") + parser.add_argument("--is-comparable-base", default="false") + args = parser.parse_args(argv) + + ctx = { + "driver_name": args.driver_name, + "machine_name": args.machine_name, + "agent_name": args.agent_name, + "pipeline_run_id": args.pipeline_run_id, + "build_url": args.build_url, + "run_date": args.run_date, + "branch_name": args.branch_name, + "version_string": args.version_string, + "commit_hash": args.commit_hash, + "commit_date": args.commit_date, + "is_comparable_base": str(args.is_comparable_base).lower() in ("1", "true", "yes"), + } + + run_row, result_rows = translate(args.input_dir, ctx) + _write_ndjson(args.out_run, [run_row]) + _write_ndjson(args.out_results, result_rows) + + print(f"Wrote 1 PerfRun row -> {args.out_run}") + print(f"Wrote {len(result_rows)} PerfBenchmarkResult row(s) -> {args.out_results}") + if not result_rows: + print("WARNING: no benchmark results were translated.", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eng/pipelines/perf/scripts/run-perf-tests.ps1 b/eng/pipelines/perf/scripts/run-perf-tests.ps1 index 22a490c13c..0517530f2b 100644 --- a/eng/pipelines/perf/scripts/run-perf-tests.ps1 +++ b/eng/pipelines/perf/scripts/run-perf-tests.ps1 @@ -23,7 +23,9 @@ param( [string]$Configuration = "Release", [string]$Framework = "net9.0", - [string]$ResultsSubdir = "perf-results" + [string]$ResultsSubdir = "perf-results", + [string]$BaselineVersion = "", + [string]$RegressionThreshold = "10" ) $ErrorActionPreference = "Stop" @@ -53,6 +55,7 @@ Write-Host " Perf project : $PerfProject" Write-Host " Configuration : $Configuration" Write-Host " Framework : $Framework" Write-Host " Results dir : $ResultsDir" +Write-Host " Baseline ver : $(if ($BaselineVersion) { $BaselineVersion } else { '' })" Write-Host " SQL_SERVER : $SqlServer" Write-Host " PERF_CLIENT_CPUS: $($env:PERF_CLIENT_CPUS)" Write-Host " PERF_SQL_CPUS : $($env:PERF_SQL_CPUS)" @@ -67,6 +70,9 @@ if ([string]::IsNullOrEmpty($SqlPassword)) { New-Item -ItemType Directory -Force -Path $ResultsDir | Out-Null +# Record VM-side run metadata (e.g. the perf VM hostname) for the agent-side Kusto translation. +"MACHINE_NAME=$env:COMPUTERNAME" | Set-Content -Path (Join-Path $ResultsDir "runinfo.env") -Encoding ASCII + $env:DOTNET_NOLOGO = "1" $env:DOTNET_CLI_TELEMETRY_OPTOUT = "1" $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE = "1" @@ -150,15 +156,32 @@ Write-Host "Wrote runner config to $RunnerConfig (Server=tcp:$SqlServer,1433; In #################################################################################################### # 4 & 5. Run the benchmarks, pinned to the reserved client CPU set. +# +# Two passes are executed so the pipeline can compare the branch under test against a released +# baseline: +# * baseline -> Microsoft.Data.SqlClient restored from NuGet.org at $BaselineVersion +# (ReferenceType=Package + CPM VersionOverride). Skipped when no baseline is given. +# * current -> Microsoft.Data.SqlClient built from the source tree in this repo (ProjectReference). +# +# Each pass runs from its own directory; its BenchmarkDotNet artifacts are collected into +# results\