diff --git a/.github/workflows/maven-configuration-warm-path.yml b/.github/workflows/maven-configuration-warm-path.yml new file mode 100644 index 0000000..a07c59d --- /dev/null +++ b/.github/workflows/maven-configuration-warm-path.yml @@ -0,0 +1,173 @@ +name: Maven configuration warm path + +on: + workflow_dispatch: + inputs: + setup-java-repository: + description: Repository containing the setup-java action + required: true + default: actions/setup-java + type: string + baseline-ref: + description: Git ref containing the baseline implementation + required: true + default: main + type: string + candidate-ref: + description: Git ref containing the candidate implementation + required: true + default: main + type: string + +permissions: + contents: read + +defaults: + run: + shell: bash + +concurrency: + group: maven-configuration-warm-path + cancel-in-progress: false + +jobs: + benchmark: + name: ${{ matrix.os }} ${{ matrix.cache }} ${{ matrix.versions.name }} ${{ matrix.toolchains }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-15-intel] + cache: [none, maven, gradle] + versions: + - name: single + java-version: "21" + - name: multiple + java-version: | + 17 + 21 + toolchains: [empty, existing] + steps: + - name: Check out benchmark repository + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + - name: Check out baseline setup-java + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: ${{ inputs.setup-java-repository }} + path: baseline + persist-credentials: false + ref: ${{ inputs.baseline-ref }} + - name: Check out candidate setup-java + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: ${{ inputs.setup-java-repository }} + path: candidate + persist-credentials: false + ref: ${{ inputs.candidate-ref }} + + - name: Record setup dist sizes + run: | + bash scripts/benchmark-maven-configuration.sh record-size baseline baseline + bash scripts/benchmark-maven-configuration.sh record-size candidate candidate + + - name: Prepare baseline iteration 1 + run: bash scripts/benchmark-maven-configuration.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}" + - name: Start baseline iteration 1 timer + run: bash scripts/benchmark-maven-configuration.sh start + - name: Run baseline iteration 1 + uses: ./baseline + with: + distribution: temurin + java-version: ${{ matrix.versions.java-version }} + cache: ${{ matrix.cache == 'none' && '' || matrix.cache }} + cache-dependency-path: benchmark/${{ matrix.cache == 'gradle' && 'build.gradle' || 'pom.xml' }} + settings-path: benchmark-maven-home + - name: Record baseline iteration 1 + run: bash scripts/benchmark-maven-configuration.sh record "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.versions.name }}" "${{ matrix.toolchains }}" baseline 1 + + - name: Prepare candidate iteration 1 + run: bash scripts/benchmark-maven-configuration.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}" + - name: Start candidate iteration 1 timer + run: bash scripts/benchmark-maven-configuration.sh start + - name: Run candidate iteration 1 + uses: ./candidate + with: + distribution: temurin + java-version: ${{ matrix.versions.java-version }} + cache: ${{ matrix.cache == 'none' && '' || matrix.cache }} + cache-dependency-path: benchmark/${{ matrix.cache == 'gradle' && 'build.gradle' || 'pom.xml' }} + settings-path: benchmark-maven-home + - name: Record candidate iteration 1 + run: bash scripts/benchmark-maven-configuration.sh record "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.versions.name }}" "${{ matrix.toolchains }}" candidate 1 + + - name: Prepare baseline iteration 2 + run: bash scripts/benchmark-maven-configuration.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}" + - name: Start baseline iteration 2 timer + run: bash scripts/benchmark-maven-configuration.sh start + - name: Run baseline iteration 2 + uses: ./baseline + with: + distribution: temurin + java-version: ${{ matrix.versions.java-version }} + cache: ${{ matrix.cache == 'none' && '' || matrix.cache }} + cache-dependency-path: benchmark/${{ matrix.cache == 'gradle' && 'build.gradle' || 'pom.xml' }} + settings-path: benchmark-maven-home + - name: Record baseline iteration 2 + run: bash scripts/benchmark-maven-configuration.sh record "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.versions.name }}" "${{ matrix.toolchains }}" baseline 2 + + - name: Prepare candidate iteration 2 + run: bash scripts/benchmark-maven-configuration.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}" + - name: Start candidate iteration 2 timer + run: bash scripts/benchmark-maven-configuration.sh start + - name: Run candidate iteration 2 + uses: ./candidate + with: + distribution: temurin + java-version: ${{ matrix.versions.java-version }} + cache: ${{ matrix.cache == 'none' && '' || matrix.cache }} + cache-dependency-path: benchmark/${{ matrix.cache == 'gradle' && 'build.gradle' || 'pom.xml' }} + settings-path: benchmark-maven-home + - name: Record candidate iteration 2 + run: bash scripts/benchmark-maven-configuration.sh record "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.versions.name }}" "${{ matrix.toolchains }}" candidate 2 + + - name: Prepare baseline iteration 3 + run: bash scripts/benchmark-maven-configuration.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}" + - name: Start baseline iteration 3 timer + run: bash scripts/benchmark-maven-configuration.sh start + - name: Run baseline iteration 3 + uses: ./baseline + with: + distribution: temurin + java-version: ${{ matrix.versions.java-version }} + cache: ${{ matrix.cache == 'none' && '' || matrix.cache }} + cache-dependency-path: benchmark/${{ matrix.cache == 'gradle' && 'build.gradle' || 'pom.xml' }} + settings-path: benchmark-maven-home + - name: Record baseline iteration 3 + run: bash scripts/benchmark-maven-configuration.sh record "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.versions.name }}" "${{ matrix.toolchains }}" baseline 3 + + - name: Prepare candidate iteration 3 + run: bash scripts/benchmark-maven-configuration.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}" + - name: Start candidate iteration 3 timer + run: bash scripts/benchmark-maven-configuration.sh start + - name: Run candidate iteration 3 + uses: ./candidate + with: + distribution: temurin + java-version: ${{ matrix.versions.java-version }} + cache: ${{ matrix.cache == 'none' && '' || matrix.cache }} + cache-dependency-path: benchmark/${{ matrix.cache == 'gradle' && 'build.gradle' || 'pom.xml' }} + settings-path: benchmark-maven-home + - name: Record candidate iteration 3 + run: bash scripts/benchmark-maven-configuration.sh record "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.versions.name }}" "${{ matrix.toolchains }}" candidate 3 + + - name: Summarize benchmark + run: bash scripts/benchmark-maven-configuration.sh summarize "$GITHUB_STEP_SUMMARY" + - name: Upload raw benchmark data + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: maven-config-${{ matrix.os }}-${{ matrix.cache }}-${{ matrix.versions.name }}-${{ matrix.toolchains }} + path: .benchmark-results/ + if-no-files-found: error + retention-days: 30 diff --git a/README.md b/README.md index d1c4774..f862abf 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,12 @@ The report job writes a Markdown summary and uploads raw JSON and CSV files. Ben The **Focused cache restore** workflow isolates the setup step for comparing v4 with `main`. It uses a pinned Temurin JDK from the hosted runner tool cache, seeds a synthetic 160 MiB dependency cache for both versions and a 9 MiB wrapper cache for `main`, and runs no Maven command. Warm measurement jobs therefore contain no JDK or Maven Central downloads; they measure JDK discovery and Actions cache restoration only. +### Maven configuration warm path + +The **Maven configuration warm path** workflow compares two `actions/setup-java` refs in the action's own Maven configuration path. It checks out a configurable setup-java repository, runs baseline and candidate refs on Linux, Windows, and macOS, and covers Maven cache, Gradle cache, no-cache, single-version, multi-version, empty toolchains, and existing toolchains scenarios. + +Each matrix entry alternates three warm in-job setup runs for the baseline and candidate implementations, then reports median and p95 setup time. The workflow also records `dist/setup/index.js`, total `dist/setup` JavaScript bytes, and the JavaScript chunk files containing the XML parser. + ## Reading results The summary reports medians for: @@ -55,4 +61,6 @@ Network throughput, hosted-runner image changes, upstream artifact availability, ```bash npm test +bash -n scripts/*.sh +shellcheck scripts/*.sh ``` diff --git a/scripts/benchmark-maven-configuration.sh b/scripts/benchmark-maven-configuration.sh new file mode 100755 index 0000000..ed6b33c --- /dev/null +++ b/scripts/benchmark-maven-configuration.sh @@ -0,0 +1,163 @@ +#!/usr/bin/env bash + +set -euo pipefail + +command=${1:?command is required} + +benchmark_home="$PWD/benchmark-maven-home" +results_dir="$PWD/.benchmark-results" +results_file="$results_dir/maven-configuration-timings.csv" +sizes_file="$results_dir/maven-configuration-sizes.csv" + +case "$command" in + prepare) + cache=${2:?cache profile is required} + toolchains_profile=${3:?toolchains profile is required} + + rm -rf "$benchmark_home" + mkdir -p "$benchmark_home" benchmark + printf '\n' > benchmark/pom.xml + printf 'plugins { id("java") }\n' > benchmark/build.gradle + + if [ "$toolchains_profile" = "existing" ]; then + cat > "$benchmark_home/toolchains.xml" <<'XML' + + + foo + + preserved + + + /opt/foo + + + +XML + elif [ "$toolchains_profile" != "empty" ]; then + echo "Unsupported toolchains profile: $toolchains_profile" >&2 + exit 1 + fi + + case "$cache" in + none | maven | gradle) ;; + *) + echo "Unsupported cache profile: $cache" >&2 + exit 1 + ;; + esac + ;; + start) + mkdir -p "$results_dir" + node -e "require('fs').writeFileSync('.benchmark-start', String(Date.now()))" + ;; + record) + os=${2:?os is required} + cache=${3:?cache profile is required} + versions=${4:?versions profile is required} + toolchains_profile=${5:?toolchains profile is required} + implementation=${6:?implementation is required} + iteration=${7:?iteration is required} + + started=$(cat .benchmark-start) + finished=$(node -e "process.stdout.write(String(Date.now()))") + elapsed=$((finished - started)) + mkdir -p "$results_dir" + printf '%s,%s,%s,%s,%s,%s,%s\n' \ + "$os" "$cache" "$versions" "$toolchains_profile" "$implementation" "$iteration" "$elapsed" \ + >> "$results_file" + ;; + record-size) + implementation=${2:?implementation is required} + action_path=${3:?action path is required} + + mkdir -p "$results_dir" + index_bytes=$(node -e "const fs=require('fs'); process.stdout.write(String(fs.statSync(process.argv[1]).size))" "$action_path/dist/setup/index.js") + js_bytes=$(node -e "const fs=require('fs'); const path=require('path'); let total=0; for (const entry of fs.readdirSync(process.argv[1])) { if (entry.endsWith('.js')) total += fs.statSync(path.join(process.argv[1], entry)).size; } process.stdout.write(String(total));" "$action_path/dist/setup") + chunk_count=$(find "$action_path/dist/setup" -maxdepth 1 -name '*.js' | wc -l | tr -d ' ') + xml_parser_chunks=$(grep -Rsl "fast-xml-parser" "$action_path/dist/setup"/*.js 2>/dev/null | xargs -n 1 basename 2>/dev/null | paste -sd ';' - || true) + printf '%s,%s,%s,%s\n' \ + "$implementation" "$index_bytes" "$js_bytes" "${xml_parser_chunks:-none} ($chunk_count js files)" \ + >> "$sizes_file" + ;; + summarize) + summary_file=${2:?summary file is required} + + node --input-type=module - "$results_file" "$sizes_file" "$summary_file" <<'NODE' +import fs from 'node:fs'; + +const [, , resultsFile, sizesFile, summaryFile] = process.argv; + +const percentile = (values, percentileValue) => { + const sorted = [...values].sort((left, right) => left - right); + const index = Math.ceil((percentileValue / 100) * sorted.length) - 1; + return sorted[Math.max(0, Math.min(index, sorted.length - 1))]; +}; + +const rows = fs + .readFileSync(resultsFile, 'utf8') + .trim() + .split('\n') + .filter(Boolean) + .map(line => { + const [os, cache, versions, toolchains, implementation, iteration, elapsed] = + line.split(','); + return { + os, + cache, + versions, + toolchains, + implementation, + iteration, + elapsed: Number(elapsed) + }; + }); + +const groups = new Map(); +for (const row of rows) { + const key = [row.os, row.cache, row.versions, row.toolchains, row.implementation].join(','); + const values = groups.get(key) ?? []; + values.push(row.elapsed); + groups.set(key, values); +} + +const lines = [ + '## Maven configuration warm-path benchmark', + '', + '| OS | Cache | Versions | Toolchains | Implementation | Runs | Median (ms) | p95 (ms) |', + '| --- | --- | --- | --- | --- | ---: | ---: | ---: |' +]; + +for (const [key, values] of [...groups.entries()].sort()) { + const [os, cache, versions, toolchains, implementation] = key.split(','); + lines.push( + `| ${os} | ${cache} | ${versions} | ${toolchains} | ${implementation} | ${values.length} | ${percentile(values, 50)} | ${percentile(values, 95)} |` + ); +} + +if (fs.existsSync(sizesFile)) { + lines.push( + '', + '## setup entry/chunk sizes', + '', + '| Implementation | dist/setup/index.js bytes | dist/setup JS bytes | XML parser chunk location |', + '| --- | ---: | ---: | --- |' + ); + for (const line of fs.readFileSync(sizesFile, 'utf8').trim().split('\n')) { + if (!line) continue; + const [implementation, indexBytes, jsBytes, xmlParserChunks] = line.split(','); + lines.push( + `| ${implementation} | ${indexBytes} | ${jsBytes} | ${xmlParserChunks} |` + ); + } +} + +fs.appendFileSync(summaryFile, `${lines.join('\n')}\n`); +NODE + ;; + *) + echo "Unsupported command: $command" >&2 + exit 1 + ;; +esac