Add Maven configuration warm-path benchmark

.github/workflows/maven-configuration-warm-path.yml

@@ -0,0 +1,173 @@
+name: Maven configuration warm path
+
+on:
+  workflow_dispatch:
+    inputs:
+      setup-java-repository:
+        description: Repository containing the setup-java action
+        required: true
+        default: actions/setup-java
+        type: string
+      baseline-ref:
+        description: Git ref containing the baseline implementation
+        required: true
+        default: main
+        type: string
+      candidate-ref:
+        description: Git ref containing the candidate implementation
+        required: true
+        default: main
+        type: string
+
+permissions:
+  contents: read
+
+defaults:
+  run:
+    shell: bash
+
+concurrency:
+  group: maven-configuration-warm-path
+  cancel-in-progress: false
+
+jobs:
+  benchmark:
+    name: ${{ matrix.os }} ${{ matrix.cache }} ${{ matrix.versions.name }} ${{ matrix.toolchains }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-15-intel]
+        cache: [none, maven, gradle]
+        versions:
+          - name: single
+            java-version: "21"
+          - name: multiple
+            java-version: |
+              17
+              21
+        toolchains: [empty, existing]
+    steps:
+      - name: Check out benchmark repository
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          persist-credentials: false
+      - name: Check out baseline setup-java
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          repository: ${{ inputs.setup-java-repository }}
+          path: baseline
+          persist-credentials: false
+          ref: ${{ inputs.baseline-ref }}
+      - name: Check out candidate setup-java
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          repository: ${{ inputs.setup-java-repository }}
+          path: candidate
+          persist-credentials: false
+          ref: ${{ inputs.candidate-ref }}
+
+      - name: Record setup dist sizes
+        run: |
+          bash scripts/benchmark-maven-configuration.sh record-size baseline baseline
+          bash scripts/benchmark-maven-configuration.sh record-size candidate candidate
+
+      - name: Prepare baseline iteration 1
+        run: bash scripts/benchmark-maven-configuration.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}"
+      - name: Start baseline iteration 1 timer
+        run: bash scripts/benchmark-maven-configuration.sh start
+      - name: Run baseline iteration 1
+        uses: ./baseline
+        with:
+          distribution: temurin
+          java-version: ${{ matrix.versions.java-version }}
+          cache: ${{ matrix.cache == 'none' && '' || matrix.cache }}
+          cache-dependency-path: benchmark/${{ matrix.cache == 'gradle' && 'build.gradle' || 'pom.xml' }}
+          settings-path: benchmark-maven-home
+      - name: Record baseline iteration 1
+        run: bash scripts/benchmark-maven-configuration.sh record "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.versions.name }}" "${{ matrix.toolchains }}" baseline 1
+
+      - name: Prepare candidate iteration 1
+        run: bash scripts/benchmark-maven-configuration.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}"
+      - name: Start candidate iteration 1 timer
+        run: bash scripts/benchmark-maven-configuration.sh start
+      - name: Run candidate iteration 1
+        uses: ./candidate
+        with:
+          distribution: temurin
+          java-version: ${{ matrix.versions.java-version }}
+          cache: ${{ matrix.cache == 'none' && '' || matrix.cache }}
+          cache-dependency-path: benchmark/${{ matrix.cache == 'gradle' && 'build.gradle' || 'pom.xml' }}
+          settings-path: benchmark-maven-home
+      - name: Record candidate iteration 1
+        run: bash scripts/benchmark-maven-configuration.sh record "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.versions.name }}" "${{ matrix.toolchains }}" candidate 1
+
+      - name: Prepare baseline iteration 2
+        run: bash scripts/benchmark-maven-configuration.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}"
+      - name: Start baseline iteration 2 timer
+        run: bash scripts/benchmark-maven-configuration.sh start
+      - name: Run baseline iteration 2
+        uses: ./baseline
+        with:
+          distribution: temurin
+          java-version: ${{ matrix.versions.java-version }}
+          cache: ${{ matrix.cache == 'none' && '' || matrix.cache }}
+          cache-dependency-path: benchmark/${{ matrix.cache == 'gradle' && 'build.gradle' || 'pom.xml' }}
+          settings-path: benchmark-maven-home
+      - name: Record baseline iteration 2
+        run: bash scripts/benchmark-maven-configuration.sh record "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.versions.name }}" "${{ matrix.toolchains }}" baseline 2
+
+      - name: Prepare candidate iteration 2
+        run: bash scripts/benchmark-maven-configuration.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}"
+      - name: Start candidate iteration 2 timer
+        run: bash scripts/benchmark-maven-configuration.sh start
+      - name: Run candidate iteration 2
+        uses: ./candidate
+        with:
+          distribution: temurin
+          java-version: ${{ matrix.versions.java-version }}
+          cache: ${{ matrix.cache == 'none' && '' || matrix.cache }}
+          cache-dependency-path: benchmark/${{ matrix.cache == 'gradle' && 'build.gradle' || 'pom.xml' }}
+          settings-path: benchmark-maven-home
+      - name: Record candidate iteration 2
+        run: bash scripts/benchmark-maven-configuration.sh record "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.versions.name }}" "${{ matrix.toolchains }}" candidate 2
+
+      - name: Prepare baseline iteration 3
+        run: bash scripts/benchmark-maven-configuration.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}"
+      - name: Start baseline iteration 3 timer
+        run: bash scripts/benchmark-maven-configuration.sh start
+      - name: Run baseline iteration 3
+        uses: ./baseline
+        with:
+          distribution: temurin
+          java-version: ${{ matrix.versions.java-version }}
+          cache: ${{ matrix.cache == 'none' && '' || matrix.cache }}
+          cache-dependency-path: benchmark/${{ matrix.cache == 'gradle' && 'build.gradle' || 'pom.xml' }}
+          settings-path: benchmark-maven-home
+      - name: Record baseline iteration 3
+        run: bash scripts/benchmark-maven-configuration.sh record "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.versions.name }}" "${{ matrix.toolchains }}" baseline 3
+
+      - name: Prepare candidate iteration 3
+        run: bash scripts/benchmark-maven-configuration.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}"
+      - name: Start candidate iteration 3 timer
+        run: bash scripts/benchmark-maven-configuration.sh start
+      - name: Run candidate iteration 3
+        uses: ./candidate
+        with:
+          distribution: temurin
+          java-version: ${{ matrix.versions.java-version }}
+          cache: ${{ matrix.cache == 'none' && '' || matrix.cache }}
+          cache-dependency-path: benchmark/${{ matrix.cache == 'gradle' && 'build.gradle' || 'pom.xml' }}
+          settings-path: benchmark-maven-home
+      - name: Record candidate iteration 3
+        run: bash scripts/benchmark-maven-configuration.sh record "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.versions.name }}" "${{ matrix.toolchains }}" candidate 3
+
+      - name: Summarize benchmark
+        run: bash scripts/benchmark-maven-configuration.sh summarize "$GITHUB_STEP_SUMMARY"
+      - name: Upload raw benchmark data
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: maven-config-${{ matrix.os }}-${{ matrix.cache }}-${{ matrix.versions.name }}-${{ matrix.toolchains }}
+          path: .benchmark-results/
+          if-no-files-found: error
+          retention-days: 30

README.md

@@ -37,6 +37,12 @@ The report job writes a Markdown summary and uploads raw JSON and CSV files. Ben
 
 The **Focused cache restore** workflow isolates the setup step for comparing v4 with `main`. It uses a pinned Temurin JDK from the hosted runner tool cache, seeds a synthetic 160 MiB dependency cache for both versions and a 9 MiB wrapper cache for `main`, and runs no Maven command. Warm measurement jobs therefore contain no JDK or Maven Central downloads; they measure JDK discovery and Actions cache restoration only.
 
+### Maven configuration warm path
+
+The **Maven configuration warm path** workflow compares two `actions/setup-java` refs in the action's own Maven configuration path. It checks out a configurable setup-java repository, runs baseline and candidate refs on Linux, Windows, and macOS, and covers Maven cache, Gradle cache, no-cache, single-version, multi-version, empty toolchains, and existing toolchains scenarios.
+
+Each matrix entry alternates three warm in-job setup runs for the baseline and candidate implementations, then reports median and p95 setup time. The workflow also records `dist/setup/index.js`, total `dist/setup` JavaScript bytes, and the JavaScript chunk files containing the XML parser.
+
 ## Reading results
 
 The summary reports medians for:
@@ -55,4 +61,6 @@ Network throughput, hosted-runner image changes, upstream artifact availability,
 
 ```bash
 npm test
+bash -n scripts/*.sh
+shellcheck scripts/*.sh
 ```

scripts/benchmark-maven-configuration.sh

@@ -0,0 +1,163 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+command=${1:?command is required}
+
+benchmark_home="$PWD/benchmark-maven-home"
+results_dir="$PWD/.benchmark-results"
+results_file="$results_dir/maven-configuration-timings.csv"
+sizes_file="$results_dir/maven-configuration-sizes.csv"
+
+case "$command" in
+  prepare)
+    cache=${2:?cache profile is required}
+    toolchains_profile=${3:?toolchains profile is required}
+
+    rm -rf "$benchmark_home"
+    mkdir -p "$benchmark_home" benchmark
+    printf '<project />\n' > benchmark/pom.xml
+    printf 'plugins { id("java") }\n' > benchmark/build.gradle
+
+    if [ "$toolchains_profile" = "existing" ]; then
+      cat > "$benchmark_home/toolchains.xml" <<'XML'
+<toolchains xmlns="http://maven.apache.org/TOOLCHAINS/1.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/TOOLCHAINS/1.0.0 http://maven.apache.org/xsd/toolchains-1.0.0.xsd">
+  <toolchain>
+    <type>foo</type>
+    <provides>
+      <custom>preserved</custom>
+    </provides>
+    <configuration>
+      <fooHome>/opt/foo</fooHome>
+    </configuration>
+  </toolchain>
+</toolchains>
+XML
+    elif [ "$toolchains_profile" != "empty" ]; then
+      echo "Unsupported toolchains profile: $toolchains_profile" >&2
+      exit 1
+    fi
+
+    case "$cache" in
+      none | maven | gradle) ;;
+      *)
+        echo "Unsupported cache profile: $cache" >&2
+        exit 1
+        ;;
+    esac
+    ;;
+  start)
+    mkdir -p "$results_dir"
+    node -e "require('fs').writeFileSync('.benchmark-start', String(Date.now()))"
+    ;;
+  record)
+    os=${2:?os is required}
+    cache=${3:?cache profile is required}
+    versions=${4:?versions profile is required}
+    toolchains_profile=${5:?toolchains profile is required}
+    implementation=${6:?implementation is required}
+    iteration=${7:?iteration is required}
+
+    started=$(cat .benchmark-start)
+    finished=$(node -e "process.stdout.write(String(Date.now()))")
+    elapsed=$((finished - started))
+    mkdir -p "$results_dir"
+    printf '%s,%s,%s,%s,%s,%s,%s\n' \
+      "$os" "$cache" "$versions" "$toolchains_profile" "$implementation" "$iteration" "$elapsed" \
+      >> "$results_file"
+    ;;
+  record-size)
+    implementation=${2:?implementation is required}
+    action_path=${3:?action path is required}
+
+    mkdir -p "$results_dir"
+    index_bytes=$(node -e "const fs=require('fs'); process.stdout.write(String(fs.statSync(process.argv[1]).size))" "$action_path/dist/setup/index.js")
+    js_bytes=$(node -e "const fs=require('fs'); const path=require('path'); let total=0; for (const entry of fs.readdirSync(process.argv[1])) { if (entry.endsWith('.js')) total += fs.statSync(path.join(process.argv[1], entry)).size; } process.stdout.write(String(total));" "$action_path/dist/setup")
+    chunk_count=$(find "$action_path/dist/setup" -maxdepth 1 -name '*.js' | wc -l | tr -d ' ')
+    xml_parser_chunks=$(grep -Rsl "fast-xml-parser" "$action_path/dist/setup"/*.js 2>/dev/null | xargs -n 1 basename 2>/dev/null | paste -sd ';' - || true)
+    printf '%s,%s,%s,%s\n' \
+      "$implementation" "$index_bytes" "$js_bytes" "${xml_parser_chunks:-none} ($chunk_count js files)" \
+      >> "$sizes_file"
+    ;;
+  summarize)
+    summary_file=${2:?summary file is required}
+
+    node --input-type=module - "$results_file" "$sizes_file" "$summary_file" <<'NODE'
+import fs from 'node:fs';
+
+const [, , resultsFile, sizesFile, summaryFile] = process.argv;
+
+const percentile = (values, percentileValue) => {
+  const sorted = [...values].sort((left, right) => left - right);
+  const index = Math.ceil((percentileValue / 100) * sorted.length) - 1;
+  return sorted[Math.max(0, Math.min(index, sorted.length - 1))];
+};
+
+const rows = fs
+  .readFileSync(resultsFile, 'utf8')
+  .trim()
+  .split('\n')
+  .filter(Boolean)
+  .map(line => {
+    const [os, cache, versions, toolchains, implementation, iteration, elapsed] =
+      line.split(',');
+    return {
+      os,
+      cache,
+      versions,
+      toolchains,
+      implementation,
+      iteration,
+      elapsed: Number(elapsed)
+    };
+  });
+
+const groups = new Map();
+for (const row of rows) {
+  const key = [row.os, row.cache, row.versions, row.toolchains, row.implementation].join(',');
+  const values = groups.get(key) ?? [];
+  values.push(row.elapsed);
+  groups.set(key, values);
+}
+
+const lines = [
+  '## Maven configuration warm-path benchmark',
+  '',
+  '| OS | Cache | Versions | Toolchains | Implementation | Runs | Median (ms) | p95 (ms) |',
+  '| --- | --- | --- | --- | --- | ---: | ---: | ---: |'
+];
+
+for (const [key, values] of [...groups.entries()].sort()) {
+  const [os, cache, versions, toolchains, implementation] = key.split(',');
+  lines.push(
+    `| ${os} | ${cache} | ${versions} | ${toolchains} | ${implementation} | ${values.length} | ${percentile(values, 50)} | ${percentile(values, 95)} |`
+  );
+}
+
+if (fs.existsSync(sizesFile)) {
+  lines.push(
+    '',
+    '## setup entry/chunk sizes',
+    '',
+    '| Implementation | dist/setup/index.js bytes | dist/setup JS bytes | XML parser chunk location |',
+    '| --- | ---: | ---: | --- |'
+  );
+  for (const line of fs.readFileSync(sizesFile, 'utf8').trim().split('\n')) {
+    if (!line) continue;
+    const [implementation, indexBytes, jsBytes, xmlParserChunks] = line.split(',');
+    lines.push(
+      `| ${implementation} | ${indexBytes} | ${jsBytes} | ${xmlParserChunks} |`
+    );
+  }
+}
+
+fs.appendFileSync(summaryFile, `${lines.join('\n')}\n`);
+NODE
+    ;;
+  *)
+    echo "Unsupported command: $command" >&2
+    exit 1
+    ;;
+esac