diff --git a/.github/workflows/conda-audit.yml b/.github/workflows/conda-audit.yml new file mode 100644 index 000000000..5af86fa46 --- /dev/null +++ b/.github/workflows/conda-audit.yml @@ -0,0 +1,118 @@ +--- +# N3: PR-triggered conda build + masking-immune RUNPATH audit. +# +# The OneBranch conda-build pipeline is `trigger: none` / `pr: none`, so its blocking +# audit never runs on a PR. This lightweight GitHub Actions job builds ONE real +# linux-64 conda package from the SHIPPED PyPI wheels and runs +# python -m eng.conda_tools elf on it, so a regression in the $ORIGIN climb, +# the declared conda deps (krb5/libtool/openssl), or the expected DT_NEEDED set fails +# the PR automatically -- the full-agent runtime masking cannot hide it. +# +# build.sh ALWAYS stamps the relative $ORIGIN climb onto the vendored Linux ODBC .so +# (they are not code-signed, so the patch breaks no signature; an already-baked climb +# is a byte-for-byte no-op). This gate builds one real linux-64 conda package from the +# shipped PyPI wheels and audits the resulting climb (see conda/mssql-python/build.sh). +name: conda-audit + +on: + pull_request: + paths: + - 'conda/**' + - 'eng/conda_tools/**' + - 'eng/scripts/download_mssql_python_rs_wheels.py' + - 'eng/scripts/resolve_nuget_feed.py' + - 'eng/scripts/mssql_python_build_safety.py' + - 'eng/versions/mssql-python-rs*.version' + - 'OneBranchPipelines/conda-build-pipeline.yml' + - 'OneBranchPipelines/steps/conda-build-validate-step*.yml' + - 'OneBranchPipelines/jobs/consolidate-conda-artifacts-job.yml' + - 'OneBranchPipelines/jobs/consolidate-artifacts-job.yml' + - 'OneBranchPipelines/variables/*.yml' + - 'OneBranchPipelines/conda-release-pipeline.yml' + - 'OneBranchPipelines/steps/conda-release-step.yml' + - 'OneBranchPipelines/steps/conda-publish-step.yml' + - 'tests/test_027_conda_release_metadata.py' + - 'tests/test_029_bundled_binary_audit.py' + - 'tests/test_030_pe_machine_assert.py' + - 'tests/test_033_driver_load_probe.py' + - 'tests/test_034_conda_verify_cwd.py' + - 'tests/test_035_conda_macho_assert.py' + - 'tests/test_036_conda_provenance.py' + - 'tests/test_037_rs_wheel_download.py' + - 'tests/test_038_conda_archive_limits.py' + - 'requirements.txt' + - 'setup.py' + - 'mssql_python/__init__.py' + - 'mssql_python_odbc/__init__.py' + - '.github/workflows/conda-audit.yml' + +permissions: + contents: read + +jobs: + linux-conda-audit: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.11' + + - name: Unit-test the audit + release validator + run: | + python -m pip install --quiet --require-hashes --only-binary=:all: -r conda/requirements-audit.txt + # --noconftest: tests/conftest.py imports mssql_python (the native ddbc_bindings + # extension), which is NOT built in this repackage-only gate. test_027/test_029 + # are pure conda validator/audit tests using only built-in fixtures, so skip + # conftest to avoid that unrelated import. + python -m pytest --noconftest \ + tests/test_029_bundled_binary_audit.py \ + tests/test_030_pe_machine_assert.py \ + tests/test_033_driver_load_probe.py \ + tests/test_034_conda_verify_cwd.py \ + tests/test_035_conda_macho_assert.py \ + tests/test_036_conda_provenance.py \ + tests/test_037_rs_wheel_download.py \ + tests/test_038_conda_archive_limits.py \ + tests/test_027_conda_release_metadata.py -q + + - name: Fetch the shipped linux wheels from PyPI + id: public-wheels + working-directory: ${{ github.workspace }} + run: | + set -euo pipefail + # Exact maintained release versions and PyPI digests, never an older/latest fallback. + # Fetch RS only if binding METADATA requires it, using its distribution (not NuGet) pin. + # Embedded-core public wheels are historical controls, not current-source RS proof. + python -m eng.conda_tools fetch-wheels \ + --wheel-dir "$PWD/wheels" --requirements-file "$PWD/wheel-inputs.txt" \ + --python-tag cp311 --conda-subdir linux-64 + echo "Downloaded:"; ls -1 wheels + + - name: Build + audit a linux-64 conda package + working-directory: ${{ github.workspace }} + env: + RS_REQUIRED: ${{ steps.public-wheels.outputs.rsRequired }} + run: | + set -euo pipefail + # The verified public dependency/ownership facts select the actual input set. + rsArgs=() + if [[ "$RS_REQUIRED" == "true" ]]; then + rsArgs=(--rs-wheel-dir "$PWD/wheels" --rs-version-file "$PWD/eng/versions/mssql-python-rs.version") + elif [[ "$RS_REQUIRED" != "false" ]]; then + echo "Missing verified public wheel profile"; exit 1 + fi + python -m eng.conda_tools build \ + "${rsArgs[@]}" \ + --mssql-wheel-dir "$PWD/wheels" \ + --odbc-wheel-dir "$PWD/wheels" \ + --odbc-wheel-filter "mssql_python_odbc-*.whl" \ + --recipe-root "$PWD/conda" \ + --output-dir "$RUNNER_TEMP/conda-bld" \ + --stage-dir "$RUNNER_TEMP/conda-stage" \ + --conda-subdir "linux-64" \ + --python-versions "3.11" diff --git a/OneBranchPipelines/conda-release-pipeline.yml b/OneBranchPipelines/conda-release-pipeline.yml new file mode 100644 index 000000000..a2611e809 --- /dev/null +++ b/OneBranchPipelines/conda-release-pipeline.yml @@ -0,0 +1,263 @@ +# ========================================================================================= +# OneBranch STANDALONE Conda Release Pipeline (mssql-python) +# ========================================================================================= +# Decoupled from the wheel/PyPI release. Downloads the consolidated conda packages produced +# by the STANDALONE conda-build pipeline (conda-build-pipeline.yml, artifact +# drop_ConsolidateConda_ConsolidateArtifacts) -- NOT the wheel pipeline (def 2199), which no +# longer produces any conda artifact -- enforces the release-readiness gate (conda-release- +# step: required subdirs + full Python matrix + one version), and -- when publishToConda=true -- publishes the SELF-CONTAINED +# mssql-python conda package to Anaconda.org via anaconda-client (conda-publish-step; ESRP +# has no Conda ContentType). There is a single self-contained package (it vendors the ODBC +# Driver 18 payload); no companion. When publishToConda=false (the default), the same gate +# runs and a DRY-RUN step prints the exact `anaconda upload` plan WITHOUT uploading -- so a +# validate-only run of THIS pipeline replaces the old separate dummy pipeline. +# +# TEMPLATE by mode: publishToConda=false (dry run / dummy) extends the NonOfficial template +# (no Production Service-Tree classification required, so a validate-only run passes the +# inventory-compliance gate); publishToConda=true (real release) extends the Official template +# (validated production publish -- the definition must be classified Production to +# upload). Review the dry-run output, then explicitly select publishToConda=true. +# An administrator must enable the native Exclusive lock check on 'Anaconda Publishing'. +# The production stage uses that shared resource throughout publication and recovery. +# Keep its credentials, pipeline authorizations and other checks restricted. +# The OneBranch YAML needs an actual ADO run to fully validate. +# ========================================================================================= +name: $(Year:YY)$(DayOfYear)$(Rev:.r)-CondaRelease + +trigger: none +pr: none + +parameters: + - name: publishToConda + displayName: 'Publish Conda Packages to Anaconda.org (PRODUCTION)' + type: boolean + default: false # Safety: default to a validate-only dry run. + - name: condaChannel + displayName: 'Anaconda.org channel/org to publish to' + type: string + default: 'microsoft' + values: + - 'microsoft' + - name: condaLabel + displayName: 'Anaconda.org channel label' + type: string + default: 'main' + values: + - 'main' + # Automatically resolve the exact version from the recorded wheel producer's source. + # An optional override is an additional assertion, never a replacement for that source. + - name: mssqlPythonVersion + displayName: 'Optional expected mssql-python version (empty = resolve from wheel producer)' + type: string + default: '' + +variables: + - template: /OneBranchPipelines/variables/common-variables.yml@self + - template: /OneBranchPipelines/variables/onebranch-variables.yml@self + +resources: + repositories: + - repository: templates + type: git + name: 'OneBranch.Pipelines/GovernedTemplates' + ref: 'refs/heads/main' + # The build pipeline whose consolidated conda artifact this pipeline validates + publishes. + pipelines: + - pipeline: buildPipeline + # Registered ADO definition Conda-Build-Pipeline, backed by conda-build-pipeline.yml. Its + # ConsolidateConda stage publishes drop_ConsolidateConda_ConsolidateArtifacts. + # This is not the wheel pipeline (definition 2199), which no longer builds Conda. + source: 'Conda-Build-Pipeline' + branch: main + trigger: none + +extends: + # NonOfficial for a validate-only dry run (publishToConda=false): no Production Service-Tree + # classification is required, so the inventory-compliance gate passes. Official for a real + # publish (publishToConda=true): the governed production path. + ${{ if eq(parameters.publishToConda, true) }}: + template: 'v2/OneBranch.Official.CrossPlat.yml@templates' + ${{ else }}: + template: 'v2/OneBranch.NonOfficial.CrossPlat.yml@templates' + parameters: + featureFlags: + WindowsHostVersion: + Version: '2022' + globalSdl: + baseline: + baselineFile: $(Build.SourcesDirectory)/.gdn/.gdnbaselines + suppressionSet: default + suppression: + suppressionFile: $(Build.SourcesDirectory)/.gdn/.gdnsuppress + suppressionSet: default + credscan: + enabled: true + policheck: + enabled: true + break: true + exclusionFile: '$(REPO_ROOT)/.config/PolicheckExclusions.xml' + publishLogs: + enabled: true + # Official-only heavy gates (binskim break + TSA). The NonOfficial dry run skips them, + # matching the old dummy pipeline's minimal SDL. + ${{ if eq(parameters.publishToConda, true) }}: + binskim: + enabled: true + break: true + tsa: + enabled: true + configFile: '$(REPO_ROOT)/.config/tsaoptions.json' + + stages: + - stage: CondaRelease + displayName: 'Validate & Publish Conda Release' + ${{ if eq(parameters.publishToConda, true) }}: + lockBehavior: sequential + variables: + - group: 'Anaconda Publishing' + jobs: + # Gate: prove the consolidated conda set is complete (required subdirs + + # full Python matrix + one version) before anything is published. + - job: ValidateConda + displayName: 'Validate consolidated conda packages' + pool: + type: windows + isCustom: true + name: Python-1ES-pool + demands: + - imageOverride -equals PYTHON-1ES-MMS2022 + variables: + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + steps: + - checkout: self + fetchDepth: 1 + - task: UsePythonVersion@0 + displayName: 'Use Python 3.12 for provenance verification' + inputs: + versionSpec: '3.12' + addToPath: true + - task: PowerShell@2 + name: Provenance + displayName: 'Guard: recorded Conda and upstream wheel run provenance' + env: + CONDA_BUILD_PIPELINE_ID: $(resources.pipeline.buildPipeline.pipelineID) + CONDA_BUILD_RUN_ID: $(resources.pipeline.buildPipeline.runID) + CONDA_BUILD_SOURCE_BRANCH: $(resources.pipeline.buildPipeline.sourceBranch) + CONDA_BUILD_SOURCE_COMMIT: $(resources.pipeline.buildPipeline.sourceCommit) + RELEASE_SOURCE_BRANCH: $(Build.SourceBranch) + PUBLISH_TO_CONDA: '${{ parameters.publishToConda }}' + MSSQL_PYTHON_VERSION: '${{ parameters.mssqlPythonVersion }}' + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + SYSTEM_COLLECTIONURI: $(System.CollectionUri) + SYSTEM_TEAMPROJECTID: $(System.TeamProjectId) + inputs: + targetType: 'inline' + workingDirectory: '$(Build.SourcesDirectory)' + script: | + $ErrorActionPreference = 'Stop' + python -m eng.conda_tools provenance + if ($LASTEXITCODE -ne 0) { + Write-Error "Recorded producer/wheel provenance verification failed." + exit 1 + } + + - template: /OneBranchPipelines/steps/conda-release-step.yml@self + parameters: + mssqlPythonVersion: '$(Provenance.mssqlPythonVersion)' + releaseVersions: '$(Provenance.releaseVersions)' + rsTransportVersion: '$(Provenance.rsTransportVersion)' + + # Dry run (publishToConda=false): enumerate every package and print the exact + # `anaconda upload` plan WITHOUT uploading, so a validate-only run shows precisely + # what a real publish WOULD do. Reuses the conda tree the gate already downloaded + # into $(Build.SourcesDirectory)/conda-artifacts. This replaces the old dummy pipeline. + - ${{ if eq(parameters.publishToConda, false) }}: + - task: PowerShell@2 + displayName: 'Dry-run publish plan (validate-only): enumerate packages + print the anaconda upload plan (NO upload)' + env: + CONDA_CHANNEL: '${{ parameters.condaChannel }}' + CONDA_LABEL: '${{ parameters.condaLabel }}' + inputs: + targetType: 'inline' + script: | + $ErrorActionPreference = 'Stop' + + $root = "$(Build.SourcesDirectory)/conda-artifacts/conda" + if (-not (Test-Path $root)) { + Write-Error "Consolidated conda tree not found at $root. Did the readiness gate download it?" + exit 1 + } + + $pkgs = @(Get-ChildItem -Path $root -Recurse -Include *.conda, *.tar.bz2) + if ($pkgs.Count -eq 0) { Write-Error "No conda packages found under $root."; exit 1 } + $identifierPattern = '^[A-Za-z0-9][A-Za-z0-9._-]*$' + if ($env:CONDA_CHANNEL -notmatch $identifierPattern) { + Write-Error "Invalid condaChannel '$env:CONDA_CHANNEL'. Use letters, digits, dot, underscore, or hyphen; the first character must be alphanumeric." + exit 1 + } + if ($env:CONDA_LABEL -notmatch $identifierPattern) { + Write-Error "Invalid condaLabel '$env:CONDA_LABEL'. Use letters, digits, dot, underscore, or hyphen; the first character must be alphanumeric." + exit 1 + } + + # mssql-python bindings only (the self-contained model ships no companion). + function Get-Binding($items) { @($items | Where-Object { $_.Name -like 'mssql-python-*' -and $_.Name -notlike 'mssql-python-odbc-*' }) } + $bySubdir = $pkgs | Group-Object { $_.Directory.Name } | Sort-Object Name + + Write-Host "=====================================" + Write-Host "[DRY-RUN CONDA RELEASE] discovered $($pkgs.Count) package(s) across $($bySubdir.Count) subdir(s):" + foreach ($g in $bySubdir) { + Write-Host (" {0,-14} {1} mssql-python package(s)" -f $g.Name, (Get-Binding $g.Group).Count) + } + + # A self-contained release ships ONLY mssql-python; flag (don't fail) a stray companion. + $stray = @($pkgs | Where-Object { $_.Name -like 'mssql-python-odbc-*' }) + if ($stray.Count -gt 0) { + Write-Host "##vso[task.logissue type=warning]$($stray.Count) stray mssql-python-odbc package(s) present; the self-contained model ships only mssql-python." + } + + Write-Host "`nanaconda upload PLAN (DRY-RUN - NOT executed):" + Write-Host " target channel/org : $env:CONDA_CHANNEL" + Write-Host " target label : $env:CONDA_LABEL" + foreach ($p in ($pkgs | Sort-Object FullName)) { + $subdir = Split-Path -Leaf (Split-Path -Parent $p.FullName) + $sha256 = (Get-FileHash -LiteralPath $p.FullName -Algorithm SHA256).Hash.ToLowerInvariant() + Write-Host " SHA256: $sha256 $subdir/$($p.Name)" + Write-Host " WOULD RUN: anaconda upload --user $env:CONDA_CHANNEL --label $env:CONDA_LABEL --skip-existing $subdir/$($p.Name)" + } + + Write-Host "`n=====================================" + Write-Host "DRY-RUN COMPLETE: no 'anaconda upload' was executed (publishToConda=false)." + Write-Host "Review this dry run and verify the shared publishing-resource checks before selecting publishToConda=true." + Write-Host "=====================================" + + # PRODUCTION publish (releaseJob) -- runs ONLY when publishToConda=true and + # ONLY after the ValidateConda gate succeeds, so an incomplete set is never + # uploaded. anaconda-client reads ANACONDA_API_TOKEN from the env (never the + # command line), so the token never appears in the logs. + - ${{ if eq(parameters.publishToConda, true) }}: + - job: PublishConda + displayName: 'Publish conda packages to Anaconda.org' + dependsOn: ValidateConda + timeoutInMinutes: 60 + cancelTimeoutInMinutes: 5 + templateContext: + type: releaseJob + isProduction: true + pool: + type: windows + variables: + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + mssqlPythonVersion: $[ dependencies.ValidateConda.outputs['Provenance.mssqlPythonVersion'] ] + releaseVersions: $[ dependencies.ValidateConda.outputs['Provenance.releaseVersions'] ] + rsTransportVersion: $[ dependencies.ValidateConda.outputs['Provenance.rsTransportVersion'] ] + steps: + - checkout: self + fetchDepth: 1 + - template: /OneBranchPipelines/steps/conda-publish-step.yml@self + parameters: + condaChannel: '${{ parameters.condaChannel }}' + condaLabel: '${{ parameters.condaLabel }}' + mssqlPythonVersion: '$(mssqlPythonVersion)' + releaseVersions: '$(releaseVersions)' + rsTransportVersion: '$(rsTransportVersion)' diff --git a/OneBranchPipelines/steps/conda-publish-step.yml b/OneBranchPipelines/steps/conda-publish-step.yml new file mode 100644 index 000000000..af084951d --- /dev/null +++ b/OneBranchPipelines/steps/conda-publish-step.yml @@ -0,0 +1,406 @@ +# Conda Publish Step Template +# ============================================================================ +# Review a dry run before explicitly publishing. Keep publishing credentials restricted. +# Publishes the consolidated conda packages from the STANDALONE conda-build +# pipeline (conda-build-pipeline.yml, artifact +# drop_ConsolidateConda_ConsolidateArtifacts) to an Anaconda.org channel using +# anaconda-client (`anaconda upload`). +# +# Reference pattern: azure-sdk-for-python's conda publish +# (eng/pipelines/templates/stages/archetype-conda-release.yml), which runs +# `anaconda upload --user Microsoft --skip-existing` inside a 1ES releaseJob and +# authenticates via the ANACONDA_API_TOKEN env var. ESRP has NO Conda ContentType, +# so anaconda-client is the sanctioned publish path. +# +# Two adaptations vs. azure-sdk: +# 1. Our packages are NON-noarch (native binding + driver), so they live under +# per-platform subdirs (win-64 / osx-arm64 / linux-64), NOT a single noarch +# folder. We recurse every subdir. +# 2. The self-contained mssql-python package vendors the ODBC payload, so there is +# no separate companion to order; we still refuse to publish an incomplete or +# mislabeled set. +# +# The caller MUST: +# - run this ONLY after the conda-release-step.yml readiness gate has passed, and +# - keep the entire step in a stage consuming the shared publishing variable group +# with its administrator-configured native Exclusive lock check, and +# - supply ANACONDA_API_TOKEN (variable group 'Anaconda Publishing') to the job. +# This template never puts the token on the command line (anaconda-client reads it +# from the env), so it never appears in the logs. +parameters: + # Consolidated conda artifact name (see consolidate-conda-artifacts-job.yml). + - name: condaArtifactName + type: string + default: 'drop_ConsolidateConda_ConsolidateArtifacts' + # Target Anaconda.org channel/org (e.g. 'microsoft'). Empty is rejected so a + # test pipeline can never accidentally push to the production channel. + - name: condaChannel + type: string + default: '' + # Channel label to publish under (production packages go to 'main'). + - name: condaLabel + type: string + default: 'main' + # Exact release version; required again at the credentialed publish boundary so + # this job never relies only on validation performed by a prior job. + - name: mssqlPythonVersion + type: string + default: '' + - name: releaseVersions + type: string + default: '' + - name: rsTransportVersion + type: string + default: '' + # Comma-separated subdirs a complete release MUST contain (full PyPI parity minus + # musllinux, which has no conda subdir). win-arm64 IS included -- the build enforces + # its --dry-run solve + PE-arch assert, so a missing/broken win-arm64 blocks the + # release. Guards against publishing an incomplete set even if this step is run + # standalone. Keep in sync with conda-release-step.yml. + - name: requiredSubdirs + type: string + default: 'win-64,win-arm64,osx-64,osx-arm64,linux-64,linux-aarch64' + # Comma-separated superset of subdirs allowed to appear; anything else fails. + - name: allowedSubdirs + type: string + default: 'win-64,win-arm64,osx-64,osx-arm64,linux-64,linux-aarch64' + # Expected Python matrix, mirrored from conda-release-step.yml and rechecked + # against this job's independently downloaded artifact before upload. + - name: pythonVersions + type: string + default: '3.10,3.11,3.12,3.13,3.14' + # The publisher hash lock targets CPython 3.12 / Windows x64; other runtimes reject. + - name: pythonVersion + type: string + default: '3.12' + # Optional display-name prefix (e.g. '[TEST] ' for the dummy pipeline). + - name: labelPrefix + type: string + default: '' + +steps: + - task: DownloadPipelineArtifact@2 + displayName: '${{ parameters.labelPrefix }}Download consolidated conda packages (publish)' + inputs: + buildType: 'specific' + project: '$(System.TeamProject)' + definition: $(resources.pipeline.buildPipeline.pipelineID) + buildVersionToDownload: 'specific' + buildId: $(resources.pipeline.buildPipeline.runID) + artifactName: '${{ parameters.condaArtifactName }}' + targetPath: '$(Build.SourcesDirectory)/conda-artifacts' + + - task: UsePythonVersion@0 + displayName: '${{ parameters.labelPrefix }}Use Python ${{ parameters.pythonVersion }}' + inputs: + versionSpec: '${{ parameters.pythonVersion }}' + addToPath: true + + - task: PowerShell@2 + displayName: '${{ parameters.labelPrefix }}Install anaconda-client' + env: + ANACONDA_API_TOKEN: '' + SYSTEM_ACCESSTOKEN: '' + inputs: + targetType: 'inline' + script: | + $ErrorActionPreference = 'Stop' + python -c "import sys, sysconfig; assert sys.implementation.name == 'cpython' and sys.version_info[:2] == (3, 12) and sys.platform == 'win32' and sysconfig.get_platform() == 'win-amd64', 'Publisher lock requires CPython 3.12 Windows x64'" + if ($LASTEXITCODE -ne 0) { throw "Publisher runtime does not match the reviewed hash lock." } + $lock = "$(Build.SourcesDirectory)/conda/requirements-publish.txt" + if (-not (Test-Path -LiteralPath $lock -PathType Leaf)) { throw "Publisher dependency lock is missing." } + # A fresh venv avoids trusting already-installed packages or their startup hooks. + # Only the later publishing task maps credentials into its environment. + $publisherEnvironment = Join-Path "$(Agent.TempDirectory)" ("conda-publisher-" + [guid]::NewGuid().ToString('N')) + if (Test-Path -LiteralPath $publisherEnvironment) { throw "Publisher environment already exists." } + python -m venv "$publisherEnvironment" + if ($LASTEXITCODE -ne 0) { throw "Could not create the isolated publisher environment." } + $publisherPython = Join-Path $publisherEnvironment 'Scripts\python.exe' + & $publisherPython -m pip --isolated install --disable-pip-version-check --require-hashes --only-binary=:all: ` + --index-url "https://packagefeedproxy.microsoft.io/pypi/simple/" ` + -r "$lock" + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to install the complete hash-locked publisher dependency closure." + exit 1 + } + & $publisherPython -m pip --isolated check + if ($LASTEXITCODE -ne 0) { throw "Publisher dependency closure is inconsistent." } + & $publisherPython -c "import importlib.metadata as m; assert m.version('anaconda-client') == '1.14.1'; assert m.version('zstandard') == '0.23.0'; from binstar_client.utils import get_server_api; import zstandard; print('pinned publisher APIs ready')" + if ($LASTEXITCODE -ne 0) { + Write-Error "Pinned anaconda-client version/API verification failed." + exit 1 + } + $env:ANACONDA_CLIENT_FORCE_STANDALONE = '1' + & $publisherPython -m binstar_client.scripts.cli upload --help | Out-Null + if ($LASTEXITCODE -ne 0) { + Write-Error "Pinned anaconda-client upload module is unavailable." + exit 1 + } + Write-Host "##vso[task.prependpath]$(Join-Path $publisherEnvironment 'Scripts')" + + - task: PowerShell@2 + displayName: '${{ parameters.labelPrefix }}Publish conda packages to anaconda.org/${{ parameters.condaChannel }}' + env: + # The API receives this token explicitly; the CLI inherits it after the + # competing BINSTAR_API_TOKEN is removed below. Keep it off command lines. + # Supplied by the caller from the 'Anaconda Publishing' variable group. + ANACONDA_API_TOKEN: $(ANACONDA_API_TOKEN) + ANACONDA_CLIENT_FORCE_STANDALONE: '1' + BINSTAR_CONFIG_DIR: '$(Agent.TempDirectory)/anaconda-client-$(Build.BuildId)' + # Queue-time values are surfaced as env vars rather than interpolated into the script text. + # Reading them at runtime via $env: keeps PowerShell syntax independent of operator input. + CONDA_CHANNEL: '${{ parameters.condaChannel }}' + CONDA_LABEL: '${{ parameters.condaLabel }}' + MSSQL_PYTHON_VERSION: '${{ parameters.mssqlPythonVersion }}' + RELEASE_VERSIONS: '${{ parameters.releaseVersions }}' + RS_TRANSPORT_VERSION: '${{ parameters.rsTransportVersion }}' + REQUIRED_SUBDIRS: '${{ parameters.requiredSubdirs }}' + ALLOWED_SUBDIRS: '${{ parameters.allowedSubdirs }}' + PYTHON_VERSIONS: '${{ parameters.pythonVersions }}' + inputs: + targetType: 'inline' + workingDirectory: '$(Build.SourcesDirectory)' + script: | + $ErrorActionPreference = 'Stop' + + # Refuse to publish without an explicit target channel (protects the dummy + # pipeline, whose default channel is empty, from ever hitting production). + if ([string]::IsNullOrWhiteSpace($env:CONDA_CHANNEL)) { + Write-Error "condaChannel is empty. Supply the target Anaconda.org channel/org (production = 'microsoft')." + exit 1 + } + if ([string]::IsNullOrWhiteSpace($env:CONDA_LABEL)) { + Write-Error "condaLabel is empty. Supply the Anaconda.org label to publish (production = 'main')." + exit 1 + } + if ([string]::IsNullOrWhiteSpace($env:MSSQL_PYTHON_VERSION)) { + Write-Error "mssqlPythonVersion is empty. Production publication requires an exact version." + exit 1 + } + + $identifierPattern = '^[A-Za-z0-9][A-Za-z0-9._-]*$' + if ($env:CONDA_CHANNEL -notmatch $identifierPattern) { + Write-Error "Invalid condaChannel '$env:CONDA_CHANNEL'. Use letters, digits, dot, underscore, or hyphen; the first character must be alphanumeric." + exit 1 + } + if ($env:CONDA_LABEL -notmatch $identifierPattern) { + Write-Error "Invalid condaLabel '$env:CONDA_LABEL'. Use letters, digits, dot, underscore, or hyphen; the first character must be alphanumeric." + exit 1 + } + + if ([string]::IsNullOrWhiteSpace($env:ANACONDA_API_TOKEN)) { + Write-Error "ANACONDA_API_TOKEN is not set. Add the 'Anaconda Publishing' variable group to the job." + exit 1 + } + # Remove the key: even an empty BINSTAR token overrides ANACONDA in the pinned client. + [Environment]::SetEnvironmentVariable('BINSTAR_API_TOKEN', $null, 'Process') + + if ([string]::IsNullOrWhiteSpace($env:BINSTAR_CONFIG_DIR)) { + Write-Error "BINSTAR_CONFIG_DIR is empty; refusing to use ambient Anaconda client configuration." + exit 1 + } + $clientConfigDir = Join-Path $env:BINSTAR_CONFIG_DIR 'data' + New-Item -ItemType Directory -Force -Path $clientConfigDir | Out-Null + @('url: https://api.anaconda.org', 'ssl_verify: true') | + Set-Content -Path (Join-Path $clientConfigDir 'config.yaml') -Encoding ASCII + python -c "from binstar_client.utils import get_config; c=get_config(); assert c['url']=='https://api.anaconda.org'; assert c['ssl_verify'] is True; print('Anaconda endpoint configuration verified')" + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to verify isolated Anaconda endpoint configuration." + exit 1 + } + + $root = "$(Build.SourcesDirectory)/conda-artifacts/conda" + # Revalidate every archive from THIS job's fresh download. The metadata + # gate owns package identity, platform/Python policy and exact-version checks. + $metadataGate = "$(Build.SourcesDirectory)/eng/conda_tools/release.py" + if (-not (Test-Path $metadataGate)) { + Write-Error "Conda release metadata gate not found at $metadataGate." + exit 1 + } + python -m eng.conda_tools validate ` + --release-versions ` + --root "$root" ` + --required-subdirs "$env:REQUIRED_SUBDIRS" ` + --allowed-subdirs "$env:ALLOWED_SUBDIRS" ` + --pythons "$env:PYTHON_VERSIONS" ` + --mssql-python-version "$env:MSSQL_PYTHON_VERSION" + if ($LASTEXITCODE -ne 0) { + Write-Error "Credential-boundary Conda release metadata validation failed." + exit 1 + } + + $promoter = "$(Build.SourcesDirectory)/eng/conda_tools/publication.py" + if (-not (Test-Path $promoter)) { + Write-Error "Conda promotion helper not found at $promoter." + exit 1 + } + # Upload the entire validated set, never a filename-filtered subset. + $pkgs = @(Get-ChildItem -Path $root -Recurse -File -Include *.conda, *.tar.bz2 | Sort-Object FullName) + $packagePaths = @($pkgs | ForEach-Object { $_.FullName }) + $stagingLabel = "$($env:CONDA_LABEL)_staging_$(Build.BuildId)" + python -m eng.conda_tools promote ` + --owner "$env:CONDA_CHANNEL" ` + --staging-label "$stagingLabel" ` + --target-label "$env:CONDA_LABEL" ` + --expected-version "$env:MSSQL_PYTHON_VERSION" ` + --check-local-only ` + @packagePaths + if ($LASTEXITCODE -ne 0) { + Write-Error "Local Conda release input validation failed before upload." + exit 1 + } + + # Staged-then-promoted publish: upload the FULL set to a BUILD-UNIQUE staging + # label first; only after every package lands AND its bytes are verified do we + # begin adding the public label. Failed uploads/promotions clean this run's staging + # label from attempted files. Promotion verifies each mutation and rolls back failures; + # an interrupted run is idempotent and resumes from server-verified labels. The + # staging label distinguishes retries but does not prevent overlapping writers. + # The caller's protected stage keeps the shared native lock through upload, + # promotion's snapshot/rollback, and the compensating cleanup below. + # Bound stalled writers, not just retry counts. Start the 20-minute upload + # budget at the first attempt; reserve termination time within that budget. + # Preparation and compensating cleanup have separate costs: this is not a + # guarantee that they finish inside the enclosing 60-minute job deadline. + $attemptTimeoutMs = 120000 + $uploadBudgetMs = 1200000 + $terminationTimeoutMs = 10000 + $uploadState = @{ + Clock = [System.Diagnostics.Stopwatch]::new() + AttemptedPaths = [System.Collections.Generic.List[string]]::new() + TerminationUncertain = $false + } + $pythonExecutable = (Get-Command python -CommandType Application -ErrorAction Stop | Select-Object -First 1).Source + function Invoke-Anaconda($argsList) { + for ($attempt = 1; $attempt -le 3; $attempt++) { + $remainingMs = $uploadBudgetMs - $uploadState.Clock.ElapsedMilliseconds + $waitMs = [int][Math]::Min($attemptTimeoutMs, $remainingMs - 2 * $terminationTimeoutMs) + if ($waitMs -le 0) { throw "Aggregate upload budget exhausted before attempt $attempt. No new upload was started." } + # Start-Process joins ArgumentList on PS5.1. Quote for the Windows CRT, + # including empty arguments, embedded quotes and trailing backslashes. + $quotedArgs = foreach ($value in @('-m', 'binstar_client.scripts.cli') + $argsList) { + '"' + ([string]$value -replace '(\\*)"', '$1$1\"' -replace '(\\+)$', '$1$1') + '"' + } + $logPrefix = Join-Path $env:AGENT_TEMPDIRECTORY ("conda-upload-" + [guid]::NewGuid().ToString('N')) + $stdoutPath = "$logPrefix.stdout" + $stderrPath = "$logPrefix.stderr" + $process = $null + try { + $uploadState.Clock.Start() + # Record even an ambiguous failed launch/response, but not budget-rejected files. + if (-not $uploadState.AttemptedPaths.Contains($argsList[-1])) { + $uploadState.AttemptedPaths.Add($argsList[-1]) + } + $process = Start-Process -FilePath $pythonExecutable -ArgumentList ($quotedArgs -join ' ') ` + -NoNewWindow -PassThru -RedirectStandardOutput $stdoutPath -RedirectStandardError $stderrPath + $null = $process.Handle + if ($process.WaitForExit($waitMs)) { + if ($process.ExitCode -eq 0) { return } + $failure = "exit $($process.ExitCode)" + } + else { + $failure = "timeout after $waitMs ms" + $uploadState.TerminationUncertain = $true + $killer = $null + try { + $killer = Start-Process -FilePath "$env:SystemRoot\System32\taskkill.exe" ` + -ArgumentList @('/PID', $process.Id, '/T', '/F') -NoNewWindow -PassThru + $null = $killer.Handle + if (-not $killer.WaitForExit($terminationTimeoutMs)) { + $killer.Kill() + throw "Process-tree termination command exceeded its deadline." + } + if ($killer.ExitCode -ne 0 -or -not $process.WaitForExit($terminationTimeoutMs)) { + throw "Process-tree termination could not be confirmed." + } + $uploadState.TerminationUncertain = $false + } + catch { + throw "Upload $failure; owned writer PID $($process.Id) may still be active: $_" + } + finally { + if ($null -ne $killer) { $killer.Dispose() } + } + } + } + catch { + if ($null -ne $process -and -not $uploadState.TerminationUncertain) { + $uploadState.TerminationUncertain = $true + throw "Process control failed for owned writer PID $($process.Id); termination is unconfirmed: $_" + } + throw + } + finally { + if (-not $uploadState.TerminationUncertain) { + foreach ($logPath in @($stdoutPath, $stderrPath)) { + if (Test-Path -LiteralPath $logPath) { + Write-Host "Uploader log (last 8192 bytes): $logPath" + $log = $null + try { + $log = [IO.File]::Open($logPath, 'Open', 'Read', 'ReadWrite') + $null = $log.Seek([Math]::Max(0, $log.Length - 8192), 'Begin') + $buffer = [byte[]]::new(8192) + $count = $log.Read($buffer, 0, $buffer.Length) + Write-Host ([Text.Encoding]::UTF8.GetString($buffer, 0, $count)) + } + catch { Write-Warning "Could not read uploader log '$logPath': $_" } + finally { if ($null -ne $log) { $log.Dispose() } } + Remove-Item -LiteralPath $logPath -ErrorAction Continue + } + } + } + if ($null -ne $process) { $process.Dispose() } + } + if ($attempt -lt 3) { + Write-Host "attempt $attempt failed ($failure); retrying in up to 5s ..." + $delayMs = [int][Math]::Min(5000, [Math]::Max(0, $uploadBudgetMs - $uploadState.Clock.ElapsedMilliseconds)) + Start-Sleep -Milliseconds $delayMs + } + } + throw "Failed to stage $($argsList[-1]) after 3 attempts ($failure). The public label was NOT touched." + } + + Write-Host "==== Stage: upload $($pkgs.Count) package(s) to label '$stagingLabel' ====" + try { + foreach ($p in $pkgs) { + Write-Host "Uploading $($p.Directory.Name)/$($p.Name) -> $stagingLabel ..." + Invoke-Anaconda @('upload', '--user', "$env:CONDA_CHANNEL", '--label', $stagingLabel, '--skip-existing', "$($p.FullName)") + } + + python -m eng.conda_tools promote ` + --owner "$env:CONDA_CHANNEL" ` + --staging-label "$stagingLabel" ` + --target-label "$env:CONDA_LABEL" ` + --expected-version "$env:MSSQL_PYTHON_VERSION" ` + @packagePaths + if ($LASTEXITCODE -ne 0) { + throw "Conda staging verification/promotion failed." + } + } + catch { + $publicationError = $_ + if ($uploadState.TerminationUncertain) { + Write-Warning "Writer termination is unconfirmed; staging state is uncertain. No retry, promotion or remote cleanup is safe until the owned writer is confirmed stopped. Upload logs were retained." + throw $publicationError + } + $attemptedPaths = @($uploadState.AttemptedPaths.ToArray()) + if ($attemptedPaths.Count -gt 0) { + try { + python -m eng.conda_tools promote ` + --owner "$env:CONDA_CHANNEL" ` + --staging-label "$stagingLabel" ` + --target-label "$env:CONDA_LABEL" ` + --expected-version "$env:MSSQL_PYTHON_VERSION" ` + --cleanup-staging ` + @attemptedPaths + if ($LASTEXITCODE -ne 0) { throw "Staging cleanup exited with code $LASTEXITCODE." } + } + catch { + Write-Warning "Cleanup incomplete: $_. Recover with --cleanup-staging, exact label '$stagingLabel', and the retained attempted archives; ensure no other publication or recovery is running for this target." + } + } + throw $publicationError + } + + Write-Host "" + Write-Host "Published $($pkgs.Count) verified mssql-python conda package(s) to anaconda.org/$($env:CONDA_CHANNEL) (label $($env:CONDA_LABEL)) via staging label '$stagingLabel'." diff --git a/OneBranchPipelines/steps/conda-release-step.yml b/OneBranchPipelines/steps/conda-release-step.yml new file mode 100644 index 000000000..4a8386ff0 --- /dev/null +++ b/OneBranchPipelines/steps/conda-release-step.yml @@ -0,0 +1,203 @@ +# Conda Release Readiness Step Template +# ============================================================================ +# Downloads the consolidated conda packages produced by the STANDALONE conda-build +# pipeline (conda-build-pipeline.yml, artifact +# drop_ConsolidateConda_ConsolidateArtifacts) and enforces the RELEASE-TIME hard +# gate that the build pipeline intentionally does +# NOT enforce. +# +# Why the gate lives HERE and not in the build: +# - BUILD pipeline: conda is collected BEST-EFFORT (warn-only) so a conda hiccup +# on any leg can never fail the build or block the primary wheel release. +# - RELEASE pipeline: conda completeness is GATED -- an incomplete conda set +# must never be shipped. +# +# What it enforces (all from each package's AUTHORITATIVE info/index.json, never +# folder names or bare counts -- so a mis-stamped subdir or a dropped Python +# variant cannot slip through): +# - real subdir: every package's info/index.json `subdir` is in `allowedSubdirs` +# AND equals the folder it was staged into (catches an osx-64 package copied +# into osx-arm64/, which a folder-name check cannot). +# - required subdirs: every subdir in `requiredSubdirs` (the PyPI-parity set) +# is present. +# - Python matrix: on every required subdir the mssql-python binding covers +# EVERY expected Python (`pythonVersions`) -- catches e.g. 3 of 5 win-64 +# bindings shipping against the single companion. +# - versions: all packages share one version (and match the expected version +# when `mssqlPythonVersion` is supplied). +# +# The check is implemented in eng.conda_tools.release (unit-tested by +# tests/test_027_conda_release_metadata.py), which reads the zstd-compressed +# info/index.json embedded in every .conda. +# +# win-arm64 IS in the required parity set (v1): although the arm64 Python cannot be +# import-validated on the x64 build agent, its build leg runs a BLOCKING --dry-run +# dependency solve PLUS a PE machine-type assert on the vendored arm64 .pyd/.dll, so a +# broken or wrong-arch package is never STAGED. That build leg is continueOnError (so a +# win-arm64 hiccup can't nuke the shared win-64 publish), and THIS release gate is where +# win-arm64 is enforced: a missing win-arm64 fails the gate. Its matrix is 3.12-3.14 (the +# gate's per-subdir override), since Anaconda `defaults` has no win-arm64 python for +# 3.10/3.11. musllinux has no conda subdir at all (conda Linux is glibc-only), so absent. +# +# Publishing to anaconda.org is a separate step; this template proves the exact +# downloaded artifact is complete and passes every platform-native static audit. +parameters: + # Consolidated conda artifact name (see consolidate-conda-artifacts-job.yml). + - name: condaArtifactName + type: string + default: 'drop_ConsolidateConda_ConsolidateArtifacts' + # Comma-separated subdirs that a complete release MUST contain (full PyPI parity + # minus musllinux, which has no conda subdir). win-arm64 IS included: the build + # enforces its --dry-run solve + PE-arch assert, so a missing/broken win-arm64 + # blocks the release. If an emulated leg (osx-64 / linux-aarch64) ever proves too + # flaky to gate on, drop it here — no code change needed. + - name: requiredSubdirs + type: string + default: 'win-64,win-arm64,osx-64,osx-arm64,linux-64,linux-aarch64' + # Comma-separated superset of subdirs that are ALLOWED to appear. Any discovered + # subdir outside this set fails the gate (guards against a mis-stamped subdir). + # (win-arm64 is now also in requiredSubdirs above.) + - name: allowedSubdirs + type: string + default: 'win-64,win-arm64,osx-64,osx-arm64,linux-64,linux-aarch64' + # Comma-separated Python versions the binding matrix MUST cover on every required + # subdir. The gate reads each binding's pyXY build tag from info/index.json. + - name: pythonVersions + type: string + default: '3.10,3.11,3.12,3.13,3.14' + # Optional EXACT expected versions. When set, the gate asserts every package's + # info/index.json version matches; when empty it still enforces one-version-per- + # package consistency plus the subdir / matrix / pairing checks. + - name: mssqlPythonVersion + type: string + default: '' + - name: releaseVersions + type: string + default: '' + - name: rsTransportVersion + type: string + default: '' + # The audit dependency hash targets CPython 3.12 / Windows x64, like the publisher. + - name: pythonVersion + type: string + default: '3.12' + # Optional display-name prefix (e.g. '[TEST] ' for the dummy pipeline). + - name: labelPrefix + type: string + default: '' + +steps: + - task: DownloadPipelineArtifact@2 + displayName: '${{ parameters.labelPrefix }}Download consolidated conda packages' + inputs: + buildType: 'specific' + project: '$(System.TeamProject)' + definition: $(resources.pipeline.buildPipeline.pipelineID) + buildVersionToDownload: 'specific' + buildId: $(resources.pipeline.buildPipeline.runID) + artifactName: '${{ parameters.condaArtifactName }}' + targetPath: '$(Build.SourcesDirectory)/conda-artifacts' + + - task: UsePythonVersion@0 + displayName: '${{ parameters.labelPrefix }}Use Python ${{ parameters.pythonVersion }} for release validation' + inputs: + versionSpec: '${{ parameters.pythonVersion }}' + addToPath: true + + - task: PowerShell@2 + displayName: '${{ parameters.labelPrefix }}Install hash-locked release-audit dependency' + inputs: + targetType: 'inline' + script: | + $ErrorActionPreference = 'Stop' + python -m pip --isolated install --disable-pip-version-check --force-reinstall --require-hashes --only-binary=:all: ` + --index-url "https://packagefeedproxy.microsoft.io/pypi/simple/" ` + -r "$(Build.SourcesDirectory)/conda/requirements-release.txt" + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to install hash-locked zstandard for release validation." + exit 1 + } + + # Re-run every PR #744 masking/architecture audit on the EXACT downloaded artifacts + # so a stale, partial, or wrong-architecture package can never reach publication. + - task: PowerShell@2 + displayName: '${{ parameters.labelPrefix }}Re-audit exact release artifacts (ELF + PE + Mach-O)' + inputs: + targetType: 'inline' + workingDirectory: '$(Build.SourcesDirectory)' + script: | + $ErrorActionPreference = 'Stop' + $root = "$(Build.SourcesDirectory)/conda-artifacts/conda" + if (-not (Test-Path $root)) { + Write-Error "Consolidated conda tree not found at $root." + exit 1 + } + + if (-not (Test-Path -LiteralPath "$(Build.SourcesDirectory)/eng/conda_tools/__main__.py" -PathType Leaf)) { + Write-Error "Conda audit module not found in the source checkout." + exit 1 + } + $audits = @( + @{ Name = 'Linux ELF RUNPATH/inventory'; Command = 'elf' }, + @{ Name = 'Windows PE machine/inventory'; Command = 'pe' }, + @{ Name = 'macOS Mach-O architecture/inventory'; Command = 'macho' } + ) + foreach ($audit in $audits) { + Write-Host "Running $($audit.Name) audit..." + python -m eng.conda_tools $audit.Command --root "$root" + if ($LASTEXITCODE -ne 0) { + Write-Error "$($audit.Name) audit FAILED on the exact release artifacts. Refusing to publish." + exit 1 + } + } + Write-Host "All release-boundary native artifact audits passed." + + - task: PowerShell@2 + displayName: '${{ parameters.labelPrefix }}Validate conda release readiness (metadata: subdirs + Python matrix + versions)' + env: + REQUIRED_SUBDIRS: '${{ parameters.requiredSubdirs }}' + ALLOWED_SUBDIRS: '${{ parameters.allowedSubdirs }}' + PYTHON_VERSIONS: '${{ parameters.pythonVersions }}' + MSSQL_PYTHON_VERSION: '${{ parameters.mssqlPythonVersion }}' + RELEASE_VERSIONS: '${{ parameters.releaseVersions }}' + RS_TRANSPORT_VERSION: '${{ parameters.rsTransportVersion }}' + inputs: + targetType: 'inline' + workingDirectory: '$(Build.SourcesDirectory)' + script: | + $ErrorActionPreference = 'Stop' + $root = "$(Build.SourcesDirectory)/conda-artifacts/conda" + if (-not (Test-Path $root)) { + Write-Error "Consolidated conda tree not found at $root. Was ConsolidateConda produced by the selected build run?" + exit 1 + } + + $gate = "$(Build.SourcesDirectory)/eng/conda_tools/release.py" + if (-not (Test-Path $gate)) { + Write-Error "Metadata gate script not found at $gate." + exit 1 + } + + foreach ($policy in @('REQUIRED_SUBDIRS', 'ALLOWED_SUBDIRS', 'PYTHON_VERSIONS')) { + if ([string]::IsNullOrWhiteSpace([Environment]::GetEnvironmentVariable($policy))) { + Write-Error "$policy is empty. Refusing to weaken the release matrix." + exit 1 + } + } + + $gateArgs = @( + '--root', $root, + '--required-subdirs', $env:REQUIRED_SUBDIRS, + '--allowed-subdirs', $env:ALLOWED_SUBDIRS, + '--pythons', $env:PYTHON_VERSIONS + ) + if (-not [string]::IsNullOrWhiteSpace($env:MSSQL_PYTHON_VERSION)) { + $gateArgs += @('--mssql-python-version', $env:MSSQL_PYTHON_VERSION.Trim()) + } + + python -m eng.conda_tools validate @gateArgs --release-versions + if ($LASTEXITCODE -ne 0) { + Write-Error "Conda release readiness FAILED. Refusing to proceed with an incomplete/mis-labeled/mis-paired conda set." + exit 1 + } + Write-Host "Conda set is release-ready (the publish step is gated separately)." diff --git a/README.md b/README.md index d25c4b76b..7fedbb709 100644 --- a/README.md +++ b/README.md @@ -91,12 +91,13 @@ conda install -c "" -c microsoft -c conda-forge --strict-chan conda install -c "" -c microsoft -c defaults --override-channels "mssql-python=" ``` -**Conda release status:** Publication tooling and its administrator prerequisites are -proposed separately in the [release-additions PR](https://github.com/microsoft/mssql-python/pull/720). -These native-packaging changes do not publish packages or require a particular merge order. +**Conda release status:** Validate-only release and staged publication tooling are included +in this repository; neither publishes packages automatically. Production publication +requires separately configured credentials, permissions and shared publishing-resource +checks, explicit authorization, and release qualification. Static audits and import checks do not certify SQL, certificate-verified TLS, authentication, -bulk copy, or optional features across the full matrix. Production publication remains gated -on separate release controls and qualification; validate-only success does not authorize it. +bulk copy, or optional features across the full matrix. Validate-only success does not +authorize production publication. ## Key Features ### Supported Platforms diff --git a/conda/README.md b/conda/README.md index 5087309a7..a7d0acef3 100644 --- a/conda/README.md +++ b/conda/README.md @@ -11,6 +11,47 @@ effective target platform and `--python-versions` (or auto-detects versions from target-compatible bindings). Other Python/platform inputs are ignored before metadata and RS dependency checks. Selected inputs still require matching versions, ownership, tags and native compatibility; missing required RS wheels remain fatal. +The builder validates raw wheel core ownership before staging or recipe extraction, +including unrecorded files and wheel spread-path/case aliases. ODBC never supplies +`mssql_py_core`; the selected RS provider (or historical binding) must own every core +file in RECORD. Direct recipe callers must use these validated wheel inputs. +Public and staged wheels also require one canonical root METADATA member, with no +extra nested or aliased metadata. Package audits require binding metadata and exactly +one RECORD for each installed distribution; missing or duplicate ownership records fail. +Low-level binary-format parsers remain independent of package metadata. +These checks inspect all actual METADATA/RECORD/WHEEL candidates before accepting +installed ownership, including case, backslash and normalized-path aliases that a +canonical-only metadata collector would omit. Aliased or orphan entries are rejected, +not normalized into an accepted package. + +Archive readers stream ZIP components through the selected zstandard backend (or +stream legacy `.tar.bz2` files) instead of materializing complete TAR components. +Individual files still return bounded byte buffers for the native parsers. Fixed +limits fail explicitly; there is no command-line or environment override: + +| Processing budget | Limit | +| --- | --- | +| Archive file / ZIP component bytes | 256 MiB | +| ZIP central-directory read / ZIP entries | 1 MiB / 10,000 | +| Expanded TAR stream, including padding | 1 GiB | +| Cumulative TAR member bytes | 512 MiB | +| Individual member / metadata member bytes | 128 MiB / 8 MiB | +| Cumulative TAR metadata / TAR headers | 64 MiB / 10,000 | +| Zstandard decoder window / frames per component | 64 MiB / 10,000 | + +Checks apply before member allocation and while consuming 64 KiB chunks, including +frames without a declared expanded size. The third-party zstandard reader also +receives a bounded frame-structure pass because its streaming API alone accepts +truncated frame endings. ZIP envelopes must use stored or deflate compression; +sparse TAR extensions are rejected before sparse-map processing. Wheel metadata +reads share the ZIP and metadata budgets. These are package-reader limits, not a +claim that NuGet transport downloads or total process memory have the same bounds. + +The GitHub audit job installs its complete Linux x64 / CPython 3.11 test-tool closure +from `requirements-audit.txt` with `--require-hashes --only-binary=:all:`. +`requirements-audit.in` records the reviewed direct pins and regeneration commands. +This audit-only lock is separate from the Windows publisher lock; it does not pin +the hosted runner image or every input to the later Conda build. Direct recipe builds must set `MSSQL_PYTHON_VERSION` to the exact selected code-wheel version before rendering/building. The shared orchestrator derives and supplies it @@ -81,8 +122,8 @@ running again. Automatic acceptance, including an inherited `CONDA_PLUGINS_AUTO_ACCEPT_TOS` opt-in, remains disabled. Run this existing build workflow only in a disposable isolated installation: shared-environment -ownership hardening is outside this change. Publication/provenance tooling is proposed -in a separate release-additions PR that follows this native/tooling foundation; see the +ownership hardening is outside this change. Publication/provenance tooling is included +alongside this native/tooling foundation; see the [release status and qualification caveats](../README.md#installation). Neither this native-packaging change nor validate-only success authorizes production publication. @@ -95,6 +136,11 @@ compatibility policy (`contracts.py`), binary facts (`formats/elf.py`, `pe.py`, human-readable reporting live in `__main__.py`. Conda provisioning and process execution live in `environment.py`; `build.py` sequences wheel selection, building, auditing and staging; `verify.py` keeps installed-package probes isolated. +Release matrix and Python-admissibility policy live in `release.py`, recorded +Azure DevOps source checks in `provenance.py`, source-bound component inputs and +public wheel fetching in `inputs.py`, and staged publication/recovery in `publication.py`. Both release +and native auditing use `archive.py`; release validation retains its stricter +container and index rules rather than weakening them to the generic audit policy. Each `.conda` component reader requires exactly one matching `info-*.tar.zst` or `pkg-*.tar.zst` entry. Missing components and additional matches, including duplicate @@ -108,9 +154,14 @@ python -m eng.conda_tools build --help python -m eng.conda_tools elf --root python -m eng.conda_tools pe --root --subdir win-arm64 python -m eng.conda_tools macho --root --subdir osx-arm64 +python -m eng.conda_tools validate --root +python -m eng.conda_tools provenance +python -m eng.conda_tools promote --help +python -m eng.conda_tools fetch-wheels --help +python -m eng.conda_tools probe-driver ``` -These replace the old standalone audit and build scripts. They require the source +These replace the old standalone audit, build, and release scripts. They require the source checkout; the internal tools are not installed in the driver wheel. If Python reports `No module named 'eng'`, run from the checkout root rather than an installed driver environment. Build callers set the tooling subprocess's working directory @@ -119,12 +170,105 @@ Static audits do not import the driver. Runtime verification remains in separate processes from a neutral working directory, with the required core loaded independently before API probes. +`provenance` uses the existing release pipeline environment and read-only job-token +access. `promote --check-local-only` needs neither a publishing token nor an +Anaconda client; actual publication and recovery require restricted credentials and +the shared protected release stage described below. Module placement is not a new +authorization boundary. The PowerShell upload process deadlines, attempted-file +tracking, process-tree termination, and dry-run plan remain in their pipeline tasks. +Network publication and recovery require an explicit nonblank `ANACONDA_API_TOKEN`, +not an ambient or cached login. The publishing task removes `BINSTAR_API_TOKEN` +only from its process and children so the CLI uploader uses the same reviewed credential. +User/machine environment settings and saved credentials are not modified. + +Only trusted, reviewed release revisions may be queued, including validate-only feature +branches: their YAML and Python run with `System.AccessToken`. YAML provenance checks +cannot sandbox an untrusted queued revision or protect that token from it. Administrators +must restrict pipeline editing/queueing and source access, minimize the job identity's +permissions, and separately protect publishing credentials. A dry run omits the publishing +credential group; it is not token-free execution. + +Before authorizing production use of `Anaconda Publishing`, its resource owner must +configure an ADO [Branch control check](https://learn.microsoft.com/en-us/azure/devops/pipelines/process/approvals?view=azure-devops#branch-control) +on that variable group in **Pipelines > Library > Approvals and checks**: +allow only `refs/heads/main`, require branch protection, and fail when protection +cannot be verified. This resource-side check applies to every consuming stage and +pipeline, including the branches of linked producer runs; feature-branch publication +must fail before the stage receives the publishing credential. Preserve existing +permissions and approvals, and restrict permission to administer or bypass the check. +Do not grant broader pipeline access as part of configuring it. + +The existing `publishToConda` condition omits this group for validate-only runs, so +trusted feature-branch validation remains separate from credentialed publication. +Neither a checkout-local branch check nor a secondary pinned checkout can replace +the resource-side control: editable YAML could bypass either. These external checks +are not installed by this repository; production must remain unapproved until the +resource owner verifies their configuration and enforcement. Branch control does +not provide the separate exclusive publication lock. + +### Publication serialization + +The resource owner must also enable the native **Exclusive lock** check on the same +`Anaconda Publishing` variable group. Production selects `lockBehavior: sequential` +on the enclosing `CondaRelease` stage; this setting orders waiting runs but does not +create the external check. The lock spans validation, staged uploads, the initial +label snapshot, promotion, rollback and staging cleanup until the stage ends. +All publishers and recovery paths for `microsoft/mssql-python` must consume that +same resource, across versions and target labels; separate groups would not serialize +them. Keep the group and lock behavior absent from validate-only runs. + +Recover by rerunning the original protected release stage with its original artifact +inputs, preserving the build-specific staging label and reacquiring the shared lock. +Do not run mutating `promote` or `--cleanup-staging` commands outside that protected +stage: the Python helpers do not acquire an ADO resource lock themselves. +After uncertain writer termination, first confirm the writer has stopped; a native +lock does not terminate orphaned processes or repair interrupted remote operations. +Inspect check configuration and compiled stage wiring separately from run-time lock +acquisition; configuration readback alone is not evidence that a run holds the lock. + +The release pipeline resolves AUTO from the **recorded upstream wheel commit**, not +the release checkout or a latest-version lookup. It cross-checks binding setup/runtime +versions and reads ODBC and, when explicitly required by that source, RS distribution +versions. RS's NuGet transport pin is separate. Missing or ambiguous source declarations +fail rather than select a historical profile. A supplied binding version is only an +assertion against this resolved version. + +Both readiness and the credentialed job use `validate --release-versions` to read the +verified `RELEASE_VERSIONS` JSON from the environment; `RS_TRANSPORT_VERSION` supplies +the separate transport assertion. This avoids passing JSON quoting or empty arguments +through Windows PowerShell. Explicit JSON may also follow `--release-versions` in +source-only controls. With this flag, absent versions fail closed. Without it, `validate` +remains the metadata/matrix-only inspection command and makes no source-input claim. +The source-bound gate checks each installed component's METADATA and RECORD ownership, +the binding's exact dependency pins, and the selected RS filename/WHEEL tags against +`rs-transport.json`. Raw wheel/transport hashes remain input evidence; relocated +installed native files are checked by the native audits, not falsely compared with raw +wheel hashes. No numeric RS build ID is inferred from a transport-version suffix. +Release and native auditing also check all installed core paths, not only RECORD-filtered +members. This detects unowned or cross-owned additions; it cannot recover file origin +after an overwrite of an already owned path, which is why the raw pre-extraction gate +is required. + +`fetch-wheels` uses exact current maintained versions, hash-required binary-only PyPI +downloads, and actual wheel metadata/ownership. Pip selects for its executing interpreter; +the requested Python/platform must match the binding, ODBC and selected RS wheel tags +before the command accepts the inputs. These arguments validate the download, rather +than enabling cross-target pip resolution. An authentic published embedded-core +binding is explicitly reported as a **historical published packaging control, not +current-source RS qualification**. An RS-dependent published binding requires its exact +provider and source pin; unavailable releases, malformed declarations, or mismatches +fail with no older-version or TLS fallback. Public availability does not gate the +recorded ADO artifact path. Neither a public input check nor static archive checks +establish installed SQL/runtime qualification. + The driver-load probe's sole implementation is `eng/conda_tools/driver_load_probe.py`. It separates native connection execution, pure outcome classification and command reporting with typed interfaces. Verification runs that file by absolute path with the target environment's Python. Its standard-library-only bootstrap does not require `eng` to be installed there, so the tooling namespace does not weaken installed-package isolation. +The `probe-driver` module command is a convenience diagnostic in the calling +interpreter, not a substitute for this neutral-directory installed-package check. The files remaining under `conda/` are the conventional recipe entrypoints, metadata, documentation and line-ending configuration. Shared NuGet transport, diff --git a/conda/requirements-audit.in b/conda/requirements-audit.in new file mode 100644 index 000000000..c03fcaba0 --- /dev/null +++ b/conda/requirements-audit.in @@ -0,0 +1,23 @@ +# Audit tests only: CPython 3.11, Linux x64; not publisher or driver runtime dependencies. +# Regenerate from the repo root with uv 0.12.6 and pip in PowerShell: +# $work = Join-Path $env:TEMP ([guid]::NewGuid().ToString()); New-Item -ItemType Directory $work | Out-Null +# $target = @('--python-version', '3.11', '--python-platform', 'x86_64-manylinux_2_17', '--only-binary', ':all:') +# $hashes = @('--no-config', '--no-cache', '--generate-hashes', '--no-emit-index-url', '--no-header', '--no-annotate', '--strip-extras') +# uv pip compile @hashes @target --default-index https://pypi.org/simple ` +# --constraint conda\requirements-audit.txt --output-file "$work\resolved.txt" conda\requirements-audit.in +# python -m pip --isolated download --require-hashes --only-binary=:all: --no-deps ` +# --platform manylinux_2_17_x86_64 --python-version 3.11 --implementation cp --abi cp311 ` +# --index-url https://pypi.org/simple --dest "$work\wheels" -r "$work\resolved.txt" +# uv pip compile @hashes @target --no-index --find-links "$work\wheels" ` +# --output-file "$work\target.txt" conda\requirements-audit.in +# if ($LASTEXITCODE -eq 0) { Move-Item "$work\target.txt" conda\requirements-audit.txt -Force } +# Run each command only after the preceding command succeeds. +# The first compile resolves the complete Linux closure; --no-deps avoids host marker +# evaluation during cross-platform pip download. The offline compile rechecks the closure. +# Keep reviewed versions via the constraint; remove it only for intentional dependency updates. +# A fresh output avoids retaining unrelated platform hashes. Only target wheels are accepted. +pytest==9.1.1 +# setup.py bdist_wheel is exercised by test_030's binding-wheel ownership regression. +setuptools==84.0.0 +wheel==0.48.0 +zstandard==0.23.0 diff --git a/conda/requirements-audit.txt b/conda/requirements-audit.txt new file mode 100644 index 000000000..41e15515c --- /dev/null +++ b/conda/requirements-audit.txt @@ -0,0 +1,16 @@ +iniconfig==2.3.0 \ + --hash=sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12 +packaging==26.3 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c +pluggy==1.6.0 \ + --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 +pygments==2.21.0 \ + --hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 +pytest==9.1.1 \ + --hash=sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c +setuptools==84.0.0 \ + --hash=sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670 +wheel==0.48.0 \ + --hash=sha256:3217dcc807155e45db462d7ef2431f5ddda0d7273b700d05a67b271ceb1287ab +zstandard==0.23.0 \ + --hash=sha256:fd30d9c67d13d891f2360b2a120186729c111238ac63b43dbd37a5a40670b8ca diff --git a/conda/requirements-publish.in b/conda/requirements-publish.in new file mode 100644 index 000000000..c29faa3d5 --- /dev/null +++ b/conda/requirements-publish.in @@ -0,0 +1,20 @@ +# Publisher only: CPython 3.12, Windows x64; not shared build/test dependencies. +# Reproduce from the repo root in PowerShell with that Python on PATH and uv 0.12.6: +# $work = Join-Path $env:TEMP ([guid]::NewGuid().ToString()); New-Item -ItemType Directory $work | Out-Null +# $feed = 'https://packagefeedproxy.microsoft.io/pypi/simple/' +# $target = @('--python-version', '3.12', '--python-platform', 'x86_64-pc-windows-msvc', '--only-binary', ':all:') +# $hashes = @('--no-config', '--no-cache', '--generate-hashes', '--no-emit-index-url', '--no-header', '--no-annotate', '--strip-extras') +# uv pip compile @hashes @target --default-index $feed --constraint conda\requirements-publish.txt ` +# --output-file "$work\resolved.txt" conda\requirements-publish.in +# python -m pip --isolated download --require-hashes --only-binary=:all: --index-url $feed ` +# --dest "$work\wheels" -r "$work\resolved.txt" +# uv pip compile @hashes @target --no-index --find-links "$work\wheels" ` +# --output-file "$work\target.txt" conda\requirements-publish.in +# if ($LASTEXITCODE -eq 0) { Move-Item "$work\target.txt" conda\requirements-publish.txt -Force } +# A fresh output avoids uv retaining hashes from an existing lock. +# Constraints retain reviewed versions; omit --constraint only for intentional dependency updates. +# One tested target wheel per version: unavailable wheels fail closed until reviewed regeneration. +# Verify in a fresh target venv with pip --require-hashes --only-binary=:all:. +# Keep conda\requirements-release.txt on the same zstandard target pin/hash when refreshing. +anaconda-client==1.14.1 +zstandard==0.23.0 diff --git a/conda/requirements-publish.txt b/conda/requirements-publish.txt new file mode 100644 index 000000000..3bc7d88b1 --- /dev/null +++ b/conda/requirements-publish.txt @@ -0,0 +1,132 @@ +anaconda-auth==0.15.2 \ + --hash=sha256:d3d991be95ebaac75bc22ba94a07eba8bc0b1e02295a5faa4b19963f3184cc85 +anaconda-cli-base==0.9.1 \ + --hash=sha256:6d7da904b72fd2e521725887565eb3b86305750005c2b7b2d75d465cd43d64e6 +anaconda-client==1.14.1 \ + --hash=sha256:99d6c42bb77a9a89650a67f37e63504421dbb415a331a6636ce2c9c429130004 +annotated-doc==0.0.5 \ + --hash=sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101 +annotated-types==0.8.0 \ + --hash=sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0 +anyio==4.15.0 \ + --hash=sha256:7ecd9937369ffce8bba0b5ccb9b3a9507b101b0ed50256aecfbab27e6c2acb99 +attrs==26.1.0 \ + --hash=sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309 +backports-zstd==1.7.0 \ + --hash=sha256:2ea62ba2f1a6e6c9e6dc108921f9ae881969ca72e073162fa488d0de3eb2713f +certifi==2026.7.22 \ + --hash=sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 +cffi==2.1.1 \ + --hash=sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a +charset-normalizer==3.5.1 \ + --hash=sha256:3617ac3cfd8b9888f145ad89dd6e692285834b0201c6074a5eeaad3fd4d668c2 +click==8.5.0 \ + --hash=sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360 +colorama==0.4.6 \ + --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 +conda-package-handling==2.6.0 \ + --hash=sha256:a804c58fcec6f77c750b18455fa9865eb95f63912b149133627bc50636f551df +conda-package-streaming==0.13.0 \ + --hash=sha256:806ea3f2f7ac272873242e6253d543ac8aaf224e5c76d794e5fee9c63776fb5c +cryptography==50.0.1 \ + --hash=sha256:aed8db4f6d71c51efb89530e12d9464e7bf2923d46c3205dc794a2a93f8c0648 +defusedxml==0.7.1 \ + --hash=sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61 +fastjsonschema==2.22.2 \ + --hash=sha256:0fb3915616adac85ccfdd737d26be1089845d2019819505b42d39888458f74d4 +h11==0.16.0 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 +httpcore==1.0.9 \ + --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 +httpx==0.28.1 \ + --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad +idna==3.19 \ + --hash=sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4 +jaraco-classes==3.4.0 \ + --hash=sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790 +jaraco-context==6.1.2 \ + --hash=sha256:bf8150b79a2d5d91ae48629d8b427a8f7ba0e1097dd6202a9059f29a36379535 +jaraco-functools==4.6.0 \ + --hash=sha256:99e3dc0060c5cbe8fcd1cdb36258e2a65ca40f1566b2033b12abb1bb44dd3c30 +jsonschema==4.26.0 \ + --hash=sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce +jsonschema-specifications==2025.9.1 \ + --hash=sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe +jupyter-core==5.9.1 \ + --hash=sha256:ebf87fdc6073d142e114c72c9e29a9d7ca03fad818c5d300ce2adc1fb0743407 +keyring==25.7.0 \ + --hash=sha256:be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f +markdown-it-py==4.2.0 \ + --hash=sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a +mdurl==0.1.2 \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 +more-itertools==11.1.0 \ + --hash=sha256:4b65538ae22f6fed0ce4874efd317463a7489796a0939fa66824dd542125a192 +nbformat==5.11.1 \ + --hash=sha256:cc6698fa75f4fab8755ead786317815f13a6fee3b53311c0abb1a8b51d52f7ec +packaging==26.3 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c +pkce==1.0.3 \ + --hash=sha256:55927e24c7d403b2491ebe182b95d9dcb1807643243d47e3879fbda5aad4471d +platformdirs==4.11.7 \ + --hash=sha256:8a02cb259042c79d1cd0450facc2fe6dc9d303ae7901afbe33bf8ea0b188cef6 +pycparser==3.0 \ + --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 +pydantic==2.13.5 \ + --hash=sha256:346a034f080da3755d8e9cb5e00e8b07de1d39e4f6e2c87d8ab7cafa0b269a73 +pydantic-core==2.46.5 \ + --hash=sha256:5e81740c09e310f5aa5cbd3e434a01c154d4bef93241c7877b39f211d2b78ba8 +pydantic-settings==2.15.0 \ + --hash=sha256:0ba092c291c94baceb5eff768aa0d56400a457585bc0175925a5a5510303da42 +pygments==2.21.0 \ + --hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 +pyjwt==2.13.0 \ + --hash=sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728 +python-dateutil==2.9.0.post0 \ + --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 +python-dotenv==1.2.3 \ + --hash=sha256:904552145e8bfed22162c09dab1c2b9b54fefa7b23ba780f4f26ca0316b0f0d9 +pytz==2026.3.post1 \ + --hash=sha256:dd95840dd199baea12d9cc096a1d452caa6596a1c1e4b5f3dbd1541855d5e815 +pywin32-ctypes==0.2.3 \ + --hash=sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8 +pyyaml==6.0.3 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b +readchar==4.2.2 \ + --hash=sha256:92daf7e42c52b0787e6c75d01ecfb9a94f4ceff3764958b570c1dddedd47b200 +referencing==0.37.0 \ + --hash=sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231 +requests==2.34.2 \ + --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 +requests-toolbelt==1.0.0 \ + --hash=sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06 +rich==15.0.0 \ + --hash=sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb +rpds-py==2026.6.3 \ + --hash=sha256:2c958bf94822e9290a40aaf2a822d4bc5c88099093e3948ad6c571eca9272e5f +semver==3.0.4 \ + --hash=sha256:9c824d87ba7f7ab4a1890799cec8596f15c1241cb473404ea1cb0c55e4b04746 +setuptools==84.0.0 \ + --hash=sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670 +shellingham==1.5.4 \ + --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 +tomli==2.4.1 \ + --hash=sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9 +tomlkit==0.15.1 \ + --hash=sha256:177a05aece5a8ca5266fd3c448abb47b8d352f09d477d3ca8332db4d89b24304 +tqdm==4.70.0 \ + --hash=sha256:7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953 +traitlets==5.16.1 \ + --hash=sha256:f775618166caa0396c8e337099240f2bd3e5e917d203b2e6fbe21a58d3cb1f6b +typer==0.27.2 \ + --hash=sha256:b3a5fc4342d5fc8fda8fc3010b1cf117e9249aab7fae800c2eff62fd3842d97d +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 +typing-inspection==0.4.4 \ + --hash=sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147 +urllib3==2.7.0 \ + --hash=sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897 +zstandard==0.23.0 \ + --hash=sha256:64585e1dba664dc67c7cdabd56c1e5685233fbb1fc1966cfba2a340ec0dfff7b diff --git a/conda/requirements-release.txt b/conda/requirements-release.txt new file mode 100644 index 000000000..2edae5b62 --- /dev/null +++ b/conda/requirements-release.txt @@ -0,0 +1,2 @@ +zstandard==0.23.0 \ + --hash=sha256:64585e1dba664dc67c7cdabd56c1e5685233fbb1fc1966cfba2a340ec0dfff7b diff --git a/eng/conda_tools/__main__.py b/eng/conda_tools/__main__.py index 737549c4e..a7fb32d23 100644 --- a/eng/conda_tools/__main__.py +++ b/eng/conda_tools/__main__.py @@ -5,7 +5,7 @@ import argparse import sys -from . import audit, build +from . import audit, build, inputs, provenance, publication, release from .contracts import Format _ELF_DESCRIPTION = """Masking-immune audit of the vendored Linux ODBC binaries in built conda packages. @@ -182,7 +182,7 @@ def _audit(args: argparse.Namespace, kind: Format) -> int: return 0 -def main(argv: list[str] | None = None) -> int: +def parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="python -m eng.conda_tools", description="Build or audit Conda packages from the repository root.", @@ -205,7 +205,43 @@ def main(argv: list[str] | None = None) -> int: ), kind, ) - args = parser.parse_args(argv) + release.add_arguments( + commands.add_parser( + "validate", help="Validate release metadata.", description=release.__doc__ + ) + ) + publication.add_arguments( + commands.add_parser( + "promote", help="Promote or recover staged archives.", description=publication.__doc__ + ) + ) + commands.add_parser( + "provenance", help="Verify recorded producer sources.", description=provenance.__doc__ + ) + inputs.add_arguments( + commands.add_parser( + "fetch-wheels", help="Fetch and verify exact public wheel audit inputs." + ) + ) + commands.add_parser("probe-driver", help="Run the driver-load diagnostic in this interpreter.") + return parser + + +def main(argv: list[str] | None = None) -> int: + args = parser().parse_args(argv) + if args.command == "validate": + return release.execute(args) + if args.command == "promote": + return publication.cli(args) + if args.command == "provenance": + return provenance.cli() + if args.command == "fetch-wheels": + return inputs.fetch_cli(args) + if args.command == "probe-driver": + from . import driver_load_probe + + driver_load_probe.main() + return 0 if args.command == "build": return build.execute(args) return _audit(args, args.command) diff --git a/eng/conda_tools/archive.py b/eng/conda_tools/archive.py index 9f0f1a221..3d26d2541 100644 --- a/eng/conda_tools/archive.py +++ b/eng/conda_tools/archive.py @@ -4,15 +4,33 @@ import glob import csv +import bz2 +from contextlib import contextmanager import io import json import os +import posixpath +import re import tarfile import zipfile +from collections import Counter from email.parser import BytesParser from email.policy import default from pathlib import Path -from typing import Any, Iterator, TypedDict +from typing import Any, Callable, Iterable, Iterator, Protocol, TypedDict + +_CHUNK_BYTES = 64 * 1024 +_MAX_ARCHIVE_BYTES = 256 * 1024**2 +_MAX_COMPRESSED_BYTES = 256 * 1024**2 +_MAX_EXPANDED_BYTES = 1024**3 +_MAX_MEMBER_BYTES = 128 * 1024**2 +_MAX_PAYLOAD_BYTES = 512 * 1024**2 +_MAX_METADATA_BYTES = 8 * 1024**2 +_MAX_METADATA_TOTAL = 64 * 1024**2 +_MAX_MEMBERS = 10_000 +_MAX_ZIP_MEMBERS = 10_000 +_MAX_ZIP_DIRECTORY_BYTES = 1024**2 +_MAX_ZSTD_WINDOW_BYTES = 64 * 1024**2 READ_ERRORS = ( OSError, @@ -25,6 +43,83 @@ ) +def _check_limit(size: int, limit: int, description: str) -> None: + if size < 0 or size > limit: + raise ValueError(f"Archive limit exceeded: {description} ({size}; limit {limit})") + + +class _Readable(Protocol): + def read(self, size: int = -1, /) -> bytes: ... + + +class _SeekableReader(_Readable, Protocol): + def seek(self, offset: int, whence: int = 0, /) -> int: ... + + +class _LimitedReader(io.RawIOBase): + def __init__(self, source: _Readable, limit: int, description: str): + self.source = source + self.remaining = limit + self.limit = limit + self.description = description + + def readable(self) -> bool: + return True + + def read(self, size: int = -1) -> bytes: + if size < 0: + raise ValueError("Archive stream reads must specify a bounded size") + data = self.source.read(min(size, _CHUNK_BYTES, self.remaining + 1)) + self.remaining -= len(data) + _check_limit(self.limit - self.remaining, self.limit, self.description) + return data + + def readinto(self, buffer) -> int: + data = self.read(len(buffer)) + buffer[: len(data)] = data + return len(data) + + +class _ZipInput(io.BufferedReader): + def read(self, size: int | None = -1) -> bytes: + if size is None or size < 0: + size = os.fstat(self.fileno()).st_size - self.tell() + # ZipFile reads the central directory in one allocation, before exposing entries. + _check_limit(size, _MAX_ZIP_DIRECTORY_BYTES, "ZIP directory/read bytes") + return super().read(size) + + +@contextmanager +def _open_zip(path: str | Path) -> Iterator[zipfile.ZipFile]: + with _ZipInput(open(path, "rb", buffering=0)) as source: + _check_limit(os.fstat(source.fileno()).st_size, _MAX_ARCHIVE_BYTES, "archive bytes") + with zipfile.ZipFile(source) as container: + _check_limit(len(container.infolist()), _MAX_ZIP_MEMBERS, "ZIP member count") + for member in container.infolist(): + if member.filename != member.orig_filename: + raise ValueError(f"Noncanonical ZIP member name: {member.orig_filename!r}") + if member.compress_type not in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED): + raise ValueError("Archive ZIP members must use stored or deflate compression") + _check_limit(member.compress_size, _MAX_COMPRESSED_BYTES, "compressed bytes") + _check_limit(member.file_size, _MAX_COMPRESSED_BYTES, "ZIP member bytes") + yield container + + +def _read_bytes(source: _Readable, limit: int, description: str) -> bytes: + reader = _LimitedReader(source, limit, description) + with io.BytesIO() as result: + while data := reader.read(_CHUNK_BYTES): + result.write(data) + return result.getvalue() + + +def _zip_metadata(container: zipfile.ZipFile, member: str) -> bytes: + info = container.getinfo(member) + _check_limit(info.file_size, _MAX_METADATA_BYTES, "metadata bytes") + with container.open(info) as source: + return _read_bytes(source, _MAX_METADATA_BYTES, "metadata bytes") + + class DistributionMetadata(TypedDict): name: str version: str @@ -37,6 +132,19 @@ class WheelMetadata(DistributionMetadata): tags: list[str] +def canonical_distribution_name(name: str) -> str: + return re.sub(r"[-_.]+", "-", name).lower() + + +def metadata_members(members: Iterable[str], field: str) -> list[str]: + suffix = ".dist-info/" + field.casefold() + return [ + member + for member in members + if posixpath.normpath(member.replace("\\", "/")).casefold().endswith(suffix) + ] + + def parse_distribution_metadata(data: bytes) -> DistributionMetadata: """Read installed or wheel METADATA without applying dependency policy.""" metadata = BytesParser(policy=default).parsebytes(data) @@ -71,49 +179,234 @@ def parse_record_members(data: bytes) -> list[str]: def read_wheel_metadata(path: str | Path) -> WheelMetadata: """Return metadata, actual members, declared ownership and wheel tags as facts.""" - with zipfile.ZipFile(path) as wheel: + with _open_zip(path) as wheel: names = wheel.namelist() - entries = [ - name for name in names if name.count("/") == 1 and name.endswith(".dist-info/METADATA") - ] - if len(entries) != 1: + entries = metadata_members(names, "METADATA") + if ( + len(entries) != 1 + or entries[0].count("/") != 1 + or not entries[0].endswith(".dist-info/METADATA") + ): raise ValueError("expected exactly one .dist-info/METADATA entry.") - metadata = parse_distribution_metadata(wheel.read(entries[0])) + metadata = parse_distribution_metadata(_zip_metadata(wheel, entries[0])) prefix = entries[0][: -len("METADATA")] for member in ("RECORD", "WHEEL"): - if names.count(prefix + member) != 1: + if metadata_members(names, member) != [prefix + member]: raise ValueError(f"expected exactly one {prefix}{member} entry") - records = parse_record_members(wheel.read(prefix + "RECORD")) - tags = BytesParser(policy=default).parsebytes(wheel.read(prefix + "WHEEL")) + records = parse_record_members(_zip_metadata(wheel, prefix + "RECORD")) return { **metadata, "members": [entry.filename for entry in wheel.infolist() if not entry.is_dir()], "record_members": records, - "tags": [str(tag).strip() for tag in tags.get_all("Tag", [])], + "tags": parse_wheel_tags(_zip_metadata(wheel, prefix + "WHEEL")), } -def zstd_decompress(raw: bytes) -> bytes: - """Decompress a zstandard blob, preferring the 3.14+ stdlib backend.""" - try: # Python 3.14+ +def parse_wheel_tags(data: bytes) -> list[str]: + tags = BytesParser(policy=default).parsebytes(data) + return [str(tag).strip() for tag in tags.get_all("Tag", [])] + + +def installed_metadata( + names: list[str], files: dict[str, bytes] +) -> Iterator[tuple[DistributionMetadata, list[str], list[str], str]]: + """Yield facts, present RECORD-owned paths, RECORD paths and dist-info prefix. + + Installed distributions share site-packages. A binding must not inherit its + separate provider's ownership just because those files are present beside it. + """ + counts = Counter(names) + candidates = { + field: metadata_members(names, field) for field in ("METADATA", "RECORD", "WHEEL") + } + for field, members in candidates.items(): + seen = set() + for member in members: + if posixpath.normpath(member.replace("\\", "/")) != member or not member.endswith( + ".dist-info/" + field + ): + raise ValueError(f"Noncanonical installed {field} member: {member}") + if counts[member] != 1: + if field == "METADATA": + raise ValueError(f"duplicate installed metadata: {member}") + raise ValueError(f"expected exactly one installed {field} entry: {member}") + if member.casefold() in seen: + raise ValueError(f"Aliased installed {field} entries: {member}") + seen.add(member.casefold()) + if set(metadata_members(files, "METADATA")) != set(candidates["METADATA"]): + raise ValueError("Installed METADATA contents differ from the actual payload members") + distributions = [] + for member in candidates["METADATA"]: + facts = parse_distribution_metadata(files[member]) + prefix = member[: -len("METADATA")] + directory = prefix.rstrip("/").rsplit("/", 1)[-1] + expected = ( + canonical_distribution_name(facts["name"]).replace("-", "_") + + f"-{facts['version']}.dist-info" + ) + if directory != expected: + raise ValueError(f"Noncanonical installed METADATA directory: {member}") + distributions.append((facts, prefix)) + prefixes = {prefix for _, prefix in distributions} + for field in ("RECORD", "WHEEL"): + if any(member[: -len(field)] not in prefixes for member in candidates[field]): + raise ValueError(f"Unexpected or aliased installed {field} member") + record_files = {} + for _, prefix in distributions: + record = prefix + "RECORD" + if counts[record] != 1 or record not in files: + raise ValueError(f"expected exactly one installed RECORD entry: {record}") + record_files[record] = parse_record_members(files[record]) + for facts, prefix in distributions: + root = prefix.rstrip("/").rsplit("/", 1)[0] + "/" + record = prefix + "RECORD" + records = record_files[record] + owned = set(records) + present = [ + name[len(root) :] + for name in names + if name.startswith(root) and name[len(root) :] in owned + ] + yield facts, present, records, prefix + + +def collect(root: str) -> list[str]: + return sorted( + glob.glob(os.path.join(root, "**", "*.conda"), recursive=True) + + glob.glob(os.path.join(root, "**", "*.tar.bz2"), recursive=True) + ) + + +def _require_index_object(index: object, path: str) -> dict: + if not isinstance(index, dict): + raise ValueError(f"{path}: info/index.json must contain a JSON object") + return index + + +def _required_index_string(index: dict, key: str, path: str) -> str: + value = index.get(key) + if not isinstance(value, str) or not value or value != value.strip(): + raise ValueError( + f"{path}: info/index.json field '{key}' must be a non-empty, trimmed string" + ) + return value + + +def _validated_package_identity(index: dict, path: str) -> tuple[str, str, str, str]: + name, version, subdir, build = ( + _required_index_string(index, key, path) for key in ("name", "version", "subdir", "build") + ) + extension = ".conda" if path.endswith(".conda") else ".tar.bz2" + canonical_name = f"{name}-{version}-{build}{extension}" + if Path(path).name != canonical_name: + raise ValueError( + f"{path}: must use canonical basename '{canonical_name}': " + "anaconda-client normalizes upload names from metadata." + ) + return name, version, subdir, build + + +def _read_exact(source: _Readable, size: int) -> bytes: + data = source.read(size) + if len(data) != size: + raise EOFError("Truncated zstandard frame") + return data + + +def _validate_zstd_frames(source: _Readable) -> None: + """Check frame boundaries because python-zstandard's reader accepts truncated EOF.""" + import zstandard + + reader = _LimitedReader(source, _MAX_COMPRESSED_BYTES, "compressed bytes") + frames = 0 + while magic := reader.read(4): + frames += 1 + _check_limit(frames, _MAX_MEMBERS, "zstandard frame count") + if len(magic) != 4: + raise EOFError("Truncated zstandard frame") + number = int.from_bytes(magic, "little") + if number & 0xFFFFFFF0 == 0x184D2A50: + remaining = int.from_bytes(_read_exact(reader, 4), "little") + else: + if magic != b"\x28\xb5\x2f\xfd": + raise ValueError("invalid zstd frame magic") + header = magic + _read_exact(reader, 2) + header += _read_exact(reader, zstandard.frame_header_size(header) - len(header)) + parameters = zstandard.get_frame_parameters(header) + _check_limit(parameters.window_size, _MAX_ZSTD_WINDOW_BYTES, "zstandard window bytes") + if parameters.content_size != zstandard.CONTENTSIZE_UNKNOWN: + _check_limit(parameters.content_size, _MAX_EXPANDED_BYTES, "expanded bytes") + while True: + block = int.from_bytes(_read_exact(reader, 3), "little") + kind, size = (block >> 1) & 3, block >> 3 + if kind == 3 or size > 128 * 1024: + raise ValueError("invalid zstd block header") + remaining = 1 if kind == 1 else size + while remaining: + chunk = _read_exact(reader, min(remaining, _CHUNK_BYTES)) + remaining -= len(chunk) + if block & 1: + break + remaining = 4 if parameters.has_checksum else 0 + while remaining: + chunk = _read_exact(reader, min(remaining, _CHUNK_BYTES)) + remaining -= len(chunk) + if not frames: + raise EOFError("Missing zstandard frame") + + +@contextmanager +def _zstd_stream(source: _SeekableReader, purpose: str) -> Iterator[_Readable]: + try: from compression import zstd # type: ignore except ImportError: try: - import zstandard # third-party fallback + import zstandard except ImportError as exc: raise RuntimeError( - "Unable to import 'zstandard': reading .conda (.tar.zst) payloads requires " + f"Unable to import 'zstandard': reading .conda (.tar.zst) {purpose} requires " "Python 3.14+ with compression.zstd or a working 'zstandard' install " "(pip install zstandard)." ) from exc try: - return zstandard.ZstdDecompressor().decompress(raw) + _validate_zstd_frames(source) + source.seek(0) + with io.BufferedReader( + _LimitedReader(source, _MAX_COMPRESSED_BYTES, "compressed bytes"), + buffer_size=_CHUNK_BYTES, + ) as compressed: + # Tested python-zstandard 0.23/0.25 forward a byte count to libzstd here. + with zstandard.ZstdDecompressor( + max_window_size=_MAX_ZSTD_WINDOW_BYTES + ).stream_reader( + compressed, + read_size=_CHUNK_BYTES, + read_across_frames=True, + closefd=False, + ) as decoded: + yield decoded except zstandard.ZstdError as exc: raise ValueError(str(exc)) from exc - try: - return zstd.decompress(raw) - except zstd.ZstdError as exc: - raise ValueError(str(exc)) from exc + else: + try: + with zstd.ZstdFile( + _LimitedReader(source, _MAX_COMPRESSED_BYTES, "compressed bytes"), + mode="rb", + options={ + zstd.DecompressionParameter.window_log_max: _MAX_ZSTD_WINDOW_BYTES.bit_length() + - 1 + }, + ) as decoded: + yield decoded + except zstd.ZstdError as exc: + raise ValueError(str(exc)) from exc + + +def zstd_decompress(raw: bytes) -> bytes: + """Decode a bounded byte input; archive readers use streaming instead.""" + _check_limit(len(raw), _MAX_COMPRESSED_BYTES, "compressed bytes") + with _zstd_stream(io.BytesIO(raw), "payloads") as decoded: + return _read_bytes(decoded, _MAX_EXPANDED_BYTES, "expanded bytes") def _conda_component(zf: zipfile.ZipFile, component: str) -> zipfile.ZipInfo: @@ -130,54 +423,145 @@ def _conda_component(zf: zipfile.ZipFile, component: str) -> zipfile.ZipInfo: return matches[0] -def iter_payload_members(path: str) -> Iterator[tuple[str, bytes]]: - """Yield ``(member_name, data_bytes)`` for the files in a ``.conda`` / ``.tar.bz2`` payload.""" +def decompress_index(raw: bytes) -> bytes: + """Normalize bounded release-index decoding errors without changing backends.""" + _check_limit(len(raw), _MAX_COMPRESSED_BYTES, "compressed bytes") + with _zstd_stream(io.BytesIO(raw), "metadata") as decoded: + return _read_bytes(decoded, _MAX_EXPANDED_BYTES, "expanded bytes") + + +def _member_limit(member: tarfile.TarInfo) -> tuple[int, str]: + normalized = posixpath.normpath(member.name.replace("\\", "/")).casefold() + if ( + normalized.startswith("info/") + or ".dist-info/" in normalized + or member.type + in ( + tarfile.XHDTYPE, + tarfile.XGLTYPE, + tarfile.SOLARIS_XHDTYPE, + tarfile.GNUTYPE_LONGNAME, + tarfile.GNUTYPE_LONGLINK, + ) + ): + return _MAX_METADATA_BYTES, "metadata bytes" + return _MAX_MEMBER_BYTES, "member bytes" + + +def _tar_info_type() -> type[tarfile.TarInfo]: + count, metadata_bytes = 0, 0 + + class LimitedTarInfo(tarfile.TarInfo): + @classmethod + def _frombuf( + cls, buf: bytes | bytearray, encoding: str, errors: str, **kwargs: bool + ) -> LimitedTarInfo: + nonlocal count, metadata_bytes + # Recent Python security updates route parsing through _frombuf instead. + decode = getattr(super(), "_frombuf", super().frombuf) + member = decode(buf, encoding, errors, **kwargs) + count += 1 + _check_limit(count, _MAX_MEMBERS, "TAR member count") + limit, description = _member_limit(member) + _check_limit(member.size, limit, description) + if description == "metadata bytes": + metadata_bytes += member.size + _check_limit(metadata_bytes, _MAX_METADATA_TOTAL, "cumulative metadata bytes") + return member + + frombuf = _frombuf + + def _reject_sparse(self, *args): + raise ValueError("Sparse TAR members are not supported by Conda archive readers") + + # Sparse extension parsing can allocate maps before TarFile returns a member. + _proc_sparse = _proc_gnusparse_00 = _proc_gnusparse_01 = _proc_gnusparse_10 = _reject_sparse + + return LimitedTarInfo + + +def _tar_members(contents: tarfile.TarFile) -> Iterator[tarfile.TarInfo]: + total = 0 + for member in contents: + _check_limit(member.size, *_member_limit(member)) + total += member.size + _check_limit(total, _MAX_PAYLOAD_BYTES, "cumulative member bytes") + yield member + + +@contextmanager +def _stream_tar(decoded: _Readable) -> Iterator[tarfile.TarFile]: + reader = _LimitedReader(decoded, _MAX_EXPANDED_BYTES, "expanded bytes") + with tarfile.open(fileobj=reader, mode="r|", tarinfo=_tar_info_type()) as contents: + yield contents + # Account for padding/trailing frames and detect truncated compressor footers. + while reader.read(_CHUNK_BYTES): + pass + + +@contextmanager +def _open_tar( + path: str, + select: Callable[[zipfile.ZipFile], str | zipfile.ZipInfo], +) -> Iterator[tarfile.TarFile]: if path.endswith(".conda"): - with zipfile.ZipFile(path) as zf: - blob = zstd_decompress(zf.read(_conda_component(zf, "pkg"))) - with tarfile.open(fileobj=io.BytesIO(blob)) as tf: - for m in tf.getmembers(): - if not m.isfile(): - continue - f = tf.extractfile(m) - if f is not None: - yield m.name, f.read() + with _open_zip(path) as container: + with container.open(select(container)) as source: + with _zstd_stream(source, "payloads/metadata") as decoded: + with _stream_tar(decoded) as contents: + yield contents elif path.endswith(".tar.bz2"): - with tarfile.open(path, "r:bz2") as tf: - for m in tf.getmembers(): - if not m.isfile(): - continue - f = tf.extractfile(m) - if f is not None: - yield m.name, f.read() + with open(path, "rb") as source: + _check_limit(os.fstat(source.fileno()).st_size, _MAX_ARCHIVE_BYTES, "archive bytes") + with bz2.BZ2File( + _LimitedReader(source, _MAX_COMPRESSED_BYTES, "compressed bytes") + ) as decoded: + with _stream_tar(decoded) as contents: + yield contents else: - # Fail CLOSED like read_index -- a caller that gets an unexpected extension must NOT - # receive a silently-empty iterator (a truncated/renamed package would slip through). raise ValueError(f"{path}: unrecognized conda package extension") +def iter_payload_members(path: str) -> Iterator[tuple[str, bytes]]: + """Yield regular payload members after unambiguous component selection.""" + + def select(container: zipfile.ZipFile) -> zipfile.ZipInfo: + return _conda_component(container, "pkg") + + with _open_tar(path, select) as contents: + for member in _tar_members(contents): + if member.isfile(): + source = contents.extractfile(member) + if source is not None: + yield member.name, _read_bytes(source, *_member_limit(member)) + + +def _tar_index(contents: tarfile.TarFile, path: str) -> dict: + data = None + for member in _tar_members(contents): + if member.name != "info/index.json": + continue + if data is not None or not member.isfile(): + raise ValueError(f"{path}: expected exactly one regular info/index.json member") + source = contents.extractfile(member) + if source is None: + raise ValueError(f"{path}: info/index.json is unreadable") + data = _read_bytes(source, _MAX_METADATA_BYTES, "metadata bytes") + if data is None: + raise ValueError(f"{path}: expected exactly one regular info/index.json member") + return _require_index_object(json.loads(data), path) + + def read_index(path: str) -> dict[str, Any]: - """Return the package's ``info/index.json`` as a dict. + """Read native-audit metadata without applying the stricter release-container policy.""" - RAISES on a malformed/unreadable package -- callers must NOT swallow this into a - silent "non-Linux/non-Windows, skip" (a truncated package would then slip through). - """ - if path.endswith(".conda"): - with zipfile.ZipFile(path) as zf: - blob = zstd_decompress(zf.read(_conda_component(zf, "info"))) - with tarfile.open(fileobj=io.BytesIO(blob)) as tf: - member = tf.extractfile("info/index.json") - if member is None: - raise ValueError("info/index.json missing") - index = json.load(member) - elif path.endswith(".tar.bz2"): - with tarfile.open(path, "r:bz2") as tf: - member = tf.extractfile("info/index.json") - if member is None: - raise ValueError("info/index.json missing") - index = json.load(member) - else: + def select(container: zipfile.ZipFile) -> zipfile.ZipInfo: + return _conda_component(container, "info") + + if not path.endswith((".conda", ".tar.bz2")): raise ValueError("unrecognized conda package extension") + with _open_tar(path, select) as contents: + index = _tar_index(contents, path) if not isinstance(index, dict): raise ValueError("info/index.json must be an object") depends = index.get("depends", []) @@ -186,8 +570,34 @@ def read_index(path: str) -> dict[str, Any]: return index -def collect(root: str) -> list[str]: - return sorted( - glob.glob(os.path.join(root, "**", "*.conda"), recursive=True) - + glob.glob(os.path.join(root, "**", "*.tar.bz2"), recursive=True) - ) +def read_release_index(path: str) -> dict: + """Require the canonical three-member container and one regular, object-valued index.""" + + def select(container: zipfile.ZipFile) -> str: + names = container.namelist() + if len(names) != len(set(names)): + raise ValueError(f"{path}: duplicate ZIP entries are not allowed") + info_names = [n for n in names if n.startswith("info-") and n.endswith(".tar.zst")] + if len(info_names) != 1: + raise ValueError( + f"{path}: expected exactly one info-*.tar.zst member; found {len(info_names)}" + ) + pkg_names = [n for n in names if n.startswith("pkg-") and n.endswith(".tar.zst")] + if len(pkg_names) != 1: + raise ValueError( + f"{path}: expected exactly one pkg-*.tar.zst member; found {len(pkg_names)}" + ) + stem = Path(path).stem + if set(names) != {"metadata.json", f"info-{stem}.tar.zst", f"pkg-{stem}.tar.zst"}: + raise ValueError(f"{path}: expected only canonical metadata.json/info/pkg members") + metadata = json.loads(_zip_metadata(container, "metadata.json")) + if ( + not isinstance(metadata, dict) + or type(metadata.get("conda_pkg_format_version")) is not int + or metadata["conda_pkg_format_version"] != 2 + ): + raise ValueError(f"{path}: metadata.json must declare conda_pkg_format_version 2") + return info_names[0] + + with _open_tar(path, select) as contents: + return _tar_index(contents, path) diff --git a/eng/conda_tools/audit.py b/eng/conda_tools/audit.py index 3dcb318ce..9c4d664b9 100644 --- a/eng/conda_tools/audit.py +++ b/eng/conda_tools/audit.py @@ -27,21 +27,14 @@ def _validate_payload( path: str, kind: Format, index: dict[str, Any] ) -> tuple[list[str], list[str]]: # Retain names and parsed facts, not every member's native payload bytes. - metadata: dict[str, tuple[archive.DistributionMetadata, str]] = {} - records: dict[str, list[str]] = {} + metadata_files: dict[str, bytes] = {} names: list[str] = [] def members() -> Iterator[tuple[str, bytes]]: for name, data in archive.iter_payload_members(path): names.append(name) - if name.endswith(".dist-info/METADATA"): - facts = archive.parse_distribution_metadata(data) - distribution = contracts.canonical_distribution_name(facts["name"]) - if distribution in metadata: - raise ValueError(f"duplicate installed metadata for {distribution}") - metadata[distribution] = facts, name - elif name.endswith(".dist-info/RECORD"): - records[name] = archive.parse_record_members(data) + if name.endswith((".dist-info/METADATA", ".dist-info/RECORD")): + metadata_files[name] = data yield name, data base = os.path.basename(path) @@ -52,44 +45,53 @@ def members() -> Iterator[tuple[str, bytes]]: else: macho_members = [(name, macho.macho_arches(data)) for name, data in members()] - def ownership( - distribution: str, - ) -> tuple[archive.DistributionMetadata, list[str], list[str]]: - facts, member = metadata[distribution] - root = member.rsplit("/", 2)[0] + "/" - record = member[: -len("METADATA")] + "RECORD" - owned = records.get(record, []) - relative = [name[len(root) :] for name in names if name.startswith(root)] - return facts, [name for name in relative if name in owned], owned + metadata = {} + for facts, present, owned, prefix in archive.installed_metadata(names, metadata_files): + distribution = contracts.canonical_distribution_name(facts["name"]) + if distribution in metadata: + raise ValueError(f"duplicate installed metadata for {distribution}") + metadata[distribution] = facts, present, owned, prefix rs_required = "mssql-python-rs" in metadata - if rs_required and "mssql-python" not in metadata: - raise ValueError("RS distribution is missing its binding distribution metadata") - if "mssql-python" in metadata: - binding, files, owned = ownership("mssql-python") - rs_version = contracts.binding_rs_version(binding, files, owned) - if rs_version is None and rs_required: + if kind == "elf": + errors, details = contracts.validate_elf(base, index, elf_members, rs_required) + elif kind == "pe": + errors, details = contracts.validate_pe(base, index, pe_members) + else: + errors, details = contracts.validate_macho(base, index, macho_members) + if errors: + return errors, details + if "mssql-python" not in metadata: + raise ValueError("Package is missing its binding distribution metadata") + binding, files, owned, prefix = metadata["mssql-python"] + rs_version = contracts.binding_rs_version(binding, files, owned) + if rs_version is None and rs_required: + raise ValueError( + "historical embedded-core binding must not be combined with an RS distribution" + ) + if rs_version is not None: + if not rs_required: raise ValueError( - "historical embedded-core binding must not be combined with an RS distribution" + f"binding requires mssql-python-rs=={rs_version}, but its metadata is missing" ) - if rs_version is not None: - if not rs_required: - raise ValueError( - f"binding requires mssql-python-rs=={rs_version}, but its metadata is missing" - ) - rs, files, owned = ownership("mssql-python-rs") - abi = [dep for dep in index.get("depends", []) if dep.split()[:1] == ["python_abi"]] - python_tag = abi[0].rsplit("_", 1)[-1] if len(abi) == 1 else "" - errors = contracts.validate_rs_ownership( - rs, files, owned, rs_version, python_tag, index["subdir"] - ) - if errors: - raise ValueError("; ".join(errors)) - if kind == "elf": - return contracts.validate_elf(base, index, elf_members, rs_required) - if kind == "pe": - return contracts.validate_pe(base, index, pe_members) - return contracts.validate_macho(base, index, macho_members) + rs, files, owned, _ = metadata["mssql-python-rs"] + abi = [dep for dep in index.get("depends", []) if dep.split()[:1] == ["python_abi"]] + python_tag = abi[0].rsplit("_", 1)[-1] if len(abi) == 1 else "" + errors = contracts.validate_rs_ownership( + rs, files, owned, rs_version, python_tag, index["subdir"] + ) + if errors: + raise ValueError("; ".join(errors)) + root = prefix.rsplit("/", 2)[0] + "/" + errors = contracts.validate_core_ownership( + names, + {name: component[2] for name, component in metadata.items()}, + "mssql-python-rs" if rs_version is not None else "mssql-python", + root=root, + ) + if errors: + raise ValueError("; ".join(errors)) + return [], details def audit_packages(paths: Iterable[str], kind: Format, subdir: str = "") -> AuditResult: diff --git a/eng/conda_tools/build.py b/eng/conda_tools/build.py index f7e33963d..0c4bb84f6 100644 --- a/eng/conda_tools/build.py +++ b/eng/conda_tools/build.py @@ -113,6 +113,10 @@ def gather_wheels( if not source_rs: raise ValueError(f"empty RS source version assertion: {rs_version_file}") _checked_metadata(odbc, "mssql-python-odbc", odbc_ver) + if target_subdir and not contracts.odbc_wheel_matches_target( + os.path.basename(odbc), target_subdir + ): + raise ValueError(f"{odbc}: ODBC wheel does not match target {target_subdir}") for wheel in mssql: metadata = _checked_metadata(wheel, "mssql-python", mssql_ver) try: @@ -207,7 +211,11 @@ def _checked_metadata(path: str, distribution: str, version: str) -> archive.Whe try: metadata = archive.read_wheel_metadata(path) errors = contracts.validate_distribution_identity(metadata, distribution, version) + errors.extend( + contracts.validate_wheel_metadata_members(metadata["members"], distribution, version) + ) errors.extend(contracts.validate_wheel_tags(Path(path).name, metadata["tags"])) + errors.extend(contracts.validate_wheel_core_ownership(metadata)) if errors: raise ValueError("; ".join(errors)) return metadata @@ -216,25 +224,17 @@ def _checked_metadata(path: str, distribution: str, version: str) -> archive.Whe def _select_rs_wheel(directory: str | None, version: str, python_tag: str, subdir: str) -> str: - platforms = { - "win-64": "win_amd64", - "win-arm64": "win_arm64", - "osx-64": "macosx_15_0_universal2", - "osx-arm64": "macosx_15_0_universal2", - "linux-64": "manylinux_2_34_x86_64", - "linux-aarch64": "manylinux_2_34_aarch64", - } - if not directory or subdir not in platforms or not re.fullmatch(r"cp3\d+", python_tag): + if ( + not directory + or subdir not in contracts._RS_PLATFORMS + or not re.fullmatch(r"cp3\d+", python_tag) + ): raise ValueError( "an RS-dependent binding requires --rs-wheel-dir and a normal CPython target" ) matches = [] for path in sorted(Path(directory).glob(f"mssql_python_rs-{version}-*.whl")): - py, abi, platform = path.stem.rsplit("-", 3)[1:] - compatible_python = py == python_tag and abi == python_tag - if abi == "abi3" and re.fullmatch(r"cp3\d+", py): - compatible_python = int(py[2:]) <= int(python_tag[2:]) - if compatible_python and platforms[subdir] in platform.split("."): + if contracts.rs_wheel_matches_target(path.name, python_tag, subdir): matches.append(str(path)) if len(matches) != 1: raise ValueError( diff --git a/eng/conda_tools/contracts.py b/eng/conda_tools/contracts.py index b4b624b22..e2daf0132 100644 --- a/eng/conda_tools/contracts.py +++ b/eng/conda_tools/contracts.py @@ -5,9 +5,10 @@ import os import posixpath import re -from typing import Any, Iterable, Literal, Sequence +from typing import Any, Iterable, Literal, Mapping, Sequence -from .archive import DistributionMetadata +from .archive import DistributionMetadata, WheelMetadata, metadata_members +from .archive import canonical_distribution_name as canonical_distribution_name from .formats.elf import ElfDynamicInfo, ElfFacts Format = Literal["elf", "pe", "macho"] @@ -98,10 +99,6 @@ ) -def canonical_distribution_name(name: str) -> str: - return re.sub(r"[-_.]+", "-", name).lower() - - def validate_distribution_identity( metadata: DistributionMetadata, distribution: str, version: str ) -> list[str]: @@ -122,6 +119,15 @@ def validate_distribution_identity( return errors +def validate_wheel_metadata_members( + members: Iterable[str], distribution: str, version: str +) -> list[str]: + distribution = canonical_distribution_name(distribution).replace("-", "_") + expected = f"{distribution}-{version}.dist-info/METADATA" + entries = metadata_members(members, "METADATA") + return [] if entries == [expected] else [f"Expected one matching root METADATA at {expected}."] + + def validate_wheel_tags(filename: str, tags: Sequence[str]) -> list[str]: parts = filename.removesuffix(".whl").rsplit("-", 3) if not filename.endswith(".whl") or len(parts) != 4: @@ -136,6 +142,20 @@ def validate_wheel_tags(filename: str, tags: Sequence[str]) -> list[str]: return [] if set(tags) == expected else ["WHEEL tags do not match the selected filename"] +def _wheel_platform_matches_target(platforms: str, subdir: str) -> bool: + patterns = { + "win-64": r"win_amd64", + "win-arm64": r"win_arm64", + "linux-64": r"(?:linux|manylinux(?:1|2010|2014|_\d+_\d+))_x86_64", + "linux-aarch64": r"(?:linux|manylinux(?:1|2010|2014|_\d+_\d+))_aarch64", + "osx-64": r"macosx_\d+_\d+_(?:x86_64|universal2)", + "osx-arm64": r"macosx_\d+_\d+_(?:arm64|universal2)", + } + if subdir not in patterns: + raise ValueError(f"unknown Conda wheel target: {subdir}") + return any(re.fullmatch(patterns[subdir], platform) for platform in platforms.split(".")) + + def binding_wheel_matches_target( filename: str, subdir: str, python_versions: Sequence[str] = () ) -> bool: @@ -145,25 +165,48 @@ def binding_wheel_matches_target( parts = filename.removesuffix(".whl").rsplit("-", 3) if not filename.endswith(".whl") or len(parts) != 4: raise ValueError(f"invalid binding wheel filename: {filename}") - python_tag, _, platforms = parts[1:] - if subdir: - patterns = { - "win-64": r"win_amd64", - "win-arm64": r"win_arm64", - "linux-64": r"(?:linux|manylinux(?:1|2010|2014|_\d+_\d+))_x86_64", - "linux-aarch64": r"(?:linux|manylinux(?:1|2010|2014|_\d+_\d+))_aarch64", - "osx-64": r"macosx_\d+_\d+_(?:x86_64|universal2)", - "osx-arm64": r"macosx_\d+_\d+_(?:arm64|universal2)", - } - if subdir not in patterns: - raise ValueError(f"unknown binding wheel target: {subdir}") - if not any(re.fullmatch(patterns[subdir], platform) for platform in platforms.split(".")): - return False + python_tag, abi, platforms = parts[1:] + if subdir and not _wheel_platform_matches_target(platforms, subdir): + return False if not re.fullmatch(r"cp3\d+", python_tag): raise ValueError(f"invalid binding wheel Python tag: {filename}") - return not python_versions or python_tag in { + if python_versions and python_tag not in { f"cp{version.replace('.', '')}" for version in python_versions - } + }: + return False + return abi == python_tag + + +def odbc_wheel_matches_target(filename: str, subdir: str) -> bool: + parts = filename.removesuffix(".whl").rsplit("-", 3) + if not filename.endswith(".whl") or len(parts) != 4: + raise ValueError(f"invalid ODBC wheel filename: {filename}") + python_tag, abi, platforms = parts[1:] + return ( + python_tag == "py3" and abi == "none" and _wheel_platform_matches_target(platforms, subdir) + ) + + +_RS_PLATFORMS = { + "win-64": "win_amd64", + "win-arm64": "win_arm64", + "osx-64": "macosx_15_0_universal2", + "osx-arm64": "macosx_15_0_universal2", + "linux-64": "manylinux_2_34_x86_64", + "linux-aarch64": "manylinux_2_34_aarch64", +} + + +def rs_wheel_matches_target(filename: str, python_tag: str, subdir: str) -> bool: + """Use the build's exact normal-CPython/abi3 and platform selection for RS inputs.""" + parts = filename.removesuffix(".whl").rsplit("-", 3) + if not filename.endswith(".whl") or len(parts) != 4 or not re.fullmatch(r"cp3\d+", python_tag): + return False + py, abi, platform = parts[1:] + compatible_python = py == python_tag and abi == python_tag + if abi == "abi3" and re.fullmatch(r"cp3\d+", py): + compatible_python = int(py[2:]) <= int(python_tag[2:]) + return compatible_python and _RS_PLATFORMS.get(subdir) in platform.split(".") def exact_dependency_pin(requirements: Iterable[str], distribution: str) -> str | None: @@ -195,6 +238,57 @@ def owned_core_members(members: Iterable[str], records: Iterable[str]) -> list[s return [name for name in records if name in actual and name.startswith("mssql_py_core/")] +def _core_path(member: str, root: str = "") -> str | None: + # Account for wheel spread paths and aliases on case-insensitive extraction targets. + path = posixpath.normpath(member.replace("\\", "/")).lstrip("/").casefold() + if root: + prefix = posixpath.normpath(root.replace("\\", "/")).lstrip("/").casefold() + "/" + if not path.startswith(prefix): + return None + path = path[len(prefix) :] + path = re.sub(r"^[^/]+\.data/(?:purelib|platlib)/", "", path) + return path if path.startswith("mssql_py_core/") else None + + +def validate_core_ownership( + members: Iterable[str], + ownership: Mapping[str, Iterable[str]], + provider: str, + *, + root: str = "", +) -> list[str]: + """Require every actual core file to belong only to the selected provider's RECORD.""" + core = [path for member in members if (path := _core_path(member, root)) is not None] + records = { + owner: {path for member in declared if (path := _core_path(member)) is not None} + for owner, declared in ownership.items() + } + errors = [] + if len(core) != len(set(core)): + errors.append("duplicate mssql_py_core payload paths") + for member in sorted(set(core)): + owners = {owner for owner, paths in records.items() if member in paths} + if owners != {provider}: + errors.append( + f"{member}: mssql_py_core must be owned exclusively by {provider}; " + f"RECORD owners: {sorted(owners)}" + ) + return errors + + +def validate_wheel_core_ownership(metadata: WheelMetadata) -> list[str]: + distribution = canonical_distribution_name(metadata["name"]) + provider = ( + "mssql-python" + if distribution == "mssql-python" + and exact_dependency_pin(metadata["requires_dist"], "mssql-python-rs") is None + else "mssql-python-rs" + ) + return validate_core_ownership( + metadata["members"], {distribution: metadata["record_members"]}, provider + ) + + def binding_rs_version( metadata: DistributionMetadata, members: Iterable[str], diff --git a/eng/conda_tools/inputs.py b/eng/conda_tools/inputs.py new file mode 100644 index 000000000..af7f3d93d --- /dev/null +++ b/eng/conda_tools/inputs.py @@ -0,0 +1,454 @@ +"""Source-bound component inputs and explicit published-wheel audit controls.""" + +from __future__ import annotations + +import argparse +import ast +import json +import os +from pathlib import Path +from typing import Callable +import re +import subprocess +import sys +from urllib.error import URLError +from urllib.request import urlopen + +from eng.scripts.download_mssql_python_rs_wheels import DEFAULT_FEED, PACKAGE_ID, file_sha256 + +from . import archive, contracts + + +def _is_sha256(value: object) -> bool: + return isinstance(value, str) and re.fullmatch(r"[0-9a-f]{64}", value) is not None + + +def _download_published( + versions: dict[str, str], + requirements: list[str], + wheel_dir: Path, + requirements_file: Path, +) -> dict[str, dict[str, str]]: + published = {} + for name, version in versions.items(): + try: + with urlopen(f"https://pypi.org/pypi/{name}/{version}/json", timeout=60) as response: + release = json.load(response) + except URLError as error: + raise ValueError( + f"Cannot fetch published {name}=={version} from PyPI; no fallback is allowed." + ) from error + info = release.get("info") if isinstance(release, dict) else None + urls = release.get("urls") if isinstance(release, dict) else None + if ( + not isinstance(info, dict) + or info.get("name") != name + or info.get("version") != version + or not isinstance(urls, list) + or any(not isinstance(item, dict) for item in urls) + ): + raise ValueError(f"Invalid published wheel identity or SHA256 for {name}=={version}.") + wheels = [ + item + for item in urls + if item.get("packagetype") == "bdist_wheel" and not item.get("yanked") + ] + if not wheels or any( + not isinstance(item.get("filename"), str) + or not isinstance(item.get("digests"), dict) + or not _is_sha256(item["digests"].get("sha256")) + for item in wheels + ): + raise ValueError(f"Invalid published wheel identity or SHA256 for {name}=={version}.") + hashes = {item["filename"]: item["digests"]["sha256"] for item in wheels} + if len(hashes) != len(wheels): + raise ValueError(f"Ambiguous published wheel filenames for {name}=={version}.") + published[name] = hashes + requirements.append( + f"{name}=={version} " + + " ".join(f"--hash=sha256:{digest}" for digest in sorted(set(hashes.values()))) + ) + requirements_file.write_text("\n".join(requirements) + "\n", encoding="utf-8") + subprocess.run( + [ + sys.executable, + "-m", + "pip", + "--isolated", + "download", + "--index-url", + "https://pypi.org/simple", + "--no-deps", + "--only-binary=:all:", + "--require-hashes", + "-r", + str(requirements_file), + "-d", + str(wheel_dir), + ], + check=True, + ) + return published + + +def _published_wheel( + wheel_dir: Path, + name: str, + version: str, + hashes: dict[str, str], + python_tag: str, + subdir: str, +) -> archive.WheelMetadata: + selected = list(wheel_dir.glob(f"{name.replace('-', '_')}-{version}-*.whl")) + if len(selected) != 1: + raise ValueError(f"Missing or ambiguous wheel for {name}=={version}.") + path = selected[0] + if hashes.get(path.name) != file_sha256(path): + raise ValueError(f"Downloaded wheel SHA256 disagrees with published {path.name}.") + metadata = archive.read_wheel_metadata(path) + violations = contracts.validate_distribution_identity(metadata, name, version) + violations.extend(contracts.validate_wheel_metadata_members(metadata["members"], name, version)) + violations.extend(contracts.validate_wheel_tags(path.name, metadata["tags"])) + violations.extend(contracts.validate_wheel_core_ownership(metadata)) + if name == "mssql-python": + matches = contracts.binding_wheel_matches_target( + path.name, subdir, [python_tag.removeprefix("cp")] + ) + elif name == "mssql-python-odbc": + matches = contracts.odbc_wheel_matches_target(path.name, subdir) + else: + matches = contracts.rs_wheel_matches_target(path.name, python_tag, subdir) + if not matches: + violations.append( + f"Published {name} wheel does not match the requested {python_tag} {subdir} target." + ) + if violations: + raise ValueError("; ".join(violations)) + return metadata + + +def fetch_wheels( + source_versions: dict[str, str], + wheel_dir: Path, + requirements_file: Path, + python_tag: str, + subdir: str, +) -> dict[str, str]: + """Fetch exact public inputs; the returned component set identifies the actual profile. + + A published embedded-core wheel is a historical packaging control, not proof of + the current source's separate-RS contract. Recorded producer inputs do not use + this public-profile selection. + """ + if not re.fullmatch(r"cp3\d+", python_tag): + raise ValueError(f"Expected a normal CPython target tag, not {python_tag!r}.") + wheel_dir.mkdir(parents=True, exist_ok=True) + versions = {name: source_versions[name] for name in ("mssql-python", "mssql-python-odbc")} + requirements: list[str] = [] + hashes = _download_published(versions, requirements, wheel_dir, requirements_file) + binding = _published_wheel( + wheel_dir, + "mssql-python", + versions["mssql-python"], + hashes["mssql-python"], + python_tag, + subdir, + ) + _published_wheel( + wheel_dir, + "mssql-python-odbc", + versions["mssql-python-odbc"], + hashes["mssql-python-odbc"], + python_tag, + subdir, + ) + if ( + contracts.exact_dependency_pin(binding["requires_dist"], "mssql-python-odbc") + != versions["mssql-python-odbc"] + ): + raise ValueError("Binding METADATA must require exactly the selected ODBC wheel version.") + rs_version = contracts.binding_rs_version( + binding, binding["members"], binding["record_members"] + ) + if rs_version is None: + owned = contracts.owned_core_members(binding["members"], binding["record_members"]) + violations = contracts.validate_core_layout(owned, python_tag, subdir) + else: + if source_versions.get("mssql-python-rs") != rs_version: + raise ValueError( + "Published binding RS requirement differs from the maintained source pin." + ) + versions["mssql-python-rs"] = rs_version + hashes.update( + _download_published( + {"mssql-python-rs": rs_version}, requirements, wheel_dir, requirements_file + ) + ) + rs = _published_wheel( + wheel_dir, "mssql-python-rs", rs_version, hashes["mssql-python-rs"], python_tag, subdir + ) + violations = contracts.validate_rs_ownership( + rs, rs["members"], rs["record_members"], rs_version, python_tag, subdir + ) + if violations: + raise ValueError("; ".join(violations)) + if len(list(wheel_dir.glob("*.whl"))) != len(versions): + raise ValueError("Expected exactly the selected binding, ODBC and required RS wheels.") + return versions + + +def validate_installed_inputs( + path: str, + python_tag: str, + subdir: str, + versions: dict[str, str], + receipt: dict | None, +) -> None: + """Bind installed component identities/ownership to verified producer source inputs.""" + names, files = [], {} + for name, data in archive.iter_payload_members(path): + names.append(name) + if ".dist-info/" in name: + files[name] = data + if len(names) != len(set(names)): + raise ValueError(f"{path}: duplicate payload members") + metadata = {} + root = ( + "Lib/site-packages/" + if subdir.startswith("win-") + else (f"lib/python{python_tag[2]}.{python_tag[3:]}/site-packages/") + ) + for facts, present, owned, prefix in archive.installed_metadata(names, files): + name = contracts.canonical_distribution_name(facts["name"]) + if name in metadata: + raise ValueError(f"{path}: duplicate installed metadata for {name}") + metadata[name] = facts, present, owned, prefix + if name in versions: + expected = root + f"{name.replace('-', '_')}-{versions[name]}.dist-info/" + errors = contracts.validate_distribution_identity(facts, name, versions[name]) + if ( + prefix != expected + or prefix + "RECORD" not in files + or prefix + "WHEEL" not in files + ): + errors.append(f"Expected canonical installed METADATA/RECORD/WHEEL for {name}.") + if errors: + raise ValueError(f"{path}: " + "; ".join(errors)) + if set(metadata) != set(versions): + raise ValueError( + f"{path}: installed components {sorted(metadata)} differ from source {sorted(versions)}" + ) + binding, present, owned, _ = metadata["mssql-python"] + if ( + contracts.exact_dependency_pin(binding["requires_dist"], "mssql-python-odbc") + != versions["mssql-python-odbc"] + ): + raise ValueError(f"{path}: binding does not require the source ODBC version") + rs_version = contracts.binding_rs_version( + binding, present, owned, versions.get("mssql-python-rs") + ) + if rs_version != versions.get("mssql-python-rs"): + raise ValueError(f"{path}: installed binding RS profile differs from producer source") + errors = contracts.validate_core_ownership( + names, + {name: component[2] for name, component in metadata.items()}, + "mssql-python-rs" if rs_version is not None else "mssql-python", + root=root, + ) + if errors: + raise ValueError(f"{path}: " + "; ".join(errors)) + if rs_version is None: + errors = contracts.validate_core_layout( + contracts.owned_core_members(present, owned), python_tag, subdir + ) + else: + rs, present, owned, prefix = metadata["mssql-python-rs"] + errors = contracts.validate_rs_ownership(rs, present, owned, rs_version, python_tag, subdir) + source = files.get(prefix + "conda-wheel-source.txt", b"").decode("utf-8") + filename = source.removesuffix("\n") + if receipt is None or source != filename + "\n" or filename not in receipt["wheel_sha256"]: + errors.append("Selected RS wheel is absent from the verified transport receipt.") + errors.extend( + contracts.validate_wheel_tags( + filename, archive.parse_wheel_tags(files[prefix + "WHEEL"]) + ) + ) + if not contracts.rs_wheel_matches_target(filename, python_tag, subdir): + errors.append("Selected RS wheel does not match the Conda Python/platform target.") + if errors: + raise ValueError(f"{path}: " + "; ".join(errors)) + + +def add_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--wheel-dir", type=Path, required=True) + parser.add_argument("--requirements-file", type=Path, required=True) + parser.add_argument( + "--python-tag", + required=True, + help="Expected normal CPython tag (e.g. cp311), not a pip cross-target override.", + ) + parser.add_argument( + "--conda-subdir", + required=True, + help="Expected native Conda subdir; pip downloads for the executing interpreter/host.", + ) + + +def fetch_cli(args: argparse.Namespace) -> int: + try: + source = read_release_versions(lambda path: Path(path).read_text(encoding="utf-8")) + versions = fetch_wheels( + source, args.wheel_dir, args.requirements_file, args.python_tag, args.conda_subdir + ) + rs_required = "mssql-python-rs" in versions + if output := os.environ.get("GITHUB_OUTPUT"): + with open(output, "a", encoding="utf-8") as destination: + destination.write(f"rsRequired={str(rs_required).lower()}\n") + except (*archive.READ_ERRORS, subprocess.CalledProcessError) as error: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + print("PUBLIC_WHEEL_INPUT_OK: " + json.dumps(versions, sort_keys=True)) + print( + "Published RS-dependent input contract verified; no installed Conda runtime claim." + if rs_required + else "Historical published packaging control; NOT current-source RS qualification." + ) + return 0 + + +def read_release_versions(read_source: Callable[[str], str]) -> dict[str, str]: + """Read the literals maintained by the existing wheel release process, without imports.""" + setup_source = read_source("setup.py") + versions = {} + for path, field in ( + ("setup.py", "version"), + ("mssql_python/__init__.py", "__version__"), + ("mssql_python_odbc/__init__.py", "__version__"), + ): + matches = re.findall( + rf"""(?m)^\s*{field}\s*=\s*['"]([A-Za-z0-9][A-Za-z0-9._-]*)['"]\s*,?\s*(?:#.*)?$""", + setup_source if path == "setup.py" else read_source(path), + ) + if len(matches) != 1: + raise ValueError(f"{path} must contain exactly one literal {field} release version.") + versions[path] = matches[0] + if versions["setup.py"] != versions["mssql_python/__init__.py"]: + raise ValueError( + "Binding release versions in setup.py and mssql_python/__init__.py differ." + ) + result = { + "mssql-python": versions["setup.py"], + "mssql-python-odbc": versions["mssql_python_odbc/__init__.py"], + } + if _source_requires_rs(setup_source): + path = "eng/versions/mssql-python-rs.version" + version = read_source(path).strip() + if not re.fullmatch(r"[0-9][A-Za-z0-9.!+_-]*", version): + raise ValueError(f"{path} must contain one RS distribution release version.") + result["mssql-python-rs"] = version + return result + + +def _source_requires_rs(setup_source: str) -> bool: + """Recognize only the explicit maintained dependency contract, never execute setup.py.""" + rs_dependencies = [] + try: + setup_tree = ast.parse(setup_source) + except SyntaxError: + raise ValueError( + "setup.py is not valid Python; its dependency contract cannot be read." + ) from None + declarations = [ + node.value + for node in ast.walk(setup_tree) + if isinstance(node, ast.keyword) and node.arg == "install_requires" + ] + if len(declarations) != 1: + raise ValueError("setup.py must declare exactly one explicit install_requires contract.") + dependencies = declarations[0] + if not isinstance(dependencies, (ast.List, ast.Tuple)): + raise ValueError("setup.py install_requires must be an explicit dependency list.") + for dependency in dependencies.elts: + prefix = ( + dependency.values[0] + if isinstance(dependency, ast.JoinedStr) and dependency.values + else dependency + ) + if ( + not isinstance(prefix, ast.Constant) + or not isinstance(prefix.value, str) + or not prefix.value.strip() + or ( + isinstance(dependency, ast.JoinedStr) + and not re.match(r"^[A-Za-z0-9][A-Za-z0-9._-]*[<=>!~]", prefix.value) + ) + ): + raise ValueError("setup.py dependencies must have explicit distribution names.") + if any( + isinstance(part, ast.Constant) + and isinstance(part.value, str) + and re.match(r"(?i)^mssql[-_.]+python[-_.]+rs(?:[<=>!~;\s(\[]|$)", part.value.strip()) + for part in ast.walk(dependency) + ): + rs_dependencies.append(dependency) + if rs_dependencies: + expected = ast.parse( + 'f"mssql-python-rs=={_read_mssql_python_rs_version()}"', mode="eval" + ).body + if len(rs_dependencies) != 1 or ast.dump(rs_dependencies[0]) != ast.dump(expected): + raise ValueError("setup.py must require the exact maintained RS distribution pin.") + return bool(rs_dependencies) + + +def parse_release_versions(raw: str) -> dict[str, str]: + versions = json.loads(raw) + required = {"mssql-python", "mssql-python-odbc"} + if ( + not isinstance(versions, dict) + or not required.issubset(versions) + or set(versions) - required - {"mssql-python-rs"} + or any( + not isinstance(version, str) + or re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9.!+_-]*", version) is None + for version in versions.values() + ) + ): + raise ValueError("Expected exact binding/ODBC and optional RS distribution versions.") + return versions + + +def validate_rs_transport( + receipt: object, versions: dict[str, str], transport_version: str +) -> None: + """Check the selected producer's receipt against its separately verified source pins.""" + version = versions.get("mssql-python-rs") + if version is None: + if receipt is not None or transport_version: + raise ValueError("An embedded-core producer must not supply an RS transport receipt.") + return + if not isinstance(receipt, dict): + raise ValueError("The RS producer requires its recorded rs-transport.json receipt.") + expected = { + "distribution_version": version, + "transport_version": transport_version, + "feed_url": DEFAULT_FEED, + "package_id": PACKAGE_ID, + } + if not transport_version or any(receipt.get(key) != value for key, value in expected.items()): + raise ValueError( + "RS transport receipt differs from the verified producer's source pins/feed." + ) + + wheels = receipt.get("wheel_sha256") + if not _is_sha256(receipt.get("package_sha256")) or not isinstance(wheels, dict) or not wheels: + raise ValueError("RS transport receipt requires its package and wheel SHA256 values.") + for name, digest in wheels.items(): + if ( + not isinstance(name, str) + or "/" in name + or "\\" in name + or not name.startswith(f"mssql_python_rs-{version}-") + or not name.endswith(".whl") + or not _is_sha256(digest) + ): + raise ValueError("RS transport receipt contains an invalid wheel identity or SHA256.") diff --git a/eng/conda_tools/provenance.py b/eng/conda_tools/provenance.py new file mode 100644 index 000000000..126b500e9 --- /dev/null +++ b/eng/conda_tools/provenance.py @@ -0,0 +1,233 @@ +"""Read-only verification of the recorded Conda producer and its upstream wheel run.""" + +from __future__ import annotations + +import json +import os +import re +import sys +from http.client import HTTPException +from typing import Callable +from urllib.parse import urlencode +from urllib.request import HTTPRedirectHandler, Request, build_opener + +from .inputs import read_release_versions + +_REPOSITORY_ID = "eec96f30-ec96-4910-abd6-c45a99a5c29f" +_PROJECT_ID = "c6d89619-62de-46a0-8b46-70b92a84d85e" +_CONDA_PIPELINE_ID = 2318 +_WHEEL_PIPELINE_ID = 2199 +_MAIN = "refs/heads/main" + + +class _NoRedirect(HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise ValueError("Unexpected Azure DevOps API redirect; credentials were not forwarded.") + + +def _positive_id(value: object) -> int: + if not re.fullmatch(r"[1-9][0-9]*", str(value)): + raise ValueError("Missing or invalid pipeline/run ID; expected a positive decimal integer.") + return int(str(value)) + + +def _required_env(name: str) -> str: + value = os.environ.get(name) + if not value: + raise ValueError(f"Missing required environment variable {name}.") + return value + + +def _producer_inputs(get_json: Callable[[str], dict], commit: str) -> dict: + def read_source(path: str) -> str: + query = urlencode( + { + "path": "/" + path, + "includeContent": "true", + "versionDescriptor.versionType": "commit", + "versionDescriptor.version": commit, + "$format": "json", + "api-version": "7.1", + } + ) + item = get_json(f"git/repositories/{_REPOSITORY_ID}/items?{query}") + if ( + item.get("path") != "/" + path + or item.get("commitId") != commit + or item.get("gitObjectType") != "blob" + or not isinstance(item.get("content"), str) + ): + raise ValueError(f"Missing or mismatched wheel producer source for {path} at {commit}.") + return item["content"] + + versions = read_release_versions(read_source) + result = {"versions": versions, "rsTransportVersion": ""} + if "mssql-python-rs" in versions: + path = "eng/versions/mssql-python-rs-nuget.version" + transport_version = read_source(path).strip() + if not re.fullmatch(r"[0-9][A-Za-z0-9.+_-]*", transport_version): + raise ValueError(f"{path} must contain one RS transport version.") + result["rsTransportVersion"] = transport_version + return result + + +def _verify_run(build: dict, run: dict, pipeline_id: int, run_id: int) -> dict: + if build.get("id") != run_id or build.get("definition", {}).get("id") != pipeline_id: + raise ValueError(f"Build API identity mismatch for pipeline {pipeline_id}, run {run_id}.") + if run.get("id") != run_id or run.get("pipeline", {}).get("id") != pipeline_id: + raise ValueError(f"Runs API identity mismatch for pipeline {pipeline_id}, run {run_id}.") + if build.get("status") != "completed" or build.get("result") != "succeeded": + raise ValueError(f"Build {run_id} must be completed/succeeded.") + if run.get("state") != "completed" or run.get("result") != "succeeded": + raise ValueError(f"Pipeline run {run_id} must be completed/succeeded.") + if not build.get("buildNumber") or run.get("name") != build["buildNumber"]: + raise ValueError(f"Build/run number mismatch for {run_id}.") + + branch, commit = build.get("sourceBranch"), build.get("sourceVersion") + if not isinstance(branch, str) or not branch.startswith("refs/heads/"): + raise ValueError(f"Build {run_id} has no valid source branch.") + if not isinstance(commit, str) or not re.fullmatch(r"[0-9a-fA-F]{40}", commit): + raise ValueError(f"Build {run_id} has no valid source commit.") + source = run.get("resources", {}).get("repositories", {}).get("self", {}) + if ( + source.get("refName") != branch + or source.get("version") != commit + or source.get("repository", {}).get("id") != _REPOSITORY_ID + or build.get("repository", {}).get("id") != _REPOSITORY_ID + or build.get("project", {}).get("id") != _PROJECT_ID + ): + raise ValueError(f"Build/run source repository, branch or commit mismatch for {run_id}.") + return {"pipeline": pipeline_id, "run": run_id, "branch": branch, "commit": commit} + + +def verify_provenance( + get_json: Callable[[str], dict], + *, + producer_pipeline_id: int, + producer_run_id: int, + producer_branch: str, + producer_commit: str, + release_branch: str, + publish: bool, +) -> dict: + if _positive_id(producer_pipeline_id) != _CONDA_PIPELINE_ID: + raise ValueError(f"Conda producer must be pipeline {_CONDA_PIPELINE_ID}.") + producer_run_id = _positive_id(producer_run_id) + build = get_json(f"build/builds/{producer_run_id}?api-version=7.1") + run = get_json(f"pipelines/{_CONDA_PIPELINE_ID}/runs/{producer_run_id}?api-version=7.1") + producer = _verify_run(build, run, _CONDA_PIPELINE_ID, producer_run_id) + if producer["branch"] != producer_branch or producer["commit"] != producer_commit: + raise ValueError( + "Selected Conda resource branch/commit differs from the recorded producer." + ) + + # ADO records the selected run ID in the nested pipeline.id and its build number + # in version. The authoritative Build response supplies the actual definition ID. + wheel_resource = run.get("resources", {}).get("pipelines", {}).get("buildPipeline", {}) + wheel_run_id = _positive_id(wheel_resource.get("pipeline", {}).get("id")) + wheel_build = get_json(f"build/builds/{wheel_run_id}?api-version=7.1") + if not wheel_resource.get("version") or wheel_resource["version"] != wheel_build.get( + "buildNumber" + ): + raise ValueError("Recorded upstream wheel version differs from its Build API build number.") + wheel = _verify_run( + wheel_build, + get_json(f"pipelines/{_WHEEL_PIPELINE_ID}/runs/{wheel_run_id}?api-version=7.1"), + _WHEEL_PIPELINE_ID, + wheel_run_id, + ) + reasons = [ + f"{name} source is {branch}, not {_MAIN}" + for name, branch in ( + ("release", release_branch), + ("Conda producer", producer["branch"]), + ("wheel producer", wheel["branch"]), + ) + if branch != _MAIN + ] + if publish and reasons: + raise ValueError("Production publication is ineligible: " + "; ".join(reasons)) + return { + "producer": producer, + "wheel": wheel, + **_producer_inputs(get_json, wheel["commit"]), + "productionEligible": not reasons, + "productionIneligibilityReasons": reasons, + } + + +def ado_get_json(path: str) -> dict: + """Read release evidence only from the configured SqlClientDrivers collection.""" + token = _required_env("SYSTEM_ACCESSTOKEN") + if not token or token.startswith("$("): + raise ValueError("System.AccessToken is unavailable; provenance cannot be verified.") + collection = _required_env("SYSTEM_COLLECTIONURI").rstrip("/") + if collection.lower() not in { + "https://dev.azure.com/sqlclientdrivers", + "https://sqlclientdrivers.visualstudio.com", + }: + raise ValueError("Unexpected Azure DevOps collection; refusing to send credentials.") + project = _required_env("SYSTEM_TEAMPROJECTID") + if project.lower() != _PROJECT_ID: + raise ValueError("System.TeamProjectId must identify the mssql-python release project.") + + request = Request( + f"{collection}/{project}/_apis/{path}", + headers={"Authorization": f"Bearer {token}"}, + ) + with build_opener(_NoRedirect()).open(request, timeout=60) as response: + return json.load(response) + + +def execute() -> None: + publish = _required_env("PUBLISH_TO_CONDA").lower() + if publish not in {"true", "false"}: + raise ValueError("PUBLISH_TO_CONDA must be true or false.") + result = verify_provenance( + ado_get_json, + producer_pipeline_id=_positive_id(_required_env("CONDA_BUILD_PIPELINE_ID")), + producer_run_id=_positive_id(_required_env("CONDA_BUILD_RUN_ID")), + producer_branch=_required_env("CONDA_BUILD_SOURCE_BRANCH"), + producer_commit=_required_env("CONDA_BUILD_SOURCE_COMMIT"), + release_branch=_required_env("RELEASE_SOURCE_BRANCH"), + publish=publish == "true", + ) + version = result["versions"]["mssql-python"] + expected = os.environ.get("MSSQL_PYTHON_VERSION", "") + if expected and expected != version: + raise ValueError( + "mssqlPythonVersion differs from the recorded wheel producer release version." + ) + print("VERIFIED_RECORDED_PROVENANCE: " + json.dumps(result, sort_keys=True)) + print(f"##vso[task.setvariable variable=mssqlPythonVersion;isOutput=true]{version}") + versions = json.dumps(result["versions"], sort_keys=True) + print(f"##vso[task.setvariable variable=releaseVersions;isOutput=true]{versions}") + transport = result["rsTransportVersion"] + print(f"##vso[task.setvariable variable=rsTransportVersion;isOutput=true]{transport}") + if publish == "false": + print("Validate-only: provenance verified; package readiness is gated separately.") + + +def cli() -> int: + """Keep expected CLI failures concise; imported callers still receive exceptions.""" + expected_errors = (ValueError, RuntimeError, ImportError, OSError, HTTPException) + try: + execute() + except expected_errors as error: + cause = error.__cause__ + while cause is not None: + if not isinstance(cause, expected_errors): + raise + cause = cause.__cause__ + if isinstance(error, ValueError) and not isinstance( + error, (json.JSONDecodeError, UnicodeError) + ): + message = str(error) + else: + message = "Check provenance configuration, Azure DevOps access and response format." + secret = os.environ.get("SYSTEM_ACCESSTOKEN") + if secret: + message = message.replace(secret, "[REDACTED]") + print(f"ERROR: {type(error).__name__}: " + " ".join(message.split()), file=sys.stderr) + return 1 + return 0 diff --git a/eng/conda_tools/publication.py b/eng/conda_tools/publication.py new file mode 100644 index 000000000..bba59b9dc --- /dev/null +++ b/eng/conda_tools/publication.py @@ -0,0 +1,568 @@ +"""Validate local inputs and promote or recover exact staged Conda archives. + +Use restricted publishing credentials and coordinate one publication or recovery at +a time for the same owner/package/target label. This helper does not prevent concurrent +writers; overlapping runs can interfere with labels and rollback. --check-local-only +validates archives and publication scope without credentials or an API client. + +Staging labels must use ``_staging_``; this namespace is +reserved and cannot be a public target. Recovery uses the original build's label. +Uploads happen before promotion under that build-unique staging label. This module +verifies every uploaded distribution against the local artifact, adds the public +label to the complete set, and removes the staging label only after all target-label +operations succeed. +Rollback is compensating, not atomic: only attempted additions absent from the +initial snapshot are removed. An interrupted invocation resumes from verified labels. +Local archives must use the metadata-derived canonical basename that the pinned +upload client sends; a renamed file is rejected before any upload. + +Failed uploads/promotions invoke --cleanup-staging for their attempted +archives. After a hard interruption, run this same option with the original exact +staging label and retained archives, with no overlapping publisher or recovery. Cleanup +verifies identity/SHA-256 and removes only that build's staging label, never its target label +or files. It is bounded, compensating recovery, not guaranteed cleanup after a kill. +""" + +from __future__ import annotations + +import argparse +import os +import re +import sys +import time +from dataclasses import dataclass +from functools import partial +from pathlib import Path +from typing import Any, Callable + +from eng.scripts.download_mssql_python_rs_wheels import file_sha256 + +from .archive import _validated_package_identity, read_release_index +from .release import python_tag_from_index + +_IDENTIFIER_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") +_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +_ANACONDA_API_URL = "https://api.anaconda.org" + + +@dataclass(frozen=True) +class Distribution: + path: Path + package: str + version: str + basename: str + sha256: str + + +def distribution_from_path(path: str | Path) -> Distribution: + package_path = Path(path).resolve() + if not package_path.is_file(): + raise ValueError(f"Conda package does not exist: {package_path}") + + index = read_release_index(str(package_path)) + package, version, subdir, _ = _validated_package_identity(index, str(package_path)) + python_tag_from_index(index) + if package != "mssql-python": + raise ValueError(f"Unexpected package '{package}' in {package_path.name}.") + if package_path.parent.name != subdir: + raise ValueError( + f"Package {package_path.name} is staged under '{package_path.parent.name}' " + f"but metadata subdir is '{subdir}'." + ) + + return Distribution( + path=package_path, + package=package, + version=version, + basename=f"{subdir}/{package_path.name}", + sha256=file_sha256(package_path), + ) + + +def _metadata_labels(metadata: dict[str, Any], distribution: Distribution) -> set[str]: + labels = metadata.get("labels") + if not isinstance(labels, list) or not all(isinstance(label, str) for label in labels): + raise RuntimeError( + f"Anaconda metadata for '{distribution.basename}' has no valid labels list." + ) + return set(labels) + + +def verify_distribution( + api: Any, + owner: str, + distribution: Distribution, + *, + required_label: str | None = None, + forbidden_label: str | None = None, +) -> set[str]: + metadata = api.distribution( + owner, + distribution.package, + distribution.version, + distribution.basename, + ) + if metadata.get("basename") != distribution.basename: + raise RuntimeError( + f"Anaconda returned basename '{metadata.get('basename')}' for " + f"'{distribution.basename}'." + ) + + remote_sha = str(metadata.get("sha256", "")).lower() + if not _SHA256_RE.fullmatch(remote_sha): + raise RuntimeError(f"Anaconda metadata for '{distribution.basename}' has no valid SHA-256.") + if remote_sha != distribution.sha256: + raise RuntimeError( + f"SHA-256 mismatch for '{distribution.basename}': " + f"local {distribution.sha256} != remote {remote_sha}." + ) + + labels = _metadata_labels(metadata, distribution) + if required_label is not None and required_label not in labels: + raise RuntimeError( + f"Distribution '{distribution.basename}' is missing required label " + f"'{required_label}' (labels={sorted(labels)})." + ) + if forbidden_label is not None and forbidden_label in labels: + raise RuntimeError( + f"Distribution '{distribution.basename}' still has forbidden label " + f"'{forbidden_label}' (labels={sorted(labels)})." + ) + return labels + + +def _verify_with_retry( + verify: Callable[[], set[str]], + description: str, + *, + attempts: int, + delay_seconds: float, +) -> set[str]: + last_error: Exception | None = None + for attempt in range(1, attempts + 1): + try: + return verify() + except Exception as exc: # API metadata can be briefly eventually consistent. + last_error = exc + if attempt < attempts: + print( + f"{description} attempt {attempt} failed: {type(exc).__name__}; retrying...", + flush=True, + ) + time.sleep(delay_seconds) + assert last_error is not None + raise last_error + + +def validate_release_input(expected_version: str, distributions: list[Distribution]) -> None: + if not expected_version: + raise ValueError("Expected mssql-python version is required for promotion.") + if not distributions: + raise ValueError("No Conda distributions were supplied for promotion.") + + basenames = [distribution.basename for distribution in distributions] + if len(basenames) != len(set(basenames)): + raise ValueError(f"Duplicate distribution basenames: {basenames}") + wrong_versions = sorted( + { + distribution.version + for distribution in distributions + if distribution.version != expected_version + } + ) + if wrong_versions: + raise ValueError( + f"Distribution versions {wrong_versions} do not match expected " + f"mssql-python version '{expected_version}'." + ) + + +def _require_publication( + owner: str, + staging_label: str, + target_label: str, + expected_version: str, + distributions: list[Distribution], +) -> None: + if not _IDENTIFIER_RE.fullmatch(owner): + raise ValueError("Invalid Anaconda owner/channel; use letters, digits, '.', '_' or '-'.") + for label_name, label in (("staging", staging_label), ("target", target_label)): + if not _IDENTIFIER_RE.fullmatch(label): + raise ValueError(f"Invalid {label_name} label; use letters, digits, '.', '_' or '-'.") + if staging_label == target_label: + raise ValueError("Staging and target labels must be different.") + if re.fullmatch(r".+_staging_[0-9]+", target_label): + raise ValueError("Target labels must not use the reserved staging namespace.") + if not re.fullmatch(rf"{re.escape(target_label)}_staging_[0-9]+", staging_label): + raise ValueError(f"Staging label must use '{target_label}_staging_'.") + validate_release_input(expected_version, distributions) + + +def _remove_staging_label( + api: Any, + owner: str, + distribution: Distribution, + staging_label: str, + labels: set[str], + *, + required_target: str | None, + verify_attempts: int, + delay_seconds: float, +) -> None: + cleanup_error: Exception | None = None + if staging_label in labels: + try: + api.remove_channel( + staging_label, + owner, + package=distribution.package, + version=distribution.version, + filename=distribution.basename, + ) + except Exception as exc: + cleanup_error = exc + try: + _verify_with_retry( + partial( + verify_distribution, + api, + owner, + distribution, + required_label=required_target, + forbidden_label=staging_label, + ), + f"Verify staging cleanup {distribution.basename}", + attempts=verify_attempts, + delay_seconds=delay_seconds, + ) + except Exception as verification_error: + error = RuntimeError( + f"Failed to remove staging label from '{distribution.basename}': " + f"remove={cleanup_error}; verify={verification_error}" + ) + setattr( + error, + "_cli_message", + f"Failed to remove staging label '{staging_label}' from '{distribution.basename}'" + + (f"; target label '{required_target}' must remain." if required_target else "."), + ) + raise error from verification_error + if cleanup_error is not None: + print( + f"Staging cleanup API reported an error but '{distribution.basename}' " + f"verified clean: {type(cleanup_error).__name__}", + flush=True, + ) + + +def cleanup_staging( + api: Any, + owner: str, + staging_label: str, + target_label: str, + expected_version: str, + distributions: list[Distribution], + *, + verify_attempts: int = 3, + delay_seconds: float = 5, +) -> None: + """Remove only the specified staging label from verified attempted uploads.""" + _require_publication(owner, staging_label, target_label, expected_version, distributions) + from binstar_client.errors import NotFound # type: ignore[import-not-found] + + errors: list[str] = [] + last_error: Exception | None = None + for distribution in distributions: + try: + try: + labels = _verify_with_retry( + partial(verify_distribution, api, owner, distribution), + f"Verify cleanup input {distribution.basename}", + attempts=verify_attempts, + delay_seconds=delay_seconds, + ) + except NotFound: + print(f"Cleanup: '{distribution.basename}' is not present on the server.") + continue + _remove_staging_label( + api, + owner, + distribution, + staging_label, + labels, + required_target=None, + verify_attempts=verify_attempts, + delay_seconds=delay_seconds, + ) + except Exception as exc: + # Continue compensating other attempted uploads, but report every failure. + errors.append(f"{distribution.basename}: {exc}") + last_error = exc + if errors: + error = RuntimeError(f"Staging cleanup incomplete for '{staging_label}': {errors}") + setattr( + error, + "_cli_message", + f"Staging cleanup incomplete for '{staging_label}' ({len(errors)} files); " + "rerun --cleanup-staging with the original archives.", + ) + raise error from last_error + + +def promote( + api: Any, + owner: str, + staging_label: str, + target_label: str, + expected_version: str, + distributions: list[Distribution], + *, + verify_attempts: int = 3, + delay_seconds: float = 5, +) -> None: + _require_publication(owner, staging_label, target_label, expected_version, distributions) + + def verifier( + distribution: Distribution, + *, + required_label: str | None = None, + forbidden_label: str | None = None, + ) -> Callable[[], set[str]]: + def run_verification() -> set[str]: + return verify_distribution( + api, + owner, + distribution, + required_label=required_label, + forbidden_label=forbidden_label, + ) + + return run_verification + + initial_labels: dict[str, set[str]] = {} + for distribution in distributions: + labels = _verify_with_retry( + verifier(distribution), + f"Verify staged {distribution.basename}", + attempts=verify_attempts, + delay_seconds=delay_seconds, + ) + if staging_label not in labels and target_label not in labels: + stage_error: Exception | None = None + try: + # --skip-existing does not attach a new label. An exact matching file + # left on an older staging label is recoverable: attach this build's + # staging label only after checksum/identity verification above. + api.add_channel( + staging_label, + owner, + package=distribution.package, + version=distribution.version, + filename=distribution.basename, + ) + except Exception as exc: + stage_error = exc + labels = _verify_with_retry( + verifier(distribution, required_label=staging_label), + f"Verify current staging label for {distribution.basename}", + attempts=verify_attempts, + delay_seconds=delay_seconds, + ) + if stage_error is not None: + print( + f"Staging-label API reported an error but '{distribution.basename}' " + f"verified on '{staging_label}': {type(stage_error).__name__}", + flush=True, + ) + initial_labels[distribution.basename] = labels + + attempted_additions: list[Distribution] = [] + try: + for distribution in distributions: + add_error: Exception | None = None + if target_label not in initial_labels[distribution.basename]: + attempted_additions.append(distribution) + try: + api.add_channel( + target_label, + owner, + package=distribution.package, + version=distribution.version, + filename=distribution.basename, + ) + except Exception as exc: + # The request may have reached the server before the client observed + # failure. Verify metadata below; rollback covers any label that landed. + add_error = exc + _verify_with_retry( + verifier(distribution, required_label=target_label), + f"Verify promoted {distribution.basename}", + attempts=verify_attempts, + delay_seconds=delay_seconds, + ) + if add_error is not None: + print( + f"Target-label API reported an error but '{distribution.basename}' " + f"verified on '{target_label}': {type(add_error).__name__}", + flush=True, + ) + except Exception as exc: + rollback_errors: list[str] = [] + for distribution in reversed(attempted_additions): + remove_error: Exception | None = None + try: + api.remove_channel( + target_label, + owner, + package=distribution.package, + version=distribution.version, + filename=distribution.basename, + ) + except Exception as rollback_error: + # Verify the result even when the request reported an error; it may + # have reached the server before the client observed the failure. + remove_error = rollback_error + try: + _verify_with_retry( + verifier(distribution, forbidden_label=target_label), + f"Verify rollback {distribution.basename}", + attempts=verify_attempts, + delay_seconds=delay_seconds, + ) + if remove_error is not None: + print( + f"Rollback API reported an error but '{distribution.basename}' " + f"verified without '{target_label}': {type(remove_error).__name__}", + flush=True, + ) + except Exception as verification_error: + rollback_errors.append( + f"{distribution.basename}: remove={remove_error}; " + f"verify={verification_error}" + ) + detail = f" Rollback errors: {rollback_errors}" if rollback_errors else "" + error = RuntimeError( + f"Promotion failed; rollback of newly added target labels was attempted: " + f"{exc}.{detail}" + ) + setattr( + error, + "_cli_message", + "Promotion failed; rollback of newly added target labels was attempted " + f"({len(rollback_errors)} rollback errors). Recover staging label '{staging_label}' " + "with --cleanup-staging and the original archives.", + ) + raise error from exc + + for distribution in distributions: + _remove_staging_label( + api, + owner, + distribution, + staging_label, + initial_labels[distribution.basename], + required_target=target_label, + verify_attempts=verify_attempts, + delay_seconds=delay_seconds, + ) + + +def add_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--owner", required=True) + parser.add_argument("--staging-label", required=True) + parser.add_argument("--target-label", required=True) + parser.add_argument("--expected-version", required=True) + mode = parser.add_mutually_exclusive_group() + mode.add_argument("--check-local-only", action="store_true") + mode.add_argument( + "--cleanup-staging", + action="store_true", + help="Recover attempted uploads: verify exact files and remove only this staging label.", + ) + parser.add_argument("packages", nargs="+") + + +def execute(args: argparse.Namespace) -> int: + distributions = [distribution_from_path(path) for path in args.packages] + _require_publication( + args.owner, args.staging_label, args.target_label, args.expected_version, distributions + ) + if args.check_local_only: + print( + f"LOCAL_RELEASE_INPUT_OK: verified {len(distributions)} distribution(s) " + f"for mssql-python {args.expected_version}." + ) + return 0 + + token = os.environ.get("ANACONDA_API_TOKEN") + if not token or not token.strip(): + raise ValueError("ANACONDA_API_TOKEN is required for publication or staging recovery.") + + from binstar_client.utils import get_server_api # type: ignore[import-not-found] + + api = get_server_api(token=token, config={"url": _ANACONDA_API_URL, "ssl_verify": True}) + # anaconda-client 1.14.1 does not set timeouts on its metadata/label requests. + api.session.request = partial(api.session.request, timeout=(15, 60)) + operation = cleanup_staging if args.cleanup_staging else promote + operation( + api, + args.owner, + args.staging_label, + args.target_label, + args.expected_version, + distributions, + ) + if args.cleanup_staging: + print(f"STAGING_CLEANUP_OK: verified cleanup of label '{args.staging_label}'.") + else: + print( + f"PROMOTION_OK: verified and promoted {len(distributions)} distribution(s) " + f"to {args.owner}/{args.target_label}." + ) + return 0 + + +def cli(args: argparse.Namespace) -> int: + """Report expected CLI failures; imported execute() retains its raising behavior.""" + from http.client import HTTPException + from tarfile import TarError + from zipfile import BadZipFile + + def expected_errors() -> tuple[type[Exception], ...]: + return ( + ValueError, + RuntimeError, + ImportError, + OSError, + HTTPException, + TarError, + BadZipFile, + EOFError, + getattr(sys.modules.get("binstar_client.errors"), "BinstarError", RuntimeError), + getattr(sys.modules.get("zstandard"), "ZstdError", RuntimeError), + getattr(sys.modules.get("compression.zstd"), "ZstdError", RuntimeError), + ) + + try: + return execute(args) + except expected_errors() as error: + # Only already-loaded optional clients are inspected; local-only needs none. + cause = error.__cause__ + while cause is not None: + if not isinstance(cause, expected_errors()): + raise + cause = cause.__cause__ + message = getattr( + error, "_cli_message", "Check archive access and Anaconda connectivity/permissions." + ) + if isinstance(error, ImportError): + message = "Install dependencies with: python -m pip install --require-hashes -r conda/requirements-publish.txt" + elif isinstance(error, ValueError) and not isinstance(error, (OSError, UnicodeError)): + message = str(error) + for name in ("ANACONDA_API_TOKEN", "BINSTAR_API_TOKEN"): + secret = os.environ.get(name) + if secret: + message = message.replace(secret, "[REDACTED]") + message = message.replace(repr(secret)[1:-1], "[REDACTED]") + print(f"ERROR: {type(error).__name__}: " + " ".join(message.split()), file=sys.stderr) + return 1 diff --git a/eng/conda_tools/release.py b/eng/conda_tools/release.py new file mode 100644 index 000000000..65202a583 --- /dev/null +++ b/eng/conda_tools/release.py @@ -0,0 +1,480 @@ +"""Metadata-based conda release-readiness gate. + +The release pipeline must never ship an incomplete conda set. This module reads +the AUTHORITATIVE ``info/index.json`` embedded in every ``.conda`` / ``.tar.bz2`` +(never folder names or bare counts) and validates the self-contained +``mssql-python`` package -- which vendors the ODBC Driver 18 payload, so there is +NO separate companion package: + +* every package's real ``subdir`` is in the allowed set AND matches its folder + (catches a mislabeled / mis-stamped leg); +* every archive uses its metadata-derived canonical basename; +* the only package name is ``mssql-python`` and its version matches the expected + release version (or, if none supplied, is internally consistent -- one version); +* build tags, recognized exact/bounded Python requirements, and optional canonical + normal CPython ABI pins agree on the interpreter minor; +* all Python requirements jointly admit a stable release in that minor. Supported + version syntax is numeric major/minor/patch, a0 bounds, trailing .*, comparisons + (=, ==, !=, <, <=, >, >=, ~=), comma AND and pipe OR, plus an optional build pin. + Regex, parentheses, epochs, local/dev/post versions and other syntax fail closed. +* the (required-subdir x Python) matrix is complete -- every required platform + ships a package for every expected Python, honoring any per-subdir Python + override (e.g. win-arm64 ships only 3.12-3.14). + +Exit code 0 = release-ready; non-zero = a violation was found (blocks publish). +""" + +from __future__ import annotations + +import argparse +import json +import os +import operator +import re +import sys +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path + +from . import archive, inputs + +_BINDING_NAME = "mssql-python" + +_PY_TAG_RE = re.compile(r"py(\d)(\d{1,2})") +_PY_DEP_RE = re.compile(r"python\s+(?:==?)?(\d+)\.(\d+)(?:\.(?:\d+|\*))?(?:\s+\S+)?") +_PY_RANGE_RE = re.compile(r"python\s+>=\s*(\d+)\.(\d+)(?:\.\d+)?\s*,\s*<\s*(\d+)\.(\d+)(?:\.0a0)?") +_PY_ABI_RE = re.compile(r"python_abi\s+(\d+)\.(\d+)\.\*\s+\*_cp(\d)(\d{1,2})") +_PY_BOUND_RE = re.compile(r"(==|!=|<=|>=|~=|=|<|>)?(\d+(?:\.\d+){0,2})(a0)?(\.\*)?") +_COMPARISONS = { + "==": operator.eq, + "!=": operator.ne, + "<": operator.lt, + "<=": operator.le, + ">": operator.gt, + ">=": operator.ge, +} + +# Some subdirs legitimately ship a REDUCED Python matrix. win-arm64's conda +# dependencies (cryptography, pyodbc) are published on Anaconda `defaults` only +# for Python 3.12+, so 3.10/3.11 cannot be built there -- expect 3.12-3.14 only. +_DEFAULT_SUBDIR_PYTHONS = "win-arm64=3.12,3.13,3.14" + + +@dataclass(frozen=True) +class _PythonBound: + operation: str + release: tuple[int, ...] + prerelease: bool = False + + def matches(self, candidate: tuple[int, int, int]) -> bool: + if self.operation == "*": + return True + prefix = not self.prerelease and candidate[: len(self.release)] == self.release + if self.operation in {"=", "!=prefix"}: + return prefix if self.operation == "=" else not prefix + bound = (*self.release, *((0,) * (3 - len(self.release))), -int(self.prerelease)) + if self.operation == "~=": + prefix_release = self.release[:-1] + return (*candidate, 0) >= bound and candidate[: len(prefix_release)] == prefix_release + return _COMPARISONS[self.operation]((*candidate, 0), bound) + + +def _python_requirement(dep: str) -> list[list[_PythonBound]]: + """Parse only the documented CPython release-constraint subset, not MatchSpec.""" + match = re.fullmatch(r"python(?:\s+(.+))?", dep) + if not match: + raise ValueError(f"Unsupported Python requirement syntax: {dep!r}.") + specification = match.group(1) or "*" + specification = re.sub(r"\s*(>=|<=|==|!=|~=|=|>|<|,|\|)\s*", r"\1", specification) + parts = specification.split() + if len(parts) > 2 or (len(parts) == 2 and not re.fullmatch(r"[A-Za-z0-9_.*-]+", parts[1])): + raise ValueError(f"Unsupported Python requirement syntax: {dep!r}.") + clauses = [] + for alternative in parts[0].split("|"): + bounds = [] + for atom in alternative.split(","): + if atom == "*": + bounds.append(_PythonBound("*", ())) + continue + match = _PY_BOUND_RE.fullmatch(atom) + if not match: + raise ValueError(f"Unsupported Python requirement syntax: {dep!r}.") + operation, version, prerelease, wildcard = match.groups() + release = tuple(map(int, version.split("."))) + if (prerelease and (len(release) != 3 or wildcard)) or ( + operation == "~=" and (len(release) < 2 or wildcard) + ): + raise ValueError(f"Unsupported Python requirement syntax: {dep!r}.") + # Conda treats bare trailing .* and = as prefixes, but ==3.12.* + # as exact 3.12; !=3.12.* excludes the entire prefix. + operation = operation or ("=" if wildcard else "==") + if wildcard and operation == "!=": + operation = "!=prefix" + bounds.append(_PythonBound(operation, release, bool(prerelease))) + clauses.append(bounds) + return clauses + + +def python_tag_from_index(index: dict) -> str: + """Extract the ``X.Y`` Python version a package is built for, or ``''``. + + Build tokens and every recognized exact/bounded Python requirement must agree. + Canonical normal CPython ABI pins, when present, must also agree but are not + required and do not identify a variant by themselves. Broad requirements alone + cannot identify a minor. Every Python requirement must admit a common stable + patch release within the identified minor; tags never override an exclusion. + """ + minors = set() + abi_minors = set() + requirements = [] + dependencies = index.get("depends", []) + if not isinstance(dependencies, list) or not all(isinstance(dep, str) for dep in dependencies): + raise ValueError("info/index.json field 'depends' must be a list of dependency strings.") + for build_match in _PY_TAG_RE.finditer(str(index.get("build", ""))): + minors.add(f"{build_match.group(1)}.{build_match.group(2)}") + for dep in dependencies: + if re.match(r"python(?:\s|$|[<>=!~\[])", dep.strip()): + requirements.append(_python_requirement(dep.strip())) + match = _PY_DEP_RE.fullmatch(str(dep).strip()) + if match: + minors.add(f"{match.group(1)}.{match.group(2)}") + match = _PY_RANGE_RE.fullmatch(str(dep).strip()) + if match: + major, lower_minor, upper_major, upper_minor = map(int, match.groups()) + if (upper_major, upper_minor) == (major, lower_minor + 1): + minors.add(f"{major}.{lower_minor}") + match = _PY_ABI_RE.fullmatch(str(dep).strip()) + if match: + abi_minors.add(f"{match.group(1)}.{match.group(2)}") + abi_minors.add(f"{match.group(3)}.{match.group(4)}") + if len(minors | abi_minors) > 1: + raise ValueError( + f"Conflicting Python minor metadata: build={index.get('build')!r}, " + f"depends={index.get('depends')!r}." + ) + minor = next(iter(minors), "") + if minor and requirements: + major, minor_number = map(int, minor.split(".")) + major_minor = (major, minor_number) + patches = {0} + # For the supported comparisons, truth can change only at a named patch + # boundary. Its neighbors cover every interval, including the unbounded tail. + for clauses in requirements: + for bounds in clauses: + for bound in bounds: + if len(bound.release) >= 2 and bound.release[:2] == major_minor: + patch = bound.release[2] if len(bound.release) == 3 else 0 + patches.update((max(0, patch - 1), patch, patch + 1)) + if not any( + all( + any( + all(bound.matches((*major_minor, patch)) for bound in bounds) + for bounds in clauses + ) + for clauses in requirements + ) + for patch in patches + ): + raise ValueError( + f"Conflicting Python requirements exclude all stable Python {minor} releases: " + f"{index.get('depends')!r}." + ) + return minor + + +def validate( + packages: list[dict], + required_subdirs: list[str], + allowed_subdirs: list[str], + expected_pythons: list[str], + expected_versions: dict | None = None, + subdir_pythons: dict | None = None, +) -> list[str]: + """Return a list of human-readable violation strings (empty == release-ready). + + ``packages`` is a list of dicts with keys: ``folder`` (staged subdir folder), + ``subdir`` (real info/index.json subdir), ``name``, ``version``, ``build``, + ``python`` (``X.Y`` or ``''``). + + ``subdir_pythons`` maps a subdir to the Python versions expected FOR THAT + subdir, overriding ``expected_pythons`` (e.g. win-arm64 ships only 3.12-3.14). + """ + errors: list[str] = [] + expected_versions = expected_versions or {} + subdir_pythons = subdir_pythons or {} + + for policy_name, values in ( + ("required_subdirs", required_subdirs), + ("allowed_subdirs", allowed_subdirs), + ("expected_pythons", expected_pythons), + ): + if not values or any(not value.strip() for value in values): + errors.append(f"release policy '{policy_name}' must not be empty.") + elif len(values) != len(set(values)): + errors.append(f"release policy '{policy_name}' contains duplicates: {values}.") + missing_allowed = sorted(set(required_subdirs) - set(allowed_subdirs)) + if missing_allowed: + errors.append(f"required subdirs are absent from allowed_subdirs: {missing_allowed}.") + for subdir, versions in sorted(subdir_pythons.items()): + if not versions or any(not version.strip() for version in versions): + errors.append(f"subdir Python override for '{subdir}' must not be empty.") + elif len(versions) != len(set(versions)): + errors.append(f"subdir Python override for '{subdir}' contains duplicates: {versions}.") + + # 1. Authoritative subdir must be allowed AND match the folder it was staged in. + for p in packages: + ident = f"{p['name']}-{p['version']}-{p['build']}" + if not p["version"]: + errors.append(f"{ident}: package version is missing.") + if p["subdir"] not in allowed_subdirs: + errors.append( + f"{ident}: real subdir '{p['subdir']}' is not in allowed set {allowed_subdirs}." + ) + if p["subdir"] != p["folder"]: + errors.append( + f"MISLABELED: {ident} is staged in folder '{p['folder']}' but its " + f"info/index.json subdir is '{p['subdir']}'." + ) + + # 2. Only the self-contained mssql-python package may appear; versions match + # expected (or are internally consistent -- one version per package). + seen_versions: dict = defaultdict(set) + for p in packages: + if p["name"] != _BINDING_NAME: + errors.append( + f"unexpected package name '{p['name']}' ({p['version']}); the " + f"self-contained conda package ships only '{_BINDING_NAME}'." + ) + continue + seen_versions[p["name"]].add(p["version"]) + for name, versions in seen_versions.items(): + if len(versions) > 1: + errors.append( + f"{name}: multiple versions present {sorted(versions)} " + f"(a release must ship exactly one version per package)." + ) + exp = expected_versions.get(name) + if exp is not None: + for v in versions: + if v != exp: + errors.append(f"{name}: version '{v}' != expected '{exp}'.") + + # 2b. Reject duplicate (name, version, subdir, python) keys. Two packages with + # an identical key are never legitimate -- it means one leg's package bled + # into another subdir's staging folder (the shared-output-dir hazard) or was + # staged twice. The per-subdir matrix check below collapses variants into a + # set, so a duplicate would silently MASK a genuinely missing variant; fail + # loudly on the duplicate instead. + key_folders: dict = defaultdict(list) + for p in packages: + key_folders[(p["name"], p["version"], p["subdir"], p["python"])].append(p["folder"]) + for (name, version, subdir, python), folders in sorted(key_folders.items()): + if len(folders) > 1: + errors.append( + f"DUPLICATE: {name}-{version} (subdir '{subdir}', python " + f"'{python or '-'}') appears {len(folders)}x (staged in {sorted(folders)})." + ) + + # Group by the REAL (metadata) subdir, never the folder name. + by_subdir: dict = defaultdict(list) + for p in packages: + by_subdir[p["subdir"]].append(p) + + # 3. Required subdirs must be PRESENT; every present ALLOWED subdir must ship a + # COMPLETE per-Python matrix. Validating present-but-not-required subdirs too + # (not just the required set) stops a partially built allowed subdir -- e.g. a + # half-finished win-arm64 -- from slipping through to publish just because it + # is not in the required set. + for sub in required_subdirs: + if not by_subdir.get(sub): + errors.append(f"required subdir '{sub}' is MISSING.") + + for sub in sorted(by_subdir): + if sub not in allowed_subdirs: + # Not an allowed subdir: already flagged per-package in step 1. Skip the + # matrix work so the error set stays focused on the root cause. + continue + grp = by_subdir[sub] + bindings = [p for p in grp if p["name"] == _BINDING_NAME] + if not bindings: + errors.append(f"subdir '{sub}': no {_BINDING_NAME} package.") + continue + + for p in bindings: + if not p["python"]: + errors.append( + f"{p['name']}-{p['version']}-{p['build']} in '{sub}' has no " + f"detectable Python tag (build string should carry pyXY)." + ) + sub_expected = subdir_pythons.get(sub, expected_pythons) + got_pythons = sorted({p["python"] for p in bindings if p["python"]}) + missing = [py for py in sub_expected if py not in got_pythons] + if missing: + errors.append( + f"subdir '{sub}': matrix INCOMPLETE -- missing Python {missing} " + f"(present: {got_pythons or 'none'})." + ) + # Reject EXTRA pythons too (got == expected, not just expected subset of got): an + # unsupported build (e.g. a win-arm64 3.10 that slipped in) must never publish. + extra = [py for py in got_pythons if py not in sub_expected] + if extra: + errors.append( + f"subdir '{sub}': matrix has UNSUPPORTED Python {extra} " + f"(expected exactly {sub_expected})." + ) + + return errors + + +def collect_packages(root: str) -> list[dict]: + """Read every ``.conda`` / ``.tar.bz2`` under ``root`` into package dicts.""" + paths = archive.collect(root) + packages = [] + for path in paths: + index = archive.read_release_index(path) + name, version, subdir, build = archive._validated_package_identity(index, path) + packages.append( + { + "folder": Path(path).parent.name, + "subdir": subdir, + "name": name, + "version": version, + "build": build, + "python": python_tag_from_index(index), + "path": path, + } + ) + return packages + + +def _split(value: str) -> list[str]: + values = [x.strip() for x in value.split(",")] + if any(not item for item in values): + raise ValueError("release policy entries must not be empty.") + if len(values) != len(set(values)): + raise ValueError(f"release policy contains duplicates: {values}.") + return values + + +def _parse_subdir_pythons(value: str) -> dict: + """Parse ``subdir=py,py;subdir2=py,py`` into ``{subdir: [py, ...]}``.""" + result: dict = {} + if not value.strip(): + return result + for chunk in value.split(";"): + chunk = chunk.strip() + subdir, _, pys = chunk.partition("=") + if not subdir.strip() or not pys.strip(): + raise ValueError( + f"invalid subdir Python override '{chunk}'; expected subdir=X.Y[,X.Y]." + ) + subdir = subdir.strip() + if subdir in result: + raise ValueError(f"invalid subdir Python override: duplicate subdir '{subdir}'.") + try: + result[subdir] = _split(pys) + except ValueError as exc: + raise ValueError(f"invalid subdir Python override '{chunk}': {exc}") from exc + return result + + +def add_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--root", required=True, help="Root of the consolidated conda tree.") + parser.add_argument( + "--required-subdirs", + default="win-64,win-arm64,osx-64,osx-arm64,linux-64,linux-aarch64", + ) + parser.add_argument( + "--allowed-subdirs", + default="win-64,win-arm64,osx-64,osx-arm64,linux-64,linux-aarch64", + ) + parser.add_argument("--pythons", default="3.10,3.11,3.12,3.13,3.14") + parser.add_argument( + "--subdir-pythons", + default=_DEFAULT_SUBDIR_PYTHONS, + help="Per-subdir Python overrides, e.g. 'win-arm64=3.12,3.13,3.14'.", + ) + parser.add_argument("--mssql-python-version", default=None) + parser.add_argument( + "--release-versions", + nargs="?", + const=os.environ.get("RELEASE_VERSIONS", ""), + default=None, + help="Exact component version JSON; without a value, read RELEASE_VERSIONS.", + ) + parser.add_argument( + "--rs-transport-version", default=os.environ.get("RS_TRANSPORT_VERSION", "") + ) + + +def execute(args: argparse.Namespace) -> int: + try: + required_subdirs = _split(args.required_subdirs) + allowed_subdirs = _split(args.allowed_subdirs) + expected_pythons = _split(args.pythons) + subdir_pythons = _parse_subdir_pythons(args.subdir_pythons) + packages = collect_packages(args.root) + if args.release_versions is not None: + versions = inputs.parse_release_versions(args.release_versions) + if args.mssql_python_version != versions["mssql-python"]: + raise ValueError( + "Binding release version differs from verified component versions." + ) + receipt_path = Path(args.root) / "rs-transport.json" + receipt = ( + json.loads(receipt_path.read_text(encoding="utf-8")) + if receipt_path.exists() + else None + ) + inputs.validate_rs_transport(receipt, versions, args.rs_transport_version) + except archive.READ_ERRORS as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + if not packages: + print(f"ERROR: no conda packages found under {args.root}.", file=sys.stderr) + return 1 + + expected_versions = {} + if args.mssql_python_version: + expected_versions[_BINDING_NAME] = args.mssql_python_version + + print(f"Discovered {len(packages)} conda package(s):") + for p in sorted(packages, key=lambda x: (x["subdir"], x["name"], x["python"])): + print( + f" {p['subdir']:<14} {p['name']:<18} {p['version']:<12} " + f"py={p['python'] or '-':<5} build={p['build']}" + ) + if subdir_pythons: + print(f"Per-subdir Python overrides: {subdir_pythons}") + + errors = validate( + packages, + required_subdirs=required_subdirs, + allowed_subdirs=allowed_subdirs, + expected_pythons=expected_pythons, + expected_versions=expected_versions, + subdir_pythons=subdir_pythons, + ) + + if errors: + print("\nConda release readiness FAILED:", file=sys.stderr) + for e in errors: + print(f" - {e}", file=sys.stderr) + return 1 + + if args.release_versions is not None: + try: + for package in packages: + inputs.validate_installed_inputs( + package["path"], + "cp" + package["python"].replace(".", ""), + package["subdir"], + versions, + receipt, + ) + except archive.READ_ERRORS as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + print("COMPONENT_INPUT_OK: installed components match verified producer source inputs.") + print("\nOK: metadata-validated conda set is release-ready (subdirs, Python matrix, pairing).") + return 0 diff --git a/tests/test_027_conda_release_metadata.py b/tests/test_027_conda_release_metadata.py new file mode 100644 index 000000000..2386d151c --- /dev/null +++ b/tests/test_027_conda_release_metadata.py @@ -0,0 +1,1971 @@ +"""Unit tests for the metadata-based conda release gate. + +``eng.conda_tools.release`` reads each package's authoritative +``info/index.json`` and enforces: real-subdir == folder, allowed subdirs, the +full (subdir x Python) matrix, and exact versions for the self-contained +``mssql-python`` package (which vendors the ODBC payload -- no companion). These +tests exercise the pure ``validate()`` logic with synthetic package records (no +real ``.conda`` needed) plus one optional round-trip through the metadata reader. +""" + +import argparse +import importlib.util +import io +import json +import sys +import tarfile +import types +import warnings +import zipfile +from pathlib import Path + +import pytest + +_ROOT = Path(__file__).resolve().parent.parent +_MODULE_PATH = _ROOT / "eng" / "conda_tools" / "release.py" +if not _MODULE_PATH.is_file(): + pytest.skip( + f"conda source not present ({_MODULE_PATH}); skipping conda release metadata tests", + allow_module_level=True, + ) + +from eng.conda_tools import archive, publication as promoter, release as vcr +from eng.conda_tools import __main__ as cli + + +def _metadata_main(argv): + return cli.main(["validate", *argv]) + + +def _promoter_main(argv): + return promoter.execute(cli.parser().parse_args(["promote", *argv])) + + +def _promoter_cli(argv): + return cli.main(["promote", *argv]) + + +_REQUIRED = ["win-64", "osx-64", "osx-arm64", "linux-64", "linux-aarch64"] +_ALLOWED = ["win-64", "win-arm64", "osx-64", "osx-arm64", "linux-64", "linux-aarch64"] +_PYTHONS = ["3.10", "3.11", "3.12", "3.13", "3.14"] +_MP_VER = "1.13.0" + + +@pytest.mark.parametrize( + "source_present", [False, True], ids=["wheel-tests-only", "source-checkout"] +) +def test_collection_does_not_require_pyyaml(source_present, tmp_path, monkeypatch): + original_import = __import__ + yaml_imports = [] + + def without_yaml(name, *args, **kwargs): + if name == "yaml" or name.startswith("yaml."): + yaml_imports.append(name) + raise ModuleNotFoundError("No module named 'yaml'", name=name) + return original_import(name, *args, **kwargs) + + monkeypatch.setattr("builtins.__import__", without_yaml) + spec = importlib.util.spec_from_file_location("conda_metadata_without_yaml", __file__) + module = importlib.util.module_from_spec(spec) + if not source_present: + module.__file__ = str(tmp_path / "tests" / Path(__file__).name) + + if source_present: + spec.loader.exec_module(module) + else: + with pytest.raises(pytest.skip.Exception, match="conda source not present"): + spec.loader.exec_module(module) + assert yaml_imports == [] + + +@pytest.mark.parametrize( + "command", ["validate", "provenance", "promote", "fetch-wheels", "probe-driver"] +) +def test_release_module_help_and_unknown_arguments(command): + import subprocess + + for argument, expected in (("--help", 0), ("--unknown-option", 2)): + result = subprocess.run( + [sys.executable, "-m", "eng.conda_tools", command, argument], + cwd=_ROOT, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == expected, result.stdout + result.stderr + assert "python -m eng.conda_tools" in result.stdout + result.stderr + if argument == "--help": + assert f"python -m eng.conda_tools {command}" in result.stdout + assert "Traceback" not in result.stderr + + +@pytest.mark.parametrize("failed", [False, True]) +def test_probe_driver_route_preserves_leaf_reporting_and_exit(monkeypatch, capsys, failed): + from eng.conda_tools import driver_load_probe + from eng.conda_tools.__main__ import main + + before = Path.cwd() + calls = [] + + def probe(): + calls.append(Path.cwd()) + if failed: + raise SystemExit("controlled probe failure") + print("DRIVER_LOADED (controlled probe)") + + monkeypatch.setattr(driver_load_probe, "main", probe) + if failed: + with pytest.raises(SystemExit, match="controlled probe failure"): + main(["probe-driver"]) + else: + assert main(["probe-driver"]) == 0 + assert capsys.readouterr().out == "DRIVER_LOADED (controlled probe)\n" + assert calls == [before] + assert Path.cwd() == before + + +def test_release_imports_do_not_read_inputs_or_load_runtime_clients(): + import subprocess + + code = """ +import os, pathlib, subprocess, sys, urllib.request +before = dict(os.environ), os.getcwd() +def forbidden(*args, **kwargs): + raise AssertionError("import performed an external operation") +pathlib.Path.read_text = forbidden +subprocess.run = subprocess.Popen = forbidden +urllib.request.urlopen = urllib.request.build_opener = forbidden +from eng.conda_tools import __main__, archive, inputs, provenance, publication, release +assert before == (dict(os.environ), os.getcwd()) +assert not any(name.split('.')[0] in {'mssql_python', 'mssql_python_odbc', 'mssql_py_core', + 'binstar_client'} for name in sys.modules) +print("INERT_RELEASE_IMPORTS") +""" + result = subprocess.run( + [sys.executable, "-c", code], cwd=_ROOT, capture_output=True, text=True, timeout=30 + ) + assert result.returncode == 0, result.stdout + result.stderr + assert result.stdout.strip() == "INERT_RELEASE_IMPORTS" + + +def test_present_broken_release_source_is_not_skipped(tmp_path): + import os + import subprocess + + tools = tmp_path / "eng" / "conda_tools" + tools.mkdir(parents=True) + for name in ("archive", "publication"): + (tools / f"{name}.py").write_text("", encoding="utf-8") + (tools / "release.py").write_text( + 'raise RuntimeError("BROKEN_RELEASE_SOURCE")\n', encoding="utf-8" + ) + tests = tmp_path / "tests" + tests.mkdir() + target = tests / Path(__file__).name + target.write_bytes(Path(__file__).read_bytes()) + env = {key: value for key, value in os.environ.items() if key != "PYTHONPATH"} + result = subprocess.run( + [sys.executable, "-m", "pytest", "--noconftest", "--collect-only", "-q", str(target)], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 2, result.stdout + result.stderr + assert "BROKEN_RELEASE_SOURCE" in result.stdout + result.stderr + assert "skipped" not in result.stdout + + +def _binding(subdir, py, folder=None, version=_MP_VER): + return { + "folder": folder or subdir, + "subdir": subdir, + "name": "mssql-python", + "version": version, + "build": f"py{py.replace('.', '')}_0", + "python": py, + } + + +def _healthy_set(): + """A complete release: the self-contained mssql-python package for every + (required subdir x Python).""" + pkgs = [] + for sub in _REQUIRED: + for py in _PYTHONS: + pkgs.append(_binding(sub, py)) + return pkgs + + +def _run(pkgs, expected_versions=None): + return vcr.validate( + pkgs, + required_subdirs=_REQUIRED, + allowed_subdirs=_ALLOWED, + expected_pythons=_PYTHONS, + expected_versions=( + expected_versions if expected_versions is not None else {"mssql-python": _MP_VER} + ), + ) + + +def test_healthy_set_passes(): + assert _run(_healthy_set()) == [] + + +def test_mislabeled_subdir_fails(): + pkgs = _healthy_set() + # An osx-64 package physically staged into the osx-arm64 folder. + pkgs.append(_binding("osx-64", "3.12", folder="osx-arm64")) + errors = _run(pkgs) + assert any("MISLABELED" in e for e in errors) + + +def test_missing_python_variant_on_win64_fails(): + # This is the exact 8e7f217f regression: drop a win-64 binding; presence-pairing + # against the single companion used to pass, metadata matrix must now fail. + pkgs = [p for p in _healthy_set() if not (p["subdir"] == "win-64" and p["python"] == "3.12")] + errors = _run(pkgs) + assert any("win-64" in e and "INCOMPLETE" in e and "3.12" in e for e in errors) + + +def test_stray_companion_package_fails(): + # The self-contained model ships ONLY mssql-python; a stray companion package + # (the old separate mssql-python-odbc) must now be rejected as unexpected. + pkgs = _healthy_set() + pkgs.append( + { + "folder": "linux-64", + "subdir": "linux-64", + "name": "mssql-python-odbc", + "version": "18.6.2.1", + "build": "0", + "python": "", + } + ) + errors = _run(pkgs) + assert any("unexpected package name" in e and "mssql-python-odbc" in e for e in errors) + + +def test_unexpected_subdir_fails(): + pkgs = _healthy_set() + pkgs.append(_binding("linux-ppc64le", "3.12")) + errors = _run(pkgs) + assert any("linux-ppc64le" in e and "allowed" in e for e in errors) + + +def test_version_mismatch_fails(): + pkgs = _healthy_set() + pkgs.append(_binding("linux-64", "3.14", version="9.9.9")) # stray wrong-version binding + # remove the correct 3.14 to avoid duplicate-python noise masking the version check + pkgs = [ + p + for p in pkgs + if not (p["subdir"] == "linux-64" and p["python"] == "3.14" and p["version"] == _MP_VER) + ] + errors = _run(pkgs) + assert any("version" in e.lower() for e in errors) + + +def test_missing_package_version_fails_without_expected_version(): + packages = _healthy_set() + packages[0]["version"] = "" + + errors = _run(packages, expected_versions={}) + + assert any("package version is missing" in error for error in errors) + + +def test_multiple_versions_same_package_fails(): + pkgs = _healthy_set() + pkgs.append(_binding("linux-64", "3.10", version="1.12.0", folder="linux-64")) + errors = _run(pkgs, expected_versions={}) # no expected -> consistency check must still fail + assert any("multiple versions" in e for e in errors) + + +def test_missing_required_subdir_fails(): + pkgs = [p for p in _healthy_set() if p["subdir"] != "linux-aarch64"] + errors = _run(pkgs) + assert any("linux-aarch64" in e and "MISSING" in e for e in errors) + + +def test_duplicate_package_fails(): + # The identical package staged twice (same name/version/subdir/python) -- e.g. a + # leg's package collected twice from a shared output dir. A set-based matrix + # check would silently absorb it; the gate must reject the duplicate outright so + # it can never mask a genuinely missing variant. + pkgs = _healthy_set() + pkgs.append(_binding("linux-64", "3.12")) # exact duplicate of an existing entry + errors = _run(pkgs) + assert any("DUPLICATE" in e and "linux-64" in e and "3.12" in e for e in errors) + + +def test_present_allowed_subdir_partial_matrix_fails(): + # win-arm64 is ALLOWED but not REQUIRED. If it shows up only partially built it + # must still fail the gate, else a half-finished allowed subdir slips to publish + # simply because it is not in the required set. + pkgs = _healthy_set() + pkgs.append(_binding("win-arm64", "3.10")) # only one of five Pythons + errors = _run(pkgs) + assert any("win-arm64" in e and "INCOMPLETE" in e for e in errors) + + +def test_win_arm64_reduced_matrix_passes_with_override(): + # win-arm64 legitimately ships only 3.12-3.14 (Anaconda `defaults` has no + # cryptography/pyodbc for 3.10/3.11). With the per-subdir override it must PASS. + pkgs = _healthy_set() + for py in ["3.12", "3.13", "3.14"]: + pkgs.append(_binding("win-arm64", py)) + errors = vcr.validate( + pkgs, + required_subdirs=_REQUIRED, + allowed_subdirs=_ALLOWED, + expected_pythons=_PYTHONS, + expected_versions={"mssql-python": _MP_VER}, + subdir_pythons={"win-arm64": ["3.12", "3.13", "3.14"]}, + ) + assert errors == [] + + +def test_win_arm64_reduced_matrix_still_fails_when_incomplete(): + # Even with the reduced expectation, a missing 3.13 must still fail. + pkgs = _healthy_set() + for py in ["3.12", "3.14"]: + pkgs.append(_binding("win-arm64", py)) + errors = vcr.validate( + pkgs, + required_subdirs=_REQUIRED, + allowed_subdirs=_ALLOWED, + expected_pythons=_PYTHONS, + expected_versions={"mssql-python": _MP_VER}, + subdir_pythons={"win-arm64": ["3.12", "3.13", "3.14"]}, + ) + assert any("win-arm64" in e and "INCOMPLETE" in e and "3.13" in e for e in errors) + + +def test_extra_unsupported_python_fails(): + # got == expected (not just expected subset of got): an EXTRA python beyond the expected + # set (e.g. a win-arm64 3.10 that slipped in) must fail even though the expected 3.12-3.14 + # are all present. + pkgs = _healthy_set() + for py in ["3.12", "3.13", "3.14"]: + pkgs.append(_binding("win-arm64", py)) + pkgs.append(_binding("win-arm64", "3.10")) # unsupported extra + errors = vcr.validate( + pkgs, + required_subdirs=_REQUIRED, + allowed_subdirs=_ALLOWED, + expected_pythons=_PYTHONS, + expected_versions={"mssql-python": _MP_VER}, + subdir_pythons={"win-arm64": ["3.12", "3.13", "3.14"]}, + ) + assert any("win-arm64" in e and "UNSUPPORTED" in e and "3.10" in e for e in errors) + + +def test_v1_required_includes_win_arm64_passes(): + # v1 parity: win-arm64 is REQUIRED. A complete set (win-arm64 at 3.12-3.14 via the + # per-subdir override) passes. + required = ["win-64", "win-arm64", "osx-64", "osx-arm64", "linux-64", "linux-aarch64"] + pkgs = _healthy_set() + for py in ["3.12", "3.13", "3.14"]: + pkgs.append(_binding("win-arm64", py)) + errors = vcr.validate( + pkgs, + required_subdirs=required, + allowed_subdirs=_ALLOWED, + expected_pythons=_PYTHONS, + expected_versions={"mssql-python": _MP_VER}, + subdir_pythons={"win-arm64": ["3.12", "3.13", "3.14"]}, + ) + assert errors == [] + + +def test_v1_missing_win_arm64_now_fails(): + # With win-arm64 REQUIRED, a set that omits it must now fail (it silently passed before). + required = ["win-64", "win-arm64", "osx-64", "osx-arm64", "linux-64", "linux-aarch64"] + errors = vcr.validate( + _healthy_set(), # the original required parity, MINUS win-arm64 + required_subdirs=required, + allowed_subdirs=_ALLOWED, + expected_pythons=_PYTHONS, + expected_versions={"mssql-python": _MP_VER}, + subdir_pythons={"win-arm64": ["3.12", "3.13", "3.14"]}, + ) + assert any("win-arm64" in e and "MISSING" in e for e in errors) + + +def test_default_subdir_pythons_reduces_win_arm64(): + # The shipped CLI default carries the win-arm64 reduction so the pipeline needs + # no extra flag; parsing it yields the expected mapping. + assert vcr._parse_subdir_pythons(vcr._DEFAULT_SUBDIR_PYTHONS) == { + "win-arm64": ["3.12", "3.13", "3.14"] + } + + +def test_parse_subdir_pythons(): + assert vcr._parse_subdir_pythons("") == {} + assert vcr._parse_subdir_pythons("a=3.10;b=3.11,3.12") == { + "a": ["3.10"], + "b": ["3.11", "3.12"], + } + + +@pytest.mark.parametrize( + "value", + [ + "win-arm64", + "=3.12", + "win-arm64=", + "win-arm64=3.12;win-arm64=3.13", + "win-arm64=3.12; win-arm64 =3.12", + "win-arm64=3.12,", + "win-arm64=3.12,3.12", + "win-arm64=3.12;", + ], +) +def test_parse_subdir_pythons_rejects_malformed_policy(value): + with pytest.raises(ValueError, match="invalid subdir Python override"): + vcr._parse_subdir_pythons(value) + + +@pytest.mark.parametrize( + "policy, message", + [ + ({"required_subdirs": []}, "required_subdirs"), + ({"allowed_subdirs": []}, "allowed_subdirs"), + ({"expected_pythons": []}, "expected_pythons"), + ({"allowed_subdirs": ["win-64"]}, "absent from allowed"), + ({"required_subdirs": _REQUIRED * 2}, "duplicates"), + ({"allowed_subdirs": _ALLOWED * 2}, "duplicates"), + ({"expected_pythons": _PYTHONS * 2}, "duplicates"), + ({"subdir_pythons": {"win-64": []}}, "must not be empty"), + ({"required_subdirs": [*_REQUIRED, ""]}, "must not be empty"), + ({"allowed_subdirs": [*_ALLOWED, ""]}, "must not be empty"), + ({"expected_pythons": [*_PYTHONS, ""]}, "must not be empty"), + ({"subdir_pythons": {"win-64": ["3.12", ""]}}, "must not be empty"), + ({"subdir_pythons": {"win-64": ["3.12", "3.12"]}}, "duplicates"), + ], +) +def test_release_policy_cannot_disable_required_matrix(policy, message): + arguments = dict( + required_subdirs=_REQUIRED, allowed_subdirs=_ALLOWED, expected_pythons=_PYTHONS + ) + assert any(message in error for error in vcr.validate(_healthy_set(), **(arguments | policy))) + + +@pytest.mark.parametrize( + "index, expected", + [ + ({"build": "py311_0"}, "3.11"), + ({"build": "py310h1a2b3c_0"}, "3.10"), + ({"build": "0", "depends": ["python 3.12.* *_cpython"]}, "3.12"), + ({"build": "0", "depends": ["python >=3.12,<3.13.0a0"]}, "3.12"), + ({"depends": ["python >=3.12.1, <3.13"]}, "3.12"), + ({"depends": ["python ==3.12.*"]}, "3.12"), + ({"depends": ["python >=3.12"]}, ""), + ({"depends": ["python >=3.12,<3.14.0a0"]}, ""), + ({"depends": ["python 3.12,<3.14"]}, ""), + ({"depends": ["python <3.12"]}, ""), + ({"depends": ["python >=3.12,<3.13|>=3.13"]}, ""), + ({"depends": ["python_abi 3.12.* *_cp312"]}, ""), + ({"build": "0"}, ""), + ], +) +def test_python_tag_from_index(index, expected): + assert vcr.python_tag_from_index(index) == expected + + +_PYTHON_REQUIREMENT_CASES = [ + ("python >=3.13", False), + ("python <3.12", False), + ("python >3.12", True), + ("python <=3.12", True), + ("python <=3.11.99", False), + ("python !=3.12", True), + ("python !=3.12.*", False), + ("python ==3.12.*", True), + ("python ==3.12.*,!=3.12", False), + ("python =3.12", True), + ("python =3.12.2", True), + ("python 3.12.*", True), + ("python 3.12", True), + ("python 3.11.*|3.12.* *_cpython", True), + ("python ~=3.12", True), + ("python ~=3.12.2", True), + ("python ~=3.13", False), + ("python >=3.13|<3.12", False), + ("python >=3.13|>=3.12.2,<3.13", True), + ("python >=3.12.2,<3.12.3", True), + ("python >3.12.2,<3.12.3", False), + ("python >=3.12.2,!=3.12.2,<3.12.3", False), + ("python >=3.12.1000000,<3.13", True), + ("python >=3.12.2,<3.13", True), + ("python >=3.12,<3.13.0a0", True), + ("python >=3.12.0a0,<3.13.0a0", True), + ("python >=3.13.0a0", False), + ("python =3.12.0a0", False), + ("python <3.12.0a0", False), + ("python <3.12.1a0,>3.12", False), + ("python >=3.12.1a0,<3.12.2", True), + ("python >=3.10", True), + ("python *", True), + ("python", True), +] + + +@pytest.mark.parametrize("requirement,allowed", _PYTHON_REQUIREMENT_CASES) +def test_python_requirements_admit_a_stable_patch(requirement, allowed): + index = {"build": "py312_0", "depends": [requirement]} + if allowed: + assert vcr.python_tag_from_index(index) == "3.12" + else: + with pytest.raises(ValueError, match="Conflicting"): + vcr.python_tag_from_index(index) + + +@pytest.mark.parametrize( + "requirements,allowed", + [ + (["python >=3.12.2", "python <=3.12.1"], False), + (["python 3.12.*", "python !=3.12.*"], False), + (["python =3.12", "python !=3.12"], True), + (["python >=3.12.2,<3.12.5", "python !=3.12.2,!=3.12.3,!=3.12.4"], False), + (["python <3.12.3|>=3.13", "python >=3.12.3,<3.13"], False), + ], +) +def test_repeated_python_requirements_are_satisfied_jointly(requirements, allowed): + index = {"build": "py312_0", "depends": requirements} + if allowed: + assert vcr.python_tag_from_index(index) == "3.12" + else: + with pytest.raises(ValueError, match="Conflicting"): + vcr.python_tag_from_index(index) + + +@pytest.mark.parametrize( + "requirement", + [ + "python >=3.12,", + "python >=3.12||<3.13", + "python (>=3.12,<3.13)|3.14.*", + "python ^3[.]12.*$", + "python >=1!3.12", + "python ==3.12.0+local", + "python >=3.12.0dev0", + "python >=3.12.0post1", + "python 3.*.2", + "python~=3.12", + "python[version='3.12.*']", + ], +) +def test_unsupported_python_requirements_fail_closed(requirement): + with pytest.raises(ValueError, match="Unsupported Python requirement syntax"): + vcr.python_tag_from_index({"build": "py312_0", "depends": [requirement]}) + + +@pytest.mark.parametrize("dependencies", ["python >=3.13", {"python": ">=3.13"}, [None]]) +def test_malformed_dependencies_cannot_bypass_python_requirements(dependencies): + with pytest.raises(ValueError, match="'depends' must be a list"): + vcr.python_tag_from_index({"build": "py312_0", "depends": dependencies}) + + +def _zstd_available(): + try: + from compression import zstd # noqa: F401 # py3.14+ + + return True + except Exception: + try: + import zstandard # noqa: F401 + + return True + except Exception: + return False + + +def _compress_zstd(data): + try: + from compression import zstd + except ImportError: + import zstandard + + return zstandard.ZstdCompressor().compress(data) + return zstd.compress(data) + + +@pytest.mark.skipif(not _zstd_available(), reason="no zstandard backend available") +def test_read_index_json_roundtrip(tmp_path): + got = archive.read_release_index(str(_write_release_archive(tmp_path, extension=".conda"))) + assert got["subdir"] == "win-64" + assert vcr.python_tag_from_index(got) == "3.12" + + +@pytest.mark.skipif(not _zstd_available(), reason="no zstandard backend available") +def test_read_index_json_rejects_multiple_info_payloads(tmp_path): + import zipfile + + conda_path = tmp_path / "ambiguous.conda" + with zipfile.ZipFile(conda_path, "w") as container: + container.writestr("info-first.tar.zst", b"first") + container.writestr("info-second.tar.zst", b"second") + + with pytest.raises(ValueError, match="exactly one info-.*found 2"): + archive.read_release_index(str(conda_path)) + + +def test_index_json_must_be_an_object(): + with pytest.raises(ValueError, match="must contain a JSON object"): + archive._require_index_object([], "package.conda") + + +@pytest.mark.parametrize("value", [None, 123, "", " 1.13.0", "1.13.0 "]) +def test_required_index_fields_must_be_trimmed_strings(value): + with pytest.raises(ValueError, match="must be a non-empty, trimmed string"): + archive._required_index_string({"version": value}, "version", "package.conda") + + +def test_stdlib_zstd_data_error_does_not_fall_back(monkeypatch): + class CorruptFrameError(Exception): + pass + + compression = types.ModuleType("compression") + compression.zstd = types.SimpleNamespace( + ZstdFile=lambda *_args, **_kwargs: (_ for _ in ()).throw( + CorruptFrameError("corrupt frame") + ), + DecompressionParameter=types.SimpleNamespace(window_log_max=100), + ZstdError=CorruptFrameError, + ) + fallback = types.ModuleType("zstandard") + fallback.ZstdDecompressor = lambda: (_ for _ in ()).throw( + AssertionError("third-party fallback must not run after a data error") + ) + monkeypatch.setitem(sys.modules, "compression", compression) + monkeypatch.setitem(sys.modules, "zstandard", fallback) + + with pytest.raises(ValueError, match="corrupt frame") as caught: + archive.decompress_index(b"not zstd") + assert isinstance(caught.value.__cause__, CorruptFrameError) + + +def test_zstd_missing_backends_raise_clear_error(monkeypatch): + real_import = __import__ + + def _missing_backends(name, *args, **kwargs): + if name in {"compression", "zstandard"}: + raise ImportError(f"blocked {name}") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr("builtins.__import__", _missing_backends) + + with pytest.raises(RuntimeError, match="reading .conda.*requires"): + archive.decompress_index(b"data") + + +class _FakeAnacondaApi: + def __init__(self, distributions): + self.distributions = distributions + self.calls = [] + self.fail_add_after_apply = None + self.fail_remove_after_apply = None + + def distribution(self, owner, package, version, basename): + self.calls.append(("distribution", owner, package, version, basename)) + metadata = self.distributions[basename] + return {**metadata, "labels": list(metadata["labels"])} + + def add_channel(self, label, owner, *, package, version, filename): + self.calls.append(("add", label, owner, package, version, filename)) + labels = self.distributions[filename]["labels"] + if label not in labels: + labels.append(label) + if filename == self.fail_add_after_apply: + raise RuntimeError("simulated add failure after server update") + + def remove_channel(self, label, owner, *, package, version, filename): + self.calls.append(("remove", label, owner, package, version, filename)) + labels = self.distributions[filename]["labels"] + if label in labels: + labels.remove(label) + if (label, filename) == self.fail_remove_after_apply: + raise RuntimeError("simulated remove failure after server update") + + +def _distribution(subdir, filename, version="1.13.0"): + return promoter.Distribution( + path=Path(filename), + package="mssql-python", + version=version, + basename=f"{subdir}/{filename}", + sha256=(filename.encode().hex() + "0" * 64)[:64], + ) + + +_STAGING = "main_staging_123" + + +def _api_for(distributions, labels=(_STAGING,)): + return _FakeAnacondaApi( + { + distribution.basename: { + "basename": distribution.basename, + "sha256": distribution.sha256, + "labels": list(labels), + } + for distribution in distributions + } + ) + + +def _promote(api, distributions, *, staging=_STAGING, version="1.13.0"): + return promoter.promote( + api, + "microsoft", + staging, + "main", + version, + distributions, + verify_attempts=1, + delay_seconds=0, + ) + + +def test_verify_distribution_uses_full_subdir_basename_and_requires_sha_and_label(): + distribution = _distribution("win-64", "mssql-python-1.13.0-py312_0.conda") + api = _api_for([distribution]) + + labels = promoter.verify_distribution(api, "microsoft", distribution, required_label=_STAGING) + + assert labels == {_STAGING} + assert api.calls[-1][-1] == distribution.basename + + api.distributions[distribution.basename]["sha256"] = "bad" + with pytest.raises(RuntimeError, match="no valid SHA-256"): + promoter.verify_distribution(api, "microsoft", distribution) + + +def test_promote_verifies_all_files_then_cleans_staging_label(): + distributions = [ + _distribution("win-64", "mssql-python-1.13.0-py312_0.conda"), + _distribution("linux-64", "mssql-python-1.13.0-py313_0.conda"), + ] + api = _api_for(distributions) + + _promote(api, distributions) + + assert all(api.distributions[item.basename]["labels"] == ["main"] for item in distributions) + assert [call[0] for call in api.calls].count("add") == 2 + assert [call[0] for call in api.calls].count("remove") == 2 + + +def test_promote_is_idempotent_after_partial_staging_cleanup(): + first = _distribution("win-64", "mssql-python-1.13.0-py312_0.conda") + second = _distribution("linux-64", "mssql-python-1.13.0-py313_0.conda") + api = _api_for([first, second], labels=(_STAGING, "main")) + api.distributions[first.basename]["labels"] = ["main"] + + _promote(api, [first, second]) + + assert api.distributions[first.basename]["labels"] == ["main"] + assert api.distributions[second.basename]["labels"] == ["main"] + assert not any(call[0] == "add" for call in api.calls) + + +def test_promote_recovers_matching_file_left_on_old_staging_label(): + distribution = _distribution("linux-64", "mssql-python-1.13.0-py313_0.conda") + api = _api_for([distribution], labels=("main_staging_old",)) + + _promote(api, [distribution], staging=_STAGING) + + assert api.distributions[distribution.basename]["labels"] == ["main_staging_old", "main"] + add_labels = [call[1] for call in api.calls if call[0] == "add"] + assert add_labels == [_STAGING, "main"] + + +def test_promote_accepts_ambiguous_add_failure_when_label_landed(): + first = _distribution("win-64", "mssql-python-1.13.0-py312_0.conda") + second = _distribution("linux-64", "mssql-python-1.13.0-py313_0.conda") + api = _api_for([first, second]) + api.fail_add_after_apply = second.basename + + _promote(api, [first, second]) + + assert all(api.distributions[item.basename]["labels"] == ["main"] for item in (first, second)) + + +def test_promote_accepts_ambiguous_staging_cleanup_when_label_was_removed(): + distribution = _distribution("linux-64", "mssql-python-1.13.0-py313_0.conda") + api = _api_for([distribution]) + api.fail_remove_after_apply = (_STAGING, distribution.basename) + + _promote(api, [distribution]) + + assert api.distributions[distribution.basename]["labels"] == ["main"] + + +@pytest.mark.parametrize("rollback_reply_lost", [False, True]) +def test_promote_rolls_back_partial_label_promotion(rollback_reply_lost): + first = _distribution("win-64", "mssql-python-1.13.0-py312_0.conda") + second = _distribution("linux-64", "mssql-python-1.13.0-py313_0.conda") + api = _api_for([first, second]) + api.fail_add_after_apply = second.basename + if rollback_reply_lost: + api.fail_remove_after_apply = ("main", second.basename) + + original_distribution = api.distribution + hide_target_once = {second.basename} + + def fail_second_target_verification(owner, package, version, basename): + metadata = original_distribution(owner, package, version, basename) + if basename in hide_target_once and "main" in metadata["labels"]: + hide_target_once.remove(basename) + metadata["labels"].remove("main") + return metadata + + api.distribution = fail_second_target_verification + + with pytest.raises(RuntimeError, match="rollback of newly added target labels was attempted"): + _promote(api, [first, second]) + + assert all(api.distributions[item.basename]["labels"] == [_STAGING] for item in (first, second)) + + +def test_rollback_never_removes_unattempted_or_preexisting_target_labels(): + previous = _distribution("win-64", "previous.conda") + failed = _distribution("linux-64", "failed.conda") + untouched = _distribution("osx-64", "untouched.conda") + api = _api_for([previous, failed, untouched]) + api.distributions[previous.basename]["labels"].append("main") + original_distribution = api.distribution + hide_target_once = {failed.basename} + + def hide_failed_add(owner, package, version, basename): + metadata = original_distribution(owner, package, version, basename) + if basename in hide_target_once and "main" in metadata["labels"]: + hide_target_once.remove(basename) + metadata["labels"].remove("main") + return metadata + + api.distribution = hide_failed_add + with pytest.raises(RuntimeError, match="rollback"): + _promote(api, [previous, failed, untouched]) + + removed = [call[-1] for call in api.calls if call[:2] == ("remove", "main")] + assert removed == [failed.basename] + assert api.distributions[previous.basename]["labels"] == [_STAGING, "main"] + assert api.distributions[untouched.basename]["labels"] == [_STAGING] + + +@pytest.mark.parametrize("next_version", ["1.13.0", "1.14.0"]) +def test_retry_after_rollback_preserves_other_staging_labels(next_version): + distribution = _distribution("win-64", "package.conda") + api = _api_for([distribution], labels=(_STAGING, "main_staging_124")) + original_read = api.distribution + fail_a_once = [True] + + def fail_target_verification_once(owner, package, version, basename): + metadata = original_read(owner, package, version, basename) + if fail_a_once[0] and "main" in metadata["labels"]: + fail_a_once[0] = False + metadata["labels"].remove("main") + return metadata + + api.distribution = fail_target_verification_once + with pytest.raises(RuntimeError, match="rollback"): + _promote(api, [distribution], staging=_STAGING) + assert "main" not in api.distributions[distribution.basename]["labels"] + next_distribution = distribution + if next_version != "1.13.0": + next_distribution = _distribution("win-64", "next.conda", version=next_version) + api.distributions.update( + _api_for([next_distribution], labels=("main_staging_124",)).distributions + ) + _promote(api, [next_distribution], staging="main_staging_124", version=next_version) + expected = [_STAGING, "main"] if next_distribution is distribution else ["main"] + assert api.distributions[next_distribution.basename]["labels"] == expected + + +def test_promote_rejects_wrong_release_version_before_api_mutation(): + distribution = _distribution("linux-64", "mssql-python-9.9.9-py313_0.conda", version="9.9.9") + api = _api_for([distribution]) + + with pytest.raises(ValueError, match="do not match expected"): + _promote(api, [distribution]) + + assert api.calls == [] + + +@pytest.mark.parametrize( + "field,value,message", + [ + ("sha256", None, "no valid SHA-256"), + ("sha256", "0" * 64, "SHA-256 mismatch"), + ("basename", "same.conda", "basename"), + ("basename", "../win-64/same.conda", "basename"), + ("labels", None, "labels list"), + ("labels", "main", "labels list"), + ("labels", [123], "labels list"), + ("labels", ["main_staging_1"], "missing required label"), + ], +) +def test_remote_identity_checksum_and_labels_fail_closed(field, value, message): + distribution = _distribution("win-64", "same.conda") + api = _api_for([distribution]) + metadata = api.distributions[distribution.basename] + metadata[field] = value + api.distribution = lambda *_args: metadata + with pytest.raises(RuntimeError, match=message): + promoter.verify_distribution(api, "microsoft", distribution, required_label="main") + + +def test_same_filename_on_different_platforms_is_not_a_duplicate(): + distributions = [ + _distribution("win-64", "same.conda"), + _distribution("linux-64", "same.conda"), + ] + api = _api_for(distributions) + _promote(api, distributions) + assert {call[-1] for call in api.calls} == {"win-64/same.conda", "linux-64/same.conda"} + + +_INVALID_PUBLICATION_SCOPES = [ + ("../org", _STAGING, "main"), + ("microsoft", "", "main"), + ("microsoft", _STAGING, "../main"), + ("microsoft", "main", "main"), + ("microsoft", "main", "stable"), + ("microsoft", "stable", "main"), + ("microsoft", "main", _STAGING), + ("microsoft", _STAGING, "stable"), + ("microsoft", "main_staging_", "main"), + ("microsoft", "main_staging_old", "main"), + ("microsoft", "main_staging_123_extra", "main"), + ("microsoft", "main_staging_123\n", "main"), + ("microsoft", "releaseX1_staging_123", "release.1"), + ("microsoft", f"{_STAGING}_staging_456", _STAGING), +] + + +@pytest.mark.parametrize("owner,staging,target", _INVALID_PUBLICATION_SCOPES) +@pytest.mark.parametrize("operation", [promoter.promote, promoter.cleanup_staging]) +def test_invalid_publication_scope_fails_before_any_remote_read(owner, staging, target, operation): + distribution = _distribution("win-64", "package.conda") + api = _api_for([distribution]) + with pytest.raises(ValueError): + operation(api, owner, staging, target, "1.13.0", [distribution]) + assert api.calls == [] + + +def test_empty_and_duplicate_promotion_inputs_are_rejected(): + distribution = _distribution("win-64", "package.conda") + with pytest.raises(ValueError, match="No Conda distributions"): + promoter.validate_release_input("1.13.0", []) + with pytest.raises(ValueError, match="Duplicate distribution"): + promoter.validate_release_input("1.13.0", [distribution, distribution]) + + +def test_partial_upload_cannot_start_public_label_promotion(): + first = _distribution("win-64", "first.conda") + absent = _distribution("linux-64", "absent.conda") + api = _api_for([first]) + with pytest.raises(KeyError): + _promote(api, [first, absent]) + assert not any(call[0] == "add" for call in api.calls) + + +@pytest.fixture +def cleanup_not_found(monkeypatch): + class NotFound(Exception): + pass + + errors = types.ModuleType("binstar_client.errors") + errors.NotFound = NotFound + monkeypatch.setitem(sys.modules, "binstar_client.errors", errors) + return NotFound + + +@pytest.mark.parametrize("reply_lost", [False, True]) +def test_failed_upload_cleanup_preserves_public_and_other_staging_labels( + cleanup_not_found, reply_lost +): + first = _distribution("win-64", "same.conda") + absent = _distribution("linux-64", "same.conda") + api = _api_for([first], labels=(_STAGING, "main", "another_build")) + if reply_lost: + api.fail_remove_after_apply = (_STAGING, first.basename) + original_read = api.distribution + + def read(owner, package, version, basename): + if basename == absent.basename: + raise cleanup_not_found("upload never landed") + return original_read(owner, package, version, basename) + + api.distribution = read + promoter.cleanup_staging( + api, "microsoft", _STAGING, "main", "1.13.0", [first, absent], verify_attempts=1 + ) + assert api.distributions[first.basename]["labels"] == ["main", "another_build"] + assert [call for call in api.calls if call[0] != "distribution"] == [ + ("remove", _STAGING, "microsoft", first.package, first.version, first.basename) + ] + + +@pytest.mark.parametrize("failure", ["checksum", "identity", "timeout", "remove"]) +def test_cleanup_reports_failures_but_continues_other_verified_files(cleanup_not_found, failure): + failed = _distribution("win-64", "failed.conda") + good = _distribution("linux-64", "good.conda") + api = _api_for([failed, good], labels=(_STAGING, "main")) + if failure == "checksum": + api.distributions[failed.basename]["sha256"] = "0" * 64 + elif failure == "identity": + api.distributions[failed.basename]["basename"] = "wrong/failed.conda" + elif failure == "timeout": + original_read = api.distribution + + def read(owner, package, version, basename): + if basename == failed.basename: + raise TimeoutError("metadata unavailable") + return original_read(owner, package, version, basename) + + api.distribution = read + else: + original_remove = api.remove_channel + + def remove(*args, **kwargs): + if kwargs["filename"] == failed.basename: + raise TimeoutError("remove did not reach server") + original_remove(*args, **kwargs) + + api.remove_channel = remove + with pytest.raises(RuntimeError, match="Staging cleanup incomplete.*failed.conda") as caught: + promoter.cleanup_staging( + api, "microsoft", _STAGING, "main", "1.13.0", [failed, good], verify_attempts=1 + ) + assert "(1 files)" in caught.value._cli_message and _STAGING in caught.value._cli_message + assert "--cleanup-staging" in caught.value._cli_message + assert api.distributions[failed.basename]["labels"] == [_STAGING, "main"] + assert api.distributions[good.basename]["labels"] == ["main"] + assert not any(call[0] == "add" or call[:2] == ("remove", "main") for call in api.calls) + + +@pytest.mark.parametrize("target", ["main", "stable", "release.1"]) +@pytest.mark.parametrize("operation", [promoter.promote, promoter.cleanup_staging]) +def test_generated_staging_scope_recovers_idempotently(cleanup_not_found, target, operation): + distribution = _distribution("win-64", "package.conda") + staging = f"{target}_staging_900" + retained = {"main", "stable", target, f"{target}_staging_901"} + api = _api_for([distribution], labels=[staging, *sorted(retained)]) + for _ in range(2): + operation(api, "microsoft", staging, target, "1.13.0", [distribution], verify_attempts=1) + assert set(api.distributions[distribution.basename]["labels"]) == retained + assert [call for call in api.calls if call[0] != "distribution"] == [ + ( + "remove", + staging, + "microsoft", + distribution.package, + distribution.version, + distribution.basename, + ) + ] + + +@pytest.mark.parametrize("success_on_last_attempt", [True, False]) +def test_eventual_consistency_retry_count_and_delay_are_bounded( + monkeypatch, success_on_last_attempt +): + attempts, delays = [], [] + monkeypatch.setattr(promoter.time, "sleep", delays.append) + + def verify(): + attempts.append(1) + if success_on_last_attempt and len(attempts) == 3: + return {"main"} + raise TimeoutError("metadata request timed out") + + if success_on_last_attempt: + assert promoter._verify_with_retry(verify, "metadata", attempts=3, delay_seconds=5) == { + "main" + } + else: + with pytest.raises(TimeoutError): + promoter._verify_with_retry(verify, "metadata", attempts=3, delay_seconds=5) + assert len(attempts) == 3 + assert delays == [5, 5] + + +def test_rollback_failure_is_reported_and_does_not_erase_prior_good_membership(): + previous = _distribution("win-64", "previous.conda") + failed = _distribution("linux-64", "failed.conda") + api = _api_for([previous, failed]) + api.distributions[previous.basename]["labels"].append("main") + original_read = api.distribution + hide_target_once = {failed.basename} + + def fail_verification(owner, package, version, basename): + metadata = original_read(owner, package, version, basename) + if basename in hide_target_once and "main" in metadata["labels"]: + hide_target_once.remove(basename) + metadata["labels"].remove("main") + return metadata + + def fail_remove(*_args, **_kwargs): + raise TimeoutError("rollback did not reach server") + + api.distribution = fail_verification + api.remove_channel = fail_remove + with pytest.raises( + RuntimeError, match="Rollback errors.*rollback did not reach server" + ) as caught: + _promote(api, [previous, failed]) + assert "(1 rollback errors)" in caught.value._cli_message + assert _STAGING in caught.value._cli_message + assert "rollback did not reach server" not in caught.value._cli_message + assert "main" in api.distributions[previous.basename]["labels"] + assert "main" in api.distributions[failed.basename]["labels"] + + +def test_interrupted_promotion_is_recoverable_but_not_atomic(): + first = _distribution("win-64", "first.conda") + second = _distribution("linux-64", "second.conda") + api = _api_for([first, second]) + original_add = api.add_channel + + def interrupt_after_add(*args, **kwargs): + original_add(*args, **kwargs) + raise KeyboardInterrupt("simulated process interruption") + + api.add_channel = interrupt_after_add + with pytest.raises(KeyboardInterrupt): + _promote(api, [first, second]) + assert "main" in api.distributions[first.basename]["labels"] + assert "main" not in api.distributions[second.basename]["labels"] + api.add_channel = original_add + _promote(api, [first, second]) + assert all(api.distributions[item.basename]["labels"] == ["main"] for item in (first, second)) + + +def _write_release_archive( + tmp_path, *, folder="win-64", extension=".tar.bz2", payload=b"canonical payload", **overrides +): + directory = tmp_path / folder + directory.mkdir(exist_ok=True) + index = { + "name": "mssql-python", + "version": "1.13.0", + "build": "py312_0", + "subdir": folder, + "depends": ["python >=3.12,<3.13.0a0"], + **overrides, + } + path = directory / f"{index['name']}-{index['version']}-{index['build']}{extension}" + data = json.dumps(index).encode() + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode="w" if extension == ".conda" else "w:bz2") as contents: + member = tarfile.TarInfo("info/index.json") + member.size = len(data) + contents.addfile(member, io.BytesIO(data)) + if extension == ".conda": + compress = _compress_zstd + payload_tar = io.BytesIO() + with tarfile.open(fileobj=payload_tar, mode="w") as archive: + member = tarfile.TarInfo("payload-marker.txt") + member.size = len(payload) + archive.addfile(member, io.BytesIO(payload)) + with zipfile.ZipFile(path, "w") as archive: + archive.writestr("metadata.json", json.dumps({"conda_pkg_format_version": 2})) + archive.writestr(f"info-{path.stem}.tar.zst", compress(buffer.getvalue())) + archive.writestr(f"pkg-{path.stem}.tar.zst", compress(payload_tar.getvalue())) + else: + path.write_bytes(buffer.getvalue()) + return path + + +@pytest.mark.parametrize( + "metadata", + [{"name": "unexpected"}, {"version": ""}, {"subdir": "../win-64"}], +) +def test_untrusted_local_metadata_is_rejected(tmp_path, metadata): + path = _write_release_archive(tmp_path, **metadata) + with pytest.raises(ValueError): + promoter.distribution_from_path(path) + + +@pytest.mark.parametrize("field", ["name", "version", "subdir"]) +@pytest.mark.parametrize("value", [None, True, 123, "", " {value}", "{value} "]) +def test_promoter_rejects_normalized_metadata_before_client(tmp_path, monkeypatch, field, value): + canonical = {"name": "mssql-python", "version": _MP_VER, "subdir": "win-64"} + malformed = value.format(value=canonical[field]) if isinstance(value, str) else value + path = _write_release_archive(tmp_path, **{field: malformed}) + normalized = {**canonical, field: str(malformed).strip()} + directory = tmp_path / normalized["subdir"] + directory.mkdir(exist_ok=True) + path = path.rename(directory / f"{normalized['name']}-{normalized['version']}-py312_0.tar.bz2") + monkeypatch.setitem(sys.modules, "binstar_client.utils", None) + for mode in ([], ["--check-local-only"], ["--cleanup-staging"]): + with pytest.raises(ValueError, match=f"field '{field}'.*trimmed string"): + _promoter_main( + [ + "--owner", + "microsoft", + "--staging-label", + _STAGING, + "--target-label", + "main", + "--expected-version", + normalized["version"], + *mode, + str(path), + ] + ) + + +@pytest.mark.parametrize("extension", [".tar.bz2", ".conda"]) +@pytest.mark.parametrize("renamed", [False, True]) +def test_local_preflight_requires_canonical_archive_basename( + tmp_path, monkeypatch, extension, renamed +): + if extension == ".conda" and not _zstd_available(): + pytest.skip("no zstandard backend available") + paths = [ + _write_release_archive(tmp_path, folder=subdir, extension=extension) + for subdir in ("win-64", "linux-64") + ] + if renamed: + paths[1] = paths[1].rename(paths[1].with_name("renamed" + extension)) + if extension == ".conda": + with zipfile.ZipFile(paths[1]) as archive: + entries = [(info.filename, archive.read(info)) for info in archive.infolist()] + with zipfile.ZipFile(paths[1], "w") as archive: + for name, data in entries: + if name.startswith(("info-", "pkg-")): + name = f"{name.split('-', 1)[0]}-{paths[1].stem}.tar.zst" + archive.writestr(name, data) + with pytest.raises(ValueError, match="canonical"): + vcr.collect_packages(str(tmp_path)) + assert _metadata_main(["--root", str(tmp_path)]) == 1 + else: + assert len(vcr.collect_packages(str(tmp_path))) == 2 + monkeypatch.setattr( + promoter, "promote", lambda *_args: pytest.fail("local preflight must not publish") + ) + args = [ + "--owner", + "microsoft", + "--staging-label", + _STAGING, + "--target-label", + "main", + "--expected-version", + _MP_VER, + "--check-local-only", + *map(str, paths), + ] + if renamed: + with pytest.raises(ValueError, match="canonical"): + _promoter_main(args) + else: + assert _promoter_main(args) == 0 + assert {promoter.distribution_from_path(p).basename for p in paths} == { + f"{subdir}/mssql-python-1.13.0-py312_0{extension}" for subdir in ("win-64", "linux-64") + } + + +@pytest.mark.skipif(not _zstd_available(), reason="no zstandard backend available") +@pytest.mark.parametrize( + "mutation", + [ + "missing-pkg", + "decoy-first-pkg", + "duplicate-pkg", + "noncanonical-pkg", + "noncanonical-info", + "duplicate-metadata", + "missing-metadata", + "extra-member", + "unsupported-version", + "boolean-version", + "non-object-metadata", + ], +) +def test_conda_container_shape_rejects_before_publication(tmp_path, monkeypatch, mutation): + path = _write_release_archive(tmp_path, extension=".conda") + with zipfile.ZipFile(path) as archive: + entries = [(info.filename, archive.read(info)) for info in archive.infolist()] + pkg = next(entry for entry in entries if entry[0].startswith("pkg-")) + if mutation == "missing-pkg": + entries.remove(pkg) + elif mutation == "decoy-first-pkg": + decoy_path = _write_release_archive( + tmp_path, extension=".conda", payload=b"different audit decoy" + ) + with zipfile.ZipFile(decoy_path) as archive: + decoy = archive.read(pkg[0]) + assert decoy != pkg[1] + entries.insert(0, ("pkg-audit-decoy.tar.zst", decoy)) + elif mutation == "duplicate-pkg": + entries.append(pkg) + elif mutation.startswith("noncanonical-"): + component = mutation.split("-")[1] + entries = [ + (f"{component}-wrong.tar.zst" if name.startswith(component + "-") else name, data) + for name, data in entries + ] + elif mutation == "duplicate-metadata": + entries.append(entries[0]) + elif mutation == "missing-metadata": + entries = [entry for entry in entries if entry[0] != "metadata.json"] + elif mutation == "extra-member": + entries.append(("unexpected.txt", b"unexpected")) + else: + metadata = { + "unsupported-version": {"conda_pkg_format_version": 3}, + "boolean-version": {"conda_pkg_format_version": True}, + "non-object-metadata": [], + }[mutation] + entries[0] = ("metadata.json", json.dumps(metadata).encode()) + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message="Duplicate name", category=UserWarning) + with zipfile.ZipFile(path, "w") as archive: + for name, data in entries: + archive.writestr(name, data) + assert ( + _metadata_main( + [ + "--root", + str(tmp_path), + "--required-subdirs", + "win-64", + "--allowed-subdirs", + "win-64", + "--pythons", + "3.12", + "--mssql-python-version", + _MP_VER, + ] + ) + == 1 + ) + monkeypatch.setattr( + promoter, "promote", lambda *_args: pytest.fail("invalid archive must not publish") + ) + with pytest.raises(ValueError): + _promoter_main( + [ + "--owner", + "microsoft", + "--staging-label", + _STAGING, + "--target-label", + "main", + "--expected-version", + _MP_VER, + "--check-local-only", + str(path), + ] + ) + + +@pytest.mark.parametrize( + "build, depends, error", + [ + ("0", ["python >=3.12,<3.13.0a0", "python >=3.13,<3.14.0a0"], "Conflicting"), + ("0", ["python >=3.13,<3.14.0a0", "python >=3.12,<3.13.0a0"], "Conflicting"), + ("py312_0", ["python >=3.13,<3.14.0a0"], "Conflicting"), + ("py312_0", ["python >=3.12,<3.13.0a0", "python >=3.13,<3.14.0a0"], "Conflicting"), + ("py312_0", ["python >=3.12", "python_abi 3.13.* *_cp313"], "Conflicting"), + ("py312_0", ["python_abi 3.12.* *_cp313"], "Conflicting"), + ("0", ["python ==3.12.*", "python ==3.13.*"], "Conflicting"), + ("py312_py313_0", ["python 3.12.*"], "Conflicting"), + ("py313_py312_0", ["python 3.13.*"], "Conflicting"), + ("py312_py312h123_0", ["python 3.12.*"], ""), + ("0", ["python 3.12.* *_cpython", "python >=3.12,<3.13.0a0"], ""), + ("py312_0", ["python >=3.10", "python"], ""), + ("py312_0", ["python >=3.12,<3.13.0a0", "python_abi 3.12.* *_cp312"], ""), + ("0", ["python >=3.12", "python"], "no detectable Python tag"), + ("py312_0", ["python >=3.13"], "Conflicting"), + ("py312_0", ["python <3.12"], "Conflicting"), + ("py312_0", ["python >3.12"], ""), + ("py312_0", ["python >=3.12.2,<3.13"], ""), + ("py312_0", ["python >=3.13|<3.12"], "Conflicting"), + ("py312_0", ["python >=3.13|>=3.12.2,<3.13"], ""), + ("py312_0", ["python >=3.12.2", "python <=3.12.1"], "Conflicting"), + ("py312_0", ["python !=3.12.*"], "Conflicting"), + ("py312_0", ["python (>=3.12,<3.13)"], "Unsupported"), + ], +) +def test_metadata_agreement_across_full_archive_matrix(tmp_path, capsys, build, depends, error): + for subdir in _ALLOWED: + for py in _PYTHONS[2:] if subdir == "win-arm64" else _PYTHONS: + minor = int(py.split(".")[1]) + index = { + "build": f"py{py.replace('.', '')}_0", + "depends": [f"python >={py},<3.{minor + 1}.0a0"], + } + if (subdir, py) == ("win-64", "3.12"): + index.update(build=build, depends=depends) + _write_release_archive(tmp_path, folder=subdir, **index) + assert len(list(tmp_path.rglob("*.tar.bz2"))) == 28 + assert _metadata_main(["--root", str(tmp_path), "--mssql-python-version", _MP_VER]) == bool( + error + ) + output = capsys.readouterr() + assert error in output.err if error else "metadata-validated" in output.out + + +@pytest.mark.parametrize("requirement,allowed", _PYTHON_REQUIREMENT_CASES) +def test_promoter_checks_python_constraints_before_client( + tmp_path, monkeypatch, requirement, allowed +): + path = _write_release_archive(tmp_path, depends=[requirement]) + monkeypatch.setitem(sys.modules, "binstar_client.utils", None) + args = [ + "--owner", + "microsoft", + "--staging-label", + _STAGING, + "--target-label", + "main", + "--expected-version", + _MP_VER, + str(path), + ] + if allowed: + assert _promoter_main(["--check-local-only", *args]) == 0 + else: + for mode in ([], ["--check-local-only"], ["--cleanup-staging"]): + with pytest.raises(ValueError, match="Conflicting"): + _promoter_main([*mode, *args]) + + +def test_local_only_cli_needs_no_token_or_api_client(tmp_path, monkeypatch, capsys): + path = _write_release_archive(tmp_path) + monkeypatch.delenv("ANACONDA_API_TOKEN", raising=False) + monkeypatch.setenv("BINSTAR_API_TOKEN", "synthetic-other-token") + monkeypatch.delenv("SYSTEM_ACCESSTOKEN", raising=False) + monkeypatch.setitem(sys.modules, "binstar_client.utils", None) + assert ( + _promoter_main( + [ + "--owner", + "microsoft", + "--staging-label", + _STAGING, + "--target-label", + "main", + "--expected-version", + "1.13.0", + "--check-local-only", + str(path), + ] + ) + == 0 + ) + assert "LOCAL_RELEASE_INPUT_OK" in capsys.readouterr().out + + +@pytest.mark.parametrize("owner,staging,target", _INVALID_PUBLICATION_SCOPES) +@pytest.mark.parametrize("mode", [[], ["--cleanup-staging"], ["--check-local-only"]]) +def test_cli_rejects_invalid_scope_before_client_creation( + tmp_path, monkeypatch, owner, staging, target, mode +): + path = _write_release_archive(tmp_path) + monkeypatch.setitem(sys.modules, "binstar_client.utils", None) + with pytest.raises(ValueError): + _promoter_main( + [ + "--owner", + owner, + "--staging-label", + staging, + "--target-label", + target, + "--expected-version", + "1.13.0", + *mode, + str(path), + ] + ) + + +@pytest.mark.parametrize("cleanup", [False, True]) +@pytest.mark.parametrize("other_token", [None, "", "synthetic-other-token"]) +def test_promotion_cli_uses_bounded_api_requests(tmp_path, monkeypatch, cleanup, other_token): + path = _write_release_archive(tmp_path) + requests = [] + clients = [] + monkeypatch.setenv("ANACONDA_API_TOKEN", "synthetic-reviewed-token") + if other_token is None: + monkeypatch.delenv("BINSTAR_API_TOKEN", raising=False) + else: + monkeypatch.setenv("BINSTAR_API_TOKEN", other_token) + api = types.SimpleNamespace( + session=types.SimpleNamespace(request=lambda *args, **kwargs: requests.append(kwargs)) + ) + client = types.ModuleType("binstar_client") + utils = types.ModuleType("binstar_client.utils") + + def get_server_api(**kwargs): + clients.append(kwargs) + return api + + utils.get_server_api = get_server_api + monkeypatch.setitem(sys.modules, "binstar_client", client) + monkeypatch.setitem(sys.modules, "binstar_client.utils", utils) + monkeypatch.setattr( + promoter, + "cleanup_staging" if cleanup else "promote", + lambda api, *_args: api.session.request("GET", "https://api.anaconda.org/example"), + ) + assert ( + _promoter_main( + [ + "--owner", + "microsoft", + "--staging-label", + _STAGING, + "--target-label", + "main", + "--expected-version", + "1.13.0", + *(["--cleanup-staging"] if cleanup else []), + str(path), + ] + ) + == 0 + ) + assert requests == [{"timeout": (15, 60)}] + assert clients == [ + { + "token": "synthetic-reviewed-token", + "config": {"url": "https://api.anaconda.org", "ssl_verify": True}, + } + ] + + +@pytest.mark.parametrize("cleanup", [False, True]) +@pytest.mark.parametrize("token", [None, "", " \t "]) +def test_publisher_requires_explicit_token_before_client_import( + tmp_path, monkeypatch, capsys, cleanup, token +): + path = _write_release_archive(tmp_path) + if token is None: + monkeypatch.delenv("ANACONDA_API_TOKEN", raising=False) + else: + monkeypatch.setenv("ANACONDA_API_TOKEN", token) + monkeypatch.setenv("BINSTAR_API_TOKEN", "synthetic-other-token") + monkeypatch.setitem(sys.modules, "binstar_client.utils", None) + assert ( + _promoter_cli( + [ + "--owner", + "microsoft", + "--staging-label", + _STAGING, + "--target-label", + "main", + "--expected-version", + _MP_VER, + *(["--cleanup-staging"] if cleanup else []), + str(path), + ] + ) + == 1 + ) + output = capsys.readouterr() + assert output.out == "" + assert output.err == ( + "ERROR: ValueError: ANACONDA_API_TOKEN is required for publication or staging recovery.\n" + ) + + +def test_publication_lock_covers_the_complete_release_stage(): + pipeline = (_ROOT / "OneBranchPipelines" / "conda-release-pipeline.yml").read_text() + stage = pipeline.split(" - stage: CondaRelease\n", 1)[1] + configuration, jobs = stage.split(" jobs:\n", 1) + assert ( + " ${{ if eq(parameters.publishToConda, true) }}:\n" + " lockBehavior: sequential\n" + " variables:\n" + " - group: 'Anaconda Publishing'\n" + ) in configuration + assert pipeline.count("lockBehavior:") == 1 + assert pipeline.count("- group: 'Anaconda Publishing'") == 1 + assert "- job: ValidateConda" in jobs + assert "- job: PublishConda" in jobs + + +def test_dry_release_excludes_the_complete_publication_job(): + pipeline = (_ROOT / "OneBranchPipelines" / "conda-release-pipeline.yml").read_text() + assert " default: false # Safety: default to a validate-only dry run." in pipeline + guarded_job = pipeline.split( + " - ${{ if eq(parameters.publishToConda, true) }}:\n" + " - job: PublishConda\n", + 1, + )[1] + assert " dependsOn: ValidateConda\n" in guarded_job + assert " type: releaseJob\n isProduction: true\n" in guarded_job + template = "/OneBranchPipelines/steps/conda-publish-step.yml@self" + assert pipeline.count(template) == 1 + assert template in guarded_job + publisher = (_ROOT / "OneBranchPipelines" / "steps" / "conda-publish-step.yml").read_text() + assert "Invoke-Anaconda @('upload'" in publisher + assert "python -m eng.conda_tools promote `" in publisher + assert "--cleanup-staging `" in publisher + assert "\nstages:" not in publisher and "\njobs:" not in publisher + + +@pytest.mark.skipif(sys.platform != "win32", reason="Publishing task runs on Windows") +@pytest.mark.parametrize("token", [None, "", " \t ", "synthetic-reviewed-token"]) +@pytest.mark.parametrize("other_token", [None, "", "synthetic-other-token"]) +def test_upload_task_uses_only_reviewed_token(tmp_path, token, other_token): + import os + import shutil + import subprocess + import textwrap + + shell = shutil.which("powershell") or shutil.which("pwsh") + assert shell is not None, "The Windows publishing task requires PowerShell" + template = (_ROOT / "OneBranchPipelines" / "steps" / "conda-publish-step.yml").read_text() + start = template.index(" if ([string]::IsNullOrWhiteSpace($env:ANACONDA_API_TOKEN))") + end = template.index(" if ([string]::IsNullOrWhiteSpace($env:BINSTAR_CONFIG_DIR))") + guard = textwrap.dedent(template[start:end]) + probe = tmp_path / "inspect_environment.py" + probe.write_text( + "import os\n" + 'assert "BINSTAR_API_TOKEN" not in os.environ\n' + 'assert os.environ["ANACONDA_API_TOKEN"] == "synthetic-reviewed-token"\n' + 'print("PUBLISHER_TOKEN_OK")\n', + encoding="utf-8", + ) + env = dict(os.environ, TEST_PUBLISHER_PYTHON=sys.executable, TEST_PUBLISHER_PROBE=str(probe)) + for name, value in (("ANACONDA_API_TOKEN", token), ("BINSTAR_API_TOKEN", other_token)): + if value is None: + env.pop(name, None) + else: + env[name] = value + result = subprocess.run( + [ + shell, + "-NoProfile", + "-NonInteractive", + "-Command", + "$ErrorActionPreference = 'Stop'\n" + guard + """ +$process = Start-Process -FilePath $env:TEST_PUBLISHER_PYTHON ` + -ArgumentList ('"' + $env:TEST_PUBLISHER_PROBE + '"') -NoNewWindow -PassThru +$null = $process.Handle +$process.WaitForExit() +exit $process.ExitCode +""", + ], + env=env, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == (0 if token and token.strip() else 1), result.stderr + if token and token.strip(): + assert result.stdout.strip() == "PUBLISHER_TOKEN_OK" + else: + assert "ANACONDA_API_TOKEN is not set" in result.stderr + assert "PUBLISHER_TOKEN_OK" not in result.stdout + + +@pytest.mark.parametrize("build", ["py312_0", "0"]) +def test_metadata_cli_reads_real_archive_and_enforces_requested_matrix(tmp_path, capsys, build): + _write_release_archive(tmp_path, build=build) + assert _metadata_main(["--root", str(tmp_path), "--mssql-python-version", "1.13.0"]) == 1 + assert "MISSING" in capsys.readouterr().err + assert ( + _metadata_main( + [ + "--root", + str(tmp_path), + "--required-subdirs", + "win-64", + "--allowed-subdirs", + "win-64", + "--pythons", + "3.12", + "--mssql-python-version", + "1.13.0", + ] + ) + == 0 + ) + assert "metadata-validated" in capsys.readouterr().out + + +@pytest.mark.parametrize("corruption", ["zip", "bzip2", "info-zstd"]) +def test_metadata_cli_reports_corrupt_archives_without_tracebacks(tmp_path, corruption): + import subprocess + + if corruption == "info-zstd": + path = _write_release_archive(tmp_path, extension=".conda") + with zipfile.ZipFile(path) as package: + members = [(member.filename, package.read(member)) for member in package.infolist()] + with zipfile.ZipFile(path, "w") as package: + for name, data in members: + package.writestr( + name, b"invalid zstandard frame" if name.startswith("info-") else data + ) + else: + extension = ".conda" if corruption == "zip" else ".tar.bz2" + (tmp_path / f"broken{extension}").write_bytes(b"invalid archive") + with pytest.raises(archive.READ_ERRORS): + vcr.collect_packages(str(tmp_path)) + result = subprocess.run( + [sys.executable, "-m", "eng.conda_tools", "validate", "--root", str(tmp_path)], + cwd=_ROOT, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 1 + assert result.stderr.startswith("ERROR:") + assert len(result.stderr.splitlines()) == 1 + assert "Traceback" not in result.stderr + assert "release-ready" not in result.stdout + + +@pytest.mark.parametrize("boundary", ["collection", "receipt", "installed-inputs"]) +@pytest.mark.parametrize( + "error_type", [*archive.READ_ERRORS, PermissionError, TypeError, AssertionError] +) +def test_metadata_cli_read_errors_are_controlled_without_hiding_bugs( + tmp_path, monkeypatch, capsys, boundary, error_type +): + _write_release_archive(tmp_path) + error = error_type("archive read failed") + calls = [] + + def fail(*_args, **_kwargs): + calls.append(boundary) + raise error + + if boundary == "collection": + monkeypatch.setattr(vcr, "collect_packages", fail) + elif boundary == "receipt": + receipt = tmp_path / "rs-transport.json" + receipt.write_text("{}") + original = Path.read_text + + def read(path, *args, **kwargs): + if path == receipt: + fail() + return original(path, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", read) + else: + monkeypatch.setattr(vcr.inputs, "validate_installed_inputs", fail) + args = [ + "--root", + str(tmp_path), + "--required-subdirs", + "win-64", + "--allowed-subdirs", + "win-64", + "--pythons", + "3.12", + "--mssql-python-version", + _MP_VER, + "--release-versions", + json.dumps({"mssql-python": _MP_VER, "mssql-python-odbc": "18.6.2.1"}), + ] + if error_type in (TypeError, AssertionError): + with pytest.raises(error_type, match="archive read failed"): + _metadata_main(args) + else: + assert _metadata_main(args) == 1 + output = capsys.readouterr() + assert output.err == f"ERROR: {error}\n" + assert "COMPONENT_INPUT_OK" not in output.out and "release-ready" not in output.out + assert calls == [boundary] + + +@pytest.mark.parametrize( + "count, policy, message", + [ + (0, [], "no conda packages"), + (25, [], "win-arm64"), + (28, [], ""), + (28, ["--mssql-python-version", "wrong"], "!= expected"), + (28, ["--required-subdirs", ""], "must not be empty"), + (28, ["--required-subdirs", " , "], "must not be empty"), + (28, ["--allowed-subdirs", ""], "must not be empty"), + (28, ["--allowed-subdirs", " , "], "must not be empty"), + (28, ["--pythons", ""], "must not be empty"), + (28, ["--pythons", " , "], "must not be empty"), + (28, ["--allowed-subdirs", "invalid"], "not in allowed set"), + ], +) +def test_metadata_cli_enforces_full_default_release_policy( + tmp_path, monkeypatch, capsys, count, policy, message +): + packages = _healthy_set() + [_binding("win-arm64", py) for py in _PYTHONS[2:]] + monkeypatch.setattr(vcr, "collect_packages", lambda _root: packages[:count]) + assert _metadata_main(["--root", str(tmp_path), *policy]) == (1 if message else 0) + output = capsys.readouterr() + assert message in output.err if message else "metadata-validated" in output.out + + +@pytest.mark.parametrize( + "option,value", + [ + ("--required-subdirs", ",".join(_REQUIRED)), + ("--allowed-subdirs", ",".join(_ALLOWED)), + ("--pythons", ",".join(_PYTHONS)), + ("--subdir-pythons", "win-arm64=3.12,3.13,3.14"), + ], +) +@pytest.mark.parametrize("empty_entry", ["{value},", "{value}, ,3.14"]) +def test_metadata_cli_rejects_empty_policy_entries( + tmp_path, monkeypatch, capsys, option, value, empty_entry +): + packages = _healthy_set() + [_binding("win-arm64", py) for py in _PYTHONS[2:]] + monkeypatch.setattr(vcr, "collect_packages", lambda _root: packages) + assert _metadata_main(["--root", str(tmp_path), option, empty_entry.format(value=value)]) == 1 + assert "must not be empty" in capsys.readouterr().err + + +def test_metadata_cli_rejects_distinct_builds_in_one_matrix_cell(tmp_path, capsys): + for subdir in _ALLOWED: + for py in _PYTHONS[2:] if subdir == "win-arm64" else _PYTHONS: + _write_release_archive( + tmp_path, + folder=subdir, + build=f"py{py.replace('.', '')}_0", + depends=[f"python {py}.*"], + ) + assert _metadata_main(["--root", str(tmp_path)]) == 0 + _write_release_archive(tmp_path, build="py312_1") + assert len(list(tmp_path.rglob("*.tar.bz2"))) == 29 + assert _metadata_main(["--root", str(tmp_path)]) == 1 + assert "DUPLICATE" in capsys.readouterr().err + + +def test_staging_recovery_accepts_reply_lost_after_server_mutation(): + distribution = _distribution("win-64", "package.conda") + api = _api_for([distribution], labels=("old_staging",)) + api.fail_add_after_apply = distribution.basename + _promote(api, [distribution], staging=_STAGING) + assert api.distributions[distribution.basename]["labels"] == ["old_staging", "main"] + + +def test_failed_staging_cleanup_preserves_successful_publication(): + distribution = _distribution("win-64", "package.conda") + api = _api_for([distribution]) + + def timeout_before_remove(*_args, **_kwargs): + raise TimeoutError("cleanup did not reach server") + + api.remove_channel = timeout_before_remove + with pytest.raises(RuntimeError, match="Failed to remove staging label") as caught: + _promote(api, [distribution]) + assert ( + _STAGING in caught.value._cli_message and "'main' must remain" in caught.value._cli_message + ) + assert api.distributions[distribution.basename]["labels"] == [_STAGING, "main"] + + +def test_missing_local_archive_and_version_are_rejected(tmp_path): + with pytest.raises(ValueError, match="does not exist"): + promoter.distribution_from_path(tmp_path / "missing.conda") + with pytest.raises(ValueError, match="Expected mssql-python version"): + promoter.validate_release_input("", []) + + +@pytest.mark.parametrize("extension", [".conda", ".tar.bz2"]) +@pytest.mark.parametrize("index_kind", ["missing", "duplicate", "symlink"]) +def test_index_member_must_be_unique_regular_file(tmp_path, monkeypatch, extension, index_kind): + import zipfile + + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode="w" if extension == ".conda" else "w:bz2") as contents: + for _ in range(2 if index_kind == "duplicate" else 1): + member = tarfile.TarInfo("other.json" if index_kind == "missing" else "info/index.json") + if index_kind == "symlink": + member.type = tarfile.SYMTYPE + member.linkname = "../../outside.json" + contents.addfile(member) + path = tmp_path / ("package" + extension) + if extension == ".conda": + with zipfile.ZipFile(path, "w") as container: + container.writestr("metadata.json", json.dumps({"conda_pkg_format_version": 2})) + container.writestr("info-package.tar.zst", _compress_zstd(buffer.getvalue())) + container.writestr("pkg-package.tar.zst", b"not read by the metadata reader") + else: + path.write_bytes(buffer.getvalue()) + with pytest.raises(ValueError, match="exactly one regular info/index.json"): + archive.read_release_index(str(path)) + + +@pytest.mark.parametrize("problem", ["success", "missing", "policy"]) +def test_promoter_cli_local_preflight(tmp_path, monkeypatch, capsys, problem): + path = _write_release_archive(tmp_path) + if problem == "missing": + path.unlink() + monkeypatch.setitem(sys.modules, "binstar_client.utils", None) + args = [ + "--owner", + "microsoft", + "--staging-label", + _STAGING, + "--target-label", + "main", + "--expected-version", + "9.9.9" if problem == "policy" else _MP_VER, + "--check-local-only", + str(path), + ] + assert _promoter_cli(args) == (0 if problem == "success" else 1) + output = capsys.readouterr() + if problem == "success": + assert output.out.startswith("LOCAL_RELEASE_INPUT_OK:") and output.err == "" + else: + assert output.out == "" + assert output.err.startswith("ERROR: ValueError:") and len(output.err.splitlines()) == 1 + + +@pytest.mark.parametrize( + "error_type", + [ + ValueError, + RuntimeError, + ImportError, + PermissionError, + TimeoutError, + tarfile.ReadError, + zipfile.BadZipFile, + EOFError, + KeyError, + TypeError, + AssertionError, + "chained-bug", + ], +) +def test_promoter_cli_error_boundary(monkeypatch, capsys, error_type): + error = ( + RuntimeError("outer failure") + if error_type == "chained-bug" + else error_type("failure\nsynthetic-secret") + ) + if error_type == "chained-bug": + error.__cause__ = TypeError("unexpected bug") + + def fail(_argv): + raise error + + monkeypatch.setenv("ANACONDA_API_TOKEN", "synthetic-secret") + monkeypatch.setattr(promoter, "execute", fail) + with pytest.raises(type(error)): + promoter.execute(argparse.Namespace()) + if error_type in (KeyError, TypeError, AssertionError, "chained-bug"): + with pytest.raises(type(error)): + promoter.cli(argparse.Namespace()) + else: + assert promoter.cli(argparse.Namespace()) == 1 + output = capsys.readouterr() + assert output.err.startswith(f"ERROR: {error_type.__name__}:") + assert len(output.err.splitlines()) == 1 and "synthetic-secret" not in output.err diff --git a/tests/test_029_bundled_binary_audit.py b/tests/test_029_bundled_binary_audit.py index 4ec93c906..67c633107 100644 --- a/tests/test_029_bundled_binary_audit.py +++ b/tests/test_029_bundled_binary_audit.py @@ -187,6 +187,14 @@ def add(name, data): machine=machine, versions=("GLIBC_2.2.5", "GLIBC_2.34") ), } + native_payload = dict(native_payload) + root = "lib/python3.12/site-packages/" + if not any(name.endswith(".dist-info/METADATA") for name in native_payload): + prefix = root + "mssql_python-1.13.0.dist-info/" + native_payload[prefix + "METADATA"] = b"Name: mssql-python\nVersion: 1.13.0\n" + native_payload[prefix + "RECORD"] = "".join( + f"{name.removeprefix(root)},,\n" for name in [*native_payload, prefix + "RECORD"] + ).encode() for name, data in native_payload.items(): add(name, data) selected_distros = distros or _DISTROS_BY_SUBDIR.get(subdir, ("debian_ubuntu",)) @@ -217,6 +225,54 @@ def add(name, data): return str(p) +@pytest.mark.parametrize("entry_point", ["reader", "audit"]) +@pytest.mark.parametrize( + "layout", ["valid", "missing", "duplicate-good-last", "duplicate-bad-last"] +) +def test_installed_record_cardinality(tmp_path, entry_point, layout): + path = _make_pkg(tmp_path) + with tarfile.open(path, "r:bz2") as source: + members = [(item.name, source.extractfile(item).read()) for item in source.getmembers()] + root = "lib/python3.12/site-packages/" + prefix = root + "mssql_python_odbc-18.6.2.1.dist-info/" + members.append((prefix + "METADATA", b"Name: mssql-python-odbc\nVersion: 18.6.2.1\n")) + good = "".join( + f"{name.removeprefix(root)},,\n" + for name, _ in members + if name.startswith(root + "mssql_python_odbc") + ).encode() + bad = good + b"mssql_py_core/__init__.py,,\n" + if layout == "valid": + records = [good] + elif layout == "missing": + records = [] + else: + records = [bad, good] if layout == "duplicate-good-last" else [good, bad] + members.extend((prefix + "RECORD", record) for record in records) + with tarfile.open(path, "w:bz2") as target: + for name, data in members: + item = tarfile.TarInfo(name) + item.size = len(data) + target.addfile(item, io.BytesIO(data)) + if entry_point == "audit": + errors = audit.audit_package(path, "elf").violations + if layout == "valid": + assert errors == [] + else: + assert any("exactly one" in error and "RECORD" in error for error in errors) + return + from eng.conda_tools import archive + + payload = list(archive.iter_payload_members(path)) + names = [name for name, _ in payload] + files = {name: data for name, data in payload if ".dist-info/" in name} + if layout == "valid": + assert len(list(archive.installed_metadata(names, files))) == 2 + else: + with pytest.raises(ValueError, match="exactly one.*RECORD"): + list(archive.installed_metadata(names, files)) + + @pytest.mark.parametrize( "state", [ @@ -227,6 +283,11 @@ def add(name, data): "wrong-arch", "unowned-core", "wrong-rs-version", + "unrecorded-extra", + "odbc-cross-owned", + "binding-cross-owned", + "core-alias", + "owned-bytecode", ], ) def test_rs_private_libraries_keep_separate_ownership_and_runtime_contract(tmp_path, state): @@ -266,8 +327,26 @@ def test_rs_private_libraries_keep_separate_ownership_and_runtime_contract(tmp_p ).encode() if state == "missing-private-library": del payload[private] + elif state in {"unrecorded-extra", "owned-bytecode"}: + relative = ( + "mssql_py_core/unrecorded.py" + if state == "unrecorded-extra" + else "mssql_py_core/__pycache__/__init__.cpython-312.pyc" + ) + payload[root + relative] = b"" + if state == "owned-bytecode": + payload[rs_info + "RECORD"] += f"{relative},,\n".encode() + elif state == "core-alias": + payload[root + "MSSQL_PY_CORE/__init__.py"] = b"" + elif state in {"odbc-cross-owned", "binding-cross-owned"}: + owner = binding_info + if state == "odbc-cross-owned": + owner = root + "mssql_python_odbc-18.6.2.dist-info/" + payload[owner + "METADATA"] = b"Name: mssql-python-odbc\nVersion: 18.6.2\n" + payload[owner + "RECORD"] = b"" + payload[owner + "RECORD"] += b"mssql_py_core/__init__.py,,\n" result = audit.audit_package(_make_pkg(tmp_path, native_payload=payload), "elf") - if state == "valid": + if state in {"valid", "owned-bytecode"}: assert result.violations == [] else: assert result.violations @@ -275,6 +354,8 @@ def test_rs_private_libraries_keep_separate_ownership_and_runtime_contract(tmp_p assert any("OpenSSL 1.1" in error for error in result.violations) elif state == "missing-runpath": assert any("RUNPATH" in error for error in result.violations) + elif state in {"unrecorded-extra", "odbc-cross-owned", "binding-cross-owned", "core-alias"}: + assert any("mssql_py_core" in error for error in result.violations) # --- low-level parser ------------------------------------------------------- @@ -680,6 +761,16 @@ def add(name, data): add(_CORE_INIT, b"from .mssql_py_core import *\n") add(_BINDING, _make_elf64()) add(_CORE, _make_elf64()) + root = "lib/python3.12/site-packages/" + prefix = root + "mssql_python-1.13.0.dist-info/" + add(prefix + "METADATA", b"Name: mssql-python\nVersion: 1.13.0\n") + add( + prefix + "RECORD", + "".join( + f"{name.removeprefix(root)},,\n" + for name in (_CORE_INIT, _BINDING, _CORE, prefix + "METADATA", prefix + "RECORD") + ).encode(), + ) add( f"{_LIBDIR}/libmsodbcsql-18.6.so.2.1", _make_elf64(_GOOD_RUNPATH, needed=_DRIVER_NEEDED) ) diff --git a/tests/test_030_pe_machine_assert.py b/tests/test_030_pe_machine_assert.py index 7c38729cc..8f4a11efd 100644 --- a/tests/test_030_pe_machine_assert.py +++ b/tests/test_030_pe_machine_assert.py @@ -388,23 +388,38 @@ def read_component(): @pytest.mark.skipif(not _zstd_available(), reason="no zstandard backend available") -def test_win_arm64_arm64_binaries_pass(tmp_path): - p = _make_conda( - tmp_path, - "win-arm64", - { - _CORE_INIT: b"from .mssql_py_core import *\n", - "Lib/site-packages/mssql_python/ddbc_bindings.cp312-arm64.pyd": _fake_pe(_ARM64), - "Lib/site-packages/mssql_py_core/mssql_py_core.cp312-win_arm64.pyd": _fake_pe(_ARM64), - "Lib/site-packages/mssql_python_odbc/libs/windows/arm64/msodbcsql18.dll": _fake_pe( - _ARM64 - ), - "Lib/site-packages/mssql_python_odbc/libs/windows/arm64/mssql-auth.dll": _fake_pe( - _ARM64 - ), - }, - ) - assert audit.audit_package(p, "pe").violations == [] +@pytest.mark.parametrize("metadata_state", ["valid", "missing", "case", "backslash"]) +def test_win_arm64_binaries_require_binding_metadata(tmp_path, metadata_state): + payload = { + _CORE_INIT: b"from .mssql_py_core import *\n", + "Lib/site-packages/mssql_python/ddbc_bindings.cp312-arm64.pyd": _fake_pe(_ARM64), + "Lib/site-packages/mssql_py_core/mssql_py_core.cp312-win_arm64.pyd": _fake_pe(_ARM64), + "Lib/site-packages/mssql_python_odbc/libs/windows/arm64/msodbcsql18.dll": _fake_pe(_ARM64), + "Lib/site-packages/mssql_python_odbc/libs/windows/arm64/mssql-auth.dll": _fake_pe(_ARM64), + } + root = "Lib/site-packages/" + prefix = root + "mssql_python-1.13.0.dist-info/" + if metadata_state != "missing": + payload[prefix + "METADATA"] = b"Name: mssql-python\nVersion: 1.13.0\n" + payload[prefix + "RECORD"] = "".join( + f"{name.removeprefix(root)},,\n" for name in [*payload, prefix + "RECORD"] + ).encode() + if metadata_state != "valid": + payload = { + ( + (name.lower() if metadata_state == "case" else name.replace("/", "\\")) + if ".dist-info/" in name + else name + ): data + for name, data in payload.items() + } + errors = audit.audit_package(_make_conda(tmp_path, "win-arm64", payload), "pe").violations + if metadata_state == "valid": + assert errors == [] + elif metadata_state == "missing": + assert any("binding" in error and "metadata" in error.lower() for error in errors) + else: + assert any("Noncanonical installed METADATA" in error for error in errors) @pytest.mark.parametrize( @@ -444,6 +459,11 @@ def test_required_core_contract(tmp_path, state): payload[core.replace("312", "311")] = payload.pop(core) elif state == "abi3": payload[core.replace(".cp312-win_arm64", "")] = payload.pop(core) + prefix = "Lib/site-packages/mssql_python-1.13.0.dist-info/" + payload[prefix + "METADATA"] = b"Name: mssql-python\nVersion: 1.13.0\n" + payload[prefix + "RECORD"] = "".join( + f"{name.removeprefix('Lib/site-packages/')},,\n" for name in [*payload, prefix + "RECORD"] + ).encode() errors = audit.audit_package( _make_conda(tmp_path, "win-arm64", payload, depends=depends), "pe" ).violations diff --git a/tests/test_034_conda_verify_cwd.py b/tests/test_034_conda_verify_cwd.py index ecaa73192..ec01f2b9d 100644 --- a/tests/test_034_conda_verify_cwd.py +++ b/tests/test_034_conda_verify_cwd.py @@ -718,12 +718,118 @@ def _rs_inputs(tmp_path, platform="win_amd64", python_tags=("cp313",)): return code, odbc, links, rs +@pytest.mark.parametrize("with_rs", [False, True]) +@pytest.mark.parametrize("recorded", [False, True]) +@pytest.mark.parametrize( + "member", + [ + "mssql_py_core/__init__.py", + "mssql_py_core/unrecorded.py", + "./mssql_py_core/__init__.py", + "mssql_py_core\\__init__.py", + "MSSQL_PY_CORE/__init__.py", + "mssql_python_odbc-18.6.2.data/platlib/mssql_py_core/__init__.py", + "mssql_python_odbc-18.6.2.data/purelib/mssql_py_core/__init__.py", + ], +) +def test_odbc_core_payload_is_rejected_before_staging(tmp_path, capsys, with_rs, recorded, member): + if with_rs: + code, odbc, links, rs = _rs_inputs(tmp_path) + else: + code, odbc, links = _wheel_inputs( + tmp_path, ["mssql_python_odbc-18.6.2-py3-none-win_amd64.whl"] + ) + rs = None + _write_wheel( + next(odbc.glob("*.whl")), + payload={member: b"# benign cross-owner fixture\n"}, + record_members=None if recorded else [], + ) + with pytest.raises(SystemExit) as error: + build.gather_wheels( + str(code), + "*.whl", + str(odbc), + "*.whl", + str(links), + str(rs) if rs else None, + None, + "win-64", + "3.13", + ) + assert error.value.code == 1 + assert "mssql_py_core" in capsys.readouterr().err + assert not list(links.iterdir()) + + +def test_gather_rejects_wrong_target_odbc_even_with_a_broad_filter(tmp_path, capsys): + code, odbc, links, rs = _rs_inputs(tmp_path) + next(odbc.glob("*.whl")).unlink() + _write_wheel(odbc / "mssql_python_odbc-18.6.2-py3-none-win_arm64.whl") + with pytest.raises(SystemExit) as error: + build.gather_wheels( + str(code), "*.whl", str(odbc), "*.whl", str(links), str(rs), None, "win-64", "3.13" + ) + assert error.value.code == 1 + assert "target" in capsys.readouterr().err + assert not list(links.iterdir()) + + +@pytest.mark.parametrize("component", ["binding", "odbc", "rs"]) +@pytest.mark.parametrize( + "layout", ["valid", "extra-root", "nested", "dot", "backslash", "case", "wrong-directory"] +) +def test_gather_requires_canonical_metadata_before_staging(tmp_path, capsys, component, layout): + code, odbc, links, rs = _rs_inputs(tmp_path) + wheel = next({"binding": code, "odbc": odbc, "rs": rs}[component].glob("*.whl")) + with zipfile.ZipFile(wheel) as source: + members = [(item.filename, source.read(item)) for item in source.infolist()] + original = next(name for name, _ in members if name.endswith(".dist-info/METADATA")) + if layout == "wrong-directory": + prefix = original.removesuffix("METADATA") + members = [ + (name.replace(prefix, "unexpected-0.dist-info/", 1), data) for name, data in members + ] + elif layout != "valid": + extra = { + "extra-root": "unexpected-0.dist-info/METADATA", + "nested": "nested/unexpected-0.dist-info/METADATA", + "dot": "./" + original, + "backslash": "nested\\" + original.replace("/", "\\"), + "case": original.lower(), + }[layout] + members.append((extra, b"Name: unexpected\nVersion: 0\n")) + with zipfile.ZipFile(wheel, "w") as target: + for name, data in members: + target.writestr(name, data) + arguments = ( + str(code), + "*.whl", + str(odbc), + "*.whl", + str(links), + str(rs), + None, + "win-64", + "3.13", + ) + if layout == "valid": + assert build.gather_wheels(*arguments) == ("1.2.3", "18.6.2", "0.1.0") + else: + with pytest.raises(SystemExit) as error: + build.gather_wheels(*arguments) + assert error.value.code == 1 + assert "METADATA" in capsys.readouterr().err + assert not list(links.iterdir()) + + @pytest.mark.parametrize( ("platform", "subdir", "matches"), [ ("win_amd64", "win-64", True), ("win_arm64", "win-arm64", True), ("win_amd64", "win-arm64", False), + ("any", "win-64", False), ("manylinux_2_28_x86_64", "linux-64", True), ("manylinux_2_34_aarch64", "linux-aarch64", True), ("manylinux2014_x86_64.manylinux_2_28_x86_64", "linux-64", True), @@ -735,13 +841,24 @@ def _rs_inputs(tmp_path, platform="win_amd64", python_tags=("cp313",)): ("macosx_15_0_x86_64", "osx-arm64", False), ], ) -def test_binding_selection_uses_target_platform_tags(platform, subdir, matches): +def test_wheel_selection_uses_target_platform_tags(platform, subdir, matches): wheel = f"mssql_python-1.2.3-cp313-cp313-{platform}.whl" assert contracts.binding_wheel_matches_target(wheel, subdir, ["3.13"]) is matches assert not contracts.binding_wheel_matches_target(wheel, subdir, ["3.12"]) + odbc = f"mssql_python_odbc-18.6.2-py3-none-{platform}.whl" + assert contracts.odbc_wheel_matches_target(odbc, subdir) is matches + + +@pytest.mark.parametrize("abi", ["cp312", "cp313t", "abi3", "none"]) +def test_binding_selection_requires_the_requested_normal_abi(abi): + wheel = f"mssql_python-1.2.3-cp313-{abi}-win_amd64.whl" + assert not contracts.binding_wheel_matches_target(wheel, "win-64", ["3.12"]) + assert not contracts.binding_wheel_matches_target(wheel, "win-64", ["3.13"]) -@pytest.mark.parametrize("state", ["valid", "missing-rs", "mismatched-rs"]) +@pytest.mark.parametrize( + "state", ["valid", "missing-rs", "mismatched-rs", "odbc-core", "odbc-target"] +) def test_build_filters_consolidated_bindings_before_rs_resolution( tmp_path, monkeypatch, capsys, state ): @@ -750,6 +867,7 @@ def test_build_filters_consolidated_bindings_before_rs_resolution( for suffix in ( "1.2.3-cp310-cp310-win_arm64", "1.2.3-cp311-cp311-win_arm64", + "1.2.3-cp313-cp313t-win_arm64", "1.2.3-cp313-cp313-win_amd64", "9.9.9-cp313-cp313-manylinux_2_28_x86_64", ): @@ -759,6 +877,15 @@ def test_build_filters_consolidated_bindings_before_rs_resolution( selected_rs.unlink() elif state == "mismatched-rs": _write_wheel(selected_rs, "Name: mssql-python-rs\nVersion: 0.2.0\n") + elif state == "odbc-core": + _write_wheel( + next(odbc.glob("*.whl")), + payload={"mssql_py_core/__init__.py": b""}, + record_members=[], + ) + elif state == "odbc-target": + next(odbc.glob("*.whl")).unlink() + _write_wheel(odbc / "mssql_python_odbc-18.6.2-py3-none-win_amd64.whl") bootstrap_calls = [] def stop_before_conda(output_dir): @@ -797,7 +924,12 @@ def stop_before_conda(output_dir): assert error.value.code == 1 assert bootstrap_calls == [] assert not list(links.iterdir()) - expected = "cp313 win-arm64; found []" if state == "missing-rs" else "METADATA Version" + expected = { + "missing-rs": "cp313 win-arm64; found []", + "mismatched-rs": "METADATA Version", + "odbc-core": "mssql_py_core", + "odbc-target": "ODBC wheel does not match target", + }[state] assert expected in capsys.readouterr().err return assert str(error.value) == "input selection completed" diff --git a/tests/test_035_conda_macho_assert.py b/tests/test_035_conda_macho_assert.py index 32180fa9e..d851a9510 100644 --- a/tests/test_035_conda_macho_assert.py +++ b/tests/test_035_conda_macho_assert.py @@ -124,6 +124,13 @@ def _zstd_compress(raw: bytes) -> bytes: def _make_conda(tmp_path, subdir, payload, depends=("python_abi 3.12.* *_cp312",)): """Build a minimal .conda (info-*.tar.zst + pkg-*.tar.zst) with the given payload files.""" name = "mssql-python-1.13.0-py312_0" + payload = dict(payload) + root = "lib/python3.12/site-packages/" + prefix = root + "mssql_python-1.13.0.dist-info/" + payload[prefix + "METADATA"] = b"Name: mssql-python\nVersion: 1.13.0\n" + payload[prefix + "RECORD"] = "".join( + f"{member.removeprefix(root)},,\n" for member in [*payload, prefix + "RECORD"] + ).encode() pkg_buf = io.BytesIO() with tarfile.open(fileobj=pkg_buf, mode="w") as tf: diff --git a/tests/test_036_conda_provenance.py b/tests/test_036_conda_provenance.py new file mode 100644 index 000000000..459fce977 --- /dev/null +++ b/tests/test_036_conda_provenance.py @@ -0,0 +1,1263 @@ +"""Source-only tests of the authoritative Azure DevOps release provenance chain.""" + +import io +import json +import hashlib +import tarfile +import types +import zipfile +from pathlib import Path +from urllib.error import HTTPError +from urllib.parse import urlencode + +import pytest + +_PATH = Path(__file__).resolve().parent.parent / "eng" / "conda_tools" / "provenance.py" +if not _PATH.is_file(): + pytest.skip("Conda release sources are not shipped in wheels.", allow_module_level=True) + +from eng.conda_tools import __main__ as cli +from eng.conda_tools import contracts, inputs, provenance + + +def _records(pipeline, run, commit, branch="refs/heads/main"): + repository = {"id": "eec96f30-ec96-4910-abd6-c45a99a5c29f", "type": "azureReposGit"} + return ( + { + "id": run, + "definition": {"id": pipeline}, + "buildNumber": "26250.2" if pipeline == 2199 else "26253.2-CondaBuild", + "status": "completed", + "result": "succeeded", + "sourceBranch": branch, + "sourceVersion": commit, + "repository": repository, + "project": {"id": "c6d89619-62de-46a0-8b46-70b92a84d85e"}, + }, + { + "id": run, + "pipeline": {"id": pipeline}, + "name": "26250.2" if pipeline == 2199 else "26253.2-CondaBuild", + "state": "completed", + "result": "succeeded", + "resources": { + "repositories": { + "self": {"repository": repository, "refName": branch, "version": commit} + } + }, + }, + ) + + +@pytest.fixture +def release_sources(): + return { + "setup.py": "setup(\n version='1.15.0',\n install_requires=[],\n)\n", + "mssql_python/__init__.py": '__version__ = "1.15.0"\n', + "mssql_python_odbc/__init__.py": '__version__ = "18.6.2.1"\n', + } + + +def _component_files(name, version, rs=None): + prefix = f"{name.replace('-', '_')}-{version}.dist-info/" + metadata = f"Metadata-Version: 2.4\nName: {name}\nVersion: {version}\n" + files = {} + if name == "mssql-python": + metadata += "Requires-Dist: mssql-python-odbc==18.6.2.1\n" + files["mssql_python/__init__.py"] = b"" + if rs: + metadata += f"Requires-Dist: mssql-python-rs=={rs}\n" + if name == "mssql-python-rs" or (name == "mssql-python" and not rs): + files.update( + { + "mssql_py_core/__init__.py": b"", + "mssql_py_core/mssql_py_core.cp313-win_amd64.pyd": b"native fixture", + } + ) + if name == "mssql-python-rs": + files.update( + {path: b"private native fixture" for path in contracts.rs_private_libraries("win-64")} + ) + tag = "py3-none-win_amd64" if name == "mssql-python-odbc" else "cp313-cp313-win_amd64" + files[prefix + "METADATA"] = metadata.encode() + files[prefix + "WHEEL"] = f"Wheel-Version: 1.0\nTag: {tag}\n".encode() + files[prefix + "RECORD"] = "".join( + f"{path},,\n" for path in [*files, prefix + "RECORD"] + ).encode() + return files + + +def _source_dependencies(sources, dependencies): + sources["setup.py"] = sources["setup.py"].replace( + "install_requires=[]", f"install_requires={dependencies}" + ) + + +@pytest.mark.parametrize( + "problem", + [ + "success", + "valid-parentheses", + "unpublished", + "wrong-public-version", + "bad-digest", + "yanked-only", + "range", + "conditional", + "wrong-pin", + "missing-pin", + "duplicate-pin", + "wrong-metadata-version", + "wrong-metadata-name", + "duplicate-name", + "nested-metadata", + "duplicate-metadata", + "extra-nested-metadata", + "extra-dot-metadata", + "extra-case-metadata", + "extra-backslash-metadata", + "extra-wheel", + "valid-rs", + "new-source-legacy", + "rs-unpublished", + "rs-source-mismatch", + "rs-conditional", + "rs-binding-owns-core", + "rs-provider-unowned", + "rs-target-mismatch", + "rs-binding-target-mismatch", + "rs-binding-python-mismatch", + "rs-binding-abi-mismatch", + "rs-odbc-target-mismatch", + "rs-provider-extra-unowned", + "legacy-extra-unowned", + "odbc-owns-core", + "odbc-core-data", + "odbc-core-case", + "changed-wheel-hash", + "wheel-tag-mismatch", + *( + f"{profile}{member}-alias-{spelling}" + for profile in ("", "rs-") + for member in ("record", "wheel") + for spelling in ("dist-info-case", "name-case", "backslash", "nested") + ), + ], +) +def test_public_wheel_controls_use_actual_metadata_ownership_and_hashes( + tmp_path, monkeypatch, problem +): + versions = {"mssql-python": "1.15.0", "mssql-python-odbc": "18.6.2.1"} + with_rs = problem == "valid-rs" or problem.startswith("rs-") + source = {**versions, "mssql-python-rs": "0.2.0"} + if with_rs: + versions["mssql-python-rs"] = "0.2.0" + if problem == "rs-source-mismatch": + source["mssql-python-rs"] = "0.3.0" + wheels = {} + for name, version in versions.items(): + files = _component_files(name, version, "0.2.0" if with_rs else None) + key = next(path for path in files if path.endswith("/METADATA")) + metadata = files[key].decode() + if name == "mssql-python": + pin = "Requires-Dist: mssql-python-odbc==18.6.2.1\n" + replacement = { + "valid-parentheses": "Requires-Dist: MSSQL_python_ODBC (==18.6.2.1)\n", + "range": pin.replace("==", ">="), + "conditional": pin.rstrip() + '; python_version >= "3.10"\n', + "wrong-pin": pin.replace("18.6.2.1", "0.0.0"), + "missing-pin": "", + "duplicate-pin": pin * 2, + }.get(problem, pin) + metadata = metadata.replace(pin, replacement) + if problem == "rs-conditional": + metadata = metadata.replace("rs==0.2.0", 'rs==0.2.0; python_version >= "3.10"') + if problem == "rs-binding-owns-core": + files["mssql_py_core/__init__.py"] = b"" + if problem == "wrong-metadata-version": + metadata = metadata.replace(f"Version: {version}", "Version: 0.0.0") + elif problem == "wrong-metadata-name": + metadata = metadata.replace(f"Name: {name}", "Name: unexpected-package") + elif problem == "duplicate-name": + metadata += f"Name: {name}\n" + files[key] = metadata.encode() + if problem == "nested-metadata": + files["nested/" + key] = files.pop(key) + elif problem == "duplicate-metadata": + files["other-0.dist-info/METADATA"] = metadata.encode() + elif problem in { + "extra-nested-metadata", + "extra-dot-metadata", + "extra-case-metadata", + "extra-backslash-metadata", + }: + extra = { + "extra-nested-metadata": "nested/" + key, + "extra-dot-metadata": "./" + key, + "extra-case-metadata": key.lower(), + "extra-backslash-metadata": "nested\\" + key.replace("/", "\\"), + }[problem] + files[extra] = metadata.encode() + if name == "mssql-python-odbc" and problem == "odbc-owns-core": + files["mssql_py_core/unrecorded.py"] = b"" + elif name == "mssql-python-odbc" and problem == "odbc-core-data": + files[f"mssql_python_odbc-{version}.data/platlib/mssql_py_core/__init__.py"] = b"" + elif name == "mssql-python-odbc" and problem == "odbc-core-case": + files["MSSQL_PY_CORE/__init__.py"] = b"" + if (name == "mssql-python" and problem == "legacy-extra-unowned") or ( + name == "mssql-python-rs" and problem == "rs-provider-extra-unowned" + ): + files["mssql_py_core/unrecorded.py"] = b"" + if name == "mssql-python-rs" and problem == "rs-provider-unowned": + record = next(path for path in files if path.endswith("/RECORD")) + files[record] = b"" + tag = "py3-none-win_amd64" if name == "mssql-python-odbc" else "cp313-cp313-win_amd64" + if name == "mssql-python-rs" and problem == "rs-target-mismatch": + tag = tag.replace("313", "312") + files[next(path for path in files if path.endswith("/WHEEL"))] = ( + f"Tag: {tag}\n".encode() + ) + if name == "mssql-python" and problem in { + "rs-binding-target-mismatch", + "rs-binding-python-mismatch", + "rs-binding-abi-mismatch", + }: + tag = { + "rs-binding-target-mismatch": "cp313-cp313-win_arm64", + "rs-binding-python-mismatch": "cp312-cp312-win_amd64", + "rs-binding-abi-mismatch": "cp313-cp313t-win_amd64", + }[problem] + files[next(path for path in files if path.endswith("/WHEEL"))] = ( + f"Tag: {tag}\n".encode() + ) + if name == "mssql-python-odbc" and problem == "rs-odbc-target-mismatch": + tag = "py3-none-win_arm64" + files[next(path for path in files if path.endswith("/WHEEL"))] = ( + f"Tag: {tag}\n".encode() + ) + if problem == "wheel-tag-mismatch": + files[next(path for path in files if path.endswith("/WHEEL"))] = b"Tag: wrong\n" + if "-alias-" in problem: + field, _, spelling = problem.removeprefix("rs-").split("-", 2) + original = next(path for path in files if path.endswith(".dist-info/" + field.upper())) + alias = { + "dist-info-case": original.replace(".dist-info", ".DIST-INFO"), + "name-case": original.rsplit("/", 1)[0] + "/" + field, + "backslash": original.replace("/", "\\"), + "nested": "extra/" + original, + }[spelling] + files[alias] = files[original] + filename = f"{name.replace('-', '_')}-{version}-{tag}.whl" + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as wheel: + for path, data in files.items(): + entry = zipfile.ZipInfo() + entry.filename = path + wheel.writestr(entry, data) + wheels[name] = filename, buffer.getvalue() + calls = [] + + def fetch(url, *, timeout): + name, version = url.split("/")[-3:-1] + assert url == f"https://pypi.org/pypi/{name}/{versions[name]}/json" + assert timeout == 60 + calls.append(("GET", name)) + if problem == "unpublished" or (problem == "rs-unpublished" and name == "mssql-python-rs"): + raise HTTPError(url, 404, "not published", {}, None) + filename, data = wheels[name] + digest = hashlib.sha256(data).hexdigest() + response = { + "info": {"name": name, "version": version}, + "urls": [ + { + "filename": filename, + "packagetype": "bdist_wheel", + "yanked": problem == "yanked-only", + "digests": {"sha256": "invalid" if problem == "bad-digest" else digest}, + } + ], + } + if problem == "wrong-public-version": + response["info"]["version"] = "0.0.0" + return io.BytesIO(json.dumps(response).encode()) + + def download(argv, *, check): + assert check and argv[1:11] == [ + "-m", + "pip", + "--isolated", + "download", + "--index-url", + "https://pypi.org/simple", + "--no-deps", + "--only-binary=:all:", + "--require-hashes", + "-r", + ] + lock = Path(argv[11]).read_text() + directory = Path(argv[13]) + calls.append(("pip", lock)) + for requirement in lock.splitlines(): + name = requirement.split("==")[0] + filename, data = wheels[name] + assert f"--hash=sha256:{hashlib.sha256(data).hexdigest()}" in requirement + (directory / filename).write_bytes( + data + (b"changed" if problem == "changed-wheel-hash" else b"") + ) + if problem == "extra-wheel": + (directory / "extra.whl").touch() + + monkeypatch.setattr(inputs, "urlopen", fetch) + monkeypatch.setattr(inputs.subprocess, "run", download) + arguments = ( + source, + tmp_path / "wheels & spaces", + tmp_path / "wheel inputs.txt", + "cp313", + "win-64", + ) + if problem in {"success", "valid-parentheses", "valid-rs", "new-source-legacy"}: + assert inputs.fetch_wheels(*arguments) == versions + assert sum(call[0] == "pip" for call in calls) == (2 if with_rs else 1) + else: + with pytest.raises(ValueError): + inputs.fetch_wheels(*arguments) + if not with_rs: + assert ("GET", "mssql-python-rs") not in calls + + +@pytest.mark.parametrize( + "problem", + [ + "legacy", + "rs", + "new-source-legacy", + "missing-provider", + "wrong-odbc-pin", + "wrong-rs-pin", + "wrong-binding-version", + "missing-record", + "missing-wheel", + "wrong-root", + "duplicate-metadata", + "unowned-core", + "unrecorded-core", + "legacy-unrecorded-core", + "odbc-recorded-core", + "binding-recorded-core", + "core-path-alias", + "core-root-alias", + "rs-extra-owned", + "rs-pyc-owned", + "missing-core", + "missing-private-driver", + "missing-receipt", + "wrong-transport", + "missing-selected-wheel", + "wrong-selected-wheel", + "wrong-wheel-tags", + "wrong-wheel-target", + "duplicate-payload", + "missing-version-env", + "mismatched-binding-override", + *( + f"{profile}-{member}-alias-{spelling}" + for profile in ("legacy", "rs") + for member in ("metadata", "record", "wheel") + for spelling in ( + "dist-info-case", + "name-case", + "prefix-case", + "backslash", + "dot", + "parent", + "nested", + "dist-info-case-first", + "backslash-first", + ) + ), + ], +) +def test_source_bound_release_components(tmp_path, monkeypatch, capsys, problem): + with_rs = problem not in {"legacy", "new-source-legacy"} and not problem.startswith("legacy-") + versions = {"mssql-python": "1.15.0", "mssql-python-odbc": "18.6.2.1"} + if with_rs: + versions["mssql-python-rs"] = "0.2.0" + files = {} + for name, version in versions.items(): + files.update(_component_files(name, version, "0.2.0" if with_rs else None)) + if problem == "new-source-legacy": + versions["mssql-python-rs"] = "0.2.0" + rs_prefix = "mssql_python_rs-0.2.0.dist-info/" + binding_key = "mssql_python-1.15.0.dist-info/METADATA" + selected = "mssql_python_rs-0.2.0-cp313-cp313-win_amd64.whl" + receipt = { + "distribution_version": "0.2.0", + "transport_version": "0.2.0-dev.transport", + "feed_url": inputs.DEFAULT_FEED, + "package_id": inputs.PACKAGE_ID, + "package_sha256": "a" * 64, + "wheel_sha256": {selected: "b" * 64}, + } + if with_rs: + files[rs_prefix + "conda-wheel-source.txt"] = (selected + "\n").encode() + if "-alias-" in problem: + _, member, _, spelling = problem.split("-", 3) + first = spelling.endswith("-first") + spelling = spelling.removesuffix("-first") + original = "mssql_python-1.15.0.dist-info/" + member.upper() + alias = { + "dist-info-case": original.replace(".dist-info", ".DIST-INFO"), + "name-case": original.rsplit("/", 1)[0] + "/" + member, + "prefix-case": original.replace("mssql_python-", "MSSQL_PYTHON-"), + "backslash": original.replace("/", "\\"), + "dot": "./" + original, + "parent": "extra/../" + original, + "nested": "extra/" + original, + }[spelling] + data = files[original].replace(b"1.15.0", b"9.9.9").replace(b"cp313", b"cp310") + files = {alias: data, **files} if first else {**files, alias: data} + elif problem == "missing-provider": + files = {key: value for key, value in files.items() if not key.startswith(rs_prefix)} + elif problem in {"wrong-odbc-pin", "wrong-rs-pin", "wrong-binding-version"}: + old = { + "wrong-odbc-pin": "18.6.2.1", + "wrong-rs-pin": "rs==0.2.0", + "wrong-binding-version": "Version: 1.15.0", + }[problem] + files[binding_key] = files[binding_key].replace( + old.encode(), b"Version: 0.0.0" if problem == "wrong-binding-version" else b"0.0.0" + ) + elif problem in {"missing-record", "missing-wheel"}: + files.pop(rs_prefix + ("RECORD" if problem == "missing-record" else "WHEEL")) + elif problem == "duplicate-metadata": + files["other-0.dist-info/METADATA"] = files[binding_key] + elif problem == "unowned-core": + files[rs_prefix + "RECORD"] = b"" + elif problem in {"unrecorded-core", "legacy-unrecorded-core"}: + files["mssql_py_core/unrecorded.py"] = b"" + elif problem in {"odbc-recorded-core", "binding-recorded-core"}: + owner = ( + "mssql_python_odbc-18.6.2.1" + if problem == "odbc-recorded-core" + else "mssql_python-1.15.0" + ) + files[f"{owner}.dist-info/RECORD"] += b"mssql_py_core/__init__.py,,\n" + elif problem == "core-path-alias": + files["MSSQL_PY_CORE/__init__.py"] = b"" + elif problem in {"rs-extra-owned", "rs-pyc-owned"}: + member = ( + "mssql_py_core/extra.py" + if problem == "rs-extra-owned" + else "mssql_py_core/__pycache__/__init__.cpython-313.pyc" + ) + files[member] = b"" + files[rs_prefix + "RECORD"] += f"{member},,\n".encode() + elif problem == "missing-core": + files.pop("mssql_py_core/__init__.py") + elif problem == "missing-private-driver": + files.pop(contracts.rs_private_libraries("win-64")[0]) + elif problem == "missing-selected-wheel": + files.pop(rs_prefix + "conda-wheel-source.txt") + elif problem == "wrong-selected-wheel": + files[rs_prefix + "conda-wheel-source.txt"] = b"unexpected.whl\n" + elif problem == "wrong-wheel-tags": + files[rs_prefix + "WHEEL"] = b"Tag: cp310-cp310-win_amd64\n" + elif problem == "wrong-wheel-target": + wrong = selected.replace("313", "312") + receipt["wheel_sha256"] = {wrong: "b" * 64} + files[rs_prefix + "conda-wheel-source.txt"] = (wrong + "\n").encode() + files[rs_prefix + "WHEEL"] = b"Tag: cp312-cp312-win_amd64\n" + root = tmp_path / "conda" + package = root / "win-64" / "mssql-python-1.15.0-py313_0.tar.bz2" + package.parent.mkdir(parents=True) + index = { + "name": "mssql-python", + "version": "1.15.0", + "subdir": "win-64", + "build": "py313_0", + "depends": ["python >=3.13,<3.14.0a0", "python_abi 3.13.* *_cp313"], + } + prefix = "wrong/site-packages/" if problem == "wrong-root" else "Lib/site-packages/" + members = [ + ("info/index.json", json.dumps(index).encode()), + *((prefix + key, value) for key, value in files.items()), + ] + if problem == "duplicate-payload": + members.append(members[-1]) + elif problem == "core-root-alias": + members.append(("LIB/SITE-PACKAGES/mssql_py_core/unrecorded.py", b"")) + with tarfile.open(package, "w:bz2") as contents: + for name, data in members: + entry = tarfile.TarInfo(name) + entry.size = len(data) + contents.addfile(entry, io.BytesIO(data)) + if "mssql-python-rs" in versions and problem != "missing-receipt": + (root / "rs-transport.json").write_text(json.dumps(receipt)) + monkeypatch.setenv( + "RELEASE_VERSIONS", json.dumps(versions) if problem != "missing-version-env" else "" + ) + monkeypatch.setenv( + "RS_TRANSPORT_VERSION", + ( + "wrong" + if problem == "wrong-transport" + else "0.2.0-dev.transport" if "mssql-python-rs" in versions else "" + ), + ) + result = cli.main( + [ + "validate", + "--root", + str(root), + "--required-subdirs", + "win-64", + "--allowed-subdirs", + "win-64", + "--pythons", + "3.13", + "--mssql-python-version", + "0.0.0" if problem == "mismatched-binding-override" else "1.15.0", + "--release-versions", + ] + ) + assert result == ( + 0 if problem in {"legacy", "rs", "rs-extra-owned", "rs-pyc-owned"} else 1 + ), capsys.readouterr() + + +@pytest.mark.parametrize("with_rs", [False, True]) +def test_public_cli_reports_qualified_profile_without_importing_runtime( + tmp_path, monkeypatch, capsys, with_rs +): + versions = {"mssql-python": "1.15.0", "mssql-python-odbc": "18.6.2.1"} + if with_rs: + versions["mssql-python-rs"] = "0.2.0" + monkeypatch.setattr(inputs, "read_release_versions", lambda read: versions) + monkeypatch.setattr(inputs, "fetch_wheels", lambda *args: versions) + output = tmp_path / "outputs" + monkeypatch.setenv("GITHUB_OUTPUT", str(output)) + assert ( + cli.main( + [ + "fetch-wheels", + "--wheel-dir", + str(tmp_path), + "--requirements-file", + str(tmp_path / "pins"), + "--python-tag", + "cp313", + "--conda-subdir", + "win-64", + ] + ) + == 0 + ) + assert ("NOT current-source RS qualification" in capsys.readouterr().out) == (not with_rs) + assert output.read_text() == f"rsRequired={str(with_rs).lower()}\n" + + +@pytest.mark.parametrize("problem", ["input-error", "bug", "output-error"]) +def test_public_cli_expected_failures_do_not_hide_programming_errors( + tmp_path, monkeypatch, capsys, problem +): + monkeypatch.setattr(inputs, "read_release_versions", lambda read: {}) + + def fetch(*args): + if problem != "output-error": + raise ( + ValueError("expected input failure") + if problem == "input-error" + else TypeError("programming bug") + ) + return {} + + monkeypatch.setattr(inputs, "fetch_wheels", fetch) + monkeypatch.setenv("GITHUB_OUTPUT", str(tmp_path)) + arguments = cli.parser().parse_args( + [ + "fetch-wheels", + "--wheel-dir", + str(tmp_path), + "--requirements-file", + str(tmp_path / "pins"), + "--python-tag", + "cp313", + "--conda-subdir", + "win-64", + ] + ) + if problem == "bug": + with pytest.raises(TypeError, match="programming bug"): + inputs.fetch_cli(arguments) + else: + assert inputs.fetch_cli(arguments) == 1 + captured = capsys.readouterr() + assert "ERROR:" in captured.err + assert "PUBLIC_WHEEL_INPUT_OK" not in captured.out + + +@pytest.fixture +def chain(release_sources): + producer, producer_run = _records(2318, 174195, "a" * 40) + wheel, wheel_run = _records(2199, 173176, "b" * 40) + producer_run["resources"]["pipelines"] = { + "buildPipeline": {"pipeline": {"id": 173176}, "version": "26250.2"} + } + records = { + "build/builds/174195?api-version=7.1": producer, + "pipelines/2318/runs/174195?api-version=7.1": producer_run, + "build/builds/173176?api-version=7.1": wheel, + "pipelines/2199/runs/173176?api-version=7.1": wheel_run, + } + records.update(_source_record(path, content) for path, content in release_sources.items()) + return records + + +def _verify(chain, **kwargs): + return provenance.verify_provenance( + kwargs.pop("get_json", chain.__getitem__), + producer_pipeline_id=2318, + producer_run_id=174195, + producer_branch=kwargs.pop("producer_branch", "refs/heads/main"), + producer_commit=kwargs.pop("producer_commit", "a" * 40), + release_branch=kwargs.pop("release_branch", "refs/heads/main"), + publish=kwargs.pop("publish", True), + **kwargs, + ) + + +def test_exact_recorded_wheel_run_is_verified_not_packaging_mode_or_build_number(chain): + result = _verify(chain) + assert result["productionEligible"] is True + assert result["wheel"] == { + "pipeline": 2199, + "run": 173176, + "branch": "refs/heads/main", + "commit": "b" * 40, + } + assert result["producer"]["commit"] == "a" * 40 # Recipe and wheel commits can differ. + assert result["versions"] == {"mssql-python": "1.15.0", "mssql-python-odbc": "18.6.2.1"} + + +@pytest.mark.parametrize( + "content", + [ + "", + "version=get_version()", + "version='1.15.0'\nversion='1.15.0'", + "version='1.14.0'", + "version='$(unresolved)'", + "version='1.15.0' + '.dev1'", + ], +) +def test_missing_ambiguous_or_different_release_literal_fails(release_sources, content): + release_sources["setup.py"] = content + with pytest.raises(ValueError, match="release version|release versions"): + inputs.read_release_versions(release_sources.__getitem__) + + +@pytest.mark.parametrize("problem", ["absent", "ambiguous"]) +def test_absent_or_ambiguous_source_dependencies_cannot_be_classified_as_legacy( + release_sources, problem +): + if problem == "absent": + release_sources["setup.py"] = release_sources["setup.py"].replace( + " install_requires=[],\n", "" + ) + else: + release_sources["setup.py"] += "setup(install_requires=[])\n" + with pytest.raises(ValueError, match="exactly one explicit"): + inputs.read_release_versions(release_sources.__getitem__) + + +def test_release_source_is_read_not_executed(release_sources): + release_sources["setup.py"] += "raise AssertionError('must not execute setup.py')\n" + assert inputs.read_release_versions(release_sources.__getitem__)["mssql-python"] == "1.15.0" + + +@pytest.mark.parametrize( + "field,value", + [ + ("path", "/other.py"), + ("commitId", "a" * 40), + ("commitId", None), + ("gitObjectType", "tree"), + ("content", None), + ], +) +def test_version_source_response_must_match_exact_wheel_commit(chain, field, value): + item = next(value for key, value in chain.items() if key.startswith("git/repositories/")) + item[field] = value + with pytest.raises(ValueError, match="wheel producer source"): + _verify(chain, publish=False) + + +def test_main_conda_producer_cannot_publish_feature_branch_wheels(chain): + chain["build/builds/173176?api-version=7.1"]["sourceBranch"] = "refs/heads/feature" + wheel_run = chain["pipelines/2199/runs/173176?api-version=7.1"] + wheel_run["resources"]["repositories"]["self"]["refName"] = "refs/heads/feature" + with pytest.raises(ValueError, match="wheel producer source"): + _verify(chain) + result = _verify(chain, publish=False) + assert result["productionEligible"] is False + assert len(result["productionIneligibilityReasons"]) == 1 + + +def test_validate_only_can_inspect_feature_conda_artifacts_without_production_eligibility(chain): + chain["build/builds/174195?api-version=7.1"]["sourceBranch"] = "refs/heads/feature" + producer_run = chain["pipelines/2318/runs/174195?api-version=7.1"] + producer_run["resources"]["repositories"]["self"]["refName"] = "refs/heads/feature" + result = _verify( + chain, + producer_branch="refs/heads/feature", + release_branch="refs/heads/feature", + publish=False, + ) + assert result["productionEligible"] is False + assert len(result["productionIneligibilityReasons"]) == 2 + with pytest.raises(ValueError, match="Conda producer source"): + _verify(chain, producer_branch="refs/heads/feature") + + +@pytest.mark.parametrize("publish", [False, True]) +@pytest.mark.parametrize("run", [174195, 173176]) +@pytest.mark.parametrize( + "field,value", + [ + ("status", "inProgress"), + ("result", "partiallySucceeded"), + ("result", "failed"), + ("result", None), + ("sourceVersion", ""), + ("sourceBranch", ""), + ("id", 999), + ("definition", {"id": 999}), + ("buildNumber", "mismatched-number"), + ("repository", {"id": "wrong-repository"}), + ("project", {"id": "wrong-project"}), + ], +) +def test_invalid_or_incomplete_build_never_passes_even_in_dry_run( + chain, publish, run, field, value +): + chain[f"build/builds/{run}?api-version=7.1"][field] = value + with pytest.raises(ValueError): + _verify(chain, publish=publish) + + +@pytest.mark.parametrize("pipeline,run", [(2318, 174195), (2199, 173176)]) +@pytest.mark.parametrize( + "field,value", + [ + ("state", "inProgress"), + ("result", "failed"), + ("id", 999), + ("pipeline", {"id": 999}), + ("name", "mismatched-number"), + ("resources", {}), + ], +) +def test_invalid_authoritative_run_records_fail(chain, pipeline, run, field, value): + chain[f"pipelines/{pipeline}/runs/{run}?api-version=7.1"][field] = value + with pytest.raises(ValueError): + _verify(chain, publish=False) + + +@pytest.mark.parametrize( + "resource", + [ + {}, + {"pipeline": {"id": None}, "version": "26250.2"}, + {"pipeline": {"id": 173176}}, + {"pipeline": {"id": 173176}, "version": "173176"}, + {"pipeline": {"id": 173176}, "version": None}, + ], +) +def test_missing_or_wrong_wheel_resource_fails(chain, resource): + chain["pipelines/2318/runs/174195?api-version=7.1"]["resources"]["pipelines"] = { + "buildPipeline": resource + } + with pytest.raises(ValueError): + _verify(chain, publish=False) + + +@pytest.mark.parametrize("pipeline,run", [(2318, 174195), (2199, 173176)]) +@pytest.mark.parametrize("field,value", [("refName", "refs/heads/mismatch"), ("version", "c" * 40)]) +def test_recorded_source_branch_and_commit_must_match(chain, pipeline, run, field, value): + source = chain[f"pipelines/{pipeline}/runs/{run}?api-version=7.1"]["resources"]["repositories"] + source["self"][field] = value + with pytest.raises(ValueError, match="source repository, branch or commit mismatch"): + _verify(chain, publish=False) + + +@pytest.mark.parametrize( + "selected", [{"producer_commit": "c" * 40}, {"producer_branch": "refs/heads/mismatch"}] +) +def test_selected_producer_source_cannot_mismatch_api(chain, selected): + with pytest.raises(ValueError, match="Selected Conda resource"): + _verify(chain, **selected) + + +def test_feature_release_yaml_cannot_publish(chain): + with pytest.raises(ValueError, match="release source"): + _verify(chain, release_branch="refs/heads/feature") + + +def test_provenance_api_failure_is_not_treated_as_validate_only_success(chain): + del chain["build/builds/173176?api-version=7.1"] + with pytest.raises(KeyError): + _verify(chain, publish=False) + + +def test_api_cannot_substitute_another_run_for_recorded_upstream_version(chain): + resource = chain["pipelines/2318/runs/174195?api-version=7.1"]["resources"]["pipelines"] + resource["buildPipeline"]["pipeline"]["id"] = 173177 + chain["build/builds/173177?api-version=7.1"] = chain["build/builds/173176?api-version=7.1"] + chain["pipelines/2199/runs/173177?api-version=7.1"] = chain[ + "pipelines/2199/runs/173176?api-version=7.1" + ] + with pytest.raises(ValueError, match="Build API identity mismatch"): + _verify(chain, publish=False) + + +@pytest.fixture +def ado_env(monkeypatch): + for key, value in { + "SYSTEM_ACCESSTOKEN": "synthetic-token", + "SYSTEM_COLLECTIONURI": "https://dev.azure.com/SqlClientDrivers/", + "SYSTEM_TEAMPROJECTID": "c6d89619-62de-46a0-8b46-70b92a84d85e", + }.items(): + monkeypatch.setenv(key, value) + + +@pytest.mark.parametrize( + "key,value", + [ + ("SYSTEM_ACCESSTOKEN", ""), + ("SYSTEM_ACCESSTOKEN", "$(System.AccessToken)"), + ("SYSTEM_COLLECTIONURI", "https://untrusted.invalid/SqlClientDrivers"), + ("SYSTEM_TEAMPROJECTID", "904996cc-6198-4d39-8540-eca72bdf0b7b"), + ], +) +def test_http_client_rejects_untrusted_context_before_network(monkeypatch, ado_env, key, value): + monkeypatch.setenv(key, value) + monkeypatch.setattr( + provenance, "build_opener", lambda *_args: pytest.fail("Unexpected network call") + ) + with pytest.raises(ValueError): + provenance.ado_get_json("build/builds/174195?api-version=7.1") + + +def test_http_client_uses_only_get_bounded_timeout_and_trusted_endpoint(monkeypatch, ado_env): + calls = [] + + def open_request(request, **kwargs): + calls.append((request, kwargs)) + return io.BytesIO(b'{"id":174195}') + + monkeypatch.setattr( + provenance, "build_opener", lambda *_args: types.SimpleNamespace(open=open_request) + ) + assert provenance.ado_get_json("build/builds/174195?api-version=7.1") == {"id": 174195} + request, kwargs = calls[0] + assert request.get_method() == "GET" + assert request.full_url == ( + "https://dev.azure.com/SqlClientDrivers/c6d89619-62de-46a0-8b46-70b92a84d85e" + "/_apis/build/builds/174195?api-version=7.1" + ) + assert request.get_header("Authorization") == "Bearer synthetic-token" + assert kwargs == {"timeout": 60} + + +def test_api_redirects_cannot_forward_release_credentials(): + with pytest.raises(ValueError, match="credentials were not forwarded"): + provenance._NoRedirect().redirect_request( + None, None, 302, "redirect", {}, "https://untrusted.invalid" + ) + + +@pytest.mark.parametrize("publish", ["true", "false"]) +def test_provenance_cli_executes_same_chain_in_both_modes(monkeypatch, chain, capsys, publish): + for key, value in { + "PUBLISH_TO_CONDA": publish, + "CONDA_BUILD_PIPELINE_ID": "2318", + "CONDA_BUILD_RUN_ID": "174195", + "CONDA_BUILD_SOURCE_BRANCH": "refs/heads/main", + "CONDA_BUILD_SOURCE_COMMIT": "a" * 40, + "RELEASE_SOURCE_BRANCH": "refs/heads/main", + }.items(): + monkeypatch.setenv(key, value) + monkeypatch.setattr(provenance, "ado_get_json", chain.__getitem__) + provenance.execute() + output = capsys.readouterr().out + result = json.loads(output.splitlines()[0].split(": ", 1)[1]) + assert result["wheel"]["run"] == 173176 + assert ("Validate-only:" in output) == (publish == "false") + directive = "##vso[task.setvariable variable=releaseVersions;isOutput=true]" + assert ( + json.loads( + next( + line[len(directive) :] for line in output.splitlines() if line.startswith(directive) + ) + ) + == result["versions"] + ) + + +def test_invalid_publish_flag_is_not_treated_as_dry_run(monkeypatch): + monkeypatch.setenv("PUBLISH_TO_CONDA", "perhaps") + with pytest.raises(ValueError, match="true or false"): + provenance.execute() + + +def test_alternative_producer_definition_is_rejected(chain): + with pytest.raises(ValueError, match="must be pipeline 2318"): + provenance.verify_provenance( + chain.__getitem__, + producer_pipeline_id=2199, + producer_run_id=174195, + producer_branch="refs/heads/main", + producer_commit="a" * 40, + release_branch="refs/heads/main", + publish=False, + ) + + +@pytest.fixture +def provenance_cli_env(monkeypatch, ado_env): + values = { + "PUBLISH_TO_CONDA": "true", + "CONDA_BUILD_PIPELINE_ID": "2318", + "CONDA_BUILD_RUN_ID": "174195", + "CONDA_BUILD_SOURCE_BRANCH": "refs/heads/main", + "CONDA_BUILD_SOURCE_COMMIT": "a" * 40, + "RELEASE_SOURCE_BRANCH": "refs/heads/main", + "MSSQL_PYTHON_VERSION": "", + } + for key, value in values.items(): + monkeypatch.setenv(key, value) + return values + + +@pytest.mark.parametrize("expected", ["", "1.15.0", "1.14.0"]) +@pytest.mark.parametrize("publish", ["true", "false"]) +def test_cli_auto_version_and_override_are_producer_bound( + monkeypatch, provenance_cli_env, chain, capsys, expected, publish +): + monkeypatch.setenv("MSSQL_PYTHON_VERSION", expected) + monkeypatch.setenv("PUBLISH_TO_CONDA", publish) + monkeypatch.setattr(provenance, "ado_get_json", chain.__getitem__) + assert provenance.cli() == (1 if expected == "1.14.0" else 0) + output = capsys.readouterr() + if expected == "1.14.0": + assert output.out == "" and "differs from the recorded wheel producer" in output.err + else: + assert ( + "##vso[task.setvariable variable=mssqlPythonVersion;isOutput=true]1.15.0" in output.out + ) + assert output.err == "" + + +@pytest.mark.parametrize( + "key", + [ + "PUBLISH_TO_CONDA", + "CONDA_BUILD_PIPELINE_ID", + "CONDA_BUILD_RUN_ID", + "CONDA_BUILD_SOURCE_BRANCH", + "CONDA_BUILD_SOURCE_COMMIT", + "RELEASE_SOURCE_BRANCH", + "SYSTEM_ACCESSTOKEN", + "SYSTEM_COLLECTIONURI", + "SYSTEM_TEAMPROJECTID", + ], +) +def test_provenance_cli_missing_configuration(monkeypatch, provenance_cli_env, capsys, key): + monkeypatch.delenv(key) + monkeypatch.setattr(provenance, "build_opener", lambda *_: pytest.fail("Unexpected network")) + with pytest.raises(ValueError, match=key): + provenance.execute() + assert provenance.cli() == 1 + output = capsys.readouterr() + assert output.out == "" + assert output.err == f"ERROR: ValueError: Missing required environment variable {key}.\n" + + +@pytest.mark.parametrize("problem", ["success", "dry", "flag", "policy", "http", "json", "bug"]) +def test_provenance_cli_boundary(monkeypatch, provenance_cli_env, chain, capsys, problem): + errors = { + "http": HTTPError("https://private.invalid", 403, "private server body", {}, None), + "json": json.JSONDecodeError("private server body", "synthetic-token", 0), + "bug": TypeError("unexpected implementation bug"), + } + + def get_json(path): + if problem in errors: + raise errors[problem] + return chain[path] + + monkeypatch.setattr(provenance, "ado_get_json", get_json) + if problem in {"dry", "flag"}: + monkeypatch.setenv("PUBLISH_TO_CONDA", "false" if problem == "dry" else "invalid") + elif problem == "policy": + monkeypatch.setenv("RELEASE_SOURCE_BRANCH", "refs/heads/feature") + if problem == "bug": + with pytest.raises(TypeError): + provenance.cli() + return + success = problem in {"success", "dry"} + if not success: + with pytest.raises((ValueError, OSError)): + provenance.execute() + assert provenance.cli() == (0 if success else 1) + output = capsys.readouterr() + if success: + assert output.out.startswith("VERIFIED_RECORDED_PROVENANCE:") and output.err == "" + else: + assert output.out == "" + assert output.err.startswith("ERROR:") and len(output.err.splitlines()) == 1 + assert all( + value not in output.err + for value in ("private server body", "synthetic-token", "private.invalid") + ) + + +def _source_record(path, content): + query = urlencode( + { + "path": "/" + path, + "includeContent": "true", + "versionDescriptor.versionType": "commit", + "versionDescriptor.version": "b" * 40, + "$format": "json", + "api-version": "7.1", + } + ) + return f"git/repositories/{provenance._REPOSITORY_ID}/items?{query}", { + "path": "/" + path, + "commitId": "b" * 40, + "gitObjectType": "blob", + "content": content, + } + + +def test_rs_source_contract_reads_distribution_pin_not_transport(release_sources): + _source_dependencies(release_sources, '[f"mssql-python-rs=={_read_mssql_python_rs_version()}"]') + release_sources["eng/versions/mssql-python-rs.version"] = "0.2.0\n" + release_sources["eng/versions/mssql-python-rs-nuget.version"] = "9.9.9-dev.transport" + reads = [] + + def read_source(path): + reads.append(path) + return release_sources[path] + + assert provenance.read_release_versions(read_source) == { + "mssql-python": "1.15.0", + "mssql-python-odbc": "18.6.2.1", + "mssql-python-rs": "0.2.0", + } + assert reads == [ + "setup.py", + "mssql_python/__init__.py", + "mssql_python_odbc/__init__.py", + "eng/versions/mssql-python-rs.version", + ] + + +@pytest.mark.parametrize("version", ["", "0.2.0\n0.3.0", "$(version)", "../0.2.0", "0.2.0;bad"]) +def test_rs_distribution_pin_must_be_present_and_unambiguous(release_sources, version): + _source_dependencies(release_sources, '[f"mssql-python-rs=={_read_mssql_python_rs_version()}"]') + release_sources["eng/versions/mssql-python-rs.version"] = version + with pytest.raises(ValueError, match="RS distribution release version"): + provenance.read_release_versions(release_sources.__getitem__) + + +@pytest.mark.parametrize( + "dependencies", + [ + '["mssql-python-rs==0.2.0"]', + '[f"mssql-python-rs>={_read_mssql_python_rs_version()}"]', + '[f"mssql-python-rs=={other_version()}"]', + "[f\"mssql-python-rs=={_read_mssql_python_rs_version()}; python_version >= '3.10'\"]", + '[f"mssql-python-rs=={_read_mssql_python_rs_version()}"] * 2', + '["mssql-python-" + component]', + '[f"mssql-python-{component}=={version}"]', + '[f"{name}=={version}"]', + '[f" mssql-python-rs=={_read_mssql_python_rs_version()}"]', + '[f""]', + "[None]", + "get_dependencies()", + ], +) +def test_rs_source_contract_cannot_use_unverified_or_conditional_pins( + release_sources, dependencies +): + _source_dependencies(release_sources, dependencies) + with pytest.raises(ValueError, match="maintained RS distribution pin|explicit"): + provenance.read_release_versions(release_sources.__getitem__) + + +def test_unreadable_source_dependency_contract_never_becomes_legacy(release_sources): + release_sources["setup.py"] += "setup(install_requires=[\n" + with pytest.raises(ValueError, match="dependency contract cannot be read"): + provenance.read_release_versions(release_sources.__getitem__) + + +@pytest.mark.parametrize( + "problem", + ["valid", "missing", "wrong-commit", "transport-path", "missing-content", "forbidden"], +) +def test_recorded_rs_source_is_read_only_at_verified_wheel_commit(release_sources, chain, problem): + _source_dependencies(release_sources, '[f"mssql-python-rs=={_read_mssql_python_rs_version()}"]') + chain.update([_source_record("setup.py", release_sources["setup.py"])]) + key, record = _source_record("eng/versions/mssql-python-rs.version", "0.2.0\n") + if problem == "wrong-commit": + record["commitId"] = "c" * 40 + elif problem == "transport-path": + record["path"] = "/eng/versions/mssql-python-rs-nuget.version" + elif problem == "missing-content": + record.pop("content") + if problem != "missing": + chain[key] = record + transport_key, transport = _source_record( + "eng/versions/mssql-python-rs-nuget.version", "0.2.0-dev.transport" + ) + chain[transport_key] = transport + reads = [] + + def get_json(path): + reads.append(path) + if path == key and problem == "forbidden": + raise HTTPError("https://dev.azure.com", 403, "Forbidden", {}, None) + return chain[path] + + if problem == "valid": + result = _verify(chain, get_json=get_json) + assert result["versions"]["mssql-python-rs"] == "0.2.0" + assert result["rsTransportVersion"] == "0.2.0-dev.transport" + assert reads[-2:] == [key, transport_key] + else: + with pytest.raises((ValueError, KeyError, HTTPError)): + _verify(chain, get_json=get_json) + assert reads[-1] == key + assert len(reads) == (9 if problem == "valid" else 8) + + +@pytest.mark.parametrize("problem", ["missing", "wrong-commit", "invalid", "forbidden"]) +def test_rs_transport_source_cannot_fall_back_to_distribution_version( + release_sources, chain, problem +): + _source_dependencies(release_sources, '[f"mssql-python-rs=={_read_mssql_python_rs_version()}"]') + chain.update([_source_record("setup.py", release_sources["setup.py"])]) + chain.update([_source_record("eng/versions/mssql-python-rs.version", "0.2.0")]) + key, record = _source_record( + "eng/versions/mssql-python-rs-nuget.version", "0.2.0-dev.transport" + ) + if problem == "wrong-commit": + record["commitId"] = "c" * 40 + elif problem == "invalid": + record["content"] = "0.2.0\n0.3.0" + if problem != "missing": + chain[key] = record + + def get_json(path): + if path == key and problem == "forbidden": + raise HTTPError("https://dev.azure.com", 403, "Forbidden", {}, None) + return chain[path] + + with pytest.raises((ValueError, KeyError, HTTPError)): + _verify(chain, get_json=get_json) + + +@pytest.mark.parametrize( + "problem", + [ + "valid", + "absent", + "distribution", + "transport", + "feed", + "package", + "package-hash", + "empty-wheels", + "wheel-hash", + "wheel-version", + "wheel-path", + ], +) +def test_rs_transport_receipt_is_source_bound_without_inferred_producer_id(problem): + receipt = { + "distribution_version": "0.2.0", + "transport_version": "0.2.0-dev.transport", + "feed_url": ( + "https://pkgs.dev.azure.com/sqlclientdrivers/public/" + "_packaging/mssql-rs_Public/nuget/v3/index.json" + ), + "package_id": "mssql-python-rs-wheels", + "package_sha256": "a" * 64, + "wheel_sha256": {"mssql_python_rs-0.2.0-cp313-cp313-win_amd64.whl": "b" * 64}, + "nuspec_description": "InternalBuild; no independently verified numeric producer ID", + } + fields = { + "distribution": "distribution_version", + "transport": "transport_version", + "feed": "feed_url", + "package": "package_id", + "package-hash": "package_sha256", + } + if problem in fields: + receipt[fields[problem]] = "wrong" + elif problem == "absent": + receipt = None + elif problem == "empty-wheels": + receipt["wheel_sha256"] = {} + elif problem.startswith("wheel-"): + name, digest = next(iter(receipt["wheel_sha256"].items())) + if problem == "wheel-hash": + digest = "wrong" + elif problem == "wheel-version": + name = name.replace("0.2.0", "0.3.0") + else: + name = "../" + name + receipt["wheel_sha256"] = {name: digest} + arguments = (receipt, {"mssql-python-rs": "0.2.0"}, "0.2.0-dev.transport") + if problem == "valid": + inputs.validate_rs_transport(*arguments) + else: + with pytest.raises(ValueError, match="RS"): + inputs.validate_rs_transport(*arguments) + + +def test_verified_legacy_transport_absence_is_not_a_new_producer_fallback(): + inputs.validate_rs_transport(None, {"mssql-python": "1.15.0"}, "") + with pytest.raises(ValueError, match="embedded-core"): + inputs.validate_rs_transport({}, {"mssql-python": "1.15.0"}, "") + with pytest.raises(ValueError, match="recorded"): + inputs.validate_rs_transport(None, {"mssql-python-rs": "0.2.0"}, "") + + +@pytest.mark.parametrize( + "raw", + [ + "null", + "[]", + "{}", + '{"mssql-python":"1.15.0"}', + '{"mssql-python":"1.15.0","mssql-python-odbc":null}', + '{"mssql-python":"1.15.0","mssql-python-odbc":" 18.6.2.1"}', + '{"mssql-python":"1.15.0","mssql-python-odbc":"18.6.2.1","transport":"0.2.0-dev.x"}', + '{"mssql-python":"$(version)","mssql-python-odbc":"18.6.2.1"}', + ], +) +def test_release_output_versions_cannot_lose_or_substitute_source_assertions(raw): + with pytest.raises(ValueError, match="distribution versions"): + inputs.parse_release_versions(raw) + + +@pytest.mark.parametrize("with_rs", [False, True]) +def test_release_output_versions_preserve_explicit_old_or_new_source_contract(with_rs): + versions = {"mssql-python": "1.15.0", "mssql-python-odbc": "18.6.2.1"} + if with_rs: + versions["mssql-python-rs"] = "0.2.0" + assert inputs.parse_release_versions(json.dumps(versions)) == versions diff --git a/tests/test_038_conda_archive_limits.py b/tests/test_038_conda_archive_limits.py new file mode 100644 index 000000000..6a2846ea6 --- /dev/null +++ b/tests/test_038_conda_archive_limits.py @@ -0,0 +1,302 @@ +"""Small boundary fixtures for bounded archive reads; no native extension or database.""" + +import io +import json +import tarfile +import zipfile +from pathlib import Path + +import pytest + +if not (Path(__file__).resolve().parent.parent / "eng/conda_tools/archive.py").is_file(): + pytest.skip("Conda release sources are not shipped in wheels.", allow_module_level=True) + +from eng.conda_tools import archive + + +def _compress(data, *, unknown_size=False): + try: + from compression import zstd + except ImportError: + import zstandard + + return zstandard.ZstdCompressor( + write_content_size=not unknown_size, write_checksum=True + ).compress(data) + return zstd.compress( + data, + options={ + zstd.CompressionParameter.content_size_flag: int(not unknown_size), + zstd.CompressionParameter.checksum_flag: 1, + }, + ) + + +def _tar(files): + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode="w", format=tarfile.USTAR_FORMAT) as contents: + for name, data in files: + member = tarfile.TarInfo(name) + member.size = len(data) + contents.addfile(member, io.BytesIO(data)) + return buffer.getvalue() + + +def _package(tmp_path, extension=".conda", *, files=None, unknown_size=False): + index = json.dumps( + {"name": "mssql-python", "version": "1.15.0", "build": "py313_0", "subdir": "win-64"} + ).encode() + files = [("payload.bin", b"x" * 128)] if files is None else files + path = tmp_path / ("mssql-python-1.15.0-py313_0" + extension) + if extension == ".conda": + with zipfile.ZipFile(path, "w") as container: + container.writestr("metadata.json", json.dumps({"conda_pkg_format_version": 2})) + container.writestr( + f"info-{path.stem}.tar.zst", + _compress(_tar([("info/index.json", index)]), unknown_size=unknown_size), + ) + container.writestr( + f"pkg-{path.stem}.tar.zst", _compress(_tar(files), unknown_size=unknown_size) + ) + else: + with tarfile.open(path, "w:bz2", format=tarfile.USTAR_FORMAT) as contents: + for name, data in [("info/index.json", index), *files]: + member = tarfile.TarInfo(name) + member.size = len(data) + contents.addfile(member, io.BytesIO(data)) + return path, index + + +@pytest.mark.parametrize("extension", [".conda", ".tar.bz2"]) +@pytest.mark.parametrize("over", [False, True]) +def test_archive_file_size_boundary(tmp_path, monkeypatch, extension, over): + path, _ = _package(tmp_path, extension) + monkeypatch.setattr(archive, "_MAX_ARCHIVE_BYTES", path.stat().st_size - over, raising=False) + if over: + with pytest.raises(ValueError, match="archive bytes"): + archive.read_release_index(str(path)) + else: + assert archive.read_release_index(str(path))["name"] == "mssql-python" + + +@pytest.mark.parametrize("extension", [".conda", ".tar.bz2"]) +@pytest.mark.parametrize("over", [False, True]) +def test_member_size_boundary(tmp_path, monkeypatch, extension, over): + path, _ = _package(tmp_path, extension) + monkeypatch.setattr(archive, "_MAX_MEMBER_BYTES", 128 - over, raising=False) + if over: + with pytest.raises(ValueError, match="member bytes"): + list(archive.iter_payload_members(str(path))) + else: + assert dict(archive.iter_payload_members(str(path)))["payload.bin"] == b"x" * 128 + + +@pytest.mark.parametrize("extension", [".conda", ".tar.bz2"]) +@pytest.mark.parametrize("over", [False, True]) +def test_metadata_size_boundary(tmp_path, monkeypatch, extension, over): + path, index = _package(tmp_path, extension) + monkeypatch.setattr(archive, "_MAX_METADATA_BYTES", len(index) - over, raising=False) + if over: + with pytest.raises(ValueError, match="metadata bytes"): + archive.read_release_index(str(path)) + else: + assert archive.read_release_index(str(path))["name"] == "mssql-python" + + +@pytest.mark.parametrize("over", [False, True]) +def test_member_count_boundary(tmp_path, monkeypatch, over): + path, _ = _package(tmp_path, files=[("a", b""), ("b", b"")]) + monkeypatch.setattr(archive, "_MAX_MEMBERS", 2 - over, raising=False) + if over: + with pytest.raises(ValueError, match="member count"): + list(archive.iter_payload_members(str(path))) + else: + assert len(list(archive.iter_payload_members(str(path)))) == 2 + + +@pytest.mark.parametrize("over", [False, True]) +def test_cumulative_member_bytes_boundary(tmp_path, monkeypatch, over): + path, _ = _package(tmp_path, files=[("a", b"x" * 128), ("b", b"x" * 128)]) + monkeypatch.setattr(archive, "_MAX_PAYLOAD_BYTES", 256 - over, raising=False) + if over: + with pytest.raises(ValueError, match="cumulative member bytes"): + list(archive.iter_payload_members(str(path))) + else: + assert len(list(archive.iter_payload_members(str(path)))) == 2 + + +@pytest.mark.parametrize("unknown_size", [False, True]) +@pytest.mark.parametrize("over", [False, True]) +def test_expanded_size_includes_tar_padding(tmp_path, monkeypatch, unknown_size, over): + path, _ = _package(tmp_path, unknown_size=unknown_size) + monkeypatch.setattr(archive, "_MAX_EXPANDED_BYTES", 10240 - over, raising=False) + if over: + with pytest.raises(ValueError, match="expanded bytes"): + list(archive.iter_payload_members(str(path))) + else: + assert len(list(archive.iter_payload_members(str(path)))) == 1 + + +def test_readers_do_not_materialize_components_or_member_lists(tmp_path, monkeypatch): + path, _ = _package(tmp_path) + + def forbidden(*args, **kwargs): + raise AssertionError("whole ZIP member read or complete TAR member-list materialization") + + monkeypatch.setattr(zipfile.ZipFile, "read", forbidden) + monkeypatch.setattr(tarfile.TarFile, "getmembers", forbidden) + assert archive.read_release_index(str(path))["name"] == "mssql-python" + assert archive.read_index(str(path))["name"] == "mssql-python" + assert dict(archive.iter_payload_members(str(path)))["payload.bin"] == b"x" * 128 + + +@pytest.mark.parametrize("over", [False, True]) +def test_zip_directory_allocation_boundary(tmp_path, monkeypatch, over): + path, _ = _package(tmp_path) + with zipfile.ZipFile(path) as container: + directory_size = path.stat().st_size - 22 - container.start_dir + monkeypatch.setattr(archive, "_MAX_ZIP_DIRECTORY_BYTES", directory_size - over) + if over: + with pytest.raises(ValueError, match="ZIP directory/read bytes"): + archive.read_release_index(str(path)) + else: + assert archive.read_release_index(str(path))["version"] == "1.15.0" + + +@pytest.mark.parametrize("over", [False, True]) +def test_zip_member_count_boundary(tmp_path, monkeypatch, over): + path, _ = _package(tmp_path) + monkeypatch.setattr(archive, "_MAX_ZIP_MEMBERS", 3 - over) + if over: + with pytest.raises(ValueError, match="ZIP member count"): + archive.read_release_index(str(path)) + else: + assert archive.read_release_index(str(path))["version"] == "1.15.0" + + +@pytest.mark.parametrize("over", [False, True]) +def test_compressed_component_boundary(tmp_path, monkeypatch, over): + path, _ = _package(tmp_path) + with zipfile.ZipFile(path) as container: + limit = max(info.file_size for info in container.infolist()) + monkeypatch.setattr(archive, "_MAX_COMPRESSED_BYTES", limit - over) + if over: + with pytest.raises(ValueError, match="ZIP member bytes|compressed bytes"): + archive.read_release_index(str(path)) + else: + assert archive.read_release_index(str(path))["version"] == "1.15.0" + + +def test_zip_expansion_checked_before_component_decoder(tmp_path, monkeypatch): + path, _ = _package(tmp_path) + with zipfile.ZipFile(path) as container: + members = {name: container.read(name) for name in container.namelist()} + members[f"pkg-{path.stem}.tar.zst"] = b"x" * 4096 + with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED) as container: + for name, data in members.items(): + container.writestr(name, data) + monkeypatch.setattr(archive, "_MAX_COMPRESSED_BYTES", 4095) + with pytest.raises(ValueError, match="ZIP member bytes"): + list(archive.iter_payload_members(str(path))) + + +@pytest.mark.parametrize("over", [False, True]) +def test_cumulative_metadata_boundary(tmp_path, monkeypatch, over): + path, _ = _package( + tmp_path, + files=[("a.dist-info/METADATA", b"x" * 128), ("a.dist-info/RECORD", b"x" * 128)], + ) + monkeypatch.setattr(archive, "_MAX_METADATA_TOTAL", 256 - over) + if over: + with pytest.raises(ValueError, match="cumulative metadata bytes"): + list(archive.iter_payload_members(str(path))) + else: + assert len(list(archive.iter_payload_members(str(path)))) == 2 + + +@pytest.mark.parametrize("unknown_size", [False, True]) +@pytest.mark.parametrize("reader", [archive.zstd_decompress, archive.decompress_index]) +def test_truncated_zstd_footer_is_rejected(reader, unknown_size): + raw = _compress(b"readable data", unknown_size=unknown_size) + with pytest.raises(archive.READ_ERRORS): + reader(raw[:-1]) + + +@pytest.mark.parametrize("extension", [".conda", ".tar.bz2"]) +def test_expanded_limit_covers_metadata_stream_padding(tmp_path, monkeypatch, extension): + path, _ = _package(tmp_path, extension) + monkeypatch.setattr(archive, "_MAX_EXPANDED_BYTES", 10239) + with pytest.raises(ValueError, match="expanded bytes"): + archive.read_release_index(str(path)) + + +def test_zstd_window_limit_is_enforced(tmp_path, monkeypatch): + path, _ = _package(tmp_path, unknown_size=True) + monkeypatch.setattr(archive, "_MAX_ZSTD_WINDOW_BYTES", 1024) + with pytest.raises(ValueError, match="(?i)window|memory"): + list(archive.iter_payload_members(str(path))) + + +def test_normal_larger_decoder_window_is_not_accidentally_limited_to_kibibytes(tmp_path): + path, _ = _package(tmp_path, files=[("payload.bin", b"x" * 256 * 1024)], unknown_size=True) + assert len(dict(archive.iter_payload_members(str(path)))["payload.bin"]) == 256 * 1024 + + +@pytest.mark.parametrize("unknown_size", [False, True]) +def test_concatenated_zstd_frames_share_expanded_budget(monkeypatch, unknown_size): + raw = _compress(b"a" * 100, unknown_size=unknown_size) + _compress( + b"b" * 100, unknown_size=unknown_size + ) + monkeypatch.setattr(archive, "_MAX_EXPANDED_BYTES", 200) + assert archive.zstd_decompress(raw) == b"a" * 100 + b"b" * 100 + monkeypatch.setattr(archive, "_MAX_EXPANDED_BYTES", 199) + with pytest.raises(ValueError, match="expanded bytes"): + archive.zstd_decompress(raw) + + +def test_pax_size_checked_before_extended_header_processing(tmp_path, monkeypatch): + member = tarfile.TarInfo("PaxHeaders/oversized") + member.type = tarfile.XHDTYPE + member.size = 4096 + path, _ = _package(tmp_path) + with zipfile.ZipFile(path) as container: + members = {name: container.read(name) for name in container.namelist()} + members[f"pkg-{path.stem}.tar.zst"] = _compress(member.tobuf() + b"\0" * 10240) + with zipfile.ZipFile(path, "w") as container: + for name, data in members.items(): + container.writestr(name, data) + monkeypatch.setattr(archive, "_MAX_METADATA_BYTES", 4095) + + def forbidden(*args): + raise AssertionError("oversized PAX must fail before extension parsing") + + monkeypatch.setattr(tarfile.TarInfo, "_proc_pax", forbidden) + with pytest.raises(ValueError, match="metadata bytes"): + list(archive.iter_payload_members(str(path))) + + +@pytest.mark.parametrize("pax", [False, True]) +def test_sparse_maps_rejected_before_sparse_processing(tmp_path, pax): + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode="w") as contents: + member = tarfile.TarInfo("sparse") + if pax: + member.pax_headers = { + "GNU.sparse.major": "1", + "GNU.sparse.minor": "0", + "GNU.sparse.realsize": "128", + "GNU.sparse.name": "sparse", + } + else: + member.type = tarfile.GNUTYPE_SPARSE + member.size = 2 + contents.addfile(member, io.BytesIO(b"0\n")) + path, _ = _package(tmp_path) + with zipfile.ZipFile(path) as container: + members = {name: container.read(name) for name in container.namelist()} + members[f"pkg-{path.stem}.tar.zst"] = _compress(buffer.getvalue()) + with zipfile.ZipFile(path, "w") as container: + for name, data in members.items(): + container.writestr(name, data) + with pytest.raises(ValueError, match="Sparse TAR"): + list(archive.iter_payload_members(str(path)))