diff --git a/.github/workflows/mirror.yaml b/.github/workflows/mirror.yaml index 9cf062b7d..77ef064e9 100644 --- a/.github/workflows/mirror.yaml +++ b/.github/workflows/mirror.yaml @@ -16,6 +16,7 @@ on: - registry.k8s.io/git-sync/git-sync - registry-1.docker.io/library/golang - registry-1.docker.io/dxflrs/garage + - ghcr.io/astral-sh/uv image-index-manifest-tag: description: | The image index manifest tag, like 1.0.14 or v1.0.14 diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b907efd4..3aff72840 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ All notable changes to this project will be documented in this file. - superset: Add `6.1.0` ([#1514]). - zookeeper: Add `3.9.5` ([#1515]). - hadoop: Add `3.5.0` and `3.4.3` ([#1511]). +- airflow: Add `3.2.1` ([#1519]). ### Changed @@ -64,6 +65,7 @@ All notable changes to this project will be documented in this file. [#1514]: https://github.com/stackabletech/docker-images/pull/1514 [#1515]: https://github.com/stackabletech/docker-images/pull/1515 [#1518]: https://github.com/stackabletech/docker-images/pull/1518 +[#1519]: https://github.com/stackabletech/docker-images/pull/1519 [#1520]: https://github.com/stackabletech/docker-images/pull/1520 [#1521]: https://github.com/stackabletech/docker-images/pull/1521 [#1525]: https://github.com/stackabletech/docker-images/pull/1525 diff --git a/airflow/Dockerfile b/airflow/Dockerfile index 684199955..534cabddb 100644 --- a/airflow/Dockerfile +++ b/airflow/Dockerfile @@ -4,29 +4,29 @@ # - SecretsUsedInArgOrEnv : OPA_AUTH_MANAGER is a false positive and breaks the build. ARG GIT_SYNC_VERSION +ARG UV_VERSION # For updated versions check https://github.com/kubernetes/git-sync/releases # which should contain a image location (e.g. registry.k8s.io/git-sync/git-sync:v3.6.8) FROM oci.stackable.tech/sdp/git-sync/git-sync:${GIT_SYNC_VERSION} AS gitsync-image +FROM oci.stackable.tech/stackable/astral-sh/uv:${UV_VERSION} AS uv-image + FROM local-image/shared/statsd-exporter AS statsd_exporter-builder FROM local-image/vector AS opa-auth-manager-builder ARG OPA_AUTH_MANAGER ARG PYTHON_VERSION -ARG UV_VERSION +ARG STACKABLE_USER_UID COPY airflow/opa-auth-manager/${OPA_AUTH_MANAGER} /tmp/opa-auth-manager WORKDIR /tmp/opa-auth-manager -RUN < 0 {if (!seen[$0]++) print $0}' | tr '\n' ',' | sed 's/,$//') -python${PYTHON_VERSION} -m venv --system-site-packages /stackable/app - -source /stackable/app/bin/activate - -# Upgrade pip to the latest version -# Also install uv to get support for build constraints -pip install --no-cache-dir --upgrade pip -pip install --no-cache-dir uv==${UV_VERSION} - # Pin virtualenv due to a breaking change in 21.0.0 which is pulled # in by hatch. # See https://github.com/pypa/hatch/issues/2193 @@ -134,6 +137,7 @@ if [ -d "./airflow-core" ]; then cd airflow-core/src/airflow/ui # build front-end assets + # TODO: Consider making the pnpm version an ARG npm install -g pnpm@10.18.2 pnpm install --frozen-lockfile pnpm run build @@ -143,13 +147,14 @@ if [ -d "./airflow-core" ]; then cd ../../.. /root/.local/bin/hatch build -t wheel # First install the full apache-airflow package to get all dependencies including database drivers - uv pip install --no-cache-dir apache-airflow[${AIRFLOW_EXTRAS}]==${PRODUCT_VERSION} --constraint /tmp/constraints.txt --build-constraints /tmp/build-constraints.txt + uv pip install --python ${VIRTUAL_ENV}/bin/python apache-airflow[${AIRFLOW_EXTRAS}]==${PRODUCT_VERSION} --constraint /tmp/constraints.txt --build-constraints /tmp/build-constraints.txt # Then install the locally built core wheel to override the core package - uv pip install --no-cache-dir dist/apache_airflow_core-${PRODUCT_VERSION}-py3-none-any.whl[${AIRFLOW_EXTRAS}] --constraint /tmp/constraints.txt --build-constraints /tmp/build-constraints.txt + uv pip install --python ${VIRTUAL_ENV}/bin/python dist/apache_airflow_core-${PRODUCT_VERSION}-py3-none-any.whl[${AIRFLOW_EXTRAS}] --constraint /tmp/constraints.txt --build-constraints /tmp/build-constraints.txt else # Airflow 2.x # build front-end assets cd airflow/www + # TODO: Consider making the yarn version an ARG npm install -g yarn@1.22.22 yarn install --frozen-lockfile yarn run build @@ -158,25 +163,22 @@ else cd ../.. /root/.local/bin/hatch build -t wheel # First install the full apache-airflow package to get all dependencies including database drivers - uv pip install --no-cache-dir apache-airflow[${AIRFLOW_EXTRAS}]==${PRODUCT_VERSION} --constraint /tmp/constraints.txt --build-constraints /tmp/build-constraints.txt + uv pip install --python ${VIRTUAL_ENV}/bin/python apache-airflow[${AIRFLOW_EXTRAS}]==${PRODUCT_VERSION} --constraint /tmp/constraints.txt --build-constraints /tmp/build-constraints.txt # Then install the locally built wheel to override with patched version - uv pip install --no-cache-dir dist/apache_airflow-${PRODUCT_VERSION}-py3-none-any.whl[${AIRFLOW_EXTRAS}] --constraint /tmp/constraints.txt --build-constraints /tmp/build-constraints.txt + uv pip install --python ${VIRTUAL_ENV}/bin/python dist/apache_airflow-${PRODUCT_VERSION}-py3-none-any.whl[${AIRFLOW_EXTRAS}] --constraint /tmp/constraints.txt --build-constraints /tmp/build-constraints.txt fi # Needed for pandas S3 integration to e.g. write and read csv and parquet files to/from S3 -uv pip install --no-cache-dir s3fs==${S3FS_VERSION} cyclonedx-bom==${CYCLONEDX_BOM_VERSION} +# TODO: s3fs may already be installed via the s3fs extra and constraints. Check if this explicit install is still needed. +uv pip install --python ${VIRTUAL_ENV}/bin/python s3fs==${S3FS_VERSION} cyclonedx-bom==${CYCLONEDX_BOM_VERSION} # Needed for OIDC -uv pip install --no-cache-dir Flask_OIDC==2.2.0 Flask-OpenID==1.3.1 +uv pip install --python ${VIRTUAL_ENV}/bin/python Flask_OIDC==2.2.0 Flask-OpenID==1.3.1 -uv pip install --no-cache-dir /tmp/opa_auth_manager-0.1.0-py3-none-any.whl +uv pip install --python ${VIRTUAL_ENV}/bin/python /tmp/opa_auth_manager-0.1.0-py3-none-any.whl # Create the SBOM for Airflow # Important: All `pip install` commands must be above this line, otherwise the SBOM will be incomplete -if [ "$PRODUCT_VERSION" == "2.9.3" ] || [ "$PRODUCT_VERSION" == "3.0.6" ]; then - cyclonedx-py environment --schema-version 1.5 --outfile /tmp/sbom.json -else - cyclonedx-py environment --spec-version 1.5 --output-file /tmp/sbom.json -fi +${VIRTUAL_ENV}/bin/cyclonedx-py environment --spec-version 1.5 --output-file /tmp/sbom.json uv pip uninstall cyclonedx-bom # Break circular dependencies by removing the apache-airflow dependency from the providers @@ -218,8 +220,9 @@ LABEL name="Apache Airflow" \ description="This image is deployed by the Stackable Operator for Apache Airflow." ENV HOME=/stackable -ENV AIRFLOW_USER_HOME_DIR=/stackable -ENV PATH=$PATH:/bin:$HOME/app/bin +ENV VIRTUAL_ENV=$HOME/app +ENV AIRFLOW_USER_HOME_DIR=$HOME +ENV PATH=$HOME/app/bin:$PATH:/bin ENV AIRFLOW_HOME=$HOME/airflow COPY --from=airflow-build-image --chown=${STACKABLE_USER_UID}:0 /stackable/ ${HOME}/ @@ -233,6 +236,8 @@ COPY --from=gitsync-image --chown=${STACKABLE_USER_UID}:0 /git-sync ${HOME}/git- COPY airflow/licenses /licenses +COPY --from=uv-image --chown=${STACKABLE_USER_UID}:0 /uv /uvx /bin/ + # Update image and install needed packages RUN <=3.1.0 diff --git a/airflow/stackable/constraints/3.2.1/constraints-python3.14.txt b/airflow/stackable/constraints/3.2.1/constraints-python3.14.txt new file mode 100644 index 000000000..3a9d5b32d --- /dev/null +++ b/airflow/stackable/constraints/3.2.1/constraints-python3.14.txt @@ -0,0 +1,689 @@ + +# +# This constraints file was automatically generated on 2026-04-21T00:39:49.491047 +# via `uv pip install --resolution highest` for the "v3-2-test" branch of Airflow. +# This variant of constraints install uses the HEAD of the branch version for 'apache-airflow' but installs +# the providers from PIP-released packages at the moment of the constraint generation. +# +# Those constraints are actually those that regular users use to install released version of Airflow. +# We also use those constraints after "apache-airflow" is released and the constraints are tagged with +# "constraints-X.Y.Z" tag to build the production image for that version. +# +# This constraints file is meant to be used only in the "apache-airflow" installation command and not +# in all subsequent pip commands. By using a constraints.txt file, we ensure that solely the Airflow +# installation step is reproducible. Subsequent pip commands may install packages that would have +# been incompatible with the constraints used in Airflow reproducible installation step. Finally, pip +# commands that might change the installed version of apache-airflow should include "apache-airflow==X.Y.Z" +# in the list of install targets to prevent Airflow accidental upgrade or downgrade. +# +# Typical installation process of airflow for Python 3.14 is (with random selection of extras and custom +# dependencies added), usually consists of two steps: +# +# 1. Reproducible installation of airflow with selected providers (note constraints are used): +# +# pip install "apache-airflow[celery,cncf.kubernetes,google,amazon,snowflake]==X.Y.Z" \ +# --constraint \ +# "https://raw.githubusercontent.com/apache/airflow/constraints-X.Y.Z/constraints-3.14.txt" +# +# 2. Installing own dependencies that are potentially not matching the constraints (note constraints are not +# used, and apache-airflow==X.Y.Z is used to make sure there is no accidental airflow upgrade/downgrade. +# +# pip install "apache-airflow==X.Y.Z" "snowflake-connector-python[pandas]=N.M.O" +# +APScheduler==3.11.2 +Authlib==1.6.11 +Deprecated==1.3.1 +Events==0.5 +Flask-JWT-Extended==4.7.1 +Flask-Limiter==3.12 +Flask-Login==0.6.3 +Flask-SQLAlchemy==3.1.1 +Flask-Session==0.8.0 +Flask-WTF==1.2.2 +Flask==3.1.3 +GitPython==3.1.46 +JayDeBeApi==1.2.3 +Jinja2==3.1.6 +Mako==1.3.11 +Markdown==3.10.2 +MarkupSafe==3.0.3 +PyAthena==3.30.1 +PyGithub==2.9.1 +PyHive==0.7.0 +PyJWT==2.12.1 +PyMySQL==1.1.2 +PyNaCl==1.6.2 +PyYAML==6.0.3 +Pygments==2.20.0 +SQLAlchemy-Utils==0.42.1 +SQLAlchemy==2.0.49 +SecretStorage==3.5.0 +WTForms==3.2.1 +Werkzeug==3.1.8 +a2wsgi==1.10.10 +adal==1.2.7 +adlfs==2026.4.0 +aenum==3.1.17 +aiobotocore==3.4.0 +aiofiles==24.1.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.13.5 +aioitertools==0.13.0 +aiomysql==0.3.2 +aiosignal==1.4.0 +aiosmtplib==5.1.0 +aiosqlite==0.21.0 +airbyte-api==0.53.0 +alembic==1.18.4 +alibabacloud-adb20211201==3.7.1 +alibabacloud-credentials-api==1.0.0 +alibabacloud-credentials==1.0.8 +alibabacloud-oss-v2==1.2.5 +alibabacloud-tea-openapi==0.4.4 +alibabacloud-tea-util==0.3.14 +alibabacloud-tea==0.4.3 +alibabacloud_gateway_spi==0.0.3 +amqp==5.3.1 +annotated-doc==0.0.4 +annotated-types==0.7.0 +anyio==4.13.0 +apache-airflow-providers-airbyte==5.4.1 +apache-airflow-providers-alibaba==3.3.7 +apache-airflow-providers-amazon==9.25.0 +apache-airflow-providers-apache-drill==3.3.2 +apache-airflow-providers-apache-druid==4.5.2 +apache-airflow-providers-apache-flink==1.8.4 +apache-airflow-providers-apache-hdfs==4.11.5 +apache-airflow-providers-apache-hive==9.4.2 +apache-airflow-providers-apache-iceberg==2.0.2 +apache-airflow-providers-apache-impala==1.9.2 +apache-airflow-providers-apache-kafka==1.13.2 +apache-airflow-providers-apache-kylin==3.10.4 +apache-airflow-providers-apache-livy==4.5.5 +apache-airflow-providers-apache-pig==4.8.4 +apache-airflow-providers-apache-pinot==4.10.2 +apache-airflow-providers-apache-spark==6.0.1 +apache-airflow-providers-apache-tinkerpop==1.1.3 +apache-airflow-providers-apprise==2.3.3 +apache-airflow-providers-arangodb==2.9.4 +apache-airflow-providers-asana==2.11.3 +apache-airflow-providers-atlassian-jira==3.3.3 +apache-airflow-providers-celery==3.18.0 +apache-airflow-providers-cloudant==4.3.4 +apache-airflow-providers-cncf-kubernetes==10.16.0 +apache-airflow-providers-cohere==1.6.5 +apache-airflow-providers-common-ai==0.1.0 +apache-airflow-providers-common-compat==1.14.3 +apache-airflow-providers-common-io==1.7.2 +apache-airflow-providers-common-messaging==2.0.3 +apache-airflow-providers-common-sql==1.34.0 +apache-airflow-providers-databricks==7.12.1 +apache-airflow-providers-datadog==3.10.4 +apache-airflow-providers-dbt-cloud==4.8.1 +apache-airflow-providers-dingding==3.9.4 +apache-airflow-providers-discord==3.12.2 +apache-airflow-providers-docker==4.5.5 +apache-airflow-providers-edge3==3.4.0 +apache-airflow-providers-elasticsearch==6.5.2 +apache-airflow-providers-exasol==4.10.2 +apache-airflow-providers-fab==3.6.1 +apache-airflow-providers-facebook==3.9.4 +apache-airflow-providers-ftp==3.14.3 +apache-airflow-providers-git==0.3.1 +apache-airflow-providers-github==2.11.2 +apache-airflow-providers-google==21.1.0 +apache-airflow-providers-grpc==3.9.4 +apache-airflow-providers-hashicorp==4.5.2 +apache-airflow-providers-http==6.0.2 +apache-airflow-providers-imap==3.11.2 +apache-airflow-providers-influxdb==2.10.4 +apache-airflow-providers-informatica==0.1.3 +apache-airflow-providers-jdbc==5.4.3 +apache-airflow-providers-jenkins==4.2.5 +apache-airflow-providers-keycloak==0.7.1 +apache-airflow-providers-microsoft-azure==13.1.1 +apache-airflow-providers-microsoft-mssql==4.5.2 +apache-airflow-providers-microsoft-psrp==3.2.5 +apache-airflow-providers-microsoft-winrm==3.14.2 +apache-airflow-providers-mongo==5.3.5 +apache-airflow-providers-mysql==6.5.2 +apache-airflow-providers-neo4j==3.11.5 +apache-airflow-providers-odbc==4.12.2 +apache-airflow-providers-openai==1.7.4 +apache-airflow-providers-openfaas==3.9.4 +apache-airflow-providers-openlineage==2.14.0 +apache-airflow-providers-opensearch==1.9.0 +apache-airflow-providers-opsgenie==5.10.3 +apache-airflow-providers-oracle==4.5.3 +apache-airflow-providers-pagerduty==5.2.5 +apache-airflow-providers-papermill==3.12.3 +apache-airflow-providers-pgvector==1.7.1 +apache-airflow-providers-pinecone==2.4.4 +apache-airflow-providers-postgres==6.6.3 +apache-airflow-providers-presto==5.11.2 +apache-airflow-providers-qdrant==1.5.5 +apache-airflow-providers-redis==4.4.4 +apache-airflow-providers-salesforce==5.14.0 +apache-airflow-providers-samba==4.12.5 +apache-airflow-providers-segment==3.9.4 +apache-airflow-providers-sendgrid==4.2.2 +apache-airflow-providers-sftp==5.7.3 +apache-airflow-providers-singularity==3.9.3 +apache-airflow-providers-slack==9.10.0 +apache-airflow-providers-smtp==2.4.5 +apache-airflow-providers-snowflake==6.12.1 +apache-airflow-providers-sqlite==4.3.2 +apache-airflow-providers-ssh==5.0.0 +apache-airflow-providers-standard==1.12.3 +apache-airflow-providers-tableau==5.4.0 +apache-airflow-providers-telegram==4.9.4 +apache-airflow-providers-teradata==3.5.2 +apache-airflow-providers-trino==6.5.2 +apache-airflow-providers-vertica==4.3.2 +apache-airflow-providers-weaviate==3.3.3 +apache-airflow-providers-yandex==4.4.2 +apache-airflow-providers-ydb==2.5.2 +apache-airflow-providers-zendesk==4.11.3 +apispec==6.10.0 +apprise==1.9.9 +argcomplete==3.6.3 +asana==5.2.4 +asgiref==3.11.1 +asn1crypto==1.5.1 +asttokens==3.0.1 +async-timeout==4.0.3 +asyncpg==0.31.0 +asyncssh==2.22.0 +atlasclient==1.0.0 +atlassian-python-api==4.0.7 +attrs==26.1.0 +azure-batch==14.2.0 +azure-common==1.1.28 +azure-core==1.39.0 +azure-cosmos==4.15.0 +azure-datalake-store==0.0.53 +azure-identity==1.25.3 +azure-keyvault-secrets==4.10.0 +azure-kusto-data==6.0.3 +azure-mgmt-compute==37.2.0 +azure-mgmt-containerinstance==10.1.0 +azure-mgmt-containerregistry==15.0.0 +azure-mgmt-core==1.6.0 +azure-mgmt-cosmosdb==9.9.0 +azure-mgmt-datafactory==9.3.0 +azure-mgmt-datalake-nspkg==3.0.1 +azure-mgmt-datalake-store==0.5.0 +azure-mgmt-nspkg==3.0.2 +azure-mgmt-resource==25.0.0 +azure-mgmt-storage==24.0.1 +azure-nspkg==3.0.2 +azure-servicebus==7.14.3 +azure-storage-blob==12.28.0 +azure-storage-file-datalake==12.23.0 +azure-storage-file-share==12.24.0 +azure-synapse-artifacts==0.22.0 +azure-synapse-spark==0.7.0 +babel==2.18.0 +backoff==2.2.1 +bcrypt==5.0.0 +beautifulsoup4==4.14.3 +billiard==4.2.4 +bitarray==3.8.1 +black==26.3.1 +bleach==6.3.0 +blinker==1.9.0 +boto3==1.42.84 +botocore==1.42.84 +cachelib==0.13.0 +cachetools==6.2.6 +cadwyn==6.2.0 +cattrs==26.1.0 +celery==5.6.3 +certifi==2026.2.25 +cffi==2.0.0 +chardet==6.0.0.post1 +charset-normalizer==3.4.7 +ciso8601==2.3.3 +click-didyoumean==0.3.1 +click-plugins==1.1.1.2 +click-repl==0.3.0 +click==8.3.2 +cloudpickle==3.1.2 +cohere==5.21.1 +colorama==0.4.6 +colorlog==6.10.1 +comm==0.2.3 +confluent-kafka==2.14.0 +crcmod-plus==2.3.1 +cron_descriptor==2.0.8 +croniter==6.2.2 +cryptography==46.0.7 +curlify==3.0.0 +darabonba-core==1.0.5 +databricks-sql-connector==4.2.5 +dataclasses-json==0.6.7 +datadog==0.52.1 +db-dtypes==1.5.1 +debugpy==1.8.20 +decorator==5.2.1 +defusedxml==0.7.1 +deprecation==2.1.0 +dill==0.4.1 +distro==1.9.0 +dnspython==2.8.0 +docker==7.1.0 +docopt==0.6.2 +docstring_parser==0.18.0 +durationpy==0.10 +ecdsa==0.19.2 +elastic-transport==9.2.1 +elasticsearch==9.3.0 +email-validator==2.3.0 +entrypoints==0.4 +et_xmlfile==2.0.0 +eventlet==0.41.0 +executing==2.2.1 +facebook_business==25.0.1 +fastapi-cli==0.0.24 +fastapi==0.136.0 +fastavro==1.12.1 +fastjsonschema==2.21.2 +fastuuid==0.14.0 +filelock==3.28.0 +flask-appbuilder==5.2.0 +flask-babel==4.0.0 +flower==2.0.1 +frozenlist==1.8.0 +fsspec==2026.3.0 +future==1.0.0 +gcloud-aio-auth==5.4.4 +gcloud-aio-bigquery==7.1.0 +gcloud-aio-storage==9.6.4 +gcsfs==2026.3.0 +genai-prices==0.0.56 +gevent==26.4.0 +gitdb==4.0.12 +google-ads==30.0.0 +google-analytics-admin==0.28.0 +google-api-core==2.30.3 +google-api-python-client==2.194.0 +google-auth-httplib2==0.3.1 +google-auth-oauthlib==1.3.1 +google-auth==2.49.2 +google-cloud-aiplatform==1.148.0 +google-cloud-alloydb==0.9.0 +google-cloud-appengine-logging==1.9.0 +google-cloud-audit-log==0.5.0 +google-cloud-automl==2.19.0 +google-cloud-batch==0.21.0 +google-cloud-bigquery-datatransfer==3.22.0 +google-cloud-bigquery-storage==2.37.0 +google-cloud-bigquery==3.41.0 +google-cloud-bigtable==2.36.0 +google-cloud-build==3.36.0 +google-cloud-compute==1.47.0 +google-cloud-container==2.64.0 +google-cloud-core==2.5.1 +google-cloud-datacatalog==3.30.0 +google-cloud-dataflow-client==0.13.0 +google-cloud-dataform==0.10.0 +google-cloud-dataplex==2.18.0 +google-cloud-dataproc-metastore==1.22.0 +google-cloud-dataproc==5.27.0 +google-cloud-dlp==3.36.0 +google-cloud-kms==3.12.0 +google-cloud-language==2.20.0 +google-cloud-logging==3.15.0 +google-cloud-managedkafka==0.4.0 +google-cloud-memcache==1.15.0 +google-cloud-monitoring==2.30.0 +google-cloud-orchestration-airflow==1.20.0 +google-cloud-os-login==2.20.0 +google-cloud-pubsub==2.37.0 +google-cloud-redis==2.21.0 +google-cloud-resource-manager==1.17.0 +google-cloud-run==0.16.0 +google-cloud-secret-manager==2.27.0 +google-cloud-spanner==3.65.0 +google-cloud-speech==2.38.0 +google-cloud-storage-control==1.11.0 +google-cloud-storage-transfer==1.20.0 +google-cloud-storage==3.10.1 +google-cloud-tasks==2.22.0 +google-cloud-texttospeech==2.36.0 +google-cloud-translate==3.26.0 +google-cloud-videointelligence==2.19.0 +google-cloud-vision==3.13.0 +google-cloud-workflows==1.21.0 +google-crc32c==1.8.0 +google-genai==1.73.1 +google-resumable-media==2.8.2 +googleapis-common-protos==1.74.0 +graphviz==0.21 +greenback==1.3.0 +greenlet==3.4.0 +gremlinpython==3.8.1 +griffelib==2.0.2 +grpc-google-iam-v1==0.14.4 +grpc-interceptor==0.15.4 +grpcio-gcp==0.2.2 +grpcio-health-checking==1.80.0 +grpcio-status==1.80.0 +grpcio-tools==1.80.0 +grpcio==1.80.0 +gssapi==1.11.1 +gunicorn==25.3.0 +h11==0.16.0 +h2==4.3.0 +hdfs==2.7.3 +hf-xet==1.4.3 +hmsclient==0.1.1 +hpack==4.1.0 +httpcore==1.0.9 +httplib2==0.31.2 +httptools==0.7.1 +httpx==0.28.1 +huggingface_hub==1.11.0 +humanize==4.15.0 +hvac==2.4.0 +hyperframe==6.1.0 +ibm-cloud-sdk-core==3.24.4 +ibmcloudant==0.11.5 +idna==3.11 +ijson==3.4.0.post0 +immutabledict==4.3.1 +importlib_metadata==8.7.1 +impyla==0.22.0 +inflection==0.5.1 +influxdb-client==1.50.0 +ipykernel==7.2.0 +ipython==9.12.0 +ipython_pygments_lexers==1.1.1 +isodate==0.7.2 +itsdangerous==2.2.0 +jaraco.classes==3.4.0 +jaraco.context==6.1.2 +jaraco.functools==4.4.0 +jedi==0.19.2 +jeepney==0.9.0 +jiter==0.14.0 +jmespath==1.1.0 +joblib==1.5.3 +jpype1==1.7.0 +jsonpath-ng==1.8.0 +jsonpath-python==1.1.5 +jsonschema-specifications==2025.9.1 +jsonschema==4.26.0 +jupyter_client==8.8.0 +jupyter_core==5.9.1 +jupyterlab_pygments==0.3.0 +jwcrypto==1.5.7 +keyring==25.7.0 +kombu==5.6.2 +krb5==0.9.0 +kubernetes==35.0.0 +kubernetes_asyncio==35.0.1 +kylinpy==2.8.4 +lazy-object-proxy==1.12.0 +libcst==1.8.6 +limits==5.8.0 +linkify-it-py==2.1.0 +litellm==1.82.6 +lockfile==0.12.2 +logfire-api==4.32.1 +looker_sdk==26.6.1 +lxml==6.0.2 +lz4==4.4.5 +markdown-it-py==4.0.0 +marshmallow-sqlalchemy==1.5.0 +marshmallow==3.26.2 +matplotlib-inline==0.2.1 +mdit-py-plugins==0.5.0 +mdurl==0.1.2 +memray==1.19.3 +mergedeep==1.3.4 +methodtools==0.4.7 +microsoft-kiota-abstractions==1.10.1 +microsoft-kiota-authentication-azure==1.10.1 +microsoft-kiota-http==1.10.1 +microsoft-kiota-serialization-json==1.10.1 +microsoft-kiota-serialization-text==1.10.1 +mistune==3.2.0 +mmh3==5.2.1 +more-itertools==11.0.2 +msal-extensions==1.3.1 +msal==1.36.0 +msgpack==1.1.2 +msgraph-core==1.3.8 +msgraphfs==0.4 +msgspec==0.21.1 +msrest==0.7.1 +msrestazure==0.6.4.post1 +multi_key_dict==2.0.3 +multidict==6.7.1 +mypy_extensions==1.1.0 +mysql-connector-python==9.6.0 +mysqlclient==2.2.8 +natsort==8.4.0 +nbclient==0.10.4 +nbconvert==7.17.1 +nbformat==5.10.4 +neo4j==6.1.0 +nest-asyncio==1.6.0 +numpy==2.4.4 +oauthlib==3.3.1 +openai==2.32.0 +openlineage-integration-common==1.46.0 +openlineage-python==1.46.0 +openlineage_sql==1.46.0 +openpyxl==3.1.5 +opensearch-protobufs==0.19.0 +opensearch-py==3.1.0 +opentelemetry-api==1.41.0 +opentelemetry-exporter-otlp-proto-common==1.41.0 +opentelemetry-exporter-otlp-proto-grpc==1.41.0 +opentelemetry-exporter-otlp-proto-http==1.41.0 +opentelemetry-exporter-otlp==1.41.0 +opentelemetry-exporter-prometheus==0.62b0 +opentelemetry-proto==1.41.0 +opentelemetry-resourcedetector-gcp==1.11.0a0 +opentelemetry-sdk==1.41.0 +opentelemetry-semantic-conventions==0.62b0 +opsgenie-sdk==2.1.5 +oracledb==3.4.2 +ordered-set==4.1.0 +orjson==3.11.8 +outcome==1.3.0.post0 +packaging==26.1 +pagerduty==6.2.1 +pandas-gbq==0.35.0 +pandas-stubs==3.0.0.260204 +pandas==2.3.3 +pandocfilters==1.5.1 +papermill==2.7.0 +paramiko==3.5.1 +parso==0.8.6 +pathlib_abc==0.5.2 +pathspec==1.0.4 +pbr==7.0.3 +pendulum==3.2.0 +pexpect==4.9.0 +pgvector==0.4.2 +pinecone-plugin-assistant==3.0.3 +pinecone-plugin-interface==0.0.7 +pinecone==8.1.2 +pinotdb==9.1.1 +platformdirs==4.9.6 +pluggy==1.6.0 +polars-runtime-32==1.39.3 +polars==1.39.3 +portalocker==3.2.0 +presto-python-client==0.8.4 +prison==0.2.1 +prometheus_client==0.25.0 +prompt_toolkit==3.0.52 +propcache==0.4.1 +proto-plus==1.27.2 +protobuf==6.33.6 +psutil==7.2.2 +psycopg2-binary==2.9.11 +ptyprocess==0.7.0 +pure-sasl==0.6.2 +pure_eval==0.2.3 +pyOpenSSL==26.0.0 +pyarrow==23.0.1 +pyasn1==0.6.3 +pyasn1_modules==0.4.2 +pybreaker==1.4.1 +pycountry==26.2.16 +pycparser==3.0 +pycryptodome==3.23.0 +pydantic-ai-slim==1.84.0 +pydantic-extra-types==2.11.1 +pydantic-graph==1.84.0 +pydantic-settings==2.13.1 +pydantic==2.13.1 +pydantic_core==2.46.1 +pydata-google-auth==1.9.1 +pydruid==0.6.9 +pyexasol==1.3.0 +pygtrie==2.5.0 +pyiceberg==0.11.1 +pykerberos==1.2.4 +pymongo==4.16.0 +pymssql==2.3.13 +pyodbc==5.3.0 +pyodps==0.12.6 +pyparsing==3.3.2 +pypsrp==0.9.1 +pyroaring==1.0.4 +pyspark-client==4.1.1 +pyspnego==0.12.1 +python-arango==8.3.2 +python-daemon==3.1.2 +python-dateutil==2.9.0.post0 +python-dotenv==1.2.2 +python-http-client==3.3.7 +python-jenkins==1.8.3 +python-keycloak==7.1.1 +python-ldap==3.4.5 +python-multipart==0.0.26 +python-slugify==8.0.4 +python-telegram-bot==22.7 +pytokens==0.4.1 +pytz==2026.1.post1 +pywinrm==0.5.0 +pyzmq==27.1.0 +qdrant-client==1.17.1 +reactivex==4.1.0 +redis==6.4.0 +redshift_connector==2.1.13 +referencing==0.37.0 +regex==2026.4.4 +requests-file==3.0.1 +requests-kerberos==0.15.0 +requests-oauthlib==2.0.0 +requests-toolbelt==1.0.0 +requests==2.33.1 +requests_ntlm==1.3.0 +retryhttp==1.4.0 +rich-argparse==1.7.2 +rich-toolkit==0.19.7 +rich==13.9.4 +rpds-py==0.30.0 +rsa==4.9.1 +ruamel.yaml==0.19.1 +s3fs==2026.3.0 +s3transfer==0.16.0 +sagemaker_studio==1.0.26 +scikit-learn==1.8.0 +scipy==1.17.1 +scramp==1.4.8 +scrapbook==0.5.0 +segment-analytics-python==2.3.6 +sendgrid==6.12.2 +sentry-sdk==2.58.0 +setproctitle==1.3.7 +setuptools==82.0.1 +shellingham==1.5.4 +simple-salesforce==1.12.9 +six==1.17.0 +slack_sdk==3.41.0 +smbprotocol==1.16.1 +smmap==5.0.3 +sniffio==1.3.1 +snowflake-connector-python==4.4.0 +snowflake-sqlalchemy==1.9.0 +sortedcontainers==2.4.0 +soupsieve==2.8.3 +spython==0.3.14 +sqlalchemy-bigquery==1.16.0 +sqlalchemy-spanner==1.17.3 +sqlalchemy_drill==1.1.10 +sqlparse==0.5.5 +stack-data==0.6.3 +starlette==0.52.1 +statsd==4.0.1 +std-uritemplate==2.0.8 +strictyaml==1.7.3 +structlog==25.5.0 +svcs==25.1.0 +tableauserverclient==0.40 +tabulate==0.10.0 +tenacity==9.1.4 +teradatasql==20.0.0.56 +teradatasqlalchemy==20.0.0.9 +termcolor==3.3.0 +text-unidecode==1.3 +textual==6.2.1 +threadpoolctl==3.6.0 +thrift-sasl==0.4.3 +thrift==0.16.0 +tiktoken==0.12.0 +tinycss2==1.4.0 +tokenizers==0.22.2 +tomlkit==0.14.0 +tornado==6.5.5 +tqdm==4.67.3 +traitlets==5.14.3 +trino==0.337.0 +typer==0.24.1 +types-protobuf==7.34.1.20260408 +types-requests==2.33.0.20260408 +typing-inspect==0.9.0 +typing-inspection==0.4.2 +typing_extensions==4.15.0 +tzdata==2026.1 +tzlocal==5.3.1 +uc-micro-py==2.0.0 +universal_pathlib==0.3.10 +uritemplate==4.2.0 +urllib3==2.6.3 +uuid6==2025.0.1 +uv==0.11.7 +uvicorn==0.44.0 +uvloop==0.22.1 +validators==0.35.0 +vertica-python==1.4.0 +vine==5.1.0 +watchfiles==1.1.1 +watchtower==3.4.0 +wcwidth==0.6.0 +weaviate-client==4.16.2 +webencodings==0.5.1 +websocket-client==1.8.0 +websockets==16.0 +wirerope==1.0.0 +wrapt==2.1.2 +xmltodict==1.0.4 +yandex-query-client==0.1.4 +yandexcloud==0.386.0 +yarl==1.23.0 +ydb-dbapi==0.1.20 +ydb==3.28.0 +zeep==4.3.2 +zenpy==2.0.57 +zipp==3.23.1 +zope.event==6.1 +zope.interface==8.3 +zstandard==0.25.0 diff --git a/airflow/stackable/patches/3.2.1/0001-Allow-overriding-the-logging-configuration.patch b/airflow/stackable/patches/3.2.1/0001-Allow-overriding-the-logging-configuration.patch new file mode 100644 index 000000000..d62c0607a --- /dev/null +++ b/airflow/stackable/patches/3.2.1/0001-Allow-overriding-the-logging-configuration.patch @@ -0,0 +1,71 @@ +From 271099b3cff14cb951523050e367b3c3222ead16 Mon Sep 17 00:00:00 2001 +From: Siegfried Weber +Date: Mon, 9 Mar 2026 10:11:28 +0100 +Subject: Allow overriding the logging configuration + +--- + .../src/airflow_shared/logging/structlog.py | 30 ++++++++++++++++++- + 1 file changed, 29 insertions(+), 1 deletion(-) + +diff --git a/shared/logging/src/airflow_shared/logging/structlog.py b/shared/logging/src/airflow_shared/logging/structlog.py +index d5b0b9a8bfe..649451a5502 100644 +--- a/shared/logging/src/airflow_shared/logging/structlog.py ++++ b/shared/logging/src/airflow_shared/logging/structlog.py +@@ -436,6 +436,18 @@ def structlog_processors( + return shared_processors, console, console + + ++def update_config( ++ config: dict, ++ updates: Mapping, ++): ++ for key, value in updates.items(): ++ if isinstance(value, Mapping): ++ config[key] = update_config(config.get(key, {}), value) ++ else: ++ config[key] = value ++ return config ++ ++ + def configure_logging( + *, + json_output: bool = False, +@@ -518,6 +530,19 @@ def configure_logging( + else: + PER_LOGGER_LEVELS[log] = loglevel + ++ # Use the log levels defined in stdlib_config for structlog ++ logger_configs = stdlib_config.get("loggers", {}) ++ logger_configs[""] = stdlib_config.get("root", {}) ++ for logger, logger_config in logger_configs.items(): ++ if "level" in logger_config: ++ loglevel = logger_config["level"] ++ if isinstance(loglevel, str): ++ try: ++ loglevel = NAME_TO_LEVEL[loglevel.lower()] ++ except KeyError: ++ raise ValueError(f"Invalid log level for logger {logger!r}: {loglevel!r}") from None ++ PER_LOGGER_LEVELS[logger] = loglevel ++ + shared_pre_chain, for_stdlib, for_structlog = structlog_processors( + json_output, + log_format=log_format, +@@ -564,7 +589,7 @@ def configure_logging( + + import logging.config + +- config = {**stdlib_config} ++ config = {} + config.setdefault("version", 1) + config.setdefault("disable_existing_loggers", False) + config["formatters"] = {**config.get("formatters", {})} +@@ -617,6 +642,9 @@ def configure_logging( + "propagate": True, + } + ++ # Merge stdlib_config into config and override existing values ++ update_config(config, stdlib_config) ++ + logging.config.dictConfig(config) + + if json_output: diff --git a/airflow/stackable/patches/3.2.1/patchable.toml b/airflow/stackable/patches/3.2.1/patchable.toml new file mode 100644 index 000000000..dddbef2b6 --- /dev/null +++ b/airflow/stackable/patches/3.2.1/patchable.toml @@ -0,0 +1,2 @@ +mirror = "https://github.com/stackabletech/airflow.git" +base = "e5968b88e45f2af9d1047a51188355af7cbe79f4" diff --git a/shell.nix b/shell.nix index ed1742207..a13fa9974 100644 --- a/shell.nix +++ b/shell.nix @@ -10,6 +10,8 @@ pkgs.mkShell { nodejs zizmor git-cliff + python312 + uv ]; buildInputs = with pkgs; [