Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/actions/deploy-prod/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ runs:
shell: bash
env:
# renovate: datasource=github-releases depName=devantler-tech/ksail extractVersion=^v(?<version>.+)$
KSAIL_VERSION: "7.183.1"
KSAIL_VERSION: "7.184.1"
GITHUB_TOKEN: ${{ github.token }}
run: |
.github/scripts/setup-ksail.sh
Expand Down
4 changes: 2 additions & 2 deletions .github/scripts/setup-talosctl.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,10 @@ set -euo pipefail
# inspects a single node, which reads as a fleet-wide fault rather than a stale
# pin.
# renovate: datasource=github-releases depName=siderolabs/talos extractVersion=^v(?<version>.+)$
TALOS_VERSION="1.13.9"
TALOS_VERSION="1.14.0"
# SHA-256 of talosctl-linux-amd64 for the TALOS_VERSION above, from that
# release's sha256sum.txt. UPDATE BOTH TOGETHER.
TALOSCTL_SHA256="7e1d4b7d5846964bdcf63a794e3c8161bb6ef2983d5ace58ea5322f3bf32a27e"
TALOSCTL_SHA256="2c147c4a99d124c95bd5c190fe054e0b3c93495f2243fd652ebd423adb8377c7"

asset_name="talosctl-linux-amd64"
release_base="https://github.com/siderolabs/talos/releases/download/v${TALOS_VERSION}"
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/cd.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ jobs:
shell: bash
env:
# renovate: datasource=github-releases depName=devantler-tech/ksail extractVersion=^v(?<version>.+)$
KSAIL_VERSION: "7.183.1"
KSAIL_VERSION: "7.184.1"
GITHUB_TOKEN: ${{ github.token }}
run: .github/scripts/setup-ksail.sh

Expand Down
52 changes: 49 additions & 3 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -796,7 +796,7 @@ jobs:
shell: bash
env:
# renovate: datasource=github-releases depName=devantler-tech/ksail extractVersion=^v(?<version>.+)$
KSAIL_VERSION: "7.183.1"
KSAIL_VERSION: "7.184.1"
GITHUB_TOKEN: ${{ github.token }}
run: .github/scripts/setup-ksail.sh

Expand Down Expand Up @@ -1524,7 +1524,7 @@ jobs:
shell: bash
env:
# renovate: datasource=github-releases depName=devantler-tech/ksail extractVersion=^v(?<version>.+)$
KSAIL_VERSION: "7.183.1"
KSAIL_VERSION: "7.184.1"
GITHUB_TOKEN: ${{ github.token }}
run: |
.github/scripts/setup-ksail.sh
Expand Down Expand Up @@ -1947,6 +1947,14 @@ jobs:
patches=()
for f in "${dir}"/cluster/*.yaml "${dir}/${role}"/*.yaml; do
[ -f "${f}" ] || continue
# KSail 7.184.1 consumes the AppArmor patch before rendering:
# it folds install.extraKernelArgs into the Image Factory
# schematic, removes the deprecated field, and pins the UKI
# command line. Raw talosctl has no schematic stage, so applying
# that input patch directly would validate an intermediate state.
if [ "${f}" = "talos/cluster/enable-apparmor.yaml" ]; then
continue
fi
expanded="${out}/$(echo "${f}" | tr '/' '_')"
# Allowlist-scoped substitution: only the pre-checked vars are
# expanded, so an unexpected placeholder shape survives for the
Expand All @@ -1968,6 +1976,44 @@ jobs:
cp "${base}" "${out}/${role}-patched.yaml"
fi
talosctl validate -c "${out}/${role}-patched.yaml" -m "${mode}" >/dev/null

# Validation proves the document set is accepted; these checks
# prove the migration preserved the platform-specific behavior.
local rendered="${out}/${role}-patched.yaml"
if grep -qE '^kind: (KubeFlannelCNIConfig|KubeProxyConfig)$' "${rendered}"; then
echo "::error::${dir}/${role}: default CNI or kube-proxy remained enabled"
exit 1
fi
grep -qF 'defaultRuntimeSeccompProfileEnabled: true' "${rendered}"
grep -qF 'disableManifestsDirectory: true' "${rendered}"
if [ "${dir}" = "talos" ]; then
grep -qF 'image: ghcr.io/siderolabs/kubelet:v1.36.4' "${rendered}"
fi
if [ "${dir}/${role}" = "talos/control-planes" ]; then
grep -qF 'url: https://dex.platform.devantler.tech' "${rendered}"
grep -qF 'audit-log-maxbackup: "3"' "${rendered}"
if grep -qF 'secure: false' "${rendered}"; then
echo "::error::${dir}/${role}: Longhorn mount exception leaked onto control planes"
exit 1
fi
elif [ "${dir}/${role}" = "talos/workers" ]; then
grep -qF 'destination: /var/lib/longhorn' "${rendered}"
grep -qF 'node.longhorn.io/create-default-disk: "true"' "${rendered}"
grep -qF 'secure: false' "${rendered}"
if grep -qF 'url: https://dex.platform.devantler.tech' "${rendered}"; then
echo "::error::${dir}/${role}: control-plane OIDC config leaked onto workers"
exit 1
fi
elif [ "${dir}/${role}" = "talos-local/control-planes" ]; then
grep -qF 'url: https://dex.platform.lan' "${rendered}"
if grep -qF 'node-role.kubernetes.io/control-plane: NoSchedule' "${rendered}"; then
echo "::error::${dir}/${role}: local control plane remained unschedulable"
exit 1
fi
elif grep -qF 'url: https://dex.platform.lan' "${rendered}"; then
echo "::error::${dir}/${role}: control-plane OIDC config leaked onto workers"
exit 1
fi
echo "${dir}/${role}: rendered + validated (${mode})"
done
echo "::endgroup::"
Expand Down Expand Up @@ -2172,7 +2218,7 @@ jobs:
shell: bash
env:
# renovate: datasource=github-releases depName=devantler-tech/ksail extractVersion=^v(?<version>.+)$
KSAIL_VERSION: "7.183.1"
KSAIL_VERSION: "7.184.1"
GITHUB_TOKEN: ${{ github.token }}
run: .github/scripts/setup-ksail.sh

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/dr-rebuild.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ jobs:
# deploy-prod action; renovate keeps the pin current in all places.
env:
# renovate: datasource=github-releases depName=devantler-tech/ksail extractVersion=^v(?<version>.+)$
KSAIL_VERSION: "7.183.1"
KSAIL_VERSION: "7.184.1"
GITHUB_TOKEN: ${{ github.token }}
run: |
.github/scripts/setup-ksail.sh
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/prove-scoped-package-access.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ jobs:
- name: Setup KSail for the existing encrypted pull credential
env:
# renovate: datasource=github-releases depName=devantler-tech/ksail extractVersion=^v(?<version>.+)$
KSAIL_VERSION: "7.183.1"
KSAIL_VERSION: "7.184.1"
GITHUB_TOKEN: ${{ github.token }}
run: .github/scripts/setup-ksail.sh

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/validate-image-verifier-liveness.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ jobs:
shell: bash
env:
# renovate: datasource=github-releases depName=devantler-tech/ksail extractVersion=^v(?<version>.+)$
KSAIL_VERSION: "7.183.1"
KSAIL_VERSION: "7.184.1"
GITHUB_TOKEN: ${{ github.token }}
run: .github/scripts/setup-ksail.sh

Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/validate-main.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ jobs:
shell: bash
env:
# renovate: datasource=github-releases depName=devantler-tech/ksail extractVersion=^v(?<version>.+)$
KSAIL_VERSION: "7.183.1"
KSAIL_VERSION: "7.184.1"
GITHUB_TOKEN: ${{ github.token }}
run: .github/scripts/setup-ksail.sh

Expand Down Expand Up @@ -252,7 +252,7 @@ jobs:
shell: bash
env:
# renovate: datasource=github-releases depName=devantler-tech/ksail extractVersion=^v(?<version>.+)$
KSAIL_VERSION: "7.183.1"
KSAIL_VERSION: "7.184.1"
GITHUB_TOKEN: ${{ github.token }}
run: |
.github/scripts/setup-ksail.sh
Expand Down
4 changes: 3 additions & 1 deletion AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -375,7 +375,9 @@ The platform uses a hierarchical kustomization structure: **base** configuration
- **Workaround:** fork the repository and use your own Age keys; re-encrypt every `*.enc.yaml` with your key.

### CNI Configuration
- The Talos cluster starts with its default CNI disabled (via `talos-local/cluster/disable-default-cni-and-kube-proxy.yaml`).
- The Talos cluster starts with its default CNI and kube-proxy disabled (via
`talos-local/cluster/disable-default-cni.yaml` and
`talos-local/cluster/disable-kube-proxy.yaml`).
- Nodes stay `NotReady` until Cilium is installed by KSail.
- This is expected — KSail handles CNI installation automatically.

Expand Down
2 changes: 1 addition & 1 deletion docs/dr/alerting.md
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ stays quiet by design, exactly as the old Alertmanager did.
capability against Coroot: it tails `/var/log/audit/kube/audit.log` and ships
it to the Coroot OTLP logs endpoint as the `kube-apiserver-audit` application.
Retention in Coroot follows the Coroot CR's `logsTTL` (3d in the base, 7d in
prod); the on-node file backend (`talos/cluster/enable-audit-logging.yaml`,
prod); the on-node file backend (`talos/control-planes/enable-audit-logging.yaml`,
30-day rotation) remains the resilient primary for forensics beyond that
window.

Expand Down
7 changes: 6 additions & 1 deletion docs/dr/runbook.md
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,12 @@ before upgrading.
> ```bash
> # Read the installer image KSail derived (from spec.cluster.talos.version +
> # .extensions) off a healthy, already-upgraded node, then reuse it:
> IMAGE=$(talosctl --nodes <healthy-ip> get machineconfig -o jsonpath='{.spec.machine.install.image}')
> # Talos 1.14 configs carry it in the UnattendedInstallConfig document; older
> # configs keep it in the v1alpha1 document's machine.install.image.
> IMAGE=$(talosctl --nodes <healthy-ip> read /system/state/config.yaml \
> | yq -r 'select(.kind == "UnattendedInstallConfig") | .installer.image')
> [ -n "$IMAGE" ] || IMAGE=$(talosctl --nodes <healthy-ip> read /system/state/config.yaml \
> | yq -r 'select(.machine != null) | .machine.install.image')
> talosctl --nodes <node-ip> upgrade --image "$IMAGE"
> ```
>
Expand Down
2 changes: 1 addition & 1 deletion docs/runtime-security.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ they sit inside a wider set of controls:
| Network | **Cilium + Hubble** | L3–L7 flow visibility and default-deny [CiliumNetworkPolicy](../k8s/bases/infrastructure/cluster-policies/best-practices/add-default-deny.yaml) per namespace |
| Runtime detection | **Kubescape node-agent** | Learned-behaviour anomaly detection, correlated with config/CVE/compliance posture |
| Runtime enforcement | **Tetragon** | Declarative kernel-hook policies that **terminate the offending process** (SIGKILL) on a policy match |
| Forensics | **API audit log** ([`talos/cluster/enable-audit-logging.yaml`](../talos/cluster/enable-audit-logging.yaml)) | Who-did-what record of control-plane mutations |
| Forensics | **API audit log** ([`talos/control-planes/enable-audit-logging.yaml`](../talos/control-planes/enable-audit-logging.yaml)) | Who-did-what record of control-plane mutations |

This document focuses on the two middle-to-bottom rows — the eBPF sensors.

Expand Down
77 changes: 32 additions & 45 deletions docs/rwx-storage.md
Original file line number Diff line number Diff line change
@@ -1,15 +1,17 @@
# RWX Storage with Longhorn

Longhorn provides ReadWriteMany (RWX) storage on Hetzner clusters (prod) using dedicated Hetzner Cloud Volumes attached to each worker node. It replaces the `hcloud` StorageClass as the cluster default.
Longhorn provides ReadWriteMany (RWX) storage on Hetzner clusters (prod) using
replicated data on the worker nodes' EPHEMERAL filesystems. It replaces the
`hcloud` StorageClass as the cluster default.

> **Local Docker clusters do not support Longhorn** — the iSCSI kernel modules required by Longhorn are not available in Docker-based Talos containers.

## Architecture

```
Hetzner Cloud Volume (per worker)
└── mounted at /var/lib/longhorn (Talos machine config)
└── Longhorn engine
Worker EPHEMERAL filesystem
└── /var/lib/longhorn (shared into kubelet)
└── Longhorn engine and replicas
├── longhorn StorageClass (default — RWO + RWX)
└── hcloud StorageClass (non-default — Hetzner block only)
```
Expand Down Expand Up @@ -45,42 +47,39 @@ fell behind a roll), read the derived installer image off a healthy node and
reuse it:

```bash
# The installer image KSail derived for the cluster
IMAGE=$(talosctl --nodes <healthy-IP> get machineconfig -o jsonpath='{.spec.machine.install.image}')
# The installer image KSail derived for the cluster. Talos 1.14 configs carry it
# in the UnattendedInstallConfig document; older configs keep it in the v1alpha1
# document's machine.install.image.
IMAGE=$(talosctl --nodes <healthy-IP> read /system/state/config.yaml \
| yq -r 'select(.kind == "UnattendedInstallConfig") | .installer.image')
[ -n "$IMAGE" ] || IMAGE=$(talosctl --nodes <healthy-IP> read /system/state/config.yaml \
| yq -r 'select(.machine != null) | .machine.install.image')

talosctl upgrade --nodes <IP> --image "$IMAGE" --preserve
```

### 2. Hetzner Cloud Volumes for workers
### 2. Worker EPHEMERAL storage

Each worker node needs a dedicated Hetzner Cloud Volume mounted at `/var/lib/longhorn`.
Longhorn stores replicas at `/var/lib/longhorn` on each worker's EPHEMERAL
filesystem. The `talos/workers/mount-longhorn-data.yaml` patch exposes that
host path to kubelet with the recursive shared propagation Longhorn's CSI node
plugin requires. It does not provision or mount a separate block device.

**Create and attach volumes** (repeat for each worker):
Talos 1.14 makes new EPHEMERAL volumes `noexec` by default, while Longhorn v1
executes its engine binaries below `/var/lib/longhorn`. The worker-only
`talos/workers/allow-longhorn-execution.yaml` patch disables the secure mount
bundle for EPHEMERAL on storage workers so fresh and rebuilt nodes can start
Longhorn. Control planes retain Talos's secure mount defaults.

```bash
# List servers to find worker names
hcloud server list

# Create a volume and attach it to a worker
# Do NOT use --format — Talos expects to partition/format the disk itself
# The volume appears as /dev/sdb on the worker
hcloud volume create \
--name <cluster>-worker-<n>-longhorn \
--size 50 \
--server <worker-server-name>
```

The Talos machine config patch (`talos/workers/mount-longhorn-data.yaml`) handles mounting `/dev/sdb` at `/var/lib/longhorn`.

> **Verify the device path** after attaching: on Hetzner Cloud, the first attached volume
> consistently appears as `/dev/sdb`. Confirm with `talosctl disks --nodes <worker-ip>`.
> If the volume shows a different path, update `talos/workers/mount-longhorn-data.yaml` accordingly.
Do not partition `/dev/sdb` for Longhorn: Hetzner's CSI driver dynamically
attaches Cloud Volumes at `/dev/sdb`, `/dev/sdc`, and later device names for
PVCs using the separate `hcloud` StorageClass.

## StorageClasses

| StorageClass | Default | Access Modes | Backing |
| --- | --- | --- | --- |
| `longhorn` | ✅ Yes | RWO, RWX | Longhorn on Hetzner volumes |
| `longhorn` | ✅ Yes | RWO, RWX | Replicated worker EPHEMERAL storage |
| `hcloud` | ❌ No | RWO only | Hetzner Cloud Block Storage |

### Using RWX volumes
Expand Down Expand Up @@ -140,20 +139,8 @@ See [Longhorn Talos Linux Support](https://longhorn.io/docs/advanced-resources/o

## Scaling

To change the Hetzner volume size:

```bash
# Volumes can only be resized up, not down
hcloud volume resize --size 50 <volume-id>
```

After resizing, the Hetzner block device grows immediately but the XFS partition and filesystem must be expanded:

```bash
# From a privileged pod on the worker (or via talosctl debug container):
sgdisk -e /dev/sdb # Fix GPT to use all space
growpart /dev/sdb 1 # Grow partition 1 to fill disk
xfs_growfs /var/lib/longhorn # Expand XFS filesystem online
```

Longhorn detects the additional space automatically once the filesystem is grown.
Longhorn capacity scales with the EPHEMERAL storage available across the
labelled baseline workers. Add or replace a storage worker with sufficient root
disk capacity, wait for replicas to become healthy, and only then drain the old
worker. Hetzner Cloud Volumes are independent PVC backends for the `hcloud`
StorageClass; resizing one does not add capacity to Longhorn.
Loading
Loading