From 5f1dfc7ea6889679c83a0e59521e2f1af3a84b1e Mon Sep 17 00:00:00 2001 From: Ranga Ngwerume Date: Fri, 28 Aug 2026 11:50:42 +0100 Subject: [PATCH 01/62] release: prepare 17.0.0 --- codacy/requirements.lock | 48 ++++++++++++++++++++++++++++++++++++++++ codacy/values.yaml | 8 +++---- 2 files changed, 52 insertions(+), 4 deletions(-) create mode 100644 codacy/requirements.lock diff --git a/codacy/requirements.lock b/codacy/requirements.lock new file mode 100644 index 00000000..f93f5c76 --- /dev/null +++ b/codacy/requirements.lock @@ -0,0 +1,48 @@ +dependencies: +- name: minio + repository: https://charts.helm.sh/stable + version: 5.0.33 +- name: rabbitmq + repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami + version: 7.5.7 +- name: postgresql + repository: https://raw.githubusercontent.com/bitnami/charts/defb094c658024e4aa8245622dab202874880cbc/bitnami + version: 8.6.4 +- name: log-router + repository: https://charts.codacy.com/external + version: 0.6.4 +- name: codacy-ingress-sh + repository: https://charts.codacy.com/stable + version: 0.2.3 +- name: portal-sh + repository: https://charts.codacy.com/stable + version: 1.1.8 +- name: ragnaros-sh + repository: https://charts.codacy.com/stable + version: 2.0.7 +- name: remote-provider-service-sh + repository: https://charts.codacy.com/stable + version: 1.1.4 +- name: listener-sh + repository: https://charts.codacy.com/stable + version: 1.1.10 +- name: engine-sh + repository: https://charts.codacy.com/stable + version: 1.2.1 +- name: codacy-tools-sh + repository: https://charts.codacy.com/stable + version: 1.2.21 +- name: codacy-api-sh + repository: https://charts.codacy.com/stable + version: 1.2.9 +- name: worker-manager-sh + repository: https://charts.codacy.com/stable + version: 2.0.2 +- name: crow-sh + repository: https://charts.codacy.com/stable + version: 1.0.6 +- name: codacy-spa-sh + repository: https://charts.codacy.com/stable + version: 1.2.4 +digest: sha256:a2a2f97f57abc8101b00c4cb344dc30c05c79a44b2a64a9af601f4c3868ecc32 +generated: "2026-08-28T11:45:44.653253069+01:00" diff --git a/codacy/values.yaml b/codacy/values.yaml index 343286ff..6915eecc 100644 --- a/codacy/values.yaml +++ b/codacy/values.yaml @@ -28,9 +28,9 @@ global: # Trial license. Valid for 4 users until the end of 2030 license: "lV5QUFrZhlibdCP/MN8pZo76B3jhPeL/RAk4E2AL2WZlGI8Vno/LfvAwnSHm26uY817I3qvEjfhIZFCkDi8R6PngiB+UiT+h8W0Blica8Sp+d5ngnh9HuhTG6xAIf7gAQxyJpn3boV87CKTO+5nHYSeJccMd+s8esMxoS84Z2xI=" installation: - version: development + version: v17.0.0-RC-0 documentation: - version: development + version: v17.0 # github: # enabled: "false" @@ -101,7 +101,7 @@ global: workerManager: workers: config: - imageVersion: stable + imageVersion: sh-1.2.1 javaOpts: '-XX:MinRAMPercentage=50.0 -XX:MaxRAMPercentage=90.0' inactivityTimeout: 30 @@ -318,7 +318,7 @@ rabbitmq-ha: fullnameOverride: codacy-rabbitmq-ha global: imagePullSecrets: - - docker-credentials + - docker-credentials image: repository: bitnamilegacy/rabbitmq tag: "3.8.35-debian-11-r5" From 7b0ab9f2b225c0a200d02566c2f68d37669a1b16 Mon Sep 17 00:00:00 2001 From: Ranga Ngwerume Date: Fri, 28 Aug 2026 14:04:36 +0100 Subject: [PATCH 02/62] changed rabbit mq values --- .circleci/config.yml | 3 +++ codacy/values-microk8s.yaml | 2 +- codacy/values-production.yaml | 3 +++ codacy/values.yaml | 2 +- 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5325488f..89544145 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -83,6 +83,9 @@ references: size: 10Gi rabbitmq-ha: replicaCount: 1 + image: + repository: bitnamilegacy/rabbitmq + tag: "3.8.35" ulimitNofiles: 32768 qa_automation_image: &qa_automation_image diff --git a/codacy/values-microk8s.yaml b/codacy/values-microk8s.yaml index 76505a1f..79309aa6 100644 --- a/codacy/values-microk8s.yaml +++ b/codacy/values-microk8s.yaml @@ -57,7 +57,7 @@ rabbitmq-ha: replicaCount: 1 image: repository: bitnamilegacy/rabbitmq - tag: "3.8.35-debian-11-r5" + tag: "3.8.35" resources: limits: cpu: 0.5 diff --git a/codacy/values-production.yaml b/codacy/values-production.yaml index bddaed44..3009b4e2 100644 --- a/codacy/values-production.yaml +++ b/codacy/values-production.yaml @@ -313,6 +313,9 @@ codacy-spa: rabbitmq-ha: replicaCount: 1 + image: + repository: bitnamilegacy/rabbitmq + tag: "3.8.35" rabbitmqErlangCookie: <--- erlang-cookie ---> # Generate one with `openssl rand -base64 32 | tr -dc 'a-zA-Z0-9'` persistentVolume: enabled: true diff --git a/codacy/values.yaml b/codacy/values.yaml index 6915eecc..dd8d2137 100644 --- a/codacy/values.yaml +++ b/codacy/values.yaml @@ -321,7 +321,7 @@ rabbitmq-ha: - docker-credentials image: repository: bitnamilegacy/rabbitmq - tag: "3.8.35-debian-11-r5" + tag: "3.8.35" auth: username: rabbitmq-codacy password: CHANGEME From 340cf09a9a6f539bc7223e76378d3da0e73d8a18 Mon Sep 17 00:00:00 2001 From: Ranga Ngwerume Date: Fri, 28 Aug 2026 14:12:03 +0100 Subject: [PATCH 03/62] Fixed requirements.lock --- codacy/requirements.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codacy/requirements.lock b/codacy/requirements.lock index f93f5c76..1d1d5750 100644 --- a/codacy/requirements.lock +++ b/codacy/requirements.lock @@ -3,7 +3,7 @@ dependencies: repository: https://charts.helm.sh/stable version: 5.0.33 - name: rabbitmq - repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami + repository: https://raw.githubusercontent.com/bitnami/charts/defb094c658024e4aa8245622dab202874880cbc/bitnami version: 7.5.7 - name: postgresql repository: https://raw.githubusercontent.com/bitnami/charts/defb094c658024e4aa8245622dab202874880cbc/bitnami From e2fd2b3f01c9e7dfdb0c36df177182418dd65aa2 Mon Sep 17 00:00:00 2001 From: Ranga Ngwerume Date: Fri, 28 Aug 2026 14:23:00 +0100 Subject: [PATCH 04/62] Dependencies update --- codacy/requirements.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codacy/requirements.yaml b/codacy/requirements.yaml index eb6daa18..4c61ebaf 100644 --- a/codacy/requirements.yaml +++ b/codacy/requirements.yaml @@ -7,7 +7,7 @@ dependencies: - name: rabbitmq version: 7.5.7 - repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami + repository: https://raw.githubusercontent.com/bitnami/charts/defb094c658024e4aa8245622dab202874880cbc/bitnami condition: global.rabbitmq.create alias: rabbitmq-ha From 297e074d75e6576fe06b3e41472fd08d153e4d3c Mon Sep 17 00:00:00 2001 From: Ranga Ngwerume Date: Fri, 28 Aug 2026 14:30:29 +0100 Subject: [PATCH 05/62] regenerated lock file due to digest sha --- codacy/requirements.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codacy/requirements.lock b/codacy/requirements.lock index 1d1d5750..3c802448 100644 --- a/codacy/requirements.lock +++ b/codacy/requirements.lock @@ -44,5 +44,5 @@ dependencies: - name: codacy-spa-sh repository: https://charts.codacy.com/stable version: 1.2.4 -digest: sha256:a2a2f97f57abc8101b00c4cb344dc30c05c79a44b2a64a9af601f4c3868ecc32 -generated: "2026-08-28T11:45:44.653253069+01:00" +digest: sha256:d19e971e5160470800251d2660f063c96c00de20180c13c2f7c7e90375c52862 +generated: "2026-08-28T14:29:21.804571894+01:00" From 02cad8a74336de4045c0bed6bc516a4ca20568ad Mon Sep 17 00:00:00 2001 From: Ranga Ngwerume Date: Fri, 28 Aug 2026 14:35:31 +0100 Subject: [PATCH 06/62] Rabbit MQ version pinned --- .circleci/config.yml | 2 +- codacy/values-microk8s.yaml | 2 +- codacy/values-production.yaml | 2 +- codacy/values.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 89544145..f1aa5c7d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -85,7 +85,7 @@ references: replicaCount: 1 image: repository: bitnamilegacy/rabbitmq - tag: "3.8.35" + tag: "3.8.5" ulimitNofiles: 32768 qa_automation_image: &qa_automation_image diff --git a/codacy/values-microk8s.yaml b/codacy/values-microk8s.yaml index 79309aa6..1e6435e6 100644 --- a/codacy/values-microk8s.yaml +++ b/codacy/values-microk8s.yaml @@ -57,7 +57,7 @@ rabbitmq-ha: replicaCount: 1 image: repository: bitnamilegacy/rabbitmq - tag: "3.8.35" + tag: "3.8.5" resources: limits: cpu: 0.5 diff --git a/codacy/values-production.yaml b/codacy/values-production.yaml index 3009b4e2..5d65214b 100644 --- a/codacy/values-production.yaml +++ b/codacy/values-production.yaml @@ -315,7 +315,7 @@ rabbitmq-ha: replicaCount: 1 image: repository: bitnamilegacy/rabbitmq - tag: "3.8.35" + tag: "3.8.5" rabbitmqErlangCookie: <--- erlang-cookie ---> # Generate one with `openssl rand -base64 32 | tr -dc 'a-zA-Z0-9'` persistentVolume: enabled: true diff --git a/codacy/values.yaml b/codacy/values.yaml index dd8d2137..cad01db2 100644 --- a/codacy/values.yaml +++ b/codacy/values.yaml @@ -321,7 +321,7 @@ rabbitmq-ha: - docker-credentials image: repository: bitnamilegacy/rabbitmq - tag: "3.8.35" + tag: "3.8.5" auth: username: rabbitmq-codacy password: CHANGEME From 7a974b45b80dd1266dffc87177a975a56f0be079 Mon Sep 17 00:00:00 2001 From: Ranga Ngwerume Date: Fri, 28 Aug 2026 14:39:10 +0100 Subject: [PATCH 07/62] Fixed makefile to inlcude snapshot repo --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index e3f3dfe5..54ef0e9a 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ .PHONY: setup_helm_repos setup_helm_repos: # Add legacy Bitnami snapshot repo used by requirements.yaml (apiVersion v1) + helm repo add bitnami-pre-2022 https://raw.githubusercontent.com/bitnami/charts/defb094c658024e4aa8245622dab202874880cbc/bitnami # Github tag before bitnami truncated the index. Old link didn't work anymore helm repo add stable https://charts.helm.sh/stable helm repo add codacy-stable https://charts.codacy.com/stable helm repo add codacy-unstable https://charts.codacy.com/unstable From c9485f2a483be942707cf330514acad8c68fe348 Mon Sep 17 00:00:00 2001 From: Ricardo Bernardino Date: Mon, 31 Aug 2026 09:48:38 +0100 Subject: [PATCH 08/62] Increasing the make timeout --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1a2454e5..c4e6eed8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -333,6 +333,7 @@ references: - codacy/run_skip_ssh: cmd_name: Install Codacy cmd: make -e -C .do/ deploy_to_doks_from_chartmuseum VERSION=$(cat .version) + no_output_timeout: 15m helm_push: &helm_push steps: From bf2bcb1ed4a5a00a443df8063d86ad9e6acb0769 Mon Sep 17 00:00:00 2001 From: Ricardo Bernardino Date: Mon, 31 Aug 2026 09:50:15 +0100 Subject: [PATCH 09/62] Increasing the make timeout --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c4e6eed8..af454bea 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -333,7 +333,7 @@ references: - codacy/run_skip_ssh: cmd_name: Install Codacy cmd: make -e -C .do/ deploy_to_doks_from_chartmuseum VERSION=$(cat .version) - no_output_timeout: 15m + no_output_timeout: 15m helm_push: &helm_push steps: From c5d85711d0aedc47ee78e138e85a9a2e3dbb5aa6 Mon Sep 17 00:00:00 2001 From: Ricardo Bernardino Date: Mon, 31 Aug 2026 10:00:23 +0100 Subject: [PATCH 10/62] Increasing the make timeout --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index af454bea..6a2d93e1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -330,10 +330,10 @@ references: steps: - <<: *attach_workspace - <<: *doctl_authenticate - - codacy/run_skip_ssh: - cmd_name: Install Codacy - cmd: make -e -C .do/ deploy_to_doks_from_chartmuseum VERSION=$(cat .version) + - run: + name: Install Codacy no_output_timeout: 15m + command: make -e -C .do/ deploy_to_doks_from_chartmuseum VERSION=$(cat .version) helm_push: &helm_push steps: From d8293ea80d41bdc0cbd027cdd43937310d6b54fe Mon Sep 17 00:00:00 2001 From: "circleci-app[bot]" <127350680+circleci-app[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 19:14:34 +0100 Subject: [PATCH 11/62] Increase helm deploy timeout to fix release job timeout (#834) The deploy_to_doks_release job (job 30732) timed out because the CircleCI no_output_timeout (15m) matched the helm --timeout (900s). Helm produces no output while waiting for pods to roll out, so CircleCI killed the process just as helm was about to hit its own timeout. Fix: raise no_output_timeout to 30m and helm --timeout to 1500s (25m) so helm has room to complete or fail cleanly before CircleCI intervenes. AI-Generated: true Co-authored-by: circleci-app[bot] <127350680+circleci-app[bot]@users.noreply.github.com> --- .circleci/config.yml | 2 +- .do/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6a2d93e1..1446c015 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -332,7 +332,7 @@ references: - <<: *doctl_authenticate - run: name: Install Codacy - no_output_timeout: 15m + no_output_timeout: 30m command: make -e -C .do/ deploy_to_doks_from_chartmuseum VERSION=$(cat .version) helm_push: &helm_push diff --git a/.do/Makefile b/.do/Makefile index 6ecda9cf..0f2c7c07 100644 --- a/.do/Makefile +++ b/.do/Makefile @@ -13,7 +13,7 @@ define call_helm_install eval "extra_args=($$HELM_INSTALL_EXTRA_FLAGS)"; \ helm upgrade --install ${1} ${3} \ --atomic \ - --timeout=900s \ + --timeout=1500s \ -f ../codacy/values-production.yaml \ -f values.yaml \ --namespace ${2} \ From 2b42911e071b42703caec9eae9d4db8ae25eb8b9 Mon Sep 17 00:00:00 2001 From: "circleci-app[bot]" <127350680+circleci-app[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 21:43:19 +0100 Subject: [PATCH 12/62] Clean up stale hook jobs before helm upgrade to fix BackoffLimitExceeded (#836) * Clean up stale Kubernetes jobs before helm upgrade in release deployment Previous failed deployments leave hook jobs (e.g. DB migrations) behind in the codacy-release namespace. When helm tries to run the same post-upgrade hook on the next attempt it finds an already-failed job and immediately reports BackoffLimitExceeded, causing the deploy_to_doks_release CI job to fail (see job 30795). Add a clean_jobs Makefile target that deletes all jobs in the namespace before the helm upgrade runs. The codacy-release namespace is a dedicated Helm release namespace, so any lingering jobs are stale hook artifacts from prior failed runs and are safe to remove. AI-Generated: true * Apply suggestion from @codacy-production[bot] Co-authored-by: codacy-production[bot] <61871480+codacy-production[bot]@users.noreply.github.com> * Reduce the delete jobs scope --------- Co-authored-by: circleci-app[bot] <127350680+circleci-app[bot]@users.noreply.github.com> Co-authored-by: Ricardo Bernardino Co-authored-by: codacy-production[bot] <61871480+codacy-production[bot]@users.noreply.github.com> --- .do/Makefile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.do/Makefile b/.do/Makefile index 0f2c7c07..0872a6b5 100644 --- a/.do/Makefile +++ b/.do/Makefile @@ -85,10 +85,14 @@ update_ingress_values: deploy_to_doks_infrastructure: set_cluster_context update_dependencies helm_install_infrastructure .PHONY: deploy_to_doks_from_local_dir -deploy_to_doks_from_local_dir: set_cluster_context update_dependencies helm_install_from_local_dir +deploy_to_doks_from_local_dir: set_cluster_context update_dependencies clean_jobs helm_install_from_local_dir + +.PHONY: clean_jobs +clean_jobs: + -kubectl delete jobs -l app.kubernetes.io/instance=${RELEASE_NAME} --ignore-not-found=true .PHONY: deploy_to_doks_from_chartmuseum -deploy_to_doks_from_chartmuseum: set_cluster_context setup_helm_repos helm_install_from_chart_museum +deploy_to_doks_from_chartmuseum: set_cluster_context setup_helm_repos clean_jobs helm_install_from_chart_museum .PHONY: set_cluster_context set_cluster_context: From e4c8bdda0e2e986c0267049d9a163cd95f5a4bfb Mon Sep 17 00:00:00 2001 From: "circleci-app[bot]" <127350680+circleci-app[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 23:05:36 +0100 Subject: [PATCH 13/62] Fix clean_jobs missing --namespace flag causing BackoffLimitExceeded (#837) The clean_jobs Makefile target ran kubectl delete jobs without specifying the --namespace flag, so it searched the default namespace instead of the release namespace (e.g. codacy-release). This meant stale hook jobs (DB migrations etc.) from prior failed deployments were never cleaned up, and helm's next attempt would find the existing BackoffLimitExceeded job and immediately fail again. Add --namespace ${NAMESPACE} to the kubectl delete command so it targets the correct namespace for every environment (dev, sandbox, release). AI-Generated: true Co-authored-by: circleci-app[bot] <127350680+circleci-app[bot]@users.noreply.github.com> --- .do/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.do/Makefile b/.do/Makefile index 0872a6b5..641630bd 100644 --- a/.do/Makefile +++ b/.do/Makefile @@ -89,7 +89,7 @@ deploy_to_doks_from_local_dir: set_cluster_context update_dependencies clean_job .PHONY: clean_jobs clean_jobs: - -kubectl delete jobs -l app.kubernetes.io/instance=${RELEASE_NAME} --ignore-not-found=true + -kubectl delete jobs -l app.kubernetes.io/instance=${RELEASE_NAME} --namespace ${NAMESPACE} --ignore-not-found=true .PHONY: deploy_to_doks_from_chartmuseum deploy_to_doks_from_chartmuseum: set_cluster_context setup_helm_repos clean_jobs helm_install_from_chart_museum From b6dec040bb3247a105c602312ce0f5268dd31eed Mon Sep 17 00:00:00 2001 From: "circleci-app[bot]" <127350680+circleci-app[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 01:16:54 +0100 Subject: [PATCH 14/62] Add retry logic to deploy_to_doks_from_chartmuseum for hook failures (#838) When a post-upgrade helm hook job fails with BackoffLimitExceeded, the failed job lingers in the namespace. The --atomic flag causes helm to roll back, leaving the hook job behind. On the next CI run, clean_jobs removes it before the upgrade, but the hook can fail again transiently. This change converts deploy_to_doks_from_chartmuseum from a prerequisite-chain target to a recipe that cleans up and retries the helm install once if the first attempt fails, covering transient hook failures (e.g. delete-rabbitmq-queues running before RabbitMQ is ready). AI-Generated: true Co-authored-by: circleci-app[bot] <127350680+circleci-app[bot]@users.noreply.github.com> --- .do/Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.do/Makefile b/.do/Makefile index 641630bd..ac90f231 100644 --- a/.do/Makefile +++ b/.do/Makefile @@ -92,7 +92,9 @@ clean_jobs: -kubectl delete jobs -l app.kubernetes.io/instance=${RELEASE_NAME} --namespace ${NAMESPACE} --ignore-not-found=true .PHONY: deploy_to_doks_from_chartmuseum -deploy_to_doks_from_chartmuseum: set_cluster_context setup_helm_repos clean_jobs helm_install_from_chart_museum +deploy_to_doks_from_chartmuseum: set_cluster_context setup_helm_repos + -$(MAKE) clean_jobs + $(MAKE) helm_install_from_chart_museum || ($(MAKE) clean_jobs && $(MAKE) helm_install_from_chart_museum) .PHONY: set_cluster_context set_cluster_context: From 2632250e50d47f2e2dac5f89be57a0d51ad21afa Mon Sep 17 00:00:00 2001 From: "circleci-app[bot]" <127350680+circleci-app[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 02:36:18 +0100 Subject: [PATCH 15/62] Fix fluentd lifecycle hook BackoffLimitExceeded by adding MinIO readiness wait (#839) The post-upgrade hook job was spawning 100 pods (backoffLimit: 100, restartPolicy: Never) each failing in ~9s because MinIO isn't ready immediately after a Helm upgrade, totalling ~15 minutes before BackoffLimitExceeded and causing helm --atomic to roll back. Changes: - Add a wait loop in the hook container that polls MinIO (aws s3 ls) every 10s for up to 10 minutes before attempting put-bucket-lifecycle-configuration - Reduce backoffLimit from 100 to 3 since retries are now handled within the container - Add before-hook-creation to hook-delete-policy so any stale failed job is always cleaned up before the next upgrade attempt creates a new one AI-Generated: true Co-authored-by: circleci-app[bot] <127350680+circleci-app[bot]@users.noreply.github.com> --- .../fluentd/lifecycle-police-job.yaml | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/codacy/templates/fluentd/lifecycle-police-job.yaml b/codacy/templates/fluentd/lifecycle-police-job.yaml index 68e5167f..eb32590b 100644 --- a/codacy/templates/fluentd/lifecycle-police-job.yaml +++ b/codacy/templates/fluentd/lifecycle-police-job.yaml @@ -12,9 +12,9 @@ metadata: # Minio create bucket hook doesn't define any hook weight, # any positive value will ensure that the bucket is created before this "helm.sh/hook-weight": "10" - "helm.sh/hook-delete-policy": hook-succeeded + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded spec: - backoffLimit: 100 + backoffLimit: 3 template: metadata: name: "{{ .Release.Name }}" @@ -30,14 +30,26 @@ spec: value: "{{ .Values.global.minio.accessKey }}" - name: AWS_SECRET_ACCESS_KEY value: "{{ .Values.global.minio.secretKey }}" - command: ["aws"] + command: ["/bin/sh", "-c"] args: - - s3api - - put-bucket-lifecycle-configuration - - --bucket - - {{ .Values.fluentdoperator.bucketName }} - - --endpoint-url - - http://{{ .Values.global.minio.location }}:{{ .Values.global.minio.port }} - - --lifecycle-configuration - - '{"Rules":[{"Expiration":{"Days": {{ .Values.fluentdoperator.expirationDays }} },"ID":"Delete old logs","Status":"Enabled"}]}' + - | + ENDPOINT="http://{{ .Values.global.minio.location }}:{{ .Values.global.minio.port }}" + BUCKET="{{ .Values.fluentdoperator.bucketName }}" + LIFECYCLE='{"Rules":[{"Expiration":{"Days": {{ .Values.fluentdoperator.expirationDays }} },"ID":"Delete old logs","Status":"Enabled"}]}' + MAX_WAIT=600 + ELAPSED=0 + echo "Waiting for MinIO at ${ENDPOINT}..." + until aws s3 ls --endpoint-url "${ENDPOINT}" > /dev/null 2>&1; do + if [ "${ELAPSED}" -ge "${MAX_WAIT}" ]; then + echo "Timed out waiting for MinIO after ${MAX_WAIT}s" + exit 1 + fi + sleep 10 + ELAPSED=$((ELAPSED + 10)) + done + echo "MinIO is ready. Setting lifecycle configuration on bucket ${BUCKET}..." + aws s3api put-bucket-lifecycle-configuration \ + --bucket "${BUCKET}" \ + --endpoint-url "${ENDPOINT}" \ + --lifecycle-configuration "${LIFECYCLE}" {{ end }} From 82b1310ede7140e548a19906d6fd4d9694e228a0 Mon Sep 17 00:00:00 2001 From: "circleci-app[bot]" <127350680+circleci-app[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 09:11:46 +0100 Subject: [PATCH 16/62] Fix fluentd lifecycle hook BackoffLimitExceeded: add Filter field and pre-push delete (#840) Two fixes for the persistent deploy_to_doks_release failure (job 31035): 1. codacy/templates/fluentd/lifecycle-police-job.yaml: Add "Filter":{} to the lifecycle configuration JSON. Newer MinIO versions strictly follow the AWS S3 spec which requires a Filter element in each lifecycle rule, even when empty. Without it, put-bucket-lifecycle-configuration returns a MalformedXML error immediately, causing 3 fast pod failures and BackoffLimitExceeded (the ~15 min total time is main-deployment rollout; the hook itself fails in seconds). 2. .circleci/config.yml: Delete the existing chart version from chart museum before pushing. Chart museum rejects re-uploads of the same version with 409, which failed the first pipeline run (job 31005). Adding a pre-push DELETE via the chart museum REST API (with || true to tolerate missing charts) ensures subsequent CI runs can always push the updated chart. AI-Generated: true Co-authored-by: circleci-app[bot] <127350680+circleci-app[bot]@users.noreply.github.com> --- .circleci/config.yml | 2 ++ codacy/templates/fluentd/lifecycle-police-job.yaml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1446c015..79ed6ae8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -351,6 +351,8 @@ references: helm dep build ./codacy echo "Adding 'https://charts.codacy.com/${CHANNEL}'" helm repo add --username "${CHARTS_REPO_USER}" --password "${CHARTS_REPO_PASS}" ${HELM_REPOSITORY} https://charts.codacy.com/${CHANNEL} + echo "Removing existing chart version from chart museum (if present)..." + curl -s -u "${CHARTS_REPO_USER}:${CHARTS_REPO_PASS}" -X DELETE "https://charts.codacy.com/${CHANNEL}/api/charts/codacy/$(cat .version)" || true helm push ./codacy ${HELM_REPOSITORY} - <<: *persist_to_workspace diff --git a/codacy/templates/fluentd/lifecycle-police-job.yaml b/codacy/templates/fluentd/lifecycle-police-job.yaml index eb32590b..a1fda355 100644 --- a/codacy/templates/fluentd/lifecycle-police-job.yaml +++ b/codacy/templates/fluentd/lifecycle-police-job.yaml @@ -35,7 +35,7 @@ spec: - | ENDPOINT="http://{{ .Values.global.minio.location }}:{{ .Values.global.minio.port }}" BUCKET="{{ .Values.fluentdoperator.bucketName }}" - LIFECYCLE='{"Rules":[{"Expiration":{"Days": {{ .Values.fluentdoperator.expirationDays }} },"ID":"Delete old logs","Status":"Enabled"}]}' + LIFECYCLE='{"Rules":[{"Expiration":{"Days": {{ .Values.fluentdoperator.expirationDays }} },"Filter":{},"ID":"Delete old logs","Status":"Enabled"}]}' MAX_WAIT=600 ELAPSED=0 echo "Waiting for MinIO at ${ENDPOINT}..." From f5b218f2b162089382ef0a87b9d6ff97adfb342c Mon Sep 17 00:00:00 2001 From: "circleci-app[bot]" <127350680+circleci-app[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 11:11:37 +0100 Subject: [PATCH 17/62] Fix fluentd lifecycle hook waiting for bucket existence, not just MinIO (#841) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous fix (PR #839) waited for MinIO to respond to `aws s3 ls` before calling put-bucket-lifecycle-configuration. However, MinIO can be accessible while the specific bucket doesn't exist yet (bucket creation is a separate hook). This caused rapid failures of the lifecycle config command, hitting BackoffLimitExceeded (with backoffLimit:3) long before Helm's 1500s timeout. Changes: - Wait for the specific bucket (`aws s3api head-bucket`) instead of just MinIO (`aws s3 ls`), so the lifecycle config only runs once the bucket actually exists - Extend MAX_WAIT to 1200s (fits within Helm's 1500s timeout as a single container, no Kubernetes backoff overhead) - Set backoffLimit:0 so Kubernetes doesn't add exponential backoff delays between retries — all retry logic is in-shell - Add an in-shell retry loop (up to 5 attempts) for put-bucket-lifecycle-configuration to handle transient API errors after the bucket is confirmed ready AI-Generated: true Co-authored-by: circleci-app[bot] <127350680+circleci-app[bot]@users.noreply.github.com> Co-authored-by: Ricardo Bernardino --- .../fluentd/lifecycle-police-job.yaml | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/codacy/templates/fluentd/lifecycle-police-job.yaml b/codacy/templates/fluentd/lifecycle-police-job.yaml index a1fda355..f30463c8 100644 --- a/codacy/templates/fluentd/lifecycle-police-job.yaml +++ b/codacy/templates/fluentd/lifecycle-police-job.yaml @@ -14,7 +14,7 @@ metadata: "helm.sh/hook-weight": "10" "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded spec: - backoffLimit: 3 + backoffLimit: 0 template: metadata: name: "{{ .Release.Name }}" @@ -35,21 +35,32 @@ spec: - | ENDPOINT="http://{{ .Values.global.minio.location }}:{{ .Values.global.minio.port }}" BUCKET="{{ .Values.fluentdoperator.bucketName }}" - LIFECYCLE='{"Rules":[{"Expiration":{"Days": {{ .Values.fluentdoperator.expirationDays }} },"Filter":{},"ID":"Delete old logs","Status":"Enabled"}]}' - MAX_WAIT=600 + LIFECYCLE='{"Rules":[{"Expiration":{"Days": {{ .Values.fluentdoperator.expirationDays }} },"ID":"Delete old logs","Status":"Enabled"}]}' + MAX_WAIT=1200 ELAPSED=0 - echo "Waiting for MinIO at ${ENDPOINT}..." - until aws s3 ls --endpoint-url "${ENDPOINT}" > /dev/null 2>&1; do + echo "Waiting for MinIO bucket ${BUCKET} at ${ENDPOINT}..." + until aws s3api head-bucket --bucket "${BUCKET}" --endpoint-url "${ENDPOINT}" > /dev/null 2>&1; do if [ "${ELAPSED}" -ge "${MAX_WAIT}" ]; then - echo "Timed out waiting for MinIO after ${MAX_WAIT}s" + echo "Timed out waiting for MinIO bucket after ${MAX_WAIT}s" exit 1 fi + echo "Bucket not ready yet (${ELAPSED}s elapsed), retrying in 10s..." sleep 10 ELAPSED=$((ELAPSED + 10)) done - echo "MinIO is ready. Setting lifecycle configuration on bucket ${BUCKET}..." - aws s3api put-bucket-lifecycle-configuration \ + echo "MinIO bucket ${BUCKET} is ready. Setting lifecycle configuration..." + RETRY=0 + until aws s3api put-bucket-lifecycle-configuration \ --bucket "${BUCKET}" \ --endpoint-url "${ENDPOINT}" \ - --lifecycle-configuration "${LIFECYCLE}" + --lifecycle-configuration "${LIFECYCLE}"; do + RETRY=$((RETRY + 1)) + if [ "${RETRY}" -ge 5 ]; then + echo "Failed to set lifecycle configuration after ${RETRY} attempts" + exit 1 + fi + echo "Lifecycle config failed, retrying in 10s (attempt ${RETRY}/5)..." + sleep 10 + done + echo "Lifecycle configuration set successfully." {{ end }} From 1f73fa0e4761403224b4ee317cb51ecb74e9d49e Mon Sep 17 00:00:00 2001 From: "circleci-app[bot]" <127350680+circleci-app[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 14:11:04 +0100 Subject: [PATCH 18/62] Fix minio-make-bucket-job lingering after failed upgrade (#842) The stable/minio chart (v5.0.33) uses old-style Helm labels (release: , app: minio-make-bucket-job) and has hook-delete-policy: hook-succeeded only. When the job fails, it is not deleted by Helm on rollback. clean_jobs was filtering by app.kubernetes.io/instance which doesn't exist on old-style chart resources, so the stuck job was never cleaned up. The second helm upgrade attempt would then fail with "failed to deploy codacy-minio-make-bucket-job" because the job already existed. Add a second kubectl delete with the legacy release= label selector so clean_jobs catches both new- and old-style Helm-labelled jobs. AI-Generated: true Co-authored-by: circleci-app[bot] <127350680+circleci-app[bot]@users.noreply.github.com> --- .do/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/.do/Makefile b/.do/Makefile index ac90f231..e0148553 100644 --- a/.do/Makefile +++ b/.do/Makefile @@ -90,6 +90,7 @@ deploy_to_doks_from_local_dir: set_cluster_context update_dependencies clean_job .PHONY: clean_jobs clean_jobs: -kubectl delete jobs -l app.kubernetes.io/instance=${RELEASE_NAME} --namespace ${NAMESPACE} --ignore-not-found=true + -kubectl delete jobs -l release=${RELEASE_NAME} --namespace ${NAMESPACE} --ignore-not-found=true .PHONY: deploy_to_doks_from_chartmuseum deploy_to_doks_from_chartmuseum: set_cluster_context setup_helm_repos From abee531836abdff562415086f7f0b452e36e0140 Mon Sep 17 00:00:00 2001 From: "circleci-app[bot]" <127350680+circleci-app[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 14:17:52 +0100 Subject: [PATCH 19/62] Restore Filter field in fluentd lifecycle configuration (#843) PR #841 accidentally removed the `"Filter":{}` field that PR #840 had added to the S3 lifecycle rule JSON. Newer MinIO versions strictly follow the AWS S3 spec and require a Filter element in each lifecycle rule (even when empty). Without it, put-bucket-lifecycle-configuration returns MalformedXML immediately, causing the in-shell retry loop to exhaust its 5 attempts and exit with failure, which Kubernetes surfaces as BackoffLimitExceeded on the post-upgrade hook. AI-Generated: true Co-authored-by: circleci-app[bot] <127350680+circleci-app[bot]@users.noreply.github.com> --- codacy/templates/fluentd/lifecycle-police-job.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codacy/templates/fluentd/lifecycle-police-job.yaml b/codacy/templates/fluentd/lifecycle-police-job.yaml index f30463c8..5f4621a4 100644 --- a/codacy/templates/fluentd/lifecycle-police-job.yaml +++ b/codacy/templates/fluentd/lifecycle-police-job.yaml @@ -35,7 +35,7 @@ spec: - | ENDPOINT="http://{{ .Values.global.minio.location }}:{{ .Values.global.minio.port }}" BUCKET="{{ .Values.fluentdoperator.bucketName }}" - LIFECYCLE='{"Rules":[{"Expiration":{"Days": {{ .Values.fluentdoperator.expirationDays }} },"ID":"Delete old logs","Status":"Enabled"}]}' + LIFECYCLE='{"Rules":[{"Expiration":{"Days": {{ .Values.fluentdoperator.expirationDays }} },"Filter":{},"ID":"Delete old logs","Status":"Enabled"}]}' MAX_WAIT=1200 ELAPSED=0 echo "Waiting for MinIO bucket ${BUCKET} at ${ENDPOINT}..." From 4cc6583f1c4626b2eebe1a627540e9bbf2f04e26 Mon Sep 17 00:00:00 2001 From: "circleci-app[bot]" <127350680+circleci-app[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:27:47 +0100 Subject: [PATCH 20/62] Fix tag_version job: switch from HTTPS to SSH authentication (#844) The codacy/tag_version orb job was failing with: "Invalid username or token. Password authentication is not supported for Git operations." GitHub no longer accepts token-based HTTPS push for git operations. Replace both codacy/tag_version orb calls in the release_pipeline workflow with a custom tag_version_ssh job that uses SSH key authentication, bypassing the HTTPS credential issue entirely. AI-Generated: true Co-authored-by: circleci-app[bot] <127350680+circleci-app[bot]@users.noreply.github.com> --- .circleci/config.yml | 45 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 79ed6ae8..8c710784 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -513,6 +513,47 @@ jobs: command: cat .version - <<: *persist_to_workspace + tag_version_ssh: + <<: *default_doks_image + parameters: + version: + type: string + default: "" + force: + type: boolean + default: false + steps: + - <<: *attach_workspace + - add_ssh_keys: + fingerprints: + - "df:83:d7:c7:d5:79:06:c2:3b:d1:fd:e2:a3:d1:12:c5" + - run: + name: Setup SSH + command: ssh-keyscan github.com >> ~/.ssh/known_hosts + - run: + name: Set Git Config + command: | + git config --global user.email "circleci@codacy.com" + git config --global user.name "CircleCI" + git remote set-url origin git@github.com:codacy/chart.git + - run: + name: Fetch Tags + command: git fetch --tags + - run: + name: Tag + command: | + TAG="<< parameters.version >>" + if [ -z "$TAG" ]; then + TAG=$(cat .version) + fi + if [ "<< parameters.force >>" = "true" ]; then + git tag -f "$TAG" + git push origin -f "refs/tags/$TAG" + else + git tag "$TAG" + git push origin "refs/tags/$TAG" + fi + build_docs: docker: - image: circleci/python:3.8 @@ -812,7 +853,7 @@ workflows: requires: - manual_qa_hold - manual_solutions_eng_hold - - codacy/tag_version: + - tag_version_ssh: name: tag_version context: CodacyDO requires: @@ -826,7 +867,7 @@ workflows: target_charts_repo_url: "https://charts.codacy.com/stable" requires: - tag_version - - codacy/tag_version: + - tag_version_ssh: name: tag_version_latest context: CodacyDO version: latest From 18711504a1a5ec24013ed0b7f9504de2e8542af4 Mon Sep 17 00:00:00 2001 From: Nuno Fernandes Date: Fri, 4 Sep 2026 10:09:34 +0100 Subject: [PATCH 21/62] test: The release pipeline tests now wait for the k8s images steps to be green --- .circleci/config.yml | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8c710784..83ec31e9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -554,6 +554,15 @@ jobs: git push origin "refs/tags/$TAG" fi + tests_start: + docker: + - image: codacy/ci-base:1.0.1 + working_directory: ~/workdir/ + steps: + - run: + name: All microk8s installs are green + command: echo "microk8s install jobs finished successfully - starting QA tests" + build_docs: docker: - image: circleci/python:3.8 @@ -793,20 +802,28 @@ workflows: requires: - helm_push_incubator + - tests_start: + requires: + - deploy_to_doks_release + - install_k8s-1.30_helm-3.16.3 + - install_k8s-1.31_helm-3.16.3 + - install_k8s-1.32_helm-3.19.0 + - install_k8s-1.33_helm-3.19.0 + - test_api: context: CodacyAWS requires: - - deploy_to_doks_release + - tests_start - test_apiv3: context: CodacyAWS requires: - - deploy_to_doks_release + - tests_start - test_check_cookies_expired_releases_k8s: context: CodacyAWS requires: - - deploy_to_doks_release + - tests_start - test_web: context: CodacyAWS From 5f4708e5e185056bf0098d171f9489b02d50c227 Mon Sep 17 00:00:00 2001 From: Nuno Fernandes Date: Fri, 4 Sep 2026 10:12:51 +0100 Subject: [PATCH 22/62] test: The manual QA hold now depends on the tests_start to be green --- .circleci/config.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 83ec31e9..edf6a9f3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -851,10 +851,7 @@ workflows: type: approval context: CodacyDO requires: - - install_k8s-1.30_helm-3.16.3 - - install_k8s-1.31_helm-3.16.3 - - install_k8s-1.32_helm-3.19.0 - - install_k8s-1.33_helm-3.19.0 + - tests_start # Intentionally does not depend on the QA test jobs so that a manual # approval path to the deploy remains available even if any of them fail. - manual_solutions_eng_hold: From a666cc6c3fb60a035709617bf50b3975df51ebbc Mon Sep 17 00:00:00 2001 From: Leandro Salgado Date: Thu, 15 Oct 2026 10:30:31 +0100 Subject: [PATCH 23/62] add debug --- codacy/values.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/codacy/values.yaml b/codacy/values.yaml index cad01db2..d08f8b0f 100644 --- a/codacy/values.yaml +++ b/codacy/values.yaml @@ -335,6 +335,9 @@ rabbitmq-ha: existingSecret: rabbitmq-load-definition extraConfiguration: | management.load_definitions = /app/load_definition.json + extraEnvVars: + - name: RABBITMQ_LOG + value: "debug" postgres: fullnameOverride: codacy-postgres From c00e95d8ec312da6bb42e099f7c1b4f3e6d36042 Mon Sep 17 00:00:00 2001 From: Leandro Salgado Date: Thu, 15 Oct 2026 10:51:14 +0100 Subject: [PATCH 24/62] add debug --- codacy/values.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/codacy/values.yaml b/codacy/values.yaml index d08f8b0f..b897e583 100644 --- a/codacy/values.yaml +++ b/codacy/values.yaml @@ -335,9 +335,7 @@ rabbitmq-ha: existingSecret: rabbitmq-load-definition extraConfiguration: | management.load_definitions = /app/load_definition.json - extraEnvVars: - - name: RABBITMQ_LOG - value: "debug" + log.console.level = debug postgres: fullnameOverride: codacy-postgres From e99f895c76fe95efd08ef0f3d0eb6b03df01ed70 Mon Sep 17 00:00:00 2001 From: Leandro Salgado Date: Thu, 15 Oct 2026 11:25:35 +0100 Subject: [PATCH 25/62] test highwatermark --- codacy/values.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/codacy/values.yaml b/codacy/values.yaml index b897e583..58b7117b 100644 --- a/codacy/values.yaml +++ b/codacy/values.yaml @@ -325,6 +325,10 @@ rabbitmq-ha: auth: username: rabbitmq-codacy password: CHANGEME + memoryHighWatermark: + enabled: true + type: relative + value: 0.4 prometheus: operator: enabled: false From 2f94503fa26b2d6ada4b07edcc7ca73c65cb8803 Mon Sep 17 00:00:00 2001 From: Leandro Salgado Date: Thu, 15 Oct 2026 14:31:22 +0100 Subject: [PATCH 26/62] add extra env vars --- codacy/values.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/codacy/values.yaml b/codacy/values.yaml index 58b7117b..c5b9c63c 100644 --- a/codacy/values.yaml +++ b/codacy/values.yaml @@ -340,6 +340,9 @@ rabbitmq-ha: extraConfiguration: | management.load_definitions = /app/load_definition.json log.console.level = debug + extraEnvVars: + - name: RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS + value: "+S 2:2" postgres: fullnameOverride: codacy-postgres From 0167c8898a2e6269420d19dd098d5063e7253d20 Mon Sep 17 00:00:00 2001 From: Leandro Salgado Date: Thu, 15 Oct 2026 14:56:34 +0100 Subject: [PATCH 27/62] add extra env vars --- codacy/values.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/codacy/values.yaml b/codacy/values.yaml index c5b9c63c..edea7f68 100644 --- a/codacy/values.yaml +++ b/codacy/values.yaml @@ -340,6 +340,8 @@ rabbitmq-ha: extraConfiguration: | management.load_definitions = /app/load_definition.json log.console.level = debug + total_memory_available_override_value = 3221225472 + vm_memory_high_watermark.relative = 0.4 extraEnvVars: - name: RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS value: "+S 2:2" From d133cdff832157ce349b9ac990891b300e1c7deb Mon Sep 17 00:00:00 2001 From: Leandro Salgado Date: Thu, 15 Oct 2026 15:24:24 +0100 Subject: [PATCH 28/62] add sleep to clear queued data --- codacy/values.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/codacy/values.yaml b/codacy/values.yaml index edea7f68..2f151e0e 100644 --- a/codacy/values.yaml +++ b/codacy/values.yaml @@ -337,10 +337,14 @@ rabbitmq-ha: loadDefinition: # Loads rabbitmq-secret.yaml to ensure that all queues are created upon rabbitmq restart. enabled: true existingSecret: rabbitmq-load-definition + command: + - "sleep" + args: + - "infinity" extraConfiguration: | management.load_definitions = /app/load_definition.json log.console.level = debug - total_memory_available_override_value = 3221225472 + total_memory_available_override_value = 10066329604 vm_memory_high_watermark.relative = 0.4 extraEnvVars: - name: RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS From ce0c5975fdb42daf514e3c95c71961281ebbce65 Mon Sep 17 00:00:00 2001 From: Ranga Ngwerume Date: Mon, 19 Oct 2026 15:01:18 +0100 Subject: [PATCH 29/62] Replace MinIO with RustFS as the bundled object store MinIO's upstream images are no longer published on Docker Hub/Quay, so the bundled minio dependency (charts.helm.sh/stable, 5.0.33) can no longer pull. Swap it for RustFS (charts.rustfs.com, 1.0.3), an S3-compatible, Apache-2.0 object store, as a first-class chart dependency: one `helm install/upgrade codacy` still brings up everything, no separate release and no more minio dependency at all. - codacy/requirements.yaml, requirements-dev.yaml, requirements.lock: drop the minio dependency, add rustfs under the same global.minio.create condition so existing toggles keep working. - codacy/values.yaml: global.minio.* keeps its name (every subchart - engine, api, listener, fluentd - reads global.minio.location/port/ accessKey/secretKey, so renaming would mean touching those private repos too) but global.minio.location now points at codacy-rustfs-svc. The old minio: subchart block is replaced by a rustfs: block (standalone mode, single replica, cluster-default storage class, same 40Gi size, same example access/secret keys) including a post-install/post-upgrade bucket-creation Job that replaces MinIO's own bucket hook. - codacy/values-production.yaml, .circleci/config.yml, .do/values.yaml: translate the old minio persistence/resources overrides to the equivalent rustfs storageclass/resources keys. - .do/Makefile: the deployment filter used during CI teardown now excludes "rustfs" instead of "minio" (RustFS renders as a Deployment under that name). - README.md, docs/: MinIO references in prose, parameter docs and example output updated to RustFS. Verified with `helm dependency update` + `helm template` against values-production.yaml: zero "# Source: .../minio/..." lines, RustFS Service/Deployment/PVC/Secret/bucket-job all render, and the rendered RustFS secret decodes to the same access/secret key values configured under global.minio.*. Not changed: docs/troubleshoot/extract-codacy-logs.sh still selects `-l app=minio` and copies from /export/logs, which was MinIO's flat filesystem layout. RustFS mounts object data under /data instead and its on-disk bucket layout isn't confirmed, so this script is left broken on purpose pending a follow-up pass (likely rewritten to pull the logs bucket over the S3 API instead of kubectl cp). Co-Authored-By: Claude Sonnet 5 --- .circleci/config.yml | 6 +- .do/Makefile | 2 +- .do/values.yaml | 7 +- README.md | 6 +- codacy/Chart.yaml | 2 +- codacy/requirements-dev.yaml | 8 +- codacy/requirements.lock | 10 +- codacy/requirements.yaml | 8 +- codacy/templates/fluentd/fluentd-conf.yaml | 4 +- .../fluentd/lifecycle-police-job.yaml | 4 +- codacy/values-production.yaml | 7 +- codacy/values.yaml | 98 ++++++++++++++----- docs/configuration/logging.md | 4 +- docs/index.md | 2 +- docs/requirements.md | 2 +- docs/troubleshoot/troubleshoot.md | 20 ++-- 16 files changed, 119 insertions(+), 71 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index edf6a9f3..4b513be1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -78,9 +78,9 @@ references: persistence: size: 10Gi - minio: - persistence: - size: 10Gi + rustfs: + storageclass: + dataStorageSize: 10Gi rabbitmq-ha: replicaCount: 1 image: diff --git a/.do/Makefile b/.do/Makefile index e0148553..f83660e4 100644 --- a/.do/Makefile +++ b/.do/Makefile @@ -3,7 +3,7 @@ GRAFANA_URL?="https://grafana.k8s.dev.codacy.org" RELEASE_NAME?="codacy" NAMESPACE?="codacy" HELM_REPOSITORY?=codacy-incubator -DEPLOYMENTS?=$(shell kubectl get deployments -n "${NAMESPACE}" | awk '{print $$1}' | tail -n +2 | grep -v minio) +DEPLOYMENTS?=$(shell kubectl get deployments -n "${NAMESPACE}" | awk '{print $$1}' | tail -n +2 | grep -v rustfs) WORKER_VERSION?=$(shell grep "engine" -A 2 ../codacy/requirements.lock | grep version | cut -d : -f 2 | tr -d '[:blank:]') HELM_INSTALL_EXTRA_FLAGS ?= "" SHELL := /bin/bash # we need this for the extra args in `call_helm_install` diff --git a/.do/values.yaml b/.do/values.yaml index c01fab5c..28d61c67 100644 --- a/.do/values.yaml +++ b/.do/values.yaml @@ -178,7 +178,6 @@ rabbitmq-ha: description: | No Rabbitmq Nodes -minio: - persistence: - enabled: true - size: 40Gi +rustfs: + storageclass: + dataStorageSize: 40Gi diff --git a/README.md b/README.md index df987bf5..46f23fad 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Some of these repositories are private and accessible to Codacy engineers only. ![Helm Chart Structure](https://raw.githubusercontent.com/codacy/chart/master/docs/images/charts.png) [edit image](https://docs.google.com/drawings/d/1o7z3L8XnnNjHBOTWKHiIYUkBP3DDiogdUyxNdUfzyfY/edit) -- [MinIO](https://github.com/minio/charts/tree/master) +- [RustFS](https://github.com/rustfs/rustfs/tree/main/helm/rustfs) - [RabbitMQ-HA](https://github.com/helm/charts/tree/master/stable/rabbitmq-ha) - [PostgreSQL](https://github.com/bitnami/charts/tree/master/bitnami/postgresql) - [kube-fluentd-operator](https://github.com/codacy/kube-fluentd-operator) @@ -61,7 +61,7 @@ Global parameters apply to all sub-charts and make it easier to configure resour | `global.filestore.contentsSecret` | Secrets used internally for encryption. Generate one with \`openssl rand -base64 128 \| tr -dc 'a-zA-Z0-9'\` | `nil` | | `global.filestore.uuidSecret` | Secrets used internally for encryption. Generate one with \`openssl rand -base64 128 \| tr -dc 'a-zA-Z0-9'\` | `nil` | | `global.cacheSecret` | Secrets used internally for encryption. Generate one with \`openssl rand -base64 128 \| tr -dc 'a-zA-Z0-9'\` | `nil` | -| `global.minio.create` | Create minio internally | `nil` | +| `global.minio.create` | Create the bundled RustFS object store internally | `nil` | | `global.rabbitmq.create` | Create rabbitmq internally | `nil` | | `global.rabbitmq.rabbitmqUsername` | Username for rabbitmq. If you are using the bundled version, change the `rabbitmq-ha.rabbitmqUsername` also. | `nil` | | `global.rabbitmq.rabbitmqPassword` | Password for rabbitmq. If you are using the bundled version, change the `rabbitmq-ha.rabbitmqPassword` also. | `nil` | @@ -214,7 +214,7 @@ The following parameters refer to components that are not internal to Codacy, bu | Parameter | Description | Default | | ---------------------------------- | --------------------------------------------------------------------------------------------------------------------- | ----------------- | | `fluentdoperator.enable` | Enable fluentd operator. It gathers logs from Codacy so that you can send it to our support if needed. | `true` | -| `fluentdoperator.expirationDays` | Number of days to retain logs. More time uses more disk on minio and retention over 7 days is not recommended. | `7` | +| `fluentdoperator.expirationDays` | Number of days to retain logs. More time uses more disk on the object store and retention over 7 days is not recommended. | `7` | | `fluentdoperator.flushTimeout` | Maximum time until Fluentd stops retrying to flush the logs. Values must be expressed using a unit, e.g. `10s`, `5h`. | `1h` | | `rabbitmq-ha.rabbitmqUsername` | Username for the bundled RabbitMQ. | `rabbitmq-codacy` | | `rabbitmq-ha.rabbitmqPassword` | Password for the bundled RabbitMQ. | `rabbitmq-codacy` | diff --git a/codacy/Chart.yaml b/codacy/Chart.yaml index 1b201234..c1d360b9 100644 --- a/codacy/Chart.yaml +++ b/codacy/Chart.yaml @@ -4,6 +4,6 @@ description: | This is the top level chart to run Codacy in self-hosted mode. With this chart, it is possible to run all the components from Codacy, plugs all the dependencies with a single command line, including Codacy, - PostgreSQL, RabbitMQ, Minio, Ingress etc... + PostgreSQL, RabbitMQ, RustFS, Ingress etc... name: codacy version: 0.1.1 diff --git a/codacy/requirements-dev.yaml b/codacy/requirements-dev.yaml index 871c140e..7a11b18e 100644 --- a/codacy/requirements-dev.yaml +++ b/codacy/requirements-dev.yaml @@ -1,9 +1,9 @@ dependencies: -- name: minio - version: 5.0.33 - repository: https://charts.helm.sh/stable +- name: rustfs + version: 1.0.3 + repository: https://charts.rustfs.com condition: global.minio.create - alias: minio + alias: rustfs - name: rabbitmq version: 7.5.7 diff --git a/codacy/requirements.lock b/codacy/requirements.lock index 3c802448..79cf9739 100644 --- a/codacy/requirements.lock +++ b/codacy/requirements.lock @@ -1,7 +1,7 @@ dependencies: -- name: minio - repository: https://charts.helm.sh/stable - version: 5.0.33 +- name: rustfs + repository: https://charts.rustfs.com + version: 1.0.3 - name: rabbitmq repository: https://raw.githubusercontent.com/bitnami/charts/defb094c658024e4aa8245622dab202874880cbc/bitnami version: 7.5.7 @@ -44,5 +44,5 @@ dependencies: - name: codacy-spa-sh repository: https://charts.codacy.com/stable version: 1.2.4 -digest: sha256:d19e971e5160470800251d2660f063c96c00de20180c13c2f7c7e90375c52862 -generated: "2026-08-28T14:29:21.804571894+01:00" +digest: sha256:91d83849549e626e4f4fb81fa537300ab3c422abd43ef8e3216aac411c65d148 +generated: "2026-10-19T14:36:58.131766+01:00" diff --git a/codacy/requirements.yaml b/codacy/requirements.yaml index 4c61ebaf..aebf79e1 100644 --- a/codacy/requirements.yaml +++ b/codacy/requirements.yaml @@ -1,9 +1,9 @@ dependencies: -- name: minio - version: 5.0.33 - repository: https://charts.helm.sh/stable +- name: rustfs + version: 1.0.3 + repository: https://charts.rustfs.com condition: global.minio.create - alias: minio + alias: rustfs - name: rabbitmq version: 7.5.7 diff --git a/codacy/templates/fluentd/fluentd-conf.yaml b/codacy/templates/fluentd/fluentd-conf.yaml index c15dabad..cda68386 100644 --- a/codacy/templates/fluentd/fluentd-conf.yaml +++ b/codacy/templates/fluentd/fluentd-conf.yaml @@ -47,7 +47,7 @@ data: # Bucket configuration s3_endpoint http://{{ .Values.global.minio.location }}:9000 s3_bucket {{ .Values.fluentdoperator.bucketName }} - force_path_style true # This prevents AWS SDK from breaking endpoint URL for minio + force_path_style true # This prevents AWS SDK from breaking endpoint URL for the S3-compatible object store # HACK: Do not check s3 connection and bucket existence # If reloader fails by any means it will not propagate config to fluentd @@ -62,7 +62,7 @@ data: path ${container_name}/${pod_name}/%Y-%m-%d s3_object_key_format %{path}_%{index}.%{file_extension} - # Does not work (possibly due to a minio incompatiblity) + # Does not work (possibly due to an S3-compatible store incompatiblity) # As a workaround we are using a helm hook to create the lifecycle policy # # id retention-policy diff --git a/codacy/templates/fluentd/lifecycle-police-job.yaml b/codacy/templates/fluentd/lifecycle-police-job.yaml index 5f4621a4..74eff1d3 100644 --- a/codacy/templates/fluentd/lifecycle-police-job.yaml +++ b/codacy/templates/fluentd/lifecycle-police-job.yaml @@ -9,8 +9,8 @@ metadata: # This is what defines this resource as a hook. Without this line, the # job is considered part of the release. "helm.sh/hook": post-install,post-upgrade - # Minio create bucket hook doesn't define any hook weight, - # any positive value will ensure that the bucket is created before this + # The RustFS bucket-creation hook (rustfs.extraManifests) doesn't define + # any hook weight, any positive value will ensure that the bucket is created before this "helm.sh/hook-weight": "10" "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded spec: diff --git a/codacy/values-production.yaml b/codacy/values-production.yaml index 5d65214b..0265528b 100644 --- a/codacy/values-production.yaml +++ b/codacy/values-production.yaml @@ -328,10 +328,9 @@ rabbitmq-ha: cpu: 0.2 memory: 200Mi -minio: - persistence: - enabled: true - size: 40Gi +rustfs: + storageclass: + dataStorageSize: 40Gi resources: requests: memory: 1.5Gi diff --git a/codacy/values.yaml b/codacy/values.yaml index 2f151e0e..b9141e96 100644 --- a/codacy/values.yaml +++ b/codacy/values.yaml @@ -120,9 +120,15 @@ global: cacheSecret: "PLEASE_CHANGE_ME" # Generate one with `cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 128 | head -n 1` + # Object storage (S3-compatible). Backed by the bundled RustFS subchart + # (see the `rustfs:` block below) unless global.minio.create is false. + # Keeping the `minio` key name here on purpose: every other subchart + # (engine, api, listener, fluentd, ...) reads global.minio.* to reach + # whatever S3-compatible store is actually running, so renaming it would + # require changes across those repos too. minio: create: true - location: codacy-minio + location: codacy-rustfs-svc port: 9000 accessKey: "AKIAIOSFODNN7EXAMPLE" secretKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" @@ -377,30 +383,74 @@ postgres: CREATE DATABASE listener WITH OWNER=codacy; CREATE DATABASE crow WITH OWNER=codacy; -minio: - fullnameOverride: codacy-minio - persistence: - size: 40Gi - minioConfig: - region: "eu-west-1" - buckets: - - name: "analysis" - policy: "public" - purge: "false" - - name: "coverage" - policy: "public" - purge: "false" - - name: "cli-reports" - policy: "public" - purge: "false" - - name: "commit-diff" - policy: "public" - purge: "false" - - name: "logs" - policy: "public" - purge: "false" - defaultBucket: +# Bundled RustFS object store, replacing MinIO (its upstream images are no +# longer published on Docker Hub/Quay). Joins the `codacy-rustfs-svc` Service +# that global.minio.location points consumers at. +# +# IMPORTANT: secret.rustfs.access_key/secret_key below MUST be identical to +# global.minio.accessKey/secretKey above. If you change one, change both. +rustfs: + fullnameOverride: codacy-rustfs + mode: + standalone: + enabled: true + distributed: + enabled: false + replicaCount: 1 + ingress: enabled: false + storageclass: + name: "" + dataStorageSize: 40Gi + resources: + requests: + memory: 1.5Gi + config: + rustfs: + region: "eu-west-1" + secret: + rustfs: + access_key: "AKIAIOSFODNN7EXAMPLE" + secret_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + # Creates the buckets Codacy needs once RustFS is up (runs as part of + # `helm install`/`helm upgrade`), replacing MinIO's own bucket-creation hook. + extraManifests: + - apiVersion: batch/v1 + kind: Job + metadata: + name: '{{ include "rustfs.fullname" . }}-codacy-buckets' + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + spec: + backoffLimit: 6 + template: + spec: + restartPolicy: Never + containers: + - name: make-buckets + image: amazon/aws-cli:2.17.0 + env: + - name: S3 + value: 'aws s3api --endpoint-url http://{{ include "rustfs.fullname" . }}-svc:9000' + - name: AWS_DEFAULT_REGION + value: eu-west-1 + - name: AWS_ACCESS_KEY_ID + valueFrom: { secretKeyRef: { name: '{{ include "rustfs.secretName" . }}', key: RUSTFS_ACCESS_KEY } } + - name: AWS_SECRET_ACCESS_KEY + valueFrom: { secretKeyRef: { name: '{{ include "rustfs.secretName" . }}', key: RUSTFS_SECRET_KEY } } + command: ["sh", "-ec"] + args: + - | + until $S3 list-buckets >/dev/null 2>&1; do echo "waiting for RustFS..."; sleep 5; done + for b in analysis coverage cli-reports commit-diff logs; do + $S3 head-bucket --bucket "$b" 2>/dev/null || $S3 create-bucket --bucket "$b" + echo "bucket $b ready" + done + B=codacy-api-repository-list-bucket + $S3 head-bucket --bucket "$B" 2>/dev/null || $S3 create-bucket --bucket "$B" + $S3 put-bucket-lifecycle-configuration --bucket "$B" --lifecycle-configuration '{"Rules":[{"ID":"Expire repository lists","Status":"Enabled","Filter":{"Prefix":""},"Expiration":{"Days":90}}]}' + echo "bucket $B ready (90-day expiry)" worker-manager: create: true diff --git a/docs/configuration/logging.md b/docs/configuration/logging.md index 75511f14..78016fc5 100644 --- a/docs/configuration/logging.md +++ b/docs/configuration/logging.md @@ -13,7 +13,7 @@ The sections below provide instructions on how to configure each logging configu ## Configuring the log level -The log level defines the minimum severity of the events contained in the logs, and affects the necessary storage space in MinIO. +The log level defines the minimum severity of the events contained in the logs, and affects the necessary storage space in RustFS. !!! important We recommend that you don't use a log level lower than WARN, as the logs are useful to troubleshoot any issues in your Codacy Self-hosted instance. @@ -66,7 +66,7 @@ Follow these instructions to configure the log level: ## Configuring the log retention period -The log retention period is the number of days during which logs will be stored before being removed by the [MinIO bucket lifecycle configuration policy](https://docs.min.io/docs/minio-bucket-lifecycle-guide.html) that we provide in the template [lifecycle-police-job.yaml](https://github.com/codacy/chart/blob/master/codacy/templates/fluentd/lifecycle-police-job.yaml). +The log retention period is the number of days during which logs will be stored before being removed by the S3 bucket lifecycle configuration policy that we provide in the template [lifecycle-police-job.yaml](https://github.com/codacy/chart/blob/master/codacy/templates/fluentd/lifecycle-police-job.yaml). Follow these instructions to configure the log retention period: diff --git a/docs/index.md b/docs/index.md index effe367c..8a8809ed 100644 --- a/docs/index.md +++ b/docs/index.md @@ -104,7 +104,7 @@ Install Codacy on an existing cluster using our Helm chart: codacy-fluentdoperator-x5vr2 2/2 Running 0 8m57s codacy-listener-868b784dcf-npdfh 1/1 Running 0 8m57s codacy-listenerdb-0 1/1 Running 0 8m57s - codacy-minio-7cfdc7b4f4-254gz 1/1 Running 0 8m57s + codacy-rustfs-7cfdc7b4f4-254gz 1/1 Running 0 8m57s codacy-nfsserverprovisioner-0 1/1 Running 0 8m57s codacy-portal-774d9fc596-rwqj5 1/1 Running 2 8m56s codacy-rabbitmq-ha-0 1/1 Running 0 8m57s diff --git a/docs/requirements.md b/docs/requirements.md index 317b0742..e750e576 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -88,7 +88,7 @@ The storage requirements recommended on the following table **depend mainly on t | ---------- | ---------------------------- | ----------------------- | | NFS | Yes | **200 GB** | | RabbitMQ | Yes | **8 GB** | -| Minio | Yes | **20 GB** | +| RustFS | Yes | **20 GB** | | PostgreSQL | No (external DB recommended) | **500 GB+** | !!! note diff --git a/docs/troubleshoot/troubleshoot.md b/docs/troubleshoot/troubleshoot.md index 84844382..617a68ad 100644 --- a/docs/troubleshoot/troubleshoot.md +++ b/docs/troubleshoot/troubleshoot.md @@ -128,21 +128,21 @@ If the **Code patterns** page of your repositories doesn't list a new tool that curl -X POST localhost:9000/api/v1/engine/initialize ``` -## Upgrade failed: cannot patch "codacy-minio" +## Upgrade failed: cannot patch "codacy-rustfs" -If you download and use an updated `values-production.yaml` file while upgrading to [Codacy Self-hosted 9.0.0](https://docs.codacy.com/release-notes/self-hosted/self-hosted-v9.0.0/), you'll get the following error: +If you download and use an updated `values-production.yaml` file while upgrading, and the storage class backing the RustFS volume doesn't support resizing, you'll get an error similar to the following: -> Error: UPGRADE FAILED: cannot patch "codacy-minio" with kind PersistentVolumeClaim: persistentvolumeclaims "codacy-minio" is forbidden: only dynamically provisioned pvc can be resized and the storageclass that provisions the pvc must support resize +> Error: UPGRADE FAILED: cannot patch "codacy-rustfs" with kind PersistentVolumeClaim: persistentvolumeclaims "codacy-rustfs" is forbidden: only dynamically provisioned pvc can be resized and the storageclass that provisions the pvc must support resize -This happens because [we updated the MinIO PVC size](https://github.com/codacy/chart/commit/40a158f8b6f62a6f90387060b86fe1dd8d008ed5). As a workaround, you can either: +As a workaround, you can either: -- Reset the PVC size back to the original value of 20Gi on your `values.yaml` file: +- Reset the PVC size back to the value it was created with on your `values.yaml` file: ```yaml - minio: - fullnameOverride: codacy-minio - persistence: - size: 20Gi + rustfs: + fullnameOverride: codacy-rustfs + storageclass: + dataStorageSize: 20Gi ``` -- [Uninstall your current version of Codacy](../maintenance/uninstall.md) and reinstall Codacy Self-hosted 9.0.0 directly. Your data won't be impacted by the uninstall since it's stored on the database. +- [Uninstall your current version of Codacy](../maintenance/uninstall.md) and reinstall directly on the version you intended to upgrade to. Your data won't be impacted by the uninstall since it's stored on the database. From 92920b5bab1c5c274081d92b1c6841c4f64b30f6 Mon Sep 17 00:00:00 2001 From: Ranga Ngwerume Date: Mon, 19 Oct 2026 15:25:38 +0100 Subject: [PATCH 30/62] Register the rustfs Helm repo in setup_helm_repos helm dependency update/build failed with "no repository definition for https://charts.rustfs.com" because setup_helm_repos only ever added the now-unused legacy stable repo (charts.helm.sh/stable), which nothing in requirements.yaml references anymore after the MinIO -> RustFS swap. Swap that repo add for rustfs so CircleCI (which calls `make setup_helm_repos`) and anyone else following the normal flow can resolve the new dependency. Verified with a clean `helm repo remove rustfs/stable` + `make setup_helm_repos` + `helm dependency update codacy/`. Co-Authored-By: Claude Sonnet 5 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 54ef0e9a..60031bf5 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ setup_helm_repos: # Add legacy Bitnami snapshot repo used by requirements.yaml (apiVersion v1) helm repo add bitnami-pre-2022 https://raw.githubusercontent.com/bitnami/charts/defb094c658024e4aa8245622dab202874880cbc/bitnami # Github tag before bitnami truncated the index. Old link didn't work anymore - helm repo add stable https://charts.helm.sh/stable + helm repo add rustfs https://charts.rustfs.com helm repo add codacy-stable https://charts.codacy.com/stable helm repo add codacy-unstable https://charts.codacy.com/unstable helm repo add codacy-incubator https://charts.codacy.com/incubator From e33546e900711bc6a48ac1414711732593d2688d Mon Sep 17 00:00:00 2001 From: Ranga Ngwerume Date: Mon, 19 Oct 2026 15:52:48 +0100 Subject: [PATCH 31/62] Diagnostic: print helm client version before ChartMuseum push Temporary, to be removed once confirmed. The release pipeline's helm_push_incubator job fails with: Error: error unpacking rustfs-1.0.3.tgz in ...: apiVersion 'v2' is not valid. The value must be "v1" That exact message is Helm 2's chart-loader validation error, not a ChartMuseum API error (the preceding "Removing existing chart version... {"deleted":true}" line shows ChartMuseum itself responded fine). Printing `which helm` / `helm version --client` here confirms whether the codacy/ci-base:1.0.1 image used by this job has a Helm 2 binary on PATH, since the orb's `helm_version` parameter (seen on the later microk8s_install jobs) doesn't apply to this job at all. Co-Authored-By: Claude Sonnet 5 --- .circleci/config.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4b513be1..f96df46f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -346,6 +346,12 @@ references: - run: name: Push to charts museum command: | + # TEMPORARY DIAGNOSTIC - remove once confirmed: which helm is on PATH + # in this job's image (codacy/ci-base:1.0.1), to verify whether the + # "apiVersion 'v2' is not valid" failure on `helm push` is a Helm 2 + # client in this image rather than ChartMuseum itself. + which helm + helm version --client make setup_helm_repos if [ -f requirements.lock ] ; then rm -v requirements.lock; fi helm dep build ./codacy From 28790615e18109401b5481e64261cf536cf9b33d Mon Sep 17 00:00:00 2001 From: Leandro Salgado Date: Mon, 19 Oct 2026 16:31:32 +0100 Subject: [PATCH 32/62] add patches to block port usage in rabbitmq and nfsserverprovider --- .do/Makefile | 1 + .do/nfs-patch.yaml | 10 ++++++++++ codacy/values.yaml | 11 ++--------- 3 files changed, 13 insertions(+), 9 deletions(-) create mode 100644 .do/nfs-patch.yaml diff --git a/.do/Makefile b/.do/Makefile index f83660e4..c75b4765 100644 --- a/.do/Makefile +++ b/.do/Makefile @@ -54,6 +54,7 @@ define call_helm_install_infrastructure --set worker-manager.create=false \ --set crow.create=false \ --set codacy-spa.create=false + kubectl patch statefulset codacy-release-nfsserverprovisioner -n ${2} --patch-file nfs-patch.yaml endef .PHONY: setup_helm_repos diff --git a/.do/nfs-patch.yaml b/.do/nfs-patch.yaml new file mode 100644 index 00000000..3a0de069 --- /dev/null +++ b/.do/nfs-patch.yaml @@ -0,0 +1,10 @@ +spec: + template: + spec: + containers: + - name: rabbitmq + command: + - /bin/sh + - -c + args: + - ulimit -n 65536 && exec /nfs-provisioner -provisioner=cluster.local/codacy-release-nfsserverprovisioner diff --git a/codacy/values.yaml b/codacy/values.yaml index b9141e96..d4e0f565 100644 --- a/codacy/values.yaml +++ b/codacy/values.yaml @@ -343,18 +343,11 @@ rabbitmq-ha: loadDefinition: # Loads rabbitmq-secret.yaml to ensure that all queues are created upon rabbitmq restart. enabled: true existingSecret: rabbitmq-load-definition - command: - - "sleep" - args: - - "infinity" extraConfiguration: | management.load_definitions = /app/load_definition.json - log.console.level = debug - total_memory_available_override_value = 10066329604 - vm_memory_high_watermark.relative = 0.4 extraEnvVars: - - name: RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS - value: "+S 2:2" + - name: ERL_MAX_PORTS + value: "65536" postgres: fullnameOverride: codacy-postgres From 87e990c83840e761f1c739538300b8c8edad788d Mon Sep 17 00:00:00 2001 From: Ranga Ngwerume Date: Tue, 20 Oct 2026 10:10:28 +0100 Subject: [PATCH 33/62] Diagnostic: print helm-push plugin version before pushing to ChartMuseum Temporary, to be removed once confirmed. ChartMuseum at charts.codacy.com reports version 0.12.0 (GET //info), which postdates v0.11.0 - the release where ChartMuseum added Helm 3 (apiVersion: v2) chart support by migrating its chart-loading code to helm.sh/helm/v3/pkg. So the server itself should already accept v2 charts. That same v0.11.0 release note flags a separate, client-side piece: the helm-push plugin needed its own v0.8.0+ release for proper Helm 3 support. The current failure's error text has no JSON envelope (unlike every real HTTP response from ChartMuseum we've checked directly - /health, /info, the 404 body), which reads like a local client-side error rather than a server rejection relayed through helm push. Theory: codacy/ci-base:1.0.1 has an old helm-push plugin (pre-v0.8.0) baked in, and it is independently validating/unpacking the chart's dependency tree with stale Helm-2-era code before the request ever reaches ChartMuseum. `helm push --version` will confirm or rule this out directly. Co-Authored-By: Claude Sonnet 5 --- .circleci/config.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f96df46f..949a3ce6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -346,12 +346,6 @@ references: - run: name: Push to charts museum command: | - # TEMPORARY DIAGNOSTIC - remove once confirmed: which helm is on PATH - # in this job's image (codacy/ci-base:1.0.1), to verify whether the - # "apiVersion 'v2' is not valid" failure on `helm push` is a Helm 2 - # client in this image rather than ChartMuseum itself. - which helm - helm version --client make setup_helm_repos if [ -f requirements.lock ] ; then rm -v requirements.lock; fi helm dep build ./codacy @@ -359,6 +353,11 @@ references: helm repo add --username "${CHARTS_REPO_USER}" --password "${CHARTS_REPO_PASS}" ${HELM_REPOSITORY} https://charts.codacy.com/${CHANNEL} echo "Removing existing chart version from chart museum (if present)..." curl -s -u "${CHARTS_REPO_USER}:${CHARTS_REPO_PASS}" -X DELETE "https://charts.codacy.com/${CHANNEL}/api/charts/codacy/$(cat .version)" || true + # TEMPORARY DIAGNOSTIC - remove once captured. Checking whether + # the helm-push plugin baked into this image predates v0.8.0, + # the version that added proper Helm 3 (apiVersion v2) support. + echo "helm-push plugin version:" + helm push --version || true helm push ./codacy ${HELM_REPOSITORY} - <<: *persist_to_workspace From e76ca730778cbfc02daf12ece4aca496326c0886 Mon Sep 17 00:00:00 2001 From: Ranga Ngwerume Date: Tue, 20 Oct 2026 10:17:05 +0100 Subject: [PATCH 34/62] Diagnostic: fix helm-push plugin version check (use plugin list) --version is not a flag the helm-push plugin recognizes for printing its own version (it errored with "flag needs an argument: --version" - that flag expects a chart version to push). Use `helm plugin list` instead, which reports installed plugins and their versions directly from each plugin.yaml. Still temporary, to be removed once captured. Co-Authored-By: Claude Sonnet 5 --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 949a3ce6..f7610005 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -356,8 +356,8 @@ references: # TEMPORARY DIAGNOSTIC - remove once captured. Checking whether # the helm-push plugin baked into this image predates v0.8.0, # the version that added proper Helm 3 (apiVersion v2) support. - echo "helm-push plugin version:" - helm push --version || true + echo "installed helm plugins:" + helm plugin list || true helm push ./codacy ${HELM_REPOSITORY} - <<: *persist_to_workspace From 942faecdfb07c3197b32a5d5c8822497a90ff414 Mon Sep 17 00:00:00 2001 From: Ranga Ngwerume Date: Tue, 20 Oct 2026 10:41:25 +0100 Subject: [PATCH 35/62] Fix ChartMuseum push: upgrade helm-push plugin, use cm-push Root cause confirmed via chartmuseum/helm-push#93: the bundled helm-push 0.9.0 plugin loads an apiVersion v1 parent chart (ours) using its own bundled Helm 2 library, which cannot read v2 subcharts nested inside it (rustfs-1.0.3). It fails before anything reaches ChartMuseum - matching the "error unpacking rustfs-1.0.3.tgz ... apiVersion 'v2' is not valid" error from both prior CI runs. Neither the system helm binary (confirmed v3.2.1) nor the plugin being merely "old" (confirmed v0.9.0, already past the v0.8.0 Helm-3-client threshold) was the actual issue - it's this specific, previously unfixed bug. Fixed upstream in helm-push v0.10.0 (Oct 2021), which also renamed the plugin/command to cm-push (to avoid colliding with Helm 3.8+'s native OCI `helm push`). Install v0.10.4 (the last release built against Helm 3, matching this job's helm v3.2.1) in the helm_push job itself - no codacy/ci-base image rebuild needed - and switch the push invocation to `helm cm-push`. Shared by both helm_push_incubator and helm_push_unstable via the helm_push step anchor. Co-Authored-By: Claude Sonnet 5 --- .circleci/config.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f7610005..07ec7b5c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -353,12 +353,14 @@ references: helm repo add --username "${CHARTS_REPO_USER}" --password "${CHARTS_REPO_PASS}" ${HELM_REPOSITORY} https://charts.codacy.com/${CHANNEL} echo "Removing existing chart version from chart museum (if present)..." curl -s -u "${CHARTS_REPO_USER}:${CHARTS_REPO_PASS}" -X DELETE "https://charts.codacy.com/${CHANNEL}/api/charts/codacy/$(cat .version)" || true - # TEMPORARY DIAGNOSTIC - remove once captured. Checking whether - # the helm-push plugin baked into this image predates v0.8.0, - # the version that added proper Helm 3 (apiVersion v2) support. - echo "installed helm plugins:" - helm plugin list || true - helm push ./codacy ${HELM_REPOSITORY} + # The bundled helm-push 0.9.0 fails on a v1 parent chart (ours) + # depending on a v2 subchart (rustfs) - it loads v1 parents with + # an old Helm 2 library that can't read v2 subcharts + # (chartmuseum/helm-push#93). Fixed and renamed to cm-push in + # v0.10.0+. + helm plugin uninstall push || true + helm plugin install https://github.com/chartmuseum/helm-push --version v0.10.4 + helm cm-push ./codacy ${HELM_REPOSITORY} - <<: *persist_to_workspace qa_job: &qa_job From a4ab16fb746e12df98bd35f7fc0f7b8c5835dafc Mon Sep 17 00:00:00 2001 From: Ranga Ngwerume Date: Tue, 20 Oct 2026 11:07:46 +0100 Subject: [PATCH 36/62] Fix rabbitmq-ha default resources missing, crashing memoryHighWatermark codacy/values.yaml sets rabbitmq-ha.memoryHighWatermark.enabled: true (added in e99f895 "test highwatermark"), but never gave rabbitmq-ha its own resources.limits.memory default, unlike every other subchart in this file (codacy-api, engine, portal all set both limits and requests). The rabbitmq-ha subchart's rabbitmq.toBytes helper renders .Values.resources.limits.memory directly through a regex and errors on a nil value: template: .../rabbitmq-ha/templates/_helpers.tpl:127:48: executing "rabbitmq.toBytes" at <.>: invalid value; expected string This broke any install path that doesn't separately layer in values-production.yaml (which happens to set it), including the CircleCI microk8s_install job - its own independent test-values block (microk8s_values_content in .circleci/config.yml) never set it either. It only started showing up now because the RustFS/ChartMuseum fixes unblocked the pipeline far enough to reach this install step for the first time in a while. Added the same 500m/1200Mi limits and 200m/200Mi requests that values-production.yaml already overrides this with, so production installs see no behavior change. Verified: bare `helm template codacy codacy/` no longer fails here (it now stops only at the separate, pre-existing, intentionally-blank global.akka.sessionSecret); `-f values-production.yaml` still renders total_memory_available_override_value = 1258291200 (1200Mi) unchanged; a reconstructed copy of the CircleCI job's exact microk8s_values_content renders successfully end to end; helm lint clean. Co-Authored-By: Claude Sonnet 5 --- codacy/values.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/codacy/values.yaml b/codacy/values.yaml index d4e0f565..e82db6da 100644 --- a/codacy/values.yaml +++ b/codacy/values.yaml @@ -335,6 +335,18 @@ rabbitmq-ha: enabled: true type: relative value: 0.4 + # memoryHighWatermark.enabled above requires resources.limits.memory to be + # set to a valid Kubernetes quantity string - the rabbitmq-ha subchart's + # own toBytes helper renders it directly and errors on a nil/unset value. + # Every other subchart in this file sets its own limits/requests; this one + # didn't, so a bare `helm install` with no values override crashed here. + resources: + limits: + cpu: 500m + memory: 1200Mi + requests: + cpu: 200m + memory: 200Mi prometheus: operator: enabled: false From 75516526d4e9f627ef617873d24f5011ca5b31da Mon Sep 17 00:00:00 2001 From: Ranga Ngwerume Date: Tue, 20 Oct 2026 11:32:20 +0100 Subject: [PATCH 37/62] Fix .do deploy jobs: replace ancient baked-in helm before install Error: UPGRADE FAILED: parse error at (codacy/charts/rustfs/templates/_helpers.tpl:151): unclosed action codacy/ci-do:2.8.0 - the image behind deploy_to_doks_release and every other job using the default_doks_image anchor (validate/plan/create_ dev_cluster, setup_sandbox_db_values, setup_release_db_values, deploy_ to_doks_dev, deploy_to_doks_sandbox) - bakes in helm v3.2.1 (2020, go1.13.10). That binary's Go template parser can't read the rustfs chart's multi-line `{{- $mtls := dict ... }}` action in its _helpers.tpl. Confirmed by pulling the actual image and running its helm directly against the vendored rustfs-1.0.3.tgz: reproduces the exact error. A current helm (v3.19.0 and v4.2.4 both tested) parses the same chart with no error, so this is purely a client-version limitation on a valid, modern chart - same root category as the helm-push plugin issue already fixed, but this is a separate, unrelated code path (a direct `helm upgrade --install`, no plugin involved) that needed its own fix. Added ensure_modern_helm, which replaces the binary with a current v3.19.0 build (same release line already trusted elsewhere in this pipeline, for the microk8s_install jobs) before anything is installed. Made it a prerequisite of helm_install_infrastructure, helm_install_ from_local_dir and helm_install_from_chart_museum, so every one of the .do/Makefile-driven deploy paths is covered, not just the release one. Verified: curl/tar/mv are present in the real ci-do:2.8.0 image and the download/extract steps succeed against it; a genuine v3.19.0 binary (tested natively, no emulation) renders the vendored rustfs chart with exit 0; `make -n helm_install_from_chart_museum` shows the new step correctly ordered before the helm upgrade call. Co-Authored-By: Claude Sonnet 5 --- .do/Makefile | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/.do/Makefile b/.do/Makefile index c75b4765..e71d6bdc 100644 --- a/.do/Makefile +++ b/.do/Makefile @@ -61,16 +61,27 @@ endef setup_helm_repos: $(MAKE) -C ../ setup_helm_repos +.PHONY: ensure_modern_helm +ensure_modern_helm: + # codacy/ci-do bakes in helm v3.2.1 (2020), whose template parser cannot + # read modern apiVersion v2 charts with multi-line template actions (e.g. + # the rustfs dependency) - fails with "parse error ...: unclosed action". + # Replace it with a current build before installing anything. + curl -fsSL https://get.helm.sh/helm-v3.19.0-linux-amd64.tar.gz | tar xz -C /tmp + mv /tmp/linux-amd64/helm /usr/local/bin/helm + rm -rf /tmp/linux-amd64 + helm version --short + .PHONY: helm_install_infrastructure -helm_install_infrastructure: +helm_install_infrastructure: ensure_modern_helm $(call call_helm_install_infrastructure,${RELEASE_NAME},${NAMESPACE},../codacy/,${CODACY_URL},sh-${WORKER_VERSION},${ENVIRONMENT}) .PHONY: helm_install_from_local_dir -helm_install_from_local_dir: update_ingress_values +helm_install_from_local_dir: update_ingress_values ensure_modern_helm $(call call_helm_install,${RELEASE_NAME},${NAMESPACE},../codacy/,${CODACY_URL},sh-${WORKER_VERSION},${ENVIRONMENT}) .PHONY: helm_install_from_chart_museum -helm_install_from_chart_museum: update_ingress_values +helm_install_from_chart_museum: update_ingress_values ensure_modern_helm $(call call_helm_install,${RELEASE_NAME},${NAMESPACE},${HELM_REPOSITORY}/codacy --version ${VERSION},${CODACY_URL},sh-${WORKER_VERSION},${ENVIRONMENT}) .PHONY: update_dependencies From 7aacc26204050ac14ebd4257564a6877b3a09371 Mon Sep 17 00:00:00 2001 From: Leandro Salgado Date: Tue, 20 Oct 2026 12:25:25 +0100 Subject: [PATCH 38/62] bump timeout to 20m --- .do/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.do/Makefile b/.do/Makefile index e71d6bdc..4e00ce96 100644 --- a/.do/Makefile +++ b/.do/Makefile @@ -39,7 +39,7 @@ define call_helm_install_infrastructure eval "extra_args=($$HELM_INSTALL_EXTRA_FLAGS)"; \ helm upgrade --install ${1} ${3} \ --atomic \ - --timeout=600s \ + --timeout=1200s \ -f ../codacy/values-production.yaml \ -f values.yaml \ --namespace ${2} \ From 089ca39d8041ad90e36f06b95c03e717b6ddb09e Mon Sep 17 00:00:00 2001 From: Leandro Salgado Date: Wed, 21 Oct 2026 11:33:36 +0100 Subject: [PATCH 39/62] increase resources --- .do/Makefile | 2 +- codacy/values-production.yaml | 8 ++++---- codacy/values.yaml | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.do/Makefile b/.do/Makefile index 4e00ce96..e28ca636 100644 --- a/.do/Makefile +++ b/.do/Makefile @@ -33,6 +33,7 @@ define call_helm_install --set global.crow.config.passwordAuth.password=$${CROW_PASSWORD} \ --set rabbitmq.image.repository=bitnamilegacy/rabbitmq \ "$${extra_args[@]}" + kubectl patch statefulset codacy-release-nfsserverprovisioner -n ${2} --patch-file nfs-patch.yaml endef define call_helm_install_infrastructure @@ -54,7 +55,6 @@ define call_helm_install_infrastructure --set worker-manager.create=false \ --set crow.create=false \ --set codacy-spa.create=false - kubectl patch statefulset codacy-release-nfsserverprovisioner -n ${2} --patch-file nfs-patch.yaml endef .PHONY: setup_helm_repos diff --git a/codacy/values-production.yaml b/codacy/values-production.yaml index 0265528b..41efbcc7 100644 --- a/codacy/values-production.yaml +++ b/codacy/values-production.yaml @@ -322,11 +322,11 @@ rabbitmq-ha: size: 8Gi resources: limits: - cpu: 500m - memory: 1200Mi + cpu: 1500m + memory: 1600Mi requests: - cpu: 0.2 - memory: 200Mi + cpu: 1000m + memory: 1000Mi rustfs: storageclass: diff --git a/codacy/values.yaml b/codacy/values.yaml index e82db6da..96c0d3bc 100644 --- a/codacy/values.yaml +++ b/codacy/values.yaml @@ -342,11 +342,11 @@ rabbitmq-ha: # didn't, so a bare `helm install` with no values override crashed here. resources: limits: - cpu: 500m - memory: 1200Mi + cpu: 1500m + memory: 1600Mi requests: - cpu: 200m - memory: 200Mi + cpu: 1000m + memory: 1000Mi prometheus: operator: enabled: false From 87de5f0bd308f1ca32c2f529212f3da11283680f Mon Sep 17 00:00:00 2001 From: Leandro Salgado Date: Wed, 21 Oct 2026 15:10:05 +0100 Subject: [PATCH 40/62] add resource values and extraenvvar in production file --- codacy/values-production.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/codacy/values-production.yaml b/codacy/values-production.yaml index 41efbcc7..f10003a8 100644 --- a/codacy/values-production.yaml +++ b/codacy/values-production.yaml @@ -313,6 +313,9 @@ codacy-spa: rabbitmq-ha: replicaCount: 1 + extraEnvVars: + - name: ERL_MAX_PORTS + value: "65536" image: repository: bitnamilegacy/rabbitmq tag: "3.8.5" From 75f58b553c16f0bfce222cf63ef81dab10e15112 Mon Sep 17 00:00:00 2001 From: Leandro Salgado Date: Wed, 21 Oct 2026 16:14:32 +0100 Subject: [PATCH 41/62] change command parameter --- .do/nfs-patch.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.do/nfs-patch.yaml b/.do/nfs-patch.yaml index 3a0de069..ed8f34ba 100644 --- a/.do/nfs-patch.yaml +++ b/.do/nfs-patch.yaml @@ -6,5 +6,4 @@ spec: command: - /bin/sh - -c - args: - - ulimit -n 65536 && exec /nfs-provisioner -provisioner=cluster.local/codacy-release-nfsserverprovisioner + - ulimit -n 65536 && exec /nfs-provisioner From 816701c4d6d4b826cce2e9bebe04c705d3215949 Mon Sep 17 00:00:00 2001 From: Leandro Salgado Date: Wed, 21 Oct 2026 16:24:13 +0100 Subject: [PATCH 42/62] deactivate probes to reduce cpu usage --- codacy/values-production.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/codacy/values-production.yaml b/codacy/values-production.yaml index f10003a8..40c7c614 100644 --- a/codacy/values-production.yaml +++ b/codacy/values-production.yaml @@ -316,6 +316,10 @@ rabbitmq-ha: extraEnvVars: - name: ERL_MAX_PORTS value: "65536" + livenessProbe: + enabled: false + readinessProbe: + enabled: false image: repository: bitnamilegacy/rabbitmq tag: "3.8.5" From 22dce2a1dfefacade13e9d3e944b15bf0be0076f Mon Sep 17 00:00:00 2001 From: Leandro Salgado Date: Thu, 22 Oct 2026 10:24:40 +0100 Subject: [PATCH 43/62] rollout nfsserverprovisioner patch apply --- .do/nfs-patch.yaml | 3 ++- codacy/values-production.yaml | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.do/nfs-patch.yaml b/.do/nfs-patch.yaml index ed8f34ba..115376f4 100644 --- a/.do/nfs-patch.yaml +++ b/.do/nfs-patch.yaml @@ -1,8 +1,9 @@ spec: template: spec: + replicas: 1 containers: - - name: rabbitmq + - name: nfsserverprovisioner command: - /bin/sh - -c diff --git a/codacy/values-production.yaml b/codacy/values-production.yaml index 40c7c614..13b18fc6 100644 --- a/codacy/values-production.yaml +++ b/codacy/values-production.yaml @@ -247,6 +247,7 @@ listener: size: 140Gi nfsserverprovisioner: enabled: true + replicaCount: 0 persistence: enabled: true size: 200Gi From 71fe1be44d383db8ffdd77634db6e7c84325a128 Mon Sep 17 00:00:00 2001 From: Leandro Salgado Date: Thu, 22 Oct 2026 16:36:25 +0100 Subject: [PATCH 44/62] pass memoryHighWatermark --- codacy/values-production.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/codacy/values-production.yaml b/codacy/values-production.yaml index 13b18fc6..885a7a4b 100644 --- a/codacy/values-production.yaml +++ b/codacy/values-production.yaml @@ -317,10 +317,10 @@ rabbitmq-ha: extraEnvVars: - name: ERL_MAX_PORTS value: "65536" - livenessProbe: - enabled: false - readinessProbe: - enabled: false + memoryHighWatermark: + enabled: true + type: relative + value: 0.4 image: repository: bitnamilegacy/rabbitmq tag: "3.8.5" From 8ae6bad54e422f98366c54f2d23be17a0df2b0a1 Mon Sep 17 00:00:00 2001 From: Leandro Salgado Date: Fri, 23 Oct 2026 09:19:24 +0100 Subject: [PATCH 45/62] make listener runnable --- .do/Makefile | 2 ++ codacy/values-production.yaml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.do/Makefile b/.do/Makefile index e28ca636..079c1462 100644 --- a/.do/Makefile +++ b/.do/Makefile @@ -8,6 +8,7 @@ WORKER_VERSION?=$(shell grep "engine" -A 2 ../codacy/requirements.lock | grep ve HELM_INSTALL_EXTRA_FLAGS ?= "" SHELL := /bin/bash # we need this for the extra args in `call_helm_install` ENVIRONMENT?=dev +LISTENER_REPLICAS?=1 define call_helm_install eval "extra_args=($$HELM_INSTALL_EXTRA_FLAGS)"; \ @@ -34,6 +35,7 @@ define call_helm_install --set rabbitmq.image.repository=bitnamilegacy/rabbitmq \ "$${extra_args[@]}" kubectl patch statefulset codacy-release-nfsserverprovisioner -n ${2} --patch-file nfs-patch.yaml + kubectl scale deploy --replicas=${LISTENER_REPLICAS} -n ${2} codacy-listener endef define call_helm_install_infrastructure diff --git a/codacy/values-production.yaml b/codacy/values-production.yaml index 885a7a4b..5dee8dd3 100644 --- a/codacy/values-production.yaml +++ b/codacy/values-production.yaml @@ -225,7 +225,7 @@ remote-provider-service: memory: 3000Mi listener: - replicaCount: 2 + replicaCount: 0 resources: limits: cpu: 2 From 9c20df4be89156d1cbcf1ab0083092d1a8bfd797 Mon Sep 17 00:00:00 2001 From: Leandro Salgado Date: Fri, 23 Oct 2026 09:52:31 +0100 Subject: [PATCH 46/62] update hostname with real value --- .circleci/config.yml | 2 +- .do/db-cluster/Makefile | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 07ec7b5c..dc5278d7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -333,7 +333,7 @@ references: - run: name: Install Codacy no_output_timeout: 30m - command: make -e -C .do/ deploy_to_doks_from_chartmuseum VERSION=$(cat .version) + command: make -e -C .do/ deploy_to_doks_from_chartmuseum VERSION=$(cat .version) LISTENER_REPLICAS=2 helm_push: &helm_push steps: diff --git a/.do/db-cluster/Makefile b/.do/db-cluster/Makefile index 6e243ac9..1815a8e9 100644 --- a/.do/db-cluster/Makefile +++ b/.do/db-cluster/Makefile @@ -64,6 +64,7 @@ replace_db_values: @sed -i "s/postgresqlDatabase: crow/postgresqlDatabase: \"crow?sslmode=require\"/g" ../../codacy/values-production.yaml @sed -i "s/<--- codacy-db-username --->/$(username)/g; s/<--- codacy-db-password --->/$(password)/g; s/<--- codacy-db-host --->/$(host)/g; s/<--- codacy-db-port --->/$(port)/g;" ../values.yaml @sed -i "s/postgresqlDatabase: crow/postgresqlDatabase: \"crow?sslmode=require\"/g" ../values.yaml + @sed -i "s/\.a\./\.d\./g;" ../../codacy/values-production.yaml @echo "Replaced template values for db connections" .PHONY: tf_init From c29d149108f1534732f2012aa8d37a635a4ffcd7 Mon Sep 17 00:00:00 2001 From: Leandro Salgado Date: Fri, 23 Oct 2026 11:25:34 +0100 Subject: [PATCH 47/62] cleanup --- .circleci/config.yml | 2 +- .do/values.yaml | 2 +- codacy/values-production.yaml | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dc5278d7..d48724f8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -333,7 +333,7 @@ references: - run: name: Install Codacy no_output_timeout: 30m - command: make -e -C .do/ deploy_to_doks_from_chartmuseum VERSION=$(cat .version) LISTENER_REPLICAS=2 + command: make -e -C .do/ deploy_to_doks_from_chartmuseum VERSION=$(cat .version) LISTENER_REPLICAS=1 helm_push: &helm_push steps: diff --git a/.do/values.yaml b/.do/values.yaml index 28d61c67..675e0dec 100644 --- a/.do/values.yaml +++ b/.do/values.yaml @@ -107,7 +107,7 @@ remote-provider-service: enabled: true listener: - replicaCount: 1 + replicaCount: 0 listenerdb: create: false persistence: diff --git a/codacy/values-production.yaml b/codacy/values-production.yaml index 5dee8dd3..9f9af915 100644 --- a/codacy/values-production.yaml +++ b/codacy/values-production.yaml @@ -317,6 +317,10 @@ rabbitmq-ha: extraEnvVars: - name: ERL_MAX_PORTS value: "65536" + livenessProbe: + enabled: false + readinessProbe: + enabled: false memoryHighWatermark: enabled: true type: relative From eb24007fd95fea113675e714069cf3a7846e17f3 Mon Sep 17 00:00:00 2001 From: Leandro Salgado Date: Fri, 23 Oct 2026 15:35:46 +0100 Subject: [PATCH 48/62] increase timeout --- .do/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.do/Makefile b/.do/Makefile index 079c1462..197bad6c 100644 --- a/.do/Makefile +++ b/.do/Makefile @@ -14,7 +14,7 @@ define call_helm_install eval "extra_args=($$HELM_INSTALL_EXTRA_FLAGS)"; \ helm upgrade --install ${1} ${3} \ --atomic \ - --timeout=1500s \ + --timeout=2100s \ -f ../codacy/values-production.yaml \ -f values.yaml \ --namespace ${2} \ From 239e309d7a0041194683dff07bf96194bd5bea67 Mon Sep 17 00:00:00 2001 From: Leandro Salgado Date: Fri, 23 Oct 2026 15:54:18 +0100 Subject: [PATCH 49/62] increase timeout --- .do/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.do/Makefile b/.do/Makefile index 197bad6c..a098eac2 100644 --- a/.do/Makefile +++ b/.do/Makefile @@ -14,7 +14,7 @@ define call_helm_install eval "extra_args=($$HELM_INSTALL_EXTRA_FLAGS)"; \ helm upgrade --install ${1} ${3} \ --atomic \ - --timeout=2100s \ + --timeout=2400s \ -f ../codacy/values-production.yaml \ -f values.yaml \ --namespace ${2} \ From b319c4b9c52b91d7223d680a72ed259e1a2f4347 Mon Sep 17 00:00:00 2001 From: Leandro Salgado Date: Mon, 26 Oct 2026 15:43:21 +0000 Subject: [PATCH 50/62] list objects in bucket --- codacy/values.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/codacy/values.yaml b/codacy/values.yaml index 96c0d3bc..7658082f 100644 --- a/codacy/values.yaml +++ b/codacy/values.yaml @@ -453,6 +453,7 @@ rustfs: echo "bucket $b ready" done B=codacy-api-repository-list-bucket + $S3 list-objects --bucket "$B" $S3 head-bucket --bucket "$B" 2>/dev/null || $S3 create-bucket --bucket "$B" $S3 put-bucket-lifecycle-configuration --bucket "$B" --lifecycle-configuration '{"Rules":[{"ID":"Expire repository lists","Status":"Enabled","Filter":{"Prefix":""},"Expiration":{"Days":90}}]}' echo "bucket $B ready (90-day expiry)" From c2a1fcce43c91e5386678e70d1e5cff5c65b3859 Mon Sep 17 00:00:00 2001 From: Leandro Salgado Date: Tue, 27 Oct 2026 14:47:19 +0000 Subject: [PATCH 51/62] delete all objects from bucket --- codacy/values.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/codacy/values.yaml b/codacy/values.yaml index 7658082f..51325f67 100644 --- a/codacy/values.yaml +++ b/codacy/values.yaml @@ -453,8 +453,9 @@ rustfs: echo "bucket $b ready" done B=codacy-api-repository-list-bucket - $S3 list-objects --bucket "$B" $S3 head-bucket --bucket "$B" 2>/dev/null || $S3 create-bucket --bucket "$B" + $S3 list-objects --bucket "$B" + $S3 delete-objects --bucket "$B" --delete "$($S3 list-object-versions --bucket "my-bucket" --output=json --query='{Objects: Versions[].{Key:Key,VersionId:VersionId}}')" $S3 put-bucket-lifecycle-configuration --bucket "$B" --lifecycle-configuration '{"Rules":[{"ID":"Expire repository lists","Status":"Enabled","Filter":{"Prefix":""},"Expiration":{"Days":90}}]}' echo "bucket $B ready (90-day expiry)" From bde083399b4e78c9be173917350987728d814593 Mon Sep 17 00:00:00 2001 From: Leandro Salgado Date: Tue, 27 Oct 2026 15:23:16 +0000 Subject: [PATCH 52/62] delete all objects from bucket --- codacy/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codacy/values.yaml b/codacy/values.yaml index 51325f67..c34da59e 100644 --- a/codacy/values.yaml +++ b/codacy/values.yaml @@ -455,7 +455,7 @@ rustfs: B=codacy-api-repository-list-bucket $S3 head-bucket --bucket "$B" 2>/dev/null || $S3 create-bucket --bucket "$B" $S3 list-objects --bucket "$B" - $S3 delete-objects --bucket "$B" --delete "$($S3 list-object-versions --bucket "my-bucket" --output=json --query='{Objects: Versions[].{Key:Key,VersionId:VersionId}}')" + $S3 delete-objects --bucket "$B" --delete "$($S3 list-object-versions --bucket "$B" --output=json --query='{Objects: Versions[].{Key:Key,VersionId:VersionId}}')" $S3 put-bucket-lifecycle-configuration --bucket "$B" --lifecycle-configuration '{"Rules":[{"ID":"Expire repository lists","Status":"Enabled","Filter":{"Prefix":""},"Expiration":{"Days":90}}]}' echo "bucket $B ready (90-day expiry)" From 65a375cdaf470d649f4b3d3ab6cc57ac6ef284fb Mon Sep 17 00:00:00 2001 From: Leandro Salgado Date: Tue, 27 Oct 2026 15:45:57 +0000 Subject: [PATCH 53/62] delete bucket --- codacy/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codacy/values.yaml b/codacy/values.yaml index c34da59e..e38e4ce1 100644 --- a/codacy/values.yaml +++ b/codacy/values.yaml @@ -453,9 +453,9 @@ rustfs: echo "bucket $b ready" done B=codacy-api-repository-list-bucket + $S3 delete-bucket --bucket "$B" $S3 head-bucket --bucket "$B" 2>/dev/null || $S3 create-bucket --bucket "$B" $S3 list-objects --bucket "$B" - $S3 delete-objects --bucket "$B" --delete "$($S3 list-object-versions --bucket "$B" --output=json --query='{Objects: Versions[].{Key:Key,VersionId:VersionId}}')" $S3 put-bucket-lifecycle-configuration --bucket "$B" --lifecycle-configuration '{"Rules":[{"ID":"Expire repository lists","Status":"Enabled","Filter":{"Prefix":""},"Expiration":{"Days":90}}]}' echo "bucket $B ready (90-day expiry)" From a63dd479baa4b5fec2b4b5cb0cfd8e0bf5a77fc2 Mon Sep 17 00:00:00 2001 From: Leandro Salgado Date: Tue, 27 Oct 2026 16:00:59 +0000 Subject: [PATCH 54/62] delete bucket --- codacy/values.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/codacy/values.yaml b/codacy/values.yaml index e38e4ce1..d060d120 100644 --- a/codacy/values.yaml +++ b/codacy/values.yaml @@ -453,7 +453,6 @@ rustfs: echo "bucket $b ready" done B=codacy-api-repository-list-bucket - $S3 delete-bucket --bucket "$B" $S3 head-bucket --bucket "$B" 2>/dev/null || $S3 create-bucket --bucket "$B" $S3 list-objects --bucket "$B" $S3 put-bucket-lifecycle-configuration --bucket "$B" --lifecycle-configuration '{"Rules":[{"ID":"Expire repository lists","Status":"Enabled","Filter":{"Prefix":""},"Expiration":{"Days":90}}]}' From ceb40cafe057cd267b6fc0dfff0452cf0f529f9b Mon Sep 17 00:00:00 2001 From: Leandro Salgado Date: Tue, 27 Oct 2026 16:11:15 +0000 Subject: [PATCH 55/62] change bucket name --- codacy/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codacy/values.yaml b/codacy/values.yaml index d060d120..d58b7dae 100644 --- a/codacy/values.yaml +++ b/codacy/values.yaml @@ -452,7 +452,7 @@ rustfs: $S3 head-bucket --bucket "$b" 2>/dev/null || $S3 create-bucket --bucket "$b" echo "bucket $b ready" done - B=codacy-api-repository-list-bucket + B=codacy-repository-list-bucket $S3 head-bucket --bucket "$B" 2>/dev/null || $S3 create-bucket --bucket "$B" $S3 list-objects --bucket "$B" $S3 put-bucket-lifecycle-configuration --bucket "$B" --lifecycle-configuration '{"Rules":[{"ID":"Expire repository lists","Status":"Enabled","Filter":{"Prefix":""},"Expiration":{"Days":90}}]}' From 7b1b0a9d837169a270dbb159074a5e1c08fa17a8 Mon Sep 17 00:00:00 2001 From: Leandro Salgado Date: Tue, 27 Oct 2026 16:23:11 +0000 Subject: [PATCH 56/62] test --- codacy/values.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/codacy/values.yaml b/codacy/values.yaml index d58b7dae..01f05e11 100644 --- a/codacy/values.yaml +++ b/codacy/values.yaml @@ -453,10 +453,6 @@ rustfs: echo "bucket $b ready" done B=codacy-repository-list-bucket - $S3 head-bucket --bucket "$B" 2>/dev/null || $S3 create-bucket --bucket "$B" - $S3 list-objects --bucket "$B" - $S3 put-bucket-lifecycle-configuration --bucket "$B" --lifecycle-configuration '{"Rules":[{"ID":"Expire repository lists","Status":"Enabled","Filter":{"Prefix":""},"Expiration":{"Days":90}}]}' - echo "bucket $B ready (90-day expiry)" worker-manager: create: true From 00d5c65ca8bde3dcb5398ca3d8c8cc3a32aae689 Mon Sep 17 00:00:00 2001 From: Leandro Salgado Date: Wed, 28 Oct 2026 11:05:48 +0000 Subject: [PATCH 57/62] remove nfsserverprovider replica count to zero --- .do/Makefile | 1 - codacy/values-production.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/.do/Makefile b/.do/Makefile index a098eac2..7dd4168c 100644 --- a/.do/Makefile +++ b/.do/Makefile @@ -34,7 +34,6 @@ define call_helm_install --set global.crow.config.passwordAuth.password=$${CROW_PASSWORD} \ --set rabbitmq.image.repository=bitnamilegacy/rabbitmq \ "$${extra_args[@]}" - kubectl patch statefulset codacy-release-nfsserverprovisioner -n ${2} --patch-file nfs-patch.yaml kubectl scale deploy --replicas=${LISTENER_REPLICAS} -n ${2} codacy-listener endef diff --git a/codacy/values-production.yaml b/codacy/values-production.yaml index 9f9af915..08f6d5f6 100644 --- a/codacy/values-production.yaml +++ b/codacy/values-production.yaml @@ -247,7 +247,6 @@ listener: size: 140Gi nfsserverprovisioner: enabled: true - replicaCount: 0 persistence: enabled: true size: 200Gi From ba06636dd0b8bd2984fb19e6d25e9aebf5e726ba Mon Sep 17 00:00:00 2001 From: Leandro Salgado Date: Wed, 28 Oct 2026 12:38:01 +0000 Subject: [PATCH 58/62] test running without lifecycle-creator --- .../fluentd/lifecycle-police-job.yaml | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/codacy/templates/fluentd/lifecycle-police-job.yaml b/codacy/templates/fluentd/lifecycle-police-job.yaml index 74eff1d3..4411f290 100644 --- a/codacy/templates/fluentd/lifecycle-police-job.yaml +++ b/codacy/templates/fluentd/lifecycle-police-job.yaml @@ -39,28 +39,5 @@ spec: MAX_WAIT=1200 ELAPSED=0 echo "Waiting for MinIO bucket ${BUCKET} at ${ENDPOINT}..." - until aws s3api head-bucket --bucket "${BUCKET}" --endpoint-url "${ENDPOINT}" > /dev/null 2>&1; do - if [ "${ELAPSED}" -ge "${MAX_WAIT}" ]; then - echo "Timed out waiting for MinIO bucket after ${MAX_WAIT}s" - exit 1 - fi - echo "Bucket not ready yet (${ELAPSED}s elapsed), retrying in 10s..." - sleep 10 - ELAPSED=$((ELAPSED + 10)) - done - echo "MinIO bucket ${BUCKET} is ready. Setting lifecycle configuration..." - RETRY=0 - until aws s3api put-bucket-lifecycle-configuration \ - --bucket "${BUCKET}" \ - --endpoint-url "${ENDPOINT}" \ - --lifecycle-configuration "${LIFECYCLE}"; do - RETRY=$((RETRY + 1)) - if [ "${RETRY}" -ge 5 ]; then - echo "Failed to set lifecycle configuration after ${RETRY} attempts" - exit 1 - fi - echo "Lifecycle config failed, retrying in 10s (attempt ${RETRY}/5)..." - sleep 10 - done echo "Lifecycle configuration set successfully." {{ end }} From 6f79561a62d88d4974edec226a1247d959187d16 Mon Sep 17 00:00:00 2001 From: Leandro Salgado Date: Wed, 28 Oct 2026 15:26:04 +0000 Subject: [PATCH 59/62] re-add lifecycle job --- .../fluentd/lifecycle-police-job.yaml | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/codacy/templates/fluentd/lifecycle-police-job.yaml b/codacy/templates/fluentd/lifecycle-police-job.yaml index 4411f290..74eff1d3 100644 --- a/codacy/templates/fluentd/lifecycle-police-job.yaml +++ b/codacy/templates/fluentd/lifecycle-police-job.yaml @@ -39,5 +39,28 @@ spec: MAX_WAIT=1200 ELAPSED=0 echo "Waiting for MinIO bucket ${BUCKET} at ${ENDPOINT}..." + until aws s3api head-bucket --bucket "${BUCKET}" --endpoint-url "${ENDPOINT}" > /dev/null 2>&1; do + if [ "${ELAPSED}" -ge "${MAX_WAIT}" ]; then + echo "Timed out waiting for MinIO bucket after ${MAX_WAIT}s" + exit 1 + fi + echo "Bucket not ready yet (${ELAPSED}s elapsed), retrying in 10s..." + sleep 10 + ELAPSED=$((ELAPSED + 10)) + done + echo "MinIO bucket ${BUCKET} is ready. Setting lifecycle configuration..." + RETRY=0 + until aws s3api put-bucket-lifecycle-configuration \ + --bucket "${BUCKET}" \ + --endpoint-url "${ENDPOINT}" \ + --lifecycle-configuration "${LIFECYCLE}"; do + RETRY=$((RETRY + 1)) + if [ "${RETRY}" -ge 5 ]; then + echo "Failed to set lifecycle configuration after ${RETRY} attempts" + exit 1 + fi + echo "Lifecycle config failed, retrying in 10s (attempt ${RETRY}/5)..." + sleep 10 + done echo "Lifecycle configuration set successfully." {{ end }} From 99982538935288b55798f8b3f7af45a74cdf38c7 Mon Sep 17 00:00:00 2001 From: Leandro Salgado Date: Wed, 28 Oct 2026 15:52:06 +0000 Subject: [PATCH 60/62] increase logStorageSize to 3 Gi --- codacy/values-production.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/codacy/values-production.yaml b/codacy/values-production.yaml index 08f6d5f6..02d826bd 100644 --- a/codacy/values-production.yaml +++ b/codacy/values-production.yaml @@ -342,6 +342,7 @@ rabbitmq-ha: rustfs: storageclass: dataStorageSize: 40Gi + logStorageSize: 3Gi resources: requests: memory: 1.5Gi From b46feac98506af38245602f3e504ce35f3915505 Mon Sep 17 00:00:00 2001 From: Leandro Salgado Date: Wed, 28 Oct 2026 16:21:29 +0000 Subject: [PATCH 61/62] cleanup --- codacy/values.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/codacy/values.yaml b/codacy/values.yaml index 01f05e11..d060d120 100644 --- a/codacy/values.yaml +++ b/codacy/values.yaml @@ -452,7 +452,11 @@ rustfs: $S3 head-bucket --bucket "$b" 2>/dev/null || $S3 create-bucket --bucket "$b" echo "bucket $b ready" done - B=codacy-repository-list-bucket + B=codacy-api-repository-list-bucket + $S3 head-bucket --bucket "$B" 2>/dev/null || $S3 create-bucket --bucket "$B" + $S3 list-objects --bucket "$B" + $S3 put-bucket-lifecycle-configuration --bucket "$B" --lifecycle-configuration '{"Rules":[{"ID":"Expire repository lists","Status":"Enabled","Filter":{"Prefix":""},"Expiration":{"Days":90}}]}' + echo "bucket $B ready (90-day expiry)" worker-manager: create: true From 23757acb3d8f1f111a1c48cdbd380c727cc1c9a9 Mon Sep 17 00:00:00 2001 From: Leandro Salgado Date: Wed, 28 Oct 2026 16:36:54 +0000 Subject: [PATCH 62/62] cleanup --- codacy/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codacy/values.yaml b/codacy/values.yaml index d060d120..d58b7dae 100644 --- a/codacy/values.yaml +++ b/codacy/values.yaml @@ -452,7 +452,7 @@ rustfs: $S3 head-bucket --bucket "$b" 2>/dev/null || $S3 create-bucket --bucket "$b" echo "bucket $b ready" done - B=codacy-api-repository-list-bucket + B=codacy-repository-list-bucket $S3 head-bucket --bucket "$B" 2>/dev/null || $S3 create-bucket --bucket "$B" $S3 list-objects --bucket "$B" $S3 put-bucket-lifecycle-configuration --bucket "$B" --lifecycle-configuration '{"Rules":[{"ID":"Expire repository lists","Status":"Enabled","Filter":{"Prefix":""},"Expiration":{"Days":90}}]}'