diff --git a/.circleci/config.yml b/.circleci/config.yml index dcbdb1f86..48316a7bd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -78,9 +78,9 @@ references: persistence: size: 10Gi - minio: - persistence: - size: 10Gi + rustfs: + storageclass: + dataStorageSize: 10Gi rabbitmq-ha: replicaCount: 1 ulimitNofiles: 32768 @@ -347,7 +347,14 @@ references: helm dep build ./codacy echo "Adding 'https://charts.codacy.com/${CHANNEL}'" helm repo add --username "${CHARTS_REPO_USER}" --password "${CHARTS_REPO_PASS}" ${HELM_REPOSITORY} https://charts.codacy.com/${CHANNEL} - helm push ./codacy ${HELM_REPOSITORY} + # The bundled helm-push 0.9.0 fails on a v1 parent chart (ours) + # depending on a v2 subchart (rustfs) - it loads v1 parents with + # an old Helm 2 library that can't read v2 subcharts + # (chartmuseum/helm-push#93). Fixed and renamed to cm-push in + # v0.10.0+. + helm plugin uninstall push || true + helm plugin install https://github.com/chartmuseum/helm-push --version v0.10.4 + helm cm-push ./codacy ${HELM_REPOSITORY} - <<: *persist_to_workspace qa_job: &qa_job diff --git a/.do/Makefile b/.do/Makefile index 6ecda9cf9..d2ac5932f 100644 --- a/.do/Makefile +++ b/.do/Makefile @@ -3,7 +3,7 @@ GRAFANA_URL?="https://grafana.k8s.dev.codacy.org" RELEASE_NAME?="codacy" NAMESPACE?="codacy" HELM_REPOSITORY?=codacy-incubator -DEPLOYMENTS?=$(shell kubectl get deployments -n "${NAMESPACE}" | awk '{print $$1}' | tail -n +2 | grep -v minio) +DEPLOYMENTS?=$(shell kubectl get deployments -n "${NAMESPACE}" | awk '{print $$1}' | tail -n +2 | grep -v rustfs) WORKER_VERSION?=$(shell grep "engine" -A 2 ../codacy/requirements.lock | grep version | cut -d : -f 2 | tr -d '[:blank:]') HELM_INSTALL_EXTRA_FLAGS ?= "" SHELL := /bin/bash # we need this for the extra args in `call_helm_install` @@ -60,16 +60,27 @@ endef setup_helm_repos: $(MAKE) -C ../ setup_helm_repos +.PHONY: ensure_modern_helm +ensure_modern_helm: + # codacy/ci-do bakes in helm v3.2.1 (2020), whose template parser cannot + # read modern apiVersion v2 charts with multi-line template actions (e.g. + # the rustfs dependency) - fails with "parse error ...: unclosed action". + # Replace it with a current build before installing anything. + curl -fsSL https://get.helm.sh/helm-v3.19.0-linux-amd64.tar.gz | tar xz -C /tmp + mv /tmp/linux-amd64/helm /usr/local/bin/helm + rm -rf /tmp/linux-amd64 + helm version --short + .PHONY: helm_install_infrastructure -helm_install_infrastructure: +helm_install_infrastructure: ensure_modern_helm $(call call_helm_install_infrastructure,${RELEASE_NAME},${NAMESPACE},../codacy/,${CODACY_URL},sh-${WORKER_VERSION},${ENVIRONMENT}) .PHONY: helm_install_from_local_dir -helm_install_from_local_dir: update_ingress_values +helm_install_from_local_dir: update_ingress_values ensure_modern_helm $(call call_helm_install,${RELEASE_NAME},${NAMESPACE},../codacy/,${CODACY_URL},sh-${WORKER_VERSION},${ENVIRONMENT}) .PHONY: helm_install_from_chart_museum -helm_install_from_chart_museum: update_ingress_values +helm_install_from_chart_museum: update_ingress_values ensure_modern_helm $(call call_helm_install,${RELEASE_NAME},${NAMESPACE},${HELM_REPOSITORY}/codacy --version ${VERSION},${CODACY_URL},sh-${WORKER_VERSION},${ENVIRONMENT}) .PHONY: update_dependencies diff --git a/.do/values.yaml b/.do/values.yaml index c01fab5cb..28d61c67e 100644 --- a/.do/values.yaml +++ b/.do/values.yaml @@ -178,7 +178,6 @@ rabbitmq-ha: description: | No Rabbitmq Nodes -minio: - persistence: - enabled: true - size: 40Gi +rustfs: + storageclass: + dataStorageSize: 40Gi diff --git a/Makefile b/Makefile index e3f3dfe57..00a3cac30 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: setup_helm_repos setup_helm_repos: # Add legacy Bitnami snapshot repo used by requirements.yaml (apiVersion v1) - helm repo add stable https://charts.helm.sh/stable + helm repo add rustfs https://charts.rustfs.com helm repo add codacy-stable https://charts.codacy.com/stable helm repo add codacy-unstable https://charts.codacy.com/unstable helm repo add codacy-incubator https://charts.codacy.com/incubator diff --git a/README.md b/README.md index df987bf5f..46f23fadf 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Some of these repositories are private and accessible to Codacy engineers only. ![Helm Chart Structure](https://raw.githubusercontent.com/codacy/chart/master/docs/images/charts.png) [edit image](https://docs.google.com/drawings/d/1o7z3L8XnnNjHBOTWKHiIYUkBP3DDiogdUyxNdUfzyfY/edit) -- [MinIO](https://github.com/minio/charts/tree/master) +- [RustFS](https://github.com/rustfs/rustfs/tree/main/helm/rustfs) - [RabbitMQ-HA](https://github.com/helm/charts/tree/master/stable/rabbitmq-ha) - [PostgreSQL](https://github.com/bitnami/charts/tree/master/bitnami/postgresql) - [kube-fluentd-operator](https://github.com/codacy/kube-fluentd-operator) @@ -61,7 +61,7 @@ Global parameters apply to all sub-charts and make it easier to configure resour | `global.filestore.contentsSecret` | Secrets used internally for encryption. Generate one with \`openssl rand -base64 128 \| tr -dc 'a-zA-Z0-9'\` | `nil` | | `global.filestore.uuidSecret` | Secrets used internally for encryption. Generate one with \`openssl rand -base64 128 \| tr -dc 'a-zA-Z0-9'\` | `nil` | | `global.cacheSecret` | Secrets used internally for encryption. Generate one with \`openssl rand -base64 128 \| tr -dc 'a-zA-Z0-9'\` | `nil` | -| `global.minio.create` | Create minio internally | `nil` | +| `global.minio.create` | Create the bundled RustFS object store internally | `nil` | | `global.rabbitmq.create` | Create rabbitmq internally | `nil` | | `global.rabbitmq.rabbitmqUsername` | Username for rabbitmq. If you are using the bundled version, change the `rabbitmq-ha.rabbitmqUsername` also. | `nil` | | `global.rabbitmq.rabbitmqPassword` | Password for rabbitmq. If you are using the bundled version, change the `rabbitmq-ha.rabbitmqPassword` also. | `nil` | @@ -214,7 +214,7 @@ The following parameters refer to components that are not internal to Codacy, bu | Parameter | Description | Default | | ---------------------------------- | --------------------------------------------------------------------------------------------------------------------- | ----------------- | | `fluentdoperator.enable` | Enable fluentd operator. It gathers logs from Codacy so that you can send it to our support if needed. | `true` | -| `fluentdoperator.expirationDays` | Number of days to retain logs. More time uses more disk on minio and retention over 7 days is not recommended. | `7` | +| `fluentdoperator.expirationDays` | Number of days to retain logs. More time uses more disk on the object store and retention over 7 days is not recommended. | `7` | | `fluentdoperator.flushTimeout` | Maximum time until Fluentd stops retrying to flush the logs. Values must be expressed using a unit, e.g. `10s`, `5h`. | `1h` | | `rabbitmq-ha.rabbitmqUsername` | Username for the bundled RabbitMQ. | `rabbitmq-codacy` | | `rabbitmq-ha.rabbitmqPassword` | Password for the bundled RabbitMQ. | `rabbitmq-codacy` | diff --git a/codacy/Chart.yaml b/codacy/Chart.yaml index 1b201234a..c1d360b9e 100644 --- a/codacy/Chart.yaml +++ b/codacy/Chart.yaml @@ -4,6 +4,6 @@ description: | This is the top level chart to run Codacy in self-hosted mode. With this chart, it is possible to run all the components from Codacy, plugs all the dependencies with a single command line, including Codacy, - PostgreSQL, RabbitMQ, Minio, Ingress etc... + PostgreSQL, RabbitMQ, RustFS, Ingress etc... name: codacy version: 0.1.1 diff --git a/codacy/requirements-dev.yaml b/codacy/requirements-dev.yaml index 871c140e9..7a11b18e8 100644 --- a/codacy/requirements-dev.yaml +++ b/codacy/requirements-dev.yaml @@ -1,9 +1,9 @@ dependencies: -- name: minio - version: 5.0.33 - repository: https://charts.helm.sh/stable +- name: rustfs + version: 1.0.3 + repository: https://charts.rustfs.com condition: global.minio.create - alias: minio + alias: rustfs - name: rabbitmq version: 7.5.7 diff --git a/codacy/requirements.yaml b/codacy/requirements.yaml index eb6daa185..9f3b9ad4b 100644 --- a/codacy/requirements.yaml +++ b/codacy/requirements.yaml @@ -1,9 +1,9 @@ dependencies: -- name: minio - version: 5.0.33 - repository: https://charts.helm.sh/stable +- name: rustfs + version: 1.0.3 + repository: https://charts.rustfs.com condition: global.minio.create - alias: minio + alias: rustfs - name: rabbitmq version: 7.5.7 diff --git a/codacy/templates/fluentd/fluentd-conf.yaml b/codacy/templates/fluentd/fluentd-conf.yaml index c15dabad6..cda68386f 100644 --- a/codacy/templates/fluentd/fluentd-conf.yaml +++ b/codacy/templates/fluentd/fluentd-conf.yaml @@ -47,7 +47,7 @@ data: # Bucket configuration s3_endpoint http://{{ .Values.global.minio.location }}:9000 s3_bucket {{ .Values.fluentdoperator.bucketName }} - force_path_style true # This prevents AWS SDK from breaking endpoint URL for minio + force_path_style true # This prevents AWS SDK from breaking endpoint URL for the S3-compatible object store # HACK: Do not check s3 connection and bucket existence # If reloader fails by any means it will not propagate config to fluentd @@ -62,7 +62,7 @@ data: path ${container_name}/${pod_name}/%Y-%m-%d s3_object_key_format %{path}_%{index}.%{file_extension} - # Does not work (possibly due to a minio incompatiblity) + # Does not work (possibly due to an S3-compatible store incompatiblity) # As a workaround we are using a helm hook to create the lifecycle policy # # id retention-policy diff --git a/codacy/templates/fluentd/lifecycle-police-job.yaml b/codacy/templates/fluentd/lifecycle-police-job.yaml index 68e5167fc..1b1fcbe72 100644 --- a/codacy/templates/fluentd/lifecycle-police-job.yaml +++ b/codacy/templates/fluentd/lifecycle-police-job.yaml @@ -9,8 +9,8 @@ metadata: # This is what defines this resource as a hook. Without this line, the # job is considered part of the release. "helm.sh/hook": post-install,post-upgrade - # Minio create bucket hook doesn't define any hook weight, - # any positive value will ensure that the bucket is created before this + # The RustFS bucket-creation hook (rustfs.extraManifests) doesn't define + # any hook weight, any positive value will ensure that the bucket is created before this "helm.sh/hook-weight": "10" "helm.sh/hook-delete-policy": hook-succeeded spec: diff --git a/codacy/values-production.yaml b/codacy/values-production.yaml index bddaed449..d2d41addb 100644 --- a/codacy/values-production.yaml +++ b/codacy/values-production.yaml @@ -325,10 +325,9 @@ rabbitmq-ha: cpu: 0.2 memory: 200Mi -minio: - persistence: - enabled: true - size: 40Gi +rustfs: + storageclass: + dataStorageSize: 40Gi resources: requests: memory: 1.5Gi diff --git a/codacy/values.yaml b/codacy/values.yaml index 343286ff5..cee2c79a2 100644 --- a/codacy/values.yaml +++ b/codacy/values.yaml @@ -120,9 +120,15 @@ global: cacheSecret: "PLEASE_CHANGE_ME" # Generate one with `cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 128 | head -n 1` + # Object storage (S3-compatible). Backed by the bundled RustFS subchart + # (see the `rustfs:` block below) unless global.minio.create is false. + # Keeping the `minio` key name here on purpose: every other subchart + # (engine, api, listener, fluentd, ...) reads global.minio.* to reach + # whatever S3-compatible store is actually running, so renaming it would + # require changes across those repos too. minio: create: true - location: codacy-minio + location: codacy-rustfs-svc port: 9000 accessKey: "AKIAIOSFODNN7EXAMPLE" secretKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" @@ -363,30 +369,91 @@ postgres: CREATE DATABASE listener WITH OWNER=codacy; CREATE DATABASE crow WITH OWNER=codacy; -minio: - fullnameOverride: codacy-minio - persistence: - size: 40Gi - minioConfig: - region: "eu-west-1" - buckets: - - name: "analysis" - policy: "public" - purge: "false" - - name: "coverage" - policy: "public" - purge: "false" - - name: "cli-reports" - policy: "public" - purge: "false" - - name: "commit-diff" - policy: "public" - purge: "false" - - name: "logs" - policy: "public" - purge: "false" - defaultBucket: +# Bundled RustFS object store, replacing MinIO (its upstream images are no +# longer published on Docker Hub/Quay). Joins the `codacy-rustfs-svc` Service +# that global.minio.location points consumers at. +# +# IMPORTANT: secret.rustfs.access_key/secret_key below MUST be identical to +# global.minio.accessKey/secretKey above. If you change one, change both. +rustfs: + fullnameOverride: codacy-rustfs + mode: + standalone: + enabled: true + distributed: + enabled: false + replicaCount: 1 + ingress: enabled: false + storageclass: + name: "" + dataStorageSize: 40Gi + resources: + requests: + memory: 1.5Gi + config: + rustfs: + region: "eu-west-1" + secret: + rustfs: + access_key: "AKIAIOSFODNN7EXAMPLE" + secret_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + # Creates the buckets Codacy needs once RustFS is up (runs as part of + # `helm install`/`helm upgrade`), replacing MinIO's own bucket-creation hook. + extraManifests: + - apiVersion: batch/v1 + kind: Job + metadata: + name: '{{ include "rustfs.fullname" . }}-codacy-buckets' + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + spec: + backoffLimit: 6 + template: + spec: + restartPolicy: Never + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + volumes: + - name: tmp + emptyDir: {} + containers: + - name: make-buckets + image: amazon/aws-cli:2.17.0 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: tmp + mountPath: /tmp + env: + - name: HOME + value: /tmp + - name: S3 + value: 'aws s3api --endpoint-url http://{{ include "rustfs.fullname" . }}-svc:9000' + - name: AWS_DEFAULT_REGION + value: eu-west-1 + - name: AWS_ACCESS_KEY_ID + valueFrom: { secretKeyRef: { name: '{{ include "rustfs.secretName" . }}', key: RUSTFS_ACCESS_KEY } } + - name: AWS_SECRET_ACCESS_KEY + valueFrom: { secretKeyRef: { name: '{{ include "rustfs.secretName" . }}', key: RUSTFS_SECRET_KEY } } + command: ["sh", "-ec"] + args: + - | + until $S3 list-buckets >/dev/null 2>&1; do echo "waiting for RustFS..."; sleep 5; done + for b in analysis coverage cli-reports commit-diff logs; do + $S3 head-bucket --bucket "$b" 2>/dev/null || $S3 create-bucket --bucket "$b" + echo "bucket $b ready" + done + B=codacy-api-repository-list-bucket + $S3 head-bucket --bucket "$B" 2>/dev/null || $S3 create-bucket --bucket "$B" + $S3 put-bucket-lifecycle-configuration --bucket "$B" --lifecycle-configuration '{"Rules":[{"ID":"Expire repository lists","Status":"Enabled","Filter":{"Prefix":""},"Expiration":{"Days":90}}]}' + echo "bucket $B ready (90-day expiry)" worker-manager: create: true