diff --git a/assets/optional/kube-state-metrics/00-namespace.yaml b/assets/optional/kube-state-metrics/00-namespace.yaml new file mode 100644 index 0000000000..17f727565a --- /dev/null +++ b/assets/optional/kube-state-metrics/00-namespace.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: openshift-monitoring + labels: + name: openshift-monitoring + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/audit: privileged + pod-security.kubernetes.io/warn: privileged diff --git a/assets/optional/kube-state-metrics/01-cluster-role-binding.yaml b/assets/optional/kube-state-metrics/01-cluster-role-binding.yaml new file mode 100644 index 0000000000..c22bfc34ab --- /dev/null +++ b/assets/optional/kube-state-metrics/01-cluster-role-binding.yaml @@ -0,0 +1,18 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: exporter + app.kubernetes.io/managed-by: cluster-monitoring-operator + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/part-of: openshift-monitoring + app.kubernetes.io/version: 2.19.1 + name: kube-state-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-state-metrics +subjects: +- kind: ServiceAccount + name: kube-state-metrics + namespace: openshift-monitoring diff --git a/assets/optional/kube-state-metrics/01-cluster-role.yaml b/assets/optional/kube-state-metrics/01-cluster-role.yaml new file mode 100644 index 0000000000..75cd4e5d26 --- /dev/null +++ b/assets/optional/kube-state-metrics/01-cluster-role.yaml @@ -0,0 +1,153 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: exporter + app.kubernetes.io/managed-by: cluster-monitoring-operator + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/part-of: openshift-monitoring + app.kubernetes.io/version: 2.19.1 + name: kube-state-metrics +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + - nodes + - pods + - services + - serviceaccounts + - resourcequotas + - replicationcontrollers + - limitranges + - persistentvolumeclaims + - persistentvolumes + - namespaces + - endpoints + verbs: + - list + - watch +- apiGroups: + - apps + resources: + - statefulsets + - daemonsets + - deployments + - replicasets + verbs: + - list + - watch +- apiGroups: + - batch + resources: + - cronjobs + - jobs + verbs: + - list + - watch +- apiGroups: + - autoscaling + resources: + - horizontalpodautoscalers + verbs: + - list + - watch +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +- apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - list + - watch +- apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - volumeattachments + verbs: + - list + - watch +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + - validatingwebhookconfigurations + verbs: + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + - ingressclasses + - ingresses + verbs: + - list + - watch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - list + - watch +- apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterrolebindings + - clusterroles + - rolebindings + - roles + verbs: + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - watch +- apiGroups: + - autoscaling.k8s.io + resources: + - verticalpodautoscalers + verbs: + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - gatewayclasses + - gateways + verbs: + - list + - watch diff --git a/assets/optional/kube-state-metrics/01-service-account.yaml b/assets/optional/kube-state-metrics/01-service-account.yaml new file mode 100644 index 0000000000..0bfe63ad0e --- /dev/null +++ b/assets/optional/kube-state-metrics/01-service-account.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +automountServiceAccountToken: false +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: exporter + app.kubernetes.io/managed-by: cluster-monitoring-operator + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/part-of: openshift-monitoring + app.kubernetes.io/version: 2.19.1 + name: kube-state-metrics + namespace: openshift-monitoring diff --git a/assets/optional/kube-state-metrics/02-custom-resource-state-configmap.yaml b/assets/optional/kube-state-metrics/02-custom-resource-state-configmap.yaml new file mode 100644 index 0000000000..4d0f548939 --- /dev/null +++ b/assets/optional/kube-state-metrics/02-custom-resource-state-configmap.yaml @@ -0,0 +1,544 @@ +apiVersion: v1 +data: + custom-resource-state-configmap.yaml: |- + "kind": "CustomResourceStateMetrics" + "spec": + "resources": + - "groupVersionKind": + "group": "autoscaling.k8s.io" + "kind": "VerticalPodAutoscaler" + "version": "v1" + "metrics": + - "commonLabels": null + "each": + "stateSet": + "labelName": "updatemode" + "list": + - "Off" + - "Initial" + - "Recreate" + - "Auto" + "path": + - "spec" + - "updatePolicy" + - "updateMode" + "type": "StateSet" + "help": "Update mode of the VerticalPodAutoscaler." + "labelsFromPath": + "namespace": + - "metadata" + - "namespace" + "target_api_version": + - "spec" + - "targetRef" + - "apiVersion" + "target_kind": + - "spec" + - "targetRef" + - "kind" + "target_name": + - "spec" + - "targetRef" + - "name" + "verticalpodautoscaler": + - "metadata" + - "name" + "name": "verticalpodautoscaler_spec_updatepolicy_updatemode" + - "commonLabels": + "resource": "cpu" + "unit": "cores" + "each": + "gauge": + "labelsFromPath": + "container": + - "containerName" + "path": + - "status" + - "recommendation" + - "containerRecommendations" + "valueFrom": + - "lowerBound" + - "cpu" + "type": "Gauge" + "help": "Minimum cpu resources the container can use before the VerticalPodAutoscaler updater evicts it." + "labelsFromPath": + "namespace": + - "metadata" + - "namespace" + "target_api_version": + - "spec" + - "targetRef" + - "apiVersion" + "target_kind": + - "spec" + - "targetRef" + - "kind" + "target_name": + - "spec" + - "targetRef" + - "name" + "verticalpodautoscaler": + - "metadata" + - "name" + "name": "verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound_cpu" + - "commonLabels": + "resource": "memory" + "unit": "bytes" + "each": + "gauge": + "labelsFromPath": + "container": + - "containerName" + "path": + - "status" + - "recommendation" + - "containerRecommendations" + "valueFrom": + - "lowerBound" + - "memory" + "type": "Gauge" + "help": "Minimum memory resources the container can use before the VerticalPodAutoscaler updater evicts it." + "labelsFromPath": + "namespace": + - "metadata" + - "namespace" + "target_api_version": + - "spec" + - "targetRef" + - "apiVersion" + "target_kind": + - "spec" + - "targetRef" + - "kind" + "target_name": + - "spec" + - "targetRef" + - "name" + "verticalpodautoscaler": + - "metadata" + - "name" + "name": "verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound_memory" + - "commonLabels": + "resource": "cpu" + "unit": "cores" + "each": + "gauge": + "labelsFromPath": + "container": + - "containerName" + "path": + - "status" + - "recommendation" + - "containerRecommendations" + "valueFrom": + - "upperBound" + - "cpu" + "type": "Gauge" + "help": "Maximum cpu resources the container can use before the VerticalPodAutoscaler updater evicts it." + "labelsFromPath": + "namespace": + - "metadata" + - "namespace" + "target_api_version": + - "spec" + - "targetRef" + - "apiVersion" + "target_kind": + - "spec" + - "targetRef" + - "kind" + "target_name": + - "spec" + - "targetRef" + - "name" + "verticalpodautoscaler": + - "metadata" + - "name" + "name": "verticalpodautoscaler_status_recommendation_containerrecommendations_upperbound_cpu" + - "commonLabels": + "resource": "memory" + "unit": "bytes" + "each": + "gauge": + "labelsFromPath": + "container": + - "containerName" + "path": + - "status" + - "recommendation" + - "containerRecommendations" + "valueFrom": + - "upperBound" + - "memory" + "type": "Gauge" + "help": "Maximum memory resources the container can use before the VerticalPodAutoscaler updater evicts it." + "labelsFromPath": + "namespace": + - "metadata" + - "namespace" + "target_api_version": + - "spec" + - "targetRef" + - "apiVersion" + "target_kind": + - "spec" + - "targetRef" + - "kind" + "target_name": + - "spec" + - "targetRef" + - "name" + "verticalpodautoscaler": + - "metadata" + - "name" + "name": "verticalpodautoscaler_status_recommendation_containerrecommendations_upperbound_memory" + - "commonLabels": + "resource": "cpu" + "unit": "cores" + "each": + "gauge": + "labelsFromPath": + "container": + - "containerName" + "path": + - "status" + - "recommendation" + - "containerRecommendations" + "valueFrom": + - "target" + - "cpu" + "type": "Gauge" + "help": "Target cpu resources the VerticalPodAutoscaler recommends for the container." + "labelsFromPath": + "namespace": + - "metadata" + - "namespace" + "target_api_version": + - "spec" + - "targetRef" + - "apiVersion" + "target_kind": + - "spec" + - "targetRef" + - "kind" + "target_name": + - "spec" + - "targetRef" + - "name" + "verticalpodautoscaler": + - "metadata" + - "name" + "name": "verticalpodautoscaler_status_recommendation_containerrecommendations_target_cpu" + - "commonLabels": + "resource": "memory" + "unit": "bytes" + "each": + "gauge": + "labelsFromPath": + "container": + - "containerName" + "path": + - "status" + - "recommendation" + - "containerRecommendations" + "valueFrom": + - "target" + - "memory" + "type": "Gauge" + "help": "Target memory resources the VerticalPodAutoscaler recommends for the container." + "labelsFromPath": + "namespace": + - "metadata" + - "namespace" + "target_api_version": + - "spec" + - "targetRef" + - "apiVersion" + "target_kind": + - "spec" + - "targetRef" + - "kind" + "target_name": + - "spec" + - "targetRef" + - "name" + "verticalpodautoscaler": + - "metadata" + - "name" + "name": "verticalpodautoscaler_status_recommendation_containerrecommendations_target_memory" + - "commonLabels": + "resource": "cpu" + "unit": "cores" + "each": + "gauge": + "labelsFromPath": + "container": + - "containerName" + "path": + - "status" + - "recommendation" + - "containerRecommendations" + "valueFrom": + - "uncappedTarget" + - "cpu" + "type": "Gauge" + "help": "Target cpu resources the VerticalPodAutoscaler recommends for the container ignoring bounds." + "labelsFromPath": + "namespace": + - "metadata" + - "namespace" + "target_api_version": + - "spec" + - "targetRef" + - "apiVersion" + "target_kind": + - "spec" + - "targetRef" + - "kind" + "target_name": + - "spec" + - "targetRef" + - "name" + "verticalpodautoscaler": + - "metadata" + - "name" + "name": "verticalpodautoscaler_status_recommendation_containerrecommendations_uncappedtarget_cpu" + - "commonLabels": + "resource": "memory" + "unit": "bytes" + "each": + "gauge": + "labelsFromPath": + "container": + - "containerName" + "path": + - "status" + - "recommendation" + - "containerRecommendations" + "valueFrom": + - "uncappedTarget" + - "memory" + "type": "Gauge" + "help": "Target memory resources the VerticalPodAutoscaler recommends for the container ignoring bounds." + "labelsFromPath": + "namespace": + - "metadata" + - "namespace" + "target_api_version": + - "spec" + - "targetRef" + - "apiVersion" + "target_kind": + - "spec" + - "targetRef" + - "kind" + "target_name": + - "spec" + - "targetRef" + - "name" + "verticalpodautoscaler": + - "metadata" + - "name" + "name": "verticalpodautoscaler_status_recommendation_containerrecommendations_uncappedtarget_memory" + - "commonLabels": + "resource": "cpu" + "unit": "cores" + "each": + "gauge": + "labelsFromPath": + "container": + - "containerName" + "path": + - "spec" + - "resourcePolicy" + - "containerPolicies" + "valueFrom": + - "minAllowed" + - "cpu" + "type": "Gauge" + "help": "Minimum cpu resources the VerticalPodAutoscaler can set for containers matching the name." + "labelsFromPath": + "namespace": + - "metadata" + - "namespace" + "target_api_version": + - "spec" + - "targetRef" + - "apiVersion" + "target_kind": + - "spec" + - "targetRef" + - "kind" + "target_name": + - "spec" + - "targetRef" + - "name" + "verticalpodautoscaler": + - "metadata" + - "name" + "name": "verticalpodautoscaler_spec_resourcepolicy_container_policies_minallowed_cpu" + - "commonLabels": + "resource": "memory" + "unit": "bytes" + "each": + "gauge": + "labelsFromPath": + "container": + - "containerName" + "path": + - "spec" + - "resourcePolicy" + - "containerPolicies" + "valueFrom": + - "minAllowed" + - "memory" + "type": "Gauge" + "help": "Minimum memory resources the VerticalPodAutoscaler can set for containers matching the name." + "labelsFromPath": + "namespace": + - "metadata" + - "namespace" + "target_api_version": + - "spec" + - "targetRef" + - "apiVersion" + "target_kind": + - "spec" + - "targetRef" + - "kind" + "target_name": + - "spec" + - "targetRef" + - "name" + "verticalpodautoscaler": + - "metadata" + - "name" + "name": "verticalpodautoscaler_spec_resourcepolicy_container_policies_minallowed_memory" + - "commonLabels": + "resource": "cpu" + "unit": "cores" + "each": + "gauge": + "labelsFromPath": + "container": + - "containerName" + "path": + - "spec" + - "resourcePolicy" + - "containerPolicies" + "valueFrom": + - "maxAllowed" + - "cpu" + "type": "Gauge" + "help": "Minimum cpu resources the VerticalPodAutoscaler can set for containers matching the name." + "labelsFromPath": + "namespace": + - "metadata" + - "namespace" + "target_api_version": + - "spec" + - "targetRef" + - "apiVersion" + "target_kind": + - "spec" + - "targetRef" + - "kind" + "target_name": + - "spec" + - "targetRef" + - "name" + "verticalpodautoscaler": + - "metadata" + - "name" + "name": "verticalpodautoscaler_spec_resourcepolicy_container_policies_maxallowed_cpu" + - "commonLabels": + "resource": "memory" + "unit": "bytes" + "each": + "gauge": + "labelsFromPath": + "container": + - "containerName" + "path": + - "spec" + - "resourcePolicy" + - "containerPolicies" + "valueFrom": + - "maxAllowed" + - "memory" + "type": "Gauge" + "help": "Minimum memory resources the VerticalPodAutoscaler can set for containers matching the name." + "labelsFromPath": + "namespace": + - "metadata" + - "namespace" + "target_api_version": + - "spec" + - "targetRef" + - "apiVersion" + "target_kind": + - "spec" + - "targetRef" + - "kind" + "target_name": + - "spec" + - "targetRef" + - "name" + "verticalpodautoscaler": + - "metadata" + - "name" + "name": "verticalpodautoscaler_spec_resourcepolicy_container_policies_maxallowed_memory" + - "groupVersionKind": + "group": "gateway.networking.k8s.io" + "kind": "GatewayClass" + "version": "v1" + "metrics": + - "each": + "info": + "labelsFromPath": + "accepted": + - "status" + - "conditions" + - "[type=Accepted]" + - "status" + "controller": + - "spec" + - "controllerName" + "gateway_class": + - "metadata" + - "name" + "type": "Info" + "help": "Information about GatewayClasses" + "name": "gateway_class_info" + - "groupVersionKind": + "group": "gateway.networking.k8s.io" + "kind": "Gateway" + "version": "v1" + "metrics": + - "each": + "info": + "labelsFromPath": + "gateway": + - "metadata" + - "name" + "gateway_class": + - "spec" + - "gatewayClassName" + "namespace": + - "metadata" + - "namespace" + "programmed": + - "status" + - "conditions" + - "[type=Programmed]" + - "status" + "type": "Info" + "help": "Information about Gateways" + "name": "gateway_info" +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/managed-by: cluster-monitoring-operator + app.kubernetes.io/part-of: openshift-monitoring + name: kube-state-metrics-custom-resource-state-configmap + namespace: openshift-monitoring diff --git a/assets/optional/kube-state-metrics/02-kube-rbac-proxy-secret.yaml b/assets/optional/kube-state-metrics/02-kube-rbac-proxy-secret.yaml new file mode 100644 index 0000000000..38b8df0c9c --- /dev/null +++ b/assets/optional/kube-state-metrics/02-kube-rbac-proxy-secret.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +data: {} +kind: Secret +metadata: + labels: + app.kubernetes.io/managed-by: cluster-monitoring-operator + app.kubernetes.io/part-of: openshift-monitoring + name: kube-state-metrics-kube-rbac-proxy-config + namespace: openshift-monitoring +stringData: + config.yaml: |- + "authorization": + "static": + - "path": "/metrics" + "resourceRequest": false + "verb": "get" +type: Opaque diff --git a/assets/optional/kube-state-metrics/03-deployment.yaml b/assets/optional/kube-state-metrics/03-deployment.yaml new file mode 100644 index 0000000000..37d3f657e7 --- /dev/null +++ b/assets/optional/kube-state-metrics/03-deployment.yaml @@ -0,0 +1,171 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: exporter + app.kubernetes.io/managed-by: cluster-monitoring-operator + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/part-of: openshift-monitoring + app.kubernetes.io/version: 2.19.1 + name: kube-state-metrics + namespace: openshift-monitoring +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: exporter + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/part-of: openshift-monitoring + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: kube-state-metrics + openshift.io/required-scc: restricted-v2 + target.workload.openshift.io/management: '{"effect": "PreferredDuringScheduling"}' + labels: + app.kubernetes.io/component: exporter + app.kubernetes.io/managed-by: cluster-monitoring-operator + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/part-of: openshift-monitoring + app.kubernetes.io/version: 2.19.1 + spec: + automountServiceAccountToken: true + containers: + - args: + - --host=127.0.0.1 + - --port=8081 + - --telemetry-host=127.0.0.1 + - --telemetry-port=8082 + - --custom-resource-state-config-file=/etc/kube-state-metrics/custom-resource-state-configmap.yaml + - | + --metric-denylist= + ^kube_secret_labels$, + ^kube_.+_annotations$, + ^kube_customresource_.+_annotations_info$, + ^kube_customresource_.+_labels_info$ + - --metric-labels-allowlist=pods=[*],nodes=[*],namespaces=[*],persistentvolumes=[*],persistentvolumeclaims=[*],poddisruptionbudgets=[*] + - | + --metric-denylist= + ^kube_.+_created$, + ^kube_.+_metadata_resource_version$, + ^kube_replicaset_metadata_generation$, + ^kube_replicaset_status_observed_generation$, + ^kube_pod_restart_policy$, + ^kube_pod_init_container_status_terminated$, + ^kube_pod_init_container_status_running$, + ^kube_pod_container_status_terminated$, + ^kube_pod_container_status_running$, + ^kube_pod_completion_time$, + ^kube_pod_status_scheduled$ + image: "quay.io/openshift/kube-state-metrics" + name: kube-state-metrics + resources: + requests: + cpu: 2m + memory: 80Mi + limits: + cpu: 100m + memory: 200Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /tmp + name: volume-directive-shadow + readOnly: false + - mountPath: /etc/kube-state-metrics + name: kube-state-metrics-custom-resource-state-configmap + readOnly: true + - args: + - --secure-listen-address=:8443 + - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - --upstream=http://127.0.0.1:8081/ + - --tls-cert-file=/etc/tls/private/tls.crt + - --tls-private-key-file=/etc/tls/private/tls.key + - --client-ca-file=/etc/tls/client/client-ca.crt + - --config-file=/etc/kube-rbac-policy/config.yaml + image: "quay.io/openshift/kube-rbac-proxy" + name: kube-rbac-proxy-main + ports: + - containerPort: 8443 + name: https-main + resources: + requests: + cpu: 1m + memory: 15Mi + limits: + cpu: 20m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /etc/tls/private + name: kube-state-metrics-tls + readOnly: true + - mountPath: /etc/kube-rbac-policy + name: kube-state-metrics-kube-rbac-proxy-config + readOnly: true + - mountPath: /etc/tls/client + name: metrics-client-ca + readOnly: true + - args: + - --secure-listen-address=:9443 + - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - --upstream=http://127.0.0.1:8082/ + - --tls-cert-file=/etc/tls/private/tls.crt + - --tls-private-key-file=/etc/tls/private/tls.key + - --client-ca-file=/etc/tls/client/client-ca.crt + - --config-file=/etc/kube-rbac-policy/config.yaml + image: "quay.io/openshift/kube-rbac-proxy" + name: kube-rbac-proxy-self + ports: + - containerPort: 9443 + name: https-self + resources: + requests: + cpu: 1m + memory: 15Mi + limits: + cpu: 20m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /etc/tls/private + name: kube-state-metrics-tls + readOnly: true + - mountPath: /etc/kube-rbac-policy + name: kube-state-metrics-kube-rbac-proxy-config + readOnly: true + - mountPath: /etc/tls/client + name: metrics-client-ca + readOnly: true + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + securityContext: + runAsNonRoot: true + serviceAccountName: kube-state-metrics + volumes: + - emptyDir: {} + name: volume-directive-shadow + - name: kube-state-metrics-tls + secret: + secretName: kube-state-metrics-tls + - name: kube-state-metrics-kube-rbac-proxy-config + secret: + secretName: kube-state-metrics-kube-rbac-proxy-config + - configMap: + name: kube-state-metrics-custom-resource-state-configmap + name: kube-state-metrics-custom-resource-state-configmap + - configMap: + name: metrics-client-ca + name: metrics-client-ca diff --git a/assets/optional/kube-state-metrics/04-service.yaml b/assets/optional/kube-state-metrics/04-service.yaml new file mode 100644 index 0000000000..75fddc4371 --- /dev/null +++ b/assets/optional/kube-state-metrics/04-service.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + openshift.io/description: |- + Expose kube-state-metrics `/metrics` endpoints within the cluster on the following ports: + * Port 8443 provides access to the Kubernetes resource metrics. This port is for internal use, and no other usage is guaranteed. + * Port 9443 provides access to the internal kube-state-metrics metrics. This port is for internal use, and no other usage is guaranteed. + service.beta.openshift.io/serving-cert-secret-name: kube-state-metrics-tls + labels: + app.kubernetes.io/component: exporter + app.kubernetes.io/managed-by: cluster-monitoring-operator + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/part-of: openshift-monitoring + app.kubernetes.io/version: 2.19.1 + name: kube-state-metrics + namespace: openshift-monitoring +spec: + clusterIP: None + ports: + - name: https-main + port: 8443 + targetPort: https-main + - name: https-self + port: 9443 + targetPort: https-self + selector: + app.kubernetes.io/component: exporter + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/part-of: openshift-monitoring diff --git a/assets/optional/kube-state-metrics/kustomization.aarch64.yaml b/assets/optional/kube-state-metrics/kustomization.aarch64.yaml new file mode 100644 index 0000000000..b400ac1e80 --- /dev/null +++ b/assets/optional/kube-state-metrics/kustomization.aarch64.yaml @@ -0,0 +1,7 @@ +images: + - name: quay.io/openshift/kube-rbac-proxy + newName: quay.io/openshift-release-dev/ocp-v5.0-art-dev + digest: sha256:8a74d54a45421f51cfc1d50b7fca04e177c8601cec4cf5ecfdac250e36904819 + - name: quay.io/openshift/kube-state-metrics + newName: quay.io/openshift-release-dev/ocp-v5.0-art-dev + digest: sha256:c01bcfe85cec4ea6a2c93f71793d292fd0f2be9d7176fa9b8b5fb63d3a773373 diff --git a/assets/optional/kube-state-metrics/kustomization.x86_64.yaml b/assets/optional/kube-state-metrics/kustomization.x86_64.yaml new file mode 100644 index 0000000000..a080d292b4 --- /dev/null +++ b/assets/optional/kube-state-metrics/kustomization.x86_64.yaml @@ -0,0 +1,7 @@ +images: + - name: quay.io/openshift/kube-rbac-proxy + newName: quay.io/openshift-release-dev/ocp-v5.0-art-dev + digest: sha256:b23eabd4a8578c71398ccde56be77ded55c7cbea36e592f3800347c33ca47c55 + - name: quay.io/openshift/kube-state-metrics + newName: quay.io/openshift-release-dev/ocp-v5.0-art-dev + digest: sha256:4a633470960127f9545b5509dafd8d423108edb74ca7013c9cddde4ff5b0332d diff --git a/assets/optional/kube-state-metrics/kustomization.yaml b/assets/optional/kube-state-metrics/kustomization.yaml new file mode 100644 index 0000000000..17942badc5 --- /dev/null +++ b/assets/optional/kube-state-metrics/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - 00-namespace.yaml + - 01-service-account.yaml + - 01-cluster-role.yaml + - 01-cluster-role-binding.yaml + - 02-kube-rbac-proxy-secret.yaml + - 02-custom-resource-state-configmap.yaml + - 03-deployment.yaml + - 04-service.yaml diff --git a/assets/optional/kube-state-metrics/release-kube-state-metrics-aarch64.json b/assets/optional/kube-state-metrics/release-kube-state-metrics-aarch64.json new file mode 100644 index 0000000000..da01a2f081 --- /dev/null +++ b/assets/optional/kube-state-metrics/release-kube-state-metrics-aarch64.json @@ -0,0 +1,8 @@ +{ + "release": { + "base": "5.0.0-0.nightly-arm64-2026-06-19-154904" + }, + "images": { + "kube_state_metrics": "quay.io/openshift-release-dev/ocp-v5.0-art-dev@sha256:c01bcfe85cec4ea6a2c93f71793d292fd0f2be9d7176fa9b8b5fb63d3a773373" + } +} diff --git a/assets/optional/kube-state-metrics/release-kube-state-metrics-x86_64.json b/assets/optional/kube-state-metrics/release-kube-state-metrics-x86_64.json new file mode 100644 index 0000000000..dfdae05ea2 --- /dev/null +++ b/assets/optional/kube-state-metrics/release-kube-state-metrics-x86_64.json @@ -0,0 +1,8 @@ +{ + "release": { + "base": "5.0.0-0.nightly-2026-06-19-155631" + }, + "images": { + "kube_state_metrics": "quay.io/openshift-release-dev/ocp-v5.0-art-dev@sha256:4a633470960127f9545b5509dafd8d423108edb74ca7013c9cddde4ff5b0332d" + } +} diff --git a/assets/optional/node-exporter/00-namespace.yaml b/assets/optional/node-exporter/00-namespace.yaml new file mode 100644 index 0000000000..17f727565a --- /dev/null +++ b/assets/optional/node-exporter/00-namespace.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: openshift-monitoring + labels: + name: openshift-monitoring + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/audit: privileged + pod-security.kubernetes.io/warn: privileged diff --git a/assets/optional/node-exporter/01-cluster-role-binding.yaml b/assets/optional/node-exporter/01-cluster-role-binding.yaml new file mode 100644 index 0000000000..b6790fa9b4 --- /dev/null +++ b/assets/optional/node-exporter/01-cluster-role-binding.yaml @@ -0,0 +1,18 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: exporter + app.kubernetes.io/managed-by: cluster-monitoring-operator + app.kubernetes.io/name: node-exporter + app.kubernetes.io/part-of: openshift-monitoring + app.kubernetes.io/version: 1.11.1 + name: node-exporter +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: node-exporter +subjects: +- kind: ServiceAccount + name: node-exporter + namespace: openshift-monitoring diff --git a/assets/optional/node-exporter/01-cluster-role.yaml b/assets/optional/node-exporter/01-cluster-role.yaml new file mode 100644 index 0000000000..50d7a5e755 --- /dev/null +++ b/assets/optional/node-exporter/01-cluster-role.yaml @@ -0,0 +1,31 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: exporter + app.kubernetes.io/managed-by: cluster-monitoring-operator + app.kubernetes.io/name: node-exporter + app.kubernetes.io/part-of: openshift-monitoring + app.kubernetes.io/version: 1.11.1 + name: node-exporter +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +- apiGroups: + - security.openshift.io + resourceNames: + - node-exporter + resources: + - securitycontextconstraints + verbs: + - use diff --git a/assets/optional/node-exporter/01-security-context-constraints.yaml b/assets/optional/node-exporter/01-security-context-constraints.yaml new file mode 100644 index 0000000000..1caaf72fcd --- /dev/null +++ b/assets/optional/node-exporter/01-security-context-constraints.yaml @@ -0,0 +1,22 @@ +allowHostDirVolumePlugin: true +allowHostNetwork: true +allowHostPID: true +allowHostPorts: true +allowPrivilegedContainer: true +apiVersion: security.openshift.io/v1 +kind: SecurityContextConstraints +metadata: + annotations: + kubernetes.io/description: node-exporter scc is used for the Prometheus node exporter + labels: + app.kubernetes.io/managed-by: cluster-monitoring-operator + app.kubernetes.io/part-of: openshift-monitoring + name: node-exporter +readOnlyRootFilesystem: false +runAsUser: + type: RunAsAny +seLinuxContext: + type: RunAsAny +seccompProfiles: +- runtime/default +users: [] diff --git a/assets/optional/node-exporter/01-service-account.yaml b/assets/optional/node-exporter/01-service-account.yaml new file mode 100644 index 0000000000..c3d1dc95c9 --- /dev/null +++ b/assets/optional/node-exporter/01-service-account.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +automountServiceAccountToken: false +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: exporter + app.kubernetes.io/managed-by: cluster-monitoring-operator + app.kubernetes.io/name: node-exporter + app.kubernetes.io/part-of: openshift-monitoring + app.kubernetes.io/version: 1.11.1 + name: node-exporter + namespace: openshift-monitoring diff --git a/assets/optional/node-exporter/02-accelerators-collector-configmap.yaml b/assets/optional/node-exporter/02-accelerators-collector-configmap.yaml new file mode 100644 index 0000000000..9ddda923f2 --- /dev/null +++ b/assets/optional/node-exporter/02-accelerators-collector-configmap.yaml @@ -0,0 +1,141 @@ +apiVersion: v1 +data: + config.yaml: |- + - "models": + - "modelName": "NVIDIA A800 PCIe 80GB" + "pciID": "0x20f5" + - "modelName": "NVIDIA A800 40GB PCIe active cooled" + "pciID": "0x20f6" + - "modelName": "NVIDIA AX800" + "pciID": "0x20fd" + - "modelName": "NVIDIA A100 PCIe 40GB" + "pciID": "0x20f1" + - "modelName": "NVIDIA A100 PCIe 80GB" + "pciID": "0x20b5" + - "modelName": "NVIDIA A40" + "pciID": "0x2235" + - "modelName": "NVIDIA A30" + "pciID": "0x20b7" + - "modelName": "NVIDIA A10" + "pciID": "0x2236" + - "modelName": "NVIDIA A16" + "pciID": "0x25b6" + - "modelName": "H800 NVL" + "pciID": "0x2322" + - "modelName": "NVIDIA H100 NVL" + "pciID": "0x2321" + - "modelName": "NVIDIA H100 PCIe 80GB" + "pciID": "0x2331" + - "modelName": "NVIDIA L40" + "pciID": "0x26b5" + - "modelName": "NVIDIA L40S" + "pciID": "0x26b9" + - "modelName": "NVIDIA L20 liquid cooled" + "pciID": "0x26bA" + - "modelName": "NVIDIA L4" + "pciID": "0x27b8" + - "modelName": "NVIDIA L2" + "pciID": "0x27b6" + - "modelName": "NVIDIA RTX 6000 Ada" + "pciID": "0x26b1" + - "modelName": "NVIDIA RTX 5880 Ada" + "pciID": "0x26b3" + - "modelName": "NVIDIA RTX 5000 Ada" + "pciID": "0x2231" + - "modelName": "NVIDIA RTX A6000" + "pciID": "0x2230" + - "modelName": "NVIDIA RTX A5500" + "pciID": "0x2233" + - "modelName": "NVIDIA RTX 8000 passive" + "pciID": "0x1e30" + - "modelName": "NVIDIA RTX A2000" + "pciID": "0x2531" + - "modelName": "NVIDIA A100 SXM4 40GB" + "pciID": "0x20b0" + - "modelName": "NVIDIA H800 NVL" + "pciID": "0x233a" + - "modelName": "NVIDIA H200 NVL" + "pciID": "0x233b" + - "modelName": "NVIDIA A100 SXM4 80GB" + "pciID": "0x20b2" + - "modelName": "NVIDIA A100 SXM 64GB" + "pciID": "0x20b3" + - "modelName": "NVIDIA A800 SXM4 40GB" + "pciID": "0x20bd" + - "modelName": "NVIDIA A800 SXM4 80GB" + "pciID": "0x20f3" + - "modelName": "NVIDIA RTX A1000" + "pciID": "0x25b0" + - "modelName": "Blackwell RTX PRO 6000" + "pciID": "0x2bb5" + - "modelName": "Blackwell GB100" + "pciID": "0x2941" + - "modelName": "NVIDIA H200" + "pciID": "0x2335" + "vendorID": "0x10de" + "vendorName": "NVIDIA" + - "models": + - "modelName": "AMD MI210" + "pciID": "0x740f" + - "modelName": "AMD MI250" + "pciID": "0x740c" + - "modelName": "AMD MI250X" + "pciID": "0x7408" + - "modelName": "AMD MI300" + "pciID": "0x74a0" + - "modelName": "AMD MI300X" + "pciID": "0x74a1" + - "modelName": "AMD MI325X" + "pciID": "0x74a5" + - "modelName": "AMD MI308X" + "pciID": "0x7aa2" + - "modelName": "AMD MI300X VF" + "pciID": "0x74b5" + - "modelName": "AMD MI210 VF" + "pciID": "0x7410" + "vendorID": "0x1002" + "vendorName": "AMD" + - "models": + - "modelName": "Gaudi 1" + "pciID": "0x1000" + - "modelName": "Gaudi 2" + "pciID": "0x1020" + "vendorID": "0x1da3" + "vendorName": "GAUDI" + - "models": + - "modelName": "Intel Data Center GPU Max 1550" + "pciID": "0x0bd5" + - "modelName": "Intel Data Center GPU Max 1100" + "pciID": "0x0bda" + - "modelName": "Intel Data Center GPU Flex 170" + "pciID": "0x56c0" + - "modelName": "Intel Data Center GPU Flex 140" + "pciID": "0x56c1" + - "modelName": "Intel IPU Data Path" + "pciID": "0x1452" + "vendorID": "0x8086" + "vendorName": "Intel" + - "models": + - "modelName": "Qualcomm AI 100" + "pciID": "0xa100" + - "modelName": "Qualcomm AI 80" + "pciID": "0xa080" + "vendorID": "0x17cb" + "vendorName": "Qualcomm" + - "models": + - "modelName": "Marvell OCTEON 10 CN10XXX" + "pciID": "0xb900" + "vendorID": "0x177d" + "vendorName": "Marvell" + - "models": + - "modelName": "BlueField-3 integrated ConnectX-7" + "pciID": "0xa2dc" + "vendorID": "0x15b3" + "vendorName": "Mellanox" +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/managed-by: cluster-monitoring-operator + app.kubernetes.io/part-of: openshift-monitoring + name: node-exporter-accelerators-collector-config + namespace: openshift-monitoring diff --git a/assets/optional/node-exporter/02-kube-rbac-proxy-secret.yaml b/assets/optional/node-exporter/02-kube-rbac-proxy-secret.yaml new file mode 100644 index 0000000000..e02d0bb40b --- /dev/null +++ b/assets/optional/node-exporter/02-kube-rbac-proxy-secret.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +data: {} +kind: Secret +metadata: + labels: + app.kubernetes.io/managed-by: cluster-monitoring-operator + app.kubernetes.io/part-of: openshift-monitoring + name: node-exporter-kube-rbac-proxy-config + namespace: openshift-monitoring +stringData: + config.yaml: |- + "authorization": + "static": + - "path": "/metrics" + "resourceRequest": false + "verb": "get" +type: Opaque diff --git a/assets/optional/node-exporter/03-daemonset.yaml b/assets/optional/node-exporter/03-daemonset.yaml new file mode 100644 index 0000000000..8a384935f0 --- /dev/null +++ b/assets/optional/node-exporter/03-daemonset.yaml @@ -0,0 +1,198 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app.kubernetes.io/component: exporter + app.kubernetes.io/managed-by: cluster-monitoring-operator + app.kubernetes.io/name: node-exporter + app.kubernetes.io/part-of: openshift-monitoring + app.kubernetes.io/version: 1.11.1 + name: node-exporter + namespace: openshift-monitoring +spec: + selector: + matchLabels: + app.kubernetes.io/component: exporter + app.kubernetes.io/name: node-exporter + app.kubernetes.io/part-of: openshift-monitoring + template: + metadata: + annotations: + cluster-autoscaler.kubernetes.io/enable-ds-eviction: "false" + kubectl.kubernetes.io/default-container: node-exporter + openshift.io/required-scc: node-exporter + target.workload.openshift.io/management: '{"effect": "PreferredDuringScheduling"}' + labels: + app.kubernetes.io/component: exporter + app.kubernetes.io/managed-by: cluster-monitoring-operator + app.kubernetes.io/name: node-exporter + app.kubernetes.io/part-of: openshift-monitoring + app.kubernetes.io/version: 1.11.1 + spec: + automountServiceAccountToken: true + containers: + - args: + - --web.listen-address=127.0.0.1:9101 + - --path.sysfs=/host/sys + - --path.rootfs=/host/root + - --path.procfs=/host/root/proc + - --path.udev.data=/host/root/run/udev/data + - --no-collector.wifi + - --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|run/k3s/containerd/.+|var/lib/docker/.+|var/lib/kubelet/pods/.+)($|/) + - --collector.netclass.ignored-devices=^.*$ + - --collector.netdev.device-exclude=^.*$ + - --collector.cpu.info + - --collector.textfile.directory=/var/node_exporter/textfile + - --no-collector.btrfs + command: + - /bin/sh + - -c + - | + export GOMAXPROCS=4 + # We don't take CPU affinity into account as the container doesn't have integer CPU requests. + # In case of error, fallback to the default value. + NUM_CPUS=$(grep -c '^processor' "/proc/cpuinfo" 2>/dev/null || echo "0") + if [ "$NUM_CPUS" -lt "$GOMAXPROCS" ]; then + export GOMAXPROCS="$NUM_CPUS" + fi + echo "ts=$(date --iso-8601=seconds) num_cpus=$NUM_CPUS gomaxprocs=$GOMAXPROCS" + exec /bin/node_exporter "$0" "$@" + env: + - name: DBUS_SYSTEM_BUS_ADDRESS + value: unix:path=/host/root/var/run/dbus/system_bus_socket + image: "quay.io/openshift/node-exporter" + name: node-exporter + resources: + requests: + cpu: 8m + memory: 32Mi + securityContext: {} + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /host/sys + mountPropagation: HostToContainer + name: sys + readOnly: true + - mountPath: /host/root + mountPropagation: HostToContainer + name: root + readOnly: true + - mountPath: /var/node_exporter/textfile + name: node-exporter-textfile + readOnly: true + - mountPath: /var/node_exporter/accelerators_collector_config + name: node-exporter-accelerators-collector-config + readOnly: true + workingDir: /var/node_exporter/textfile + - args: + - --secure-listen-address=0.0.0.0:9100 + - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - --upstream=http://127.0.0.1:9101/ + - --tls-cert-file=/etc/tls/private/tls.crt + - --tls-private-key-file=/etc/tls/private/tls.key + - --client-ca-file=/etc/tls/client/client-ca.crt + - --config-file=/etc/kube-rbac-policy/config.yaml + env: + - name: IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: "quay.io/openshift/kube-rbac-proxy" + name: kube-rbac-proxy + ports: + - containerPort: 9100 + hostPort: 9100 + name: https + resources: + requests: + cpu: 1m + memory: 15Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 + seccompProfile: + type: RuntimeDefault + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /etc/tls/private + name: node-exporter-tls + readOnly: true + - mountPath: /etc/kube-rbac-policy + name: node-exporter-kube-rbac-proxy-config + readOnly: true + - mountPath: /etc/tls/client + name: metrics-client-ca + readOnly: true + hostNetwork: true + hostPID: true + initContainers: + - command: + - /bin/sh + - -c + - '[[ ! -d /node_exporter/collectors/init ]] || find /node_exporter/collectors/init -perm /111 -type f -exec {} \;' + env: + - name: TMPDIR + value: /tmp + image: "quay.io/openshift/node-exporter" + name: init-textfile + resources: + requests: + cpu: 1m + memory: 1Mi + securityContext: + privileged: true + runAsUser: 0 + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /var/node_exporter/textfile + name: node-exporter-textfile + readOnly: false + - mountPath: /var/log/wtmp + name: node-exporter-wtmp + readOnly: true + workingDir: /var/node_exporter/textfile + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + securityContext: {} + serviceAccountName: node-exporter + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /sys + name: sys + - hostPath: + path: / + name: root + - emptyDir: {} + name: node-exporter-textfile + - name: node-exporter-tls + secret: + secretName: node-exporter-tls + - hostPath: + path: /var/log/wtmp + type: File + name: node-exporter-wtmp + - name: node-exporter-kube-rbac-proxy-config + secret: + secretName: node-exporter-kube-rbac-proxy-config + - configMap: + items: + - key: config.yaml + path: config.yaml + name: node-exporter-accelerators-collector-config + name: node-exporter-accelerators-collector-config + - configMap: + name: metrics-client-ca + name: metrics-client-ca + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + type: RollingUpdate diff --git a/assets/optional/node-exporter/04-service.yaml b/assets/optional/node-exporter/04-service.yaml new file mode 100644 index 0000000000..37b420ccdb --- /dev/null +++ b/assets/optional/node-exporter/04-service.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + openshift.io/description: Expose the `/metrics` endpoint on port 9100. This port is for internal use, and no other usage is guaranteed. + service.beta.openshift.io/serving-cert-secret-name: node-exporter-tls + labels: + app.kubernetes.io/component: exporter + app.kubernetes.io/managed-by: cluster-monitoring-operator + app.kubernetes.io/name: node-exporter + app.kubernetes.io/part-of: openshift-monitoring + app.kubernetes.io/version: 1.11.1 + name: node-exporter + namespace: openshift-monitoring +spec: + clusterIP: None + ports: + - name: https + port: 9100 + targetPort: https + selector: + app.kubernetes.io/component: exporter + app.kubernetes.io/name: node-exporter + app.kubernetes.io/part-of: openshift-monitoring diff --git a/assets/optional/node-exporter/kustomization.aarch64.yaml b/assets/optional/node-exporter/kustomization.aarch64.yaml new file mode 100644 index 0000000000..7686f7f2ba --- /dev/null +++ b/assets/optional/node-exporter/kustomization.aarch64.yaml @@ -0,0 +1,7 @@ +images: + - name: quay.io/openshift/kube-rbac-proxy + newName: quay.io/openshift-release-dev/ocp-v5.0-art-dev + digest: sha256:8a74d54a45421f51cfc1d50b7fca04e177c8601cec4cf5ecfdac250e36904819 + - name: quay.io/openshift/node-exporter + newName: quay.io/openshift-release-dev/ocp-v5.0-art-dev + digest: sha256:b908cd047a093dd789377c350e13832eb8dee38964b07e2f67a8c3cf1c9a5fc9 diff --git a/assets/optional/node-exporter/kustomization.x86_64.yaml b/assets/optional/node-exporter/kustomization.x86_64.yaml new file mode 100644 index 0000000000..c3c1e34433 --- /dev/null +++ b/assets/optional/node-exporter/kustomization.x86_64.yaml @@ -0,0 +1,7 @@ +images: + - name: quay.io/openshift/kube-rbac-proxy + newName: quay.io/openshift-release-dev/ocp-v5.0-art-dev + digest: sha256:b23eabd4a8578c71398ccde56be77ded55c7cbea36e592f3800347c33ca47c55 + - name: quay.io/openshift/node-exporter + newName: quay.io/openshift-release-dev/ocp-v5.0-art-dev + digest: sha256:c39c86b8b8b39ff6db9818e1a4f7dcde73d26ee427bded9c772bfe477ec020f9 diff --git a/assets/optional/node-exporter/kustomization.yaml b/assets/optional/node-exporter/kustomization.yaml new file mode 100644 index 0000000000..20b41b0f31 --- /dev/null +++ b/assets/optional/node-exporter/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - 00-namespace.yaml + - 01-service-account.yaml + - 01-cluster-role.yaml + - 01-cluster-role-binding.yaml + - 01-security-context-constraints.yaml + - 02-kube-rbac-proxy-secret.yaml + - 02-accelerators-collector-configmap.yaml + - 03-daemonset.yaml + - 04-service.yaml diff --git a/assets/optional/node-exporter/release-node-exporter-aarch64.json b/assets/optional/node-exporter/release-node-exporter-aarch64.json new file mode 100644 index 0000000000..a1e3d0ae0f --- /dev/null +++ b/assets/optional/node-exporter/release-node-exporter-aarch64.json @@ -0,0 +1,8 @@ +{ + "release": { + "base": "5.0.0-0.nightly-arm64-2026-06-19-154904" + }, + "images": { + "node_exporter": "quay.io/openshift-release-dev/ocp-v5.0-art-dev@sha256:b908cd047a093dd789377c350e13832eb8dee38964b07e2f67a8c3cf1c9a5fc9" + } +} diff --git a/assets/optional/node-exporter/release-node-exporter-x86_64.json b/assets/optional/node-exporter/release-node-exporter-x86_64.json new file mode 100644 index 0000000000..399aa1c470 --- /dev/null +++ b/assets/optional/node-exporter/release-node-exporter-x86_64.json @@ -0,0 +1,8 @@ +{ + "release": { + "base": "5.0.0-0.nightly-2026-06-19-155631" + }, + "images": { + "node_exporter": "quay.io/openshift-release-dev/ocp-v5.0-art-dev@sha256:c39c86b8b8b39ff6db9818e1a4f7dcde73d26ee427bded9c772bfe477ec020f9" + } +} diff --git a/packaging/rpm/microshift.spec b/packaging/rpm/microshift.spec index 4dcc333d51..168d8ecad0 100644 --- a/packaging/rpm/microshift.spec +++ b/packaging/rpm/microshift.spec @@ -278,6 +278,43 @@ The microshift-metrics-server-release-info package provides release information release. These files contain the list of container image references used by the metrics-server and can be used to embed those images into osbuilder blueprints or bootc containerfiles. +%package metrics-node-exporter +Summary: Prometheus node-exporter for MicroShift +ExclusiveArch: x86_64 aarch64 +Requires: microshift = %{version} + +%description metrics-node-exporter +The microshift-metrics-node-exporter package provides the Prometheus node-exporter for MicroShift. +Install this package to expose host-level hardware and OS metrics. + +%package metrics-node-exporter-release-info +Summary: Release information for node-exporter for MicroShift +BuildArch: noarch +Requires: microshift-release-info = %{version} + +%description metrics-node-exporter-release-info +The microshift-metrics-node-exporter-release-info package provides release information files for this +release. These files contain the list of container image references used by node-exporter +and can be used to embed those images into osbuilder blueprints or bootc containerfiles. +%package metrics-kube-state +Summary: Kubernetes kube-state-metrics for MicroShift +ExclusiveArch: x86_64 aarch64 +Requires: microshift = %{version} + +%description metrics-kube-state +The microshift-metrics-kube-state package provides kube-state-metrics for MicroShift. +Install this package to expose Kubernetes object state metrics via a secure endpoint. + +%package metrics-kube-state-release-info +Summary: Release information for kube-state-metrics for MicroShift +BuildArch: noarch +Requires: microshift-release-info = %{version} + +%description metrics-kube-state-release-info +The microshift-metrics-kube-state-release-info package provides release information files for this +release. These files contain the list of container image references used by kube-state-metrics +and can be used to embed those images into osbuilder blueprints or bootc containerfiles. + %package sriov Summary: SR-IOV Network Operator for MicroShift ExclusiveArch: x86_64 aarch64 @@ -641,6 +678,53 @@ cat assets/optional/metrics-server/kustomization.x86_64.yaml >> %{buildroot}/%{_ mkdir -p -m755 %{buildroot}%{_datadir}/microshift/release install -p -m644 assets/optional/metrics-server/release-metrics-server-{x86_64,aarch64}.json %{buildroot}%{_datadir}/microshift/release/ +# node-exporter +install -d -m755 %{buildroot}/%{_prefix}/lib/microshift/manifests.d/082-microshift-node-exporter +install -p -m644 assets/optional/node-exporter/00-namespace.yaml %{buildroot}/%{_prefix}/lib/microshift/manifests.d/082-microshift-node-exporter +install -p -m644 assets/optional/node-exporter/01-service-account.yaml %{buildroot}/%{_prefix}/lib/microshift/manifests.d/082-microshift-node-exporter +install -p -m644 assets/optional/node-exporter/01-cluster-role.yaml %{buildroot}/%{_prefix}/lib/microshift/manifests.d/082-microshift-node-exporter +install -p -m644 assets/optional/node-exporter/01-cluster-role-binding.yaml %{buildroot}/%{_prefix}/lib/microshift/manifests.d/082-microshift-node-exporter +install -p -m644 assets/optional/node-exporter/01-security-context-constraints.yaml %{buildroot}/%{_prefix}/lib/microshift/manifests.d/082-microshift-node-exporter +install -p -m644 assets/optional/node-exporter/02-kube-rbac-proxy-secret.yaml %{buildroot}/%{_prefix}/lib/microshift/manifests.d/082-microshift-node-exporter +install -p -m644 assets/optional/node-exporter/02-accelerators-collector-configmap.yaml %{buildroot}/%{_prefix}/lib/microshift/manifests.d/082-microshift-node-exporter +install -p -m644 assets/optional/node-exporter/03-daemonset.yaml %{buildroot}/%{_prefix}/lib/microshift/manifests.d/082-microshift-node-exporter +install -p -m644 assets/optional/node-exporter/04-service.yaml %{buildroot}/%{_prefix}/lib/microshift/manifests.d/082-microshift-node-exporter +install -p -m644 assets/optional/node-exporter/kustomization.yaml %{buildroot}/%{_prefix}/lib/microshift/manifests.d/082-microshift-node-exporter + +%ifarch %{arm} aarch64 +cat assets/optional/node-exporter/kustomization.aarch64.yaml >> %{buildroot}/%{_prefix}/lib/microshift/manifests.d/082-microshift-node-exporter/kustomization.yaml +%endif +%ifarch x86_64 +cat assets/optional/node-exporter/kustomization.x86_64.yaml >> %{buildroot}/%{_prefix}/lib/microshift/manifests.d/082-microshift-node-exporter/kustomization.yaml +%endif + +# node-exporter-release-info +mkdir -p -m755 %{buildroot}%{_datadir}/microshift/release +install -p -m644 assets/optional/node-exporter/release-node-exporter-{x86_64,aarch64}.json %{buildroot}%{_datadir}/microshift/release/ + +# kube-state-metrics +install -d -m755 %{buildroot}/%{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics +install -p -m644 assets/optional/kube-state-metrics/00-namespace.yaml %{buildroot}/%{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics +install -p -m644 assets/optional/kube-state-metrics/01-service-account.yaml %{buildroot}/%{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics +install -p -m644 assets/optional/kube-state-metrics/01-cluster-role.yaml %{buildroot}/%{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics +install -p -m644 assets/optional/kube-state-metrics/01-cluster-role-binding.yaml %{buildroot}/%{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics +install -p -m644 assets/optional/kube-state-metrics/02-kube-rbac-proxy-secret.yaml %{buildroot}/%{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics +install -p -m644 assets/optional/kube-state-metrics/02-custom-resource-state-configmap.yaml %{buildroot}/%{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics +install -p -m644 assets/optional/kube-state-metrics/03-deployment.yaml %{buildroot}/%{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics +install -p -m644 assets/optional/kube-state-metrics/04-service.yaml %{buildroot}/%{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics +install -p -m644 assets/optional/kube-state-metrics/kustomization.yaml %{buildroot}/%{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics + +%ifarch %{arm} aarch64 +cat assets/optional/kube-state-metrics/kustomization.aarch64.yaml >> %{buildroot}/%{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics/kustomization.yaml +%endif +%ifarch x86_64 +cat assets/optional/kube-state-metrics/kustomization.x86_64.yaml >> %{buildroot}/%{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics/kustomization.yaml +%endif + +# kube-state-metrics-release-info +mkdir -p -m755 %{buildroot}%{_datadir}/microshift/release +install -p -m644 assets/optional/kube-state-metrics/release-kube-state-metrics-{x86_64,aarch64}.json %{buildroot}%{_datadir}/microshift/release/ + # sriov install -d -m755 %{buildroot}/%{_prefix}/lib/microshift/manifests.d/070-microshift-sriov install -d -m755 %{buildroot}/%{_prefix}/lib/microshift/manifests.d/070-microshift-sriov/crd @@ -851,6 +935,19 @@ fi %files metrics-server-release-info %{_datadir}/microshift/release/release-metrics-server-{x86_64,aarch64}.json +%files metrics-node-exporter +%dir %{_prefix}/lib/microshift/manifests.d/082-microshift-node-exporter +%{_prefix}/lib/microshift/manifests.d/082-microshift-node-exporter/* + +%files metrics-node-exporter-release-info +%{_datadir}/microshift/release/release-node-exporter-{x86_64,aarch64}.json +%files metrics-kube-state +%dir %{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics +%{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics/* + +%files metrics-kube-state-release-info +%{_datadir}/microshift/release/release-kube-state-metrics-{x86_64,aarch64}.json + %files sriov %dir %{_prefix}/lib/microshift/manifests.d/070-microshift-sriov %dir %{_prefix}/lib/microshift/manifests.d/070-microshift-sriov/crd @@ -866,9 +963,15 @@ fi # Use Git command to generate the log and replace the VERSION string # LANG=C git log --date="format:%a %b %d %Y" --pretty="tformat:* %cd %an <%ae> VERSION%n- %s%n" packaging/rpm/microshift.spec %changelog +* Tue Jun 23 2026 Jonathan H. Cope 5.0 +- Add node-exporter as optional rpm package + * Mon Jun 22 2026 Jonathan H. Cope 5.0 - Add metrics-server as optional rpm package +* Mon Jun 22 2026 Jonathan H. Cope 5.0 +- add kube-state-metrics as optional rpm package + * Tue Jan 20 2026 Pablo Acevedo Montserrat 4.21.0 - Add multus as dependency for sriov diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index ca2b132f91..e21fba4c5b 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -311,6 +311,13 @@ func RunMicroshift(cfg *config.Config) error { klog.Errorf("Failed to provision metrics-server certs: %v", err) } }() + // Provision shared resources for optional monitoring components. + // Runs concurrently because it polls for the namespace created by kustomize. + go func() { + if err := components.ProvisionMetricsClientCA(runCtx, cfg); err != nil { + klog.Errorf("Failed to provision metrics-client-ca: %v", err) + } + }() // Watch for SIGTERM or service error to exit, now that we are ready. select { diff --git a/pkg/components/metrics.go b/pkg/components/metrics.go index 786612f896..01d85b6dbd 100644 --- a/pkg/components/metrics.go +++ b/pkg/components/metrics.go @@ -6,6 +6,8 @@ import ( "os" "time" + "k8s.io/client-go/kubernetes" + "github.com/openshift/library-go/pkg/operator/events" "github.com/openshift/library-go/pkg/operator/resource/resourceapply" "github.com/openshift/microshift/pkg/config" @@ -21,10 +23,32 @@ import ( const ( metricsServerManifestPath = "/usr/lib/microshift/manifests.d/080-microshift-metrics-server" - metricsServerNamespace = "openshift-monitoring" + metricsNamespace = "openshift-monitoring" ) -var metricsEventRecorder events.Recorder = events.NewLoggingEventRecorder("microshift-metrics-server", clock.RealClock{}) +var metricsServerEventRecorder events.Recorder = events.NewLoggingEventRecorder("microshift-metrics-server", clock.RealClock{}) + +var metricsClientCARecorder events.Recorder = events.NewLoggingEventRecorder("metrics-client-ca", clock.RealClock{}) + +var metricsClientCAConsumerPaths = []string{ + "/usr/lib/microshift/manifests.d/081-microshift-kube-state-metrics", + "/usr/lib/microshift/manifests.d/082-microshift-node-exporter", +} + +func waitForNamespace(ctx context.Context, clientset kubernetes.Interface, namespace string) error { + return wait.PollUntilContextTimeout(ctx, 2*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) { + _, err := clientset.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) + if err == nil { + return true, nil + } + if !apierrors.IsNotFound(err) { + klog.Errorf("getting namespace %s: %v", namespace, err) + return false, nil + } + klog.V(2).Infof("Waiting for namespace %s to be created by kustomize", namespace) + return false, nil + }) +} // ProvisionMetricsServerCerts provisions the TLS client certificate and kubelet // serving CA that metrics-server needs to authenticate to kubelet and verify its @@ -48,20 +72,8 @@ func ProvisionMetricsServerCerts(ctx context.Context, cfg *config.Config) error return fmt.Errorf("creating clientset: %w", err) } - err = wait.PollUntilContextTimeout(ctx, 2*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) { - _, err := clientset.CoreV1().Namespaces().Get(ctx, metricsServerNamespace, metav1.GetOptions{}) - if err == nil { - return true, nil - } - if !apierrors.IsNotFound(err) { - klog.Errorf("getting namespace %s: %v", metricsServerNamespace, err) - return false, nil - } - klog.V(2).Infof("Waiting for namespace %s to be created by kustomize", metricsServerNamespace) - return false, nil - }) - if err != nil { - return fmt.Errorf("waiting for namespace %s: %w", metricsServerNamespace, err) + if err := waitForNamespace(ctx, clientset, metricsNamespace); err != nil { + return fmt.Errorf("waiting for namespace %s: %w", metricsNamespace, err) } certsDir := cryptomaterial.CertsDirectory(config.DataDir) @@ -79,7 +91,7 @@ func ProvisionMetricsServerCerts(ctx context.Context, cfg *config.Config) error secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: "metrics-server-client-certs", - Namespace: metricsServerNamespace, + Namespace: metricsNamespace, Annotations: map[string]string{ "openshift.io/owning-component": "metrics-server", }, @@ -92,7 +104,7 @@ func ProvisionMetricsServerCerts(ctx context.Context, cfg *config.Config) error } err = wait.PollUntilContextTimeout(ctx, 2*time.Second, 1*time.Minute, true, func(ctx context.Context) (bool, error) { - _, _, err := resourceapply.ApplySecret(ctx, clientset.CoreV1(), metricsEventRecorder, secret) + _, _, err := resourceapply.ApplySecret(ctx, clientset.CoreV1(), metricsServerEventRecorder, secret) if err != nil { klog.Errorf("applying metrics-server client cert secret: %v", err) return false, nil @@ -111,7 +123,7 @@ func ProvisionMetricsServerCerts(ctx context.Context, cfg *config.Config) error cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "kubelet-serving-ca-bundle", - Namespace: metricsServerNamespace, + Namespace: metricsNamespace, Annotations: map[string]string{ "openshift.io/owning-component": "metrics-server", }, @@ -122,7 +134,7 @@ func ProvisionMetricsServerCerts(ctx context.Context, cfg *config.Config) error } err = wait.PollUntilContextTimeout(ctx, 2*time.Second, 1*time.Minute, true, func(ctx context.Context) (bool, error) { - _, _, err := resourceapply.ApplyConfigMap(ctx, clientset.CoreV1(), metricsEventRecorder, cm) + _, _, err := resourceapply.ApplyConfigMap(ctx, clientset.CoreV1(), metricsServerEventRecorder, cm) if err != nil { klog.Errorf("applying kubelet serving CA configmap: %v", err) return false, nil @@ -130,9 +142,81 @@ func ProvisionMetricsServerCerts(ctx context.Context, cfg *config.Config) error return true, nil }) if err != nil { - return fmt.Errorf("applying kubelet serving CA configmap: %v", err) + return fmt.Errorf("applying kubelet serving CA configmap: %w", err) } klog.Infof("Provisioned metrics-server kubelet client cert and CA bundle") return nil } + +// ProvisionMetricsClientCA provisions the admin-kubeconfig-signer CA that +// kube-rbac-proxy sidecars in kube-state-metrics and node-exporter use to +// verify client certificates on incoming scrape requests. The CA cannot be +// included in static manifests because it is generated at MicroShift startup +// and may be rotated; this function ensures the ConfigMap reflects the current CA. +func ProvisionMetricsClientCA(ctx context.Context, cfg *config.Config) error { + needed := false + for _, p := range metricsClientCAConsumerPaths { + exists, err := util.PathExists(p) + if err != nil { + return err + } + if exists { + needed = true + break + } + } + if !needed { + klog.V(2).Infof("No monitoring components found, skipping metrics-client-ca provisioning") + return nil + } + + kubeconfigPath := cfg.KubeConfigPath(config.KubeAdmin) + clientset, err := getKubernetesClient(kubeconfigPath) + if err != nil { + return fmt.Errorf("creating clientset: %w", err) + } + + if err := waitForNamespace(ctx, clientset, metricsNamespace); err != nil { + return fmt.Errorf("waiting for namespace %s: %w", metricsNamespace, err) + } + + certsDir := cryptomaterial.CertsDirectory(config.DataDir) + caCertPath := cryptomaterial.CACertPath(cryptomaterial.AdminKubeconfigSignerDir(certsDir)) + caPEM, err := os.ReadFile(caCertPath) + if err != nil { + return fmt.Errorf("reading admin-kubeconfig-signer CA: %w", err) + } + + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "metrics-client-ca", + Namespace: metricsNamespace, + Annotations: map[string]string{ + "openshift.io/owning-component": "Monitoring", + }, + Labels: map[string]string{ + "app.kubernetes.io/managed-by": "cluster-monitoring-operator", + "app.kubernetes.io/part-of": "openshift-monitoring", + }, + }, + Data: map[string]string{ + "client-ca.crt": string(caPEM), + }, + } + + err = wait.PollUntilContextTimeout(ctx, 2*time.Second, 1*time.Minute, true, func(ctx context.Context) (bool, error) { + _, _, err := resourceapply.ApplyConfigMap(ctx, clientset.CoreV1(), metricsClientCARecorder, cm) + if err != nil { + klog.Errorf("applying metrics-client-ca configmap: %v", err) + return false, nil + } + return true, nil + }) + if err != nil { + return fmt.Errorf("applying metrics-client-ca configmap: %w", err) + } + + klog.Infof("Provisioned metrics-client-ca configmap in %s", metricsNamespace) + return nil +} diff --git a/pkg/healthcheck/microshift_optional_workloads.go b/pkg/healthcheck/microshift_optional_workloads.go index 7772b1e36c..1d70e4b563 100644 --- a/pkg/healthcheck/microshift_optional_workloads.go +++ b/pkg/healthcheck/microshift_optional_workloads.go @@ -45,6 +45,14 @@ var optionalWorkloadPaths = map[string]optionalWorkloads{ Namespace: "openshift-monitoring", Workloads: NamespaceWorkloads{Deployments: []string{"metrics-server"}}, }, + "/usr/lib/microshift/manifests.d/081-microshift-kube-state-metrics": { + Namespace: "openshift-monitoring", + Workloads: NamespaceWorkloads{Deployments: []string{"kube-state-metrics"}}, + }, + "/usr/lib/microshift/manifests.d/082-microshift-node-exporter": { + Namespace: "openshift-monitoring", + Workloads: NamespaceWorkloads{DaemonSets: []string{"node-exporter"}}, + }, } // mergeWorkloads combines two NamespaceWorkloads into one. diff --git a/scripts/auto-rebase/assets_cluster_monitoring_operator.yaml b/scripts/auto-rebase/assets_cluster_monitoring_operator.yaml index b252b3ef12..4ecf7e21bc 100644 --- a/scripts/auto-rebase/assets_cluster_monitoring_operator.yaml +++ b/scripts/auto-rebase/assets_cluster_monitoring_operator.yaml @@ -35,3 +35,69 @@ assets: ignore: "gets generated during image rebase" - file: release-metrics-server-aarch64.json ignore: "gets generated during image rebase" + - dir: optional/kube-state-metrics/ + no_clean: True + src: cluster-monitoring-operator/assets/kube-state-metrics/ + files: + - file: 00-namespace.yaml + ignore: "MicroShift-specific, no upstream equivalent" + git_restore: True + - file: 01-cluster-role.yaml + src: cluster-role.yaml + - file: 01-cluster-role-binding.yaml + src: cluster-role-binding.yaml + - file: 01-service-account.yaml + src: service-account.yaml + - file: 02-custom-resource-state-configmap.yaml + src: custom-resource-state-configmap.yaml + - file: 02-kube-rbac-proxy-secret.yaml + src: kube-rbac-proxy-secret.yaml + - file: 03-deployment.yaml + src: deployment.yaml + - file: 04-service.yaml + src: service.yaml + - file: kustomization.yaml + ignore: "MicroShift-specific kustomization" + git_restore: True + - file: kustomization.x86_64.yaml + ignore: "gets generated during image rebase" + - file: kustomization.aarch64.yaml + ignore: "gets generated during image rebase" + - file: release-kube-state-metrics-x86_64.json + ignore: "gets generated during image rebase" + - file: release-kube-state-metrics-aarch64.json + ignore: "gets generated during image rebase" + - dir: optional/node-exporter/ + no_clean: True + src: cluster-monitoring-operator/assets/node-exporter/ + files: + - file: 00-namespace.yaml + ignore: "MicroShift-specific, no upstream equivalent" + git_restore: True + - file: 01-cluster-role.yaml + src: cluster-role.yaml + - file: 01-cluster-role-binding.yaml + src: cluster-role-binding.yaml + - file: 01-service-account.yaml + src: service-account.yaml + - file: 01-security-context-constraints.yaml + src: security-context-constraints.yaml + - file: 02-accelerators-collector-configmap.yaml + src: accelerators-collector-configmap.yaml + - file: 02-kube-rbac-proxy-secret.yaml + src: kube-rbac-proxy-secret.yaml + - file: 03-daemonset.yaml + src: daemonset.yaml + - file: 04-service.yaml + src: service.yaml + - file: kustomization.yaml + ignore: "MicroShift-specific kustomization" + git_restore: True + - file: kustomization.x86_64.yaml + ignore: "gets generated during image rebase" + - file: kustomization.aarch64.yaml + ignore: "gets generated during image rebase" + - file: release-node-exporter-x86_64.json + ignore: "gets generated during image rebase" + - file: release-node-exporter-aarch64.json + ignore: "gets generated during image rebase" diff --git a/test/bin/common.sh b/test/bin/common.sh index ef682a676f..7f1c2aeff8 100644 --- a/test/bin/common.sh +++ b/test/bin/common.sh @@ -394,6 +394,12 @@ MICROSHIFT_Y1_OPTIONAL_RPMS_LIST=( ) MICROSHIFT_OPTIONAL_RPMS_LIST=( "${MICROSHIFT_Y1_OPTIONAL_RPMS_LIST[@]}" + microshift-metrics-kube-state + microshift-metrics-kube-state-release-info + microshift-metrics-node-exporter + microshift-metrics-node-exporter-release-info + microshift-metrics-server + microshift-metrics-server-release-info ) MICROSHIFT_Y2_X86_64_RPMS_LIST=( microshift-ai-model-serving diff --git a/test/suites/optional/metrics.robot b/test/suites/optional/metrics.robot new file mode 100644 index 0000000000..68cfdef75a --- /dev/null +++ b/test/suites/optional/metrics.robot @@ -0,0 +1,124 @@ +*** Settings *** +Documentation Verify that each optional metrics exporter is running +... and producing metrics. + +Resource ../../resources/common.resource +Resource ../../resources/kubeconfig.resource +Resource ../../resources/microshift-host.resource +Resource ../../resources/oc.resource +Resource ../../resources/optional-config.resource + +Suite Setup Setup +Suite Teardown Teardown + + +*** Variables *** +${METRICS_NS} openshift-monitoring +${CLIENT_CERT} /tmp/metrics-test-client.crt +${CLIENT_KEY} /tmp/metrics-test-client.key +${SERVICE_CA} /tmp/metrics-test-service-ca.crt + + +*** Test Cases *** +Kube State Metrics Deployment Is Running + [Documentation] The kube-state-metrics deployment should be available. + Named Deployment Should Be Available kube-state-metrics ns=${METRICS_NS} + +Kube State Metrics Main Metrics Are Exported + [Documentation] The main metrics port (8443) should return kube_node_info. + Wait Until Keyword Succeeds 60s 5s + ... Metrics Endpoint Should Contain 8443 kube_node_info + +Kube State Metrics Self Metrics Are Exported + [Documentation] The self/telemetry port (9443) should return process_* metrics. + Wait Until Keyword Succeeds 60s 5s + ... Metrics Endpoint Should Contain 9443 process_cpu_seconds_total + +Node Exporter DaemonSet Is Running + [Documentation] The node-exporter daemonset should be available. + Named Daemonset Should Be Available node-exporter ns=${METRICS_NS} + +Node Exporter Metrics Are Exported + [Documentation] Port 9100 on localhost should return node_cpu_seconds_total. + Wait Until Keyword Succeeds 60s 5s + ... Node Exporter Should Contain node_cpu_seconds_total + +Metrics Server Deployment Is Running + [Documentation] The metrics-server deployment should be available. + Named Deployment Should Be Available metrics-server ns=${METRICS_NS} + +Metrics Server API Is Available + [Documentation] The v1beta1.metrics.k8s.io APIService should report Available. + Oc Wait apiservice v1beta1.metrics.k8s.io --for=condition=Available --timeout\=120s + +Metrics Server Reports Node Metrics + [Documentation] oc adm top nodes should return resource usage data. + ${out}= Run With Kubeconfig oc adm top nodes --no-headers + Should Match Regexp ${out} \\d+m + + +*** Keywords *** +Setup + [Documentation] Login, setup kubeconfig, enable metrics optionals, and extract client certs for mTLS. + Setup Suite + Setup MicroShift With Optionals + ... 080-microshift-metrics-server + ... 081-microshift-kube-state-metrics + ... 082-microshift-node-exporter + Extract Metrics Client Certs + +Teardown + [Documentation] Remove temporary client certs, restore config, and tear down suite. + Cleanup Metrics Client Certs + Teardown MicroShift With Optionals + Teardown Suite + +Extract Metrics Client Certs + [Documentation] Extract the admin kubeconfig client cert and key to temp files + ... on the remote host. These are signed by the admin-kubeconfig-signer CA, + ... which the kube-rbac-proxy sidecars trust via the metrics-client-ca ConfigMap. + ... Uses SSH (Command Should Work) because the cert files must exist on the + ... remote host where curl runs, not locally. + Command Should Work + ... oc config view --kubeconfig=/var/lib/microshift/resources/kubeadmin/kubeconfig --raw -o jsonpath\='{.users[0].user.client-certificate-data}' | base64 -d > ${CLIENT_CERT} + Command Should Work + ... oc config view --kubeconfig=/var/lib/microshift/resources/kubeadmin/kubeconfig --raw -o jsonpath\='{.users[0].user.client-key-data}' | base64 -d > ${CLIENT_KEY} + Command Should Work + ... oc get configmap signing-cabundle -n openshift-service-ca --kubeconfig\=/var/lib/microshift/resources/kubeadmin/kubeconfig -o jsonpath\='{.data.ca-bundle\\.crt}' > ${SERVICE_CA} + +Cleanup Metrics Client Certs + [Documentation] Remove temporary client cert files from the remote host. + Command Should Work rm -f ${CLIENT_CERT} ${CLIENT_KEY} ${SERVICE_CA} + +Metrics Endpoint Should Contain + [Documentation] Scrape kube-state-metrics on the given port and assert the + ... response contains the expected metric name. + [Arguments] ${port} ${expected} + ${metrics}= Scrape Metrics From kube-state-metrics ${port} + Should Contain ${metrics} ${expected} + +Node Exporter Should Contain + [Documentation] Scrape node-exporter and assert the response contains the + ... expected metric name. + [Arguments] ${expected} + ${metrics}= Scrape Metrics From node-exporter 9100 + Should Contain ${metrics} ${expected} + +Scrape Metrics From + [Documentation] Curl the named metrics Service endpoint on the given port + ... with mTLS client certs. Resolves the IP from the Endpoints object + ... to validate that the headless Service selector matches the pod. + [Arguments] ${metrics_service} ${port} + ${ep_ip}= Oc Get JsonPath + ... endpoints + ... ${METRICS_NS} + ... ${metrics_service} + ... .subsets[0].addresses[0].ip + ${ep_ip}= Get Regexp Matches ${ep_ip} \\d+\\.\\d+\\.\\d+\\.\\d+ + Length Should Be ${ep_ip} 1 + VAR ${ep_ip}= ${ep_ip}[0] + VAR ${svc_host}= ${metrics_service}.${METRICS_NS}.svc + ${stdout}= Command Should Work + ... curl -s --resolve ${svc_host}:${port}:${ep_ip} --cacert ${SERVICE_CA} --cert ${CLIENT_CERT} --key ${CLIENT_KEY} https://${svc_host}:${port}/metrics + Should Not Be Empty ${stdout} + RETURN ${stdout}