diff --git a/deploy/crd/trinocluster.crd.yaml b/deploy/crd/trinocluster.crd.yaml index 3f887d368..0ea01d06e 100644 --- a/deploy/crd/trinocluster.crd.yaml +++ b/deploy/crd/trinocluster.crd.yaml @@ -199,6 +199,12 @@ spec: required: - roleGroups type: object + customCatalogs: + description: An optional list of references to ConfigMaps that contain catalog definitions which should be deployed into the Trino instance + items: + type: string + nullable: true + type: array hiveConfigMapName: nullable: true type: string diff --git a/deploy/helm/trino-operator/crds/crds.yaml b/deploy/helm/trino-operator/crds/crds.yaml index dc18ac2cd..235bbc76f 100644 --- a/deploy/helm/trino-operator/crds/crds.yaml +++ b/deploy/helm/trino-operator/crds/crds.yaml @@ -201,6 +201,12 @@ spec: required: - roleGroups type: object + customCatalogs: + description: An optional list of references to ConfigMaps that contain catalog definitions which should be deployed into the Trino instance + items: + type: string + nullable: true + type: array hiveConfigMapName: nullable: true type: string diff --git a/deploy/manifests/crds.yaml b/deploy/manifests/crds.yaml index a7cfa4563..24c219fab 100644 --- a/deploy/manifests/crds.yaml +++ b/deploy/manifests/crds.yaml @@ -203,6 +203,12 @@ spec: required: - roleGroups type: object + customCatalogs: + description: An optional list of references to ConfigMaps that contain catalog definitions which should be deployed into the Trino instance + items: + type: string + nullable: true + type: array hiveConfigMapName: nullable: true type: string diff --git a/docs/modules/ROOT/pages/usage.adoc b/docs/modules/ROOT/pages/usage.adoc index f05d5815b..3644a5123 100644 --- a/docs/modules/ROOT/pages/usage.adoc +++ b/docs/modules/ROOT/pages/usage.adoc @@ -253,6 +253,38 @@ To access the CLI please execute: If you use self signed certificates, you also need the `--insecure` flag above which can be omitted otherwise. +=== Defining Catalogs +If you specify a Hive connection in your Trino definition, the 'hive' catalog will automatically be created for you. + +To specify additional catalogs, you can add these via the `customCatalogs` option in the TrinoCluster CRD. +Every entry in this list is treated as the name of a ConfigMap, and all these ConfigMaps will be merged into the catalog definitions which Trino is then configured with. + +To define a hive catalog and an additional postgres catalog, you would for example define the following objects: + +[source,yaml] +---- +apiVersion: trino.stackable.tech/v1alpha1 +kind: TrinoCluster +metadata: + name: simple-trino +spec: + hiveConfigMapName: simple-hive-derby + customCatalogs: + - trino-postgres + ... +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: trino-postgres +data: + postgres.properties: | + +---- + +WARNING: To avoid hidden failures the operator will fail early on missing configmaps or duplicate keys in configmaps that are referenced! + + === Test Trino with Hive and S3 Create a schema and a table for the Iris data located in S3 and query data. This assumes to have the Iris data set in the `PARQUET` format available in the S3 bucket which can be downloaded https://www.kaggle.com/gpreda/iris-dataset/version/2?select=iris.parquet[here] diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index efc71baf3..a8e9b5b7f 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -114,6 +114,9 @@ pub struct TrinoClusterSpec { pub hive_config_map_name: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub opa_config_map_name: Option, + /// An optional list of references to ConfigMaps that contain catalog definitions + /// which should be deployed into the Trino instance + pub custom_catalogs: Option>, /// A reference to a secret containing username/password for defined users #[serde(default, skip_serializing_if = "Option::is_none")] pub authentication: Option, diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 0eddc8a1b..18209a9cb 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -1,5 +1,6 @@ //! Ensures that `Pod`s are configured and running for each [`TrinoCluster`] use snafu::{OptionExt, ResultExt, Snafu}; +use stackable_operator::kube::Resource; use stackable_operator::{ builder::{ConfigMapBuilder, ContainerBuilder, ObjectMetaBuilder, PodBuilder}, client::Client, @@ -121,6 +122,15 @@ pub enum Error { InternalOperatorFailure { source: stackable_trino_crd::Error }, #[snafu(display("no coordinator pods found for discovery"))] MissingCoordinatorPods, + #[snafu(display("Failed retrieving a ConfigMap that was referenced from the cluster definition : [{config_map_name}]"))] + ConfigMapReference { + source: stackable_operator::error::Error, + config_map_name: String, + }, + #[snafu(display( + "Duplicate catalog definitions found during reconciliation: [{duplicate_catalogs:?}]" + ))] + DuplicateCatalogs { duplicate_catalogs: Vec }, } type Result = std::result::Result; @@ -168,7 +178,7 @@ pub async fn reconcile_trino( let rg_configmap = build_rolegroup_config_map(&trino, &trino_role, &rolegroup, &config)?; let rg_catalog_configmap = - build_rolegroup_catalog_config_map(&trino, &rolegroup, &config)?; + build_rolegroup_catalog_config_map(&trino, &rolegroup, &config, client).await?; let rg_stateful_set = build_rolegroup_statefulset( &trino, &trino_role, @@ -370,12 +380,14 @@ fn build_rolegroup_config_map( /// The rolegroup catalog [`ConfigMap`] configures the rolegroup catalog based on the configuration /// given by the administrator -fn build_rolegroup_catalog_config_map( +async fn build_rolegroup_catalog_config_map( trino: &TrinoCluster, rolegroup_ref: &RoleGroupRef, config: &HashMap>, + client: &Client, ) -> Result { let mut cm_hive_data = BTreeMap::new(); + let mut catalog_conflicts: Vec = Vec::new(); for (property_name_kind, config) in config { let mut transformed_config: BTreeMap> = config @@ -403,6 +415,42 @@ fn build_rolegroup_catalog_config_map( } } + let ns = trino + .meta() + .namespace + .as_ref() + .with_context(|| ObjectHasNoNamespaceSnafu {})?; + + // Add extra catalogs that have been defined + if let Some(catalog_list) = &trino.spec.custom_catalogs { + for config_map_name in catalog_list { + let config_map = client + .get::(config_map_name, Some(ns)) + .await + .with_context(|_| ConfigMapReferenceSnafu { config_map_name })?; + + if let Some(data) = config_map.data { + for (key, value) in data { + // Check if there is already a key of this name, if so, add it to the list + // of conflicts, otherwise add to config + if cm_hive_data.contains_key(&key) { + catalog_conflicts.push(key); + } else { + cm_hive_data.insert(key.clone(), value.clone()); + } + } + } + } + } + if !catalog_conflicts.is_empty() { + // There were duplicate entries, which means catalogs of the same name + // Instead of silently overwriting this, we'll fail loudly here. + return DuplicateCatalogsSnafu { + duplicate_catalogs: catalog_conflicts, + } + .fail(); + } + ConfigMapBuilder::new() .metadata( ObjectMetaBuilder::new()