From 11da381ddc9acb9716c3735cc9373e30e7088609 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Mon, 26 Jan 2026 15:31:09 +0100 Subject: [PATCH 01/26] wip --- rust/operator-binary/src/connections/database.rs | 0 rust/operator-binary/src/connections/mod.rs | 2 ++ rust/operator-binary/src/connections/queue.rs | 0 .../kuttl/versioning/30-install-airflow-cluster.yaml.j2 | 1 + 4 files changed, 3 insertions(+) create mode 100644 rust/operator-binary/src/connections/database.rs create mode 100644 rust/operator-binary/src/connections/mod.rs create mode 100644 rust/operator-binary/src/connections/queue.rs diff --git a/rust/operator-binary/src/connections/database.rs b/rust/operator-binary/src/connections/database.rs new file mode 100644 index 00000000..e69de29b diff --git a/rust/operator-binary/src/connections/mod.rs b/rust/operator-binary/src/connections/mod.rs new file mode 100644 index 00000000..f7608e18 --- /dev/null +++ b/rust/operator-binary/src/connections/mod.rs @@ -0,0 +1,2 @@ +pub mod database; +pub mod queue; diff --git a/rust/operator-binary/src/connections/queue.rs b/rust/operator-binary/src/connections/queue.rs new file mode 100644 index 00000000..e69de29b diff --git a/tests/templates/kuttl/versioning/30-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/versioning/30-install-airflow-cluster.yaml.j2 index 682fe765..f4e09d8a 100644 --- a/tests/templates/kuttl/versioning/30-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/versioning/30-install-airflow-cluster.yaml.j2 @@ -64,6 +64,7 @@ spec: default: envOverrides: &envOverrides AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: '{"conn_type": "kubernetes", "extra": {"extra__kubernetes__in_cluster": true}}' + SOME_TEST_CONCAT: 'this is my admin password: $ADMIN_PASSWORD' replicas: 1 kubernetesExecutors: envOverrides: *envOverrides From 032bf2fff0c6ccb35f26b4f5dd288fcfc7664e48 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 27 Jan 2026 17:17:10 +0100 Subject: [PATCH 02/26] added initial tests --- extra/crds.yaml | 122 ++++++++++++++++ .../src/connections/database.rs | 136 ++++++++++++++++++ rust/operator-binary/src/connections/mod.rs | 1 - rust/operator-binary/src/connections/queue.rs | 0 rust/operator-binary/src/crd/affinity.rs | 10 ++ rust/operator-binary/src/crd/mod.rs | 10 ++ rust/operator-binary/src/main.rs | 1 + 7 files changed, 279 insertions(+), 1 deletion(-) delete mode 100644 rust/operator-binary/src/connections/queue.rs diff --git a/extra/crds.yaml b/extra/crds.yaml index 355c04c8..7466617c 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -759,6 +759,66 @@ spec: Whether to load example DAGs or not; defaults to false. The examples are used in the [getting started guide](https://docs.stackable.tech/home/nightly/airflow/getting_started/). type: boolean + metadataDatabase: + description: |- + Connection information needed to construct a connection for the + mandatory backend metadata database. + oneOf: + - required: + - postgresql + - required: + - redis + - required: + - generic + properties: + generic: + properties: + uriSecret: + type: string + required: + - uriSecret + type: object + postgresql: + properties: + credentialsSecret: + type: string + databaseName: + type: string + host: + type: string + parameters: + additionalProperties: + type: string + default: {} + type: object + port: + default: 5432 + format: uint16 + maximum: 65535.0 + minimum: 0.0 + type: integer + required: + - credentialsSecret + - databaseName + - host + type: object + redis: + properties: + credentialsSecret: + type: string + host: + type: string + port: + default: 6379 + format: uint16 + maximum: 65535.0 + minimum: 0.0 + type: integer + required: + - credentialsSecret + - host + type: object + type: object vectorAggregatorConfigMapName: description: |- Name of the Vector aggregator [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery). @@ -783,6 +843,7 @@ spec: type: array required: - credentialsSecret + - metadataDatabase type: object clusterOperation: default: @@ -3930,6 +3991,66 @@ spec: Whether to load example DAGs or not; defaults to false. The examples are used in the [getting started guide](https://docs.stackable.tech/home/nightly/airflow/getting_started/). type: boolean + metadataDatabase: + description: |- + Connection information needed to construct a connection for the + mandatory backend metadata database. + oneOf: + - required: + - postgresql + - required: + - redis + - required: + - generic + properties: + generic: + properties: + uriSecret: + type: string + required: + - uriSecret + type: object + postgresql: + properties: + credentialsSecret: + type: string + databaseName: + type: string + host: + type: string + parameters: + additionalProperties: + type: string + default: {} + type: object + port: + default: 5432 + format: uint16 + maximum: 65535.0 + minimum: 0.0 + type: integer + required: + - credentialsSecret + - databaseName + - host + type: object + redis: + properties: + credentialsSecret: + type: string + host: + type: string + port: + default: 6379 + format: uint16 + maximum: 65535.0 + minimum: 0.0 + type: integer + required: + - credentialsSecret + - host + type: object + type: object vectorAggregatorConfigMapName: description: |- Name of the Vector aggregator [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery). @@ -3954,6 +4075,7 @@ spec: type: array required: - credentialsSecret + - metadataDatabase type: object clusterOperation: default: diff --git a/rust/operator-binary/src/connections/database.rs b/rust/operator-binary/src/connections/database.rs index e69de29b..6e4175f0 100644 --- a/rust/operator-binary/src/connections/database.rs +++ b/rust/operator-binary/src/connections/database.rs @@ -0,0 +1,136 @@ +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; +use stackable_operator::schemars::{self, JsonSchema}; + +#[derive(Clone, Deserialize, Debug, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum DbType { + #[serde(rename = "postgresql")] + Postgresql(PostgresqlDb), + #[serde(rename = "redis")] + Redis(RedisDb), + #[serde(rename = "generic")] + Generic(GenericDb), +} + +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct GenericDb { + pub uri_secret: String, +} + +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct PostgresqlDb { + pub host: String, + #[serde(default = "default_postgres_port")] + pub port: u16, + pub database_name: String, + pub credentials_secret: String, + #[serde(default)] + pub parameters: BTreeMap, +} + +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct RedisDb { + pub host: String, + #[serde(default = "default_redis_port")] + pub port: u16, + pub credentials_secret: String, +} + +impl DbType { + pub fn connection_string(&self, username_env: &str, password_env: &str) -> String { + match self { + DbType::Postgresql(db) => db.connection_string(username_env, password_env), + DbType::Redis(db) => db.connection_string(username_env, password_env), + DbType::Generic(db) => db.connection_string(), + } + } +} + +impl PostgresqlDb { + pub fn connection_string(&self, username_env: &str, password_env: &str) -> String { + let params = if self.parameters.is_empty() { + String::new() + } else { + let param_str: Vec = self + .parameters + .iter() + .map(|(k, v)| format!("{}={}", k, v)) + .collect(); + format!("?{}", param_str.join("&")) + }; + + format!( + "postgresql+psycopg2://${{env:{}}}:${{env:{}}}@{}:{}/{}{}", + username_env, password_env, self.host, self.port, self.database_name, params + ) + } +} + +impl RedisDb { + pub fn connection_string(&self, username_env: &str, password_env: &str) -> String { + format!( + "redis://${{env:{}}}:${{env:{}}}@{}:{}/0", + username_env, password_env, self.host, self.port + ) + } +} + +impl GenericDb { + pub fn connection_string(&self) -> String { + format!("${{secret:{}}}", self.uri_secret) + } +} + +fn default_postgres_port() -> u16 { + 5432 +} + +fn default_redis_port() -> u16 { + 6379 +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use crate::connections::database::{ + DbType, PostgresqlDb, RedisDb, default_postgres_port, default_redis_port, + }; + + #[test] + fn test_postgresql_connection() { + let db_type = DbType::Postgresql(PostgresqlDb { + host: "airflow-postgresql".to_string(), + database_name: "airflow".to_string(), + credentials_secret: "airflow-credentials".to_string(), + port: default_postgres_port(), + parameters: BTreeMap::new(), + }); + let connection_string = db_type.connection_string("ADMIN_USERNAME", "ADMIN_PASSWORD"); + + assert_eq!( + "postgresql+psycopg2://${env:ADMIN_USERNAME}:${env:ADMIN_PASSWORD}@airflow-postgresql:5432/airflow", + connection_string + ); + } + + #[test] + fn test_redis_connection() { + let db_type = DbType::Redis(RedisDb { + host: "airflow-postgresql".to_string(), + credentials_secret: "airflow-credentials".to_string(), + port: default_redis_port(), + }); + let connection_string = db_type.connection_string("ADMIN_USERNAME", "ADMIN_PASSWORD"); + + assert_eq!( + "redis://${env:ADMIN_USERNAME}:${env:ADMIN_PASSWORD}@airflow-postgresql:6379/0", + connection_string + ); + } +} diff --git a/rust/operator-binary/src/connections/mod.rs b/rust/operator-binary/src/connections/mod.rs index f7608e18..8fd0a6be 100644 --- a/rust/operator-binary/src/connections/mod.rs +++ b/rust/operator-binary/src/connections/mod.rs @@ -1,2 +1 @@ pub mod database; -pub mod queue; diff --git a/rust/operator-binary/src/connections/queue.rs b/rust/operator-binary/src/connections/queue.rs deleted file mode 100644 index e69de29b..00000000 diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index 74cc2ee7..11ccdf8f 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -72,6 +72,11 @@ mod tests { productVersion: 3.0.6 clusterConfig: credentialsSecret: airflow-credentials + metadataDatabase: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials webservers: roleGroups: default: @@ -166,6 +171,11 @@ mod tests { productVersion: 3.0.6 clusterConfig: credentialsSecret: airflow-credentials + metadataDatabase: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials webservers: roleGroups: default: diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index d22d6ea9..f2e07f79 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -47,6 +47,7 @@ use stackable_operator::{ use strum::{Display, EnumIter, EnumString, IntoEnumIterator}; use crate::{ + connections::database::DbType, crd::{ affinity::{get_affinity, get_executor_affinity}, authentication::{ @@ -250,6 +251,10 @@ pub mod versioned { /// to find out more. pub credentials_secret: String, + /// Connection information needed to construct a connection for the + /// mandatory backend metadata database. + pub metadata_database: DbType, + /// The `gitSync` settings allow configuring DAGs to mount via `git-sync`. /// Learn more in the /// [mounting DAGs documentation](DOCS_BASE_URL_PLACEHOLDER/airflow/usage-guide/mounting-dags#_via_git_sync). @@ -1112,6 +1117,11 @@ mod tests { loadExamples: true exposeConfig: true credentialsSecret: simple-airflow-credentials + metadataDatabase: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials webservers: roleGroups: default: diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 2b3ee726..cf4c6dbc 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -39,6 +39,7 @@ use crate::{ mod airflow_controller; mod config; +mod connections; mod controller_commons; mod crd; mod env_vars; From 06c61518f5d93657f70bb08fb670ad175e779fb1 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 5 Feb 2026 14:48:24 +0100 Subject: [PATCH 03/26] adapted tests to use new structs --- extra/crds.yaml | 184 ++++++++++++++---- rust/operator-binary/src/connection.rs | 142 ++++++++++++++ .../src/connections/database.rs | 76 ++------ rust/operator-binary/src/connections/mod.rs | 1 + rust/operator-binary/src/connections/queue.rs | 68 +++++++ rust/operator-binary/src/crd/affinity.rs | 9 + rust/operator-binary/src/crd/mod.rs | 13 +- rust/operator-binary/src/env_vars.rs | 54 ++--- rust/operator-binary/src/main.rs | 1 + .../08-install-airflow.yaml.j2 | 37 +++- .../10-pause-airflow.yaml.j2 | 16 +- .../cluster-operation/20-stop-airflow.yaml.j2 | 16 +- .../30-restart-airflow.yaml.j2 | 16 +- .../install-airflow-cluster.yaml.j2 | 37 +++- .../ldap/60-install-airflow-cluster.yaml.j2 | 39 +++- .../41-install-airflow-cluster.yaml.j2 | 41 +++- .../30-install-airflow-cluster.yaml.j2 | 39 +++- .../30-install-airflow-cluster.yaml.j2 | 39 +++- .../kuttl/oidc/install-airflow.yaml.j2 | 18 +- .../kuttl/opa/30-install-airflow.yaml.j2 | 26 ++- .../30-install-airflow-cluster.yaml.j2 | 37 +++- .../overrides/10-install-airflow.yaml.j2 | 40 +++- .../overrides/20-install-airflow2.yaml.j2 | 7 +- .../40-install-airflow-cluster.yaml.j2 | 39 +++- .../30-install-airflow-cluster.yaml.j2 | 37 +++- .../smoke/40-install-airflow-cluster.yaml.j2 | 39 +++- .../30-install-airflow-cluster.yaml.j2 | 39 +++- .../30-install-airflow-cluster.yaml.j2 | 26 ++- 28 files changed, 909 insertions(+), 227 deletions(-) create mode 100644 rust/operator-binary/src/connection.rs create mode 100644 rust/operator-binary/src/connections/queue.rs diff --git a/extra/crds.yaml b/extra/crds.yaml index 7466617c..96a14f63 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -39,6 +39,78 @@ spec: The celery executor. Deployed with an explicit number of replicas. properties: + celeryBrokerUrl: + description: Connection information for the celery broker queue. + oneOf: + - required: + - redis + - required: + - generic + properties: + generic: + properties: + uriSecret: + type: string + required: + - uriSecret + type: object + redis: + properties: + credentialsSecret: + type: string + host: + type: string + port: + default: 6379 + format: uint16 + maximum: 65535.0 + minimum: 0.0 + type: integer + required: + - credentialsSecret + - host + type: object + type: object + celeryResultBackend: + description: Connection information for the celery backend database. + oneOf: + - required: + - postgresql + - required: + - generic + properties: + generic: + properties: + uriSecret: + type: string + required: + - uriSecret + type: object + postgresql: + properties: + credentialsSecret: + type: string + databaseName: + type: string + host: + type: string + parameters: + additionalProperties: + type: string + default: {} + type: object + port: + default: 5432 + format: uint16 + maximum: 65535.0 + minimum: 0.0 + type: integer + required: + - credentialsSecret + - databaseName + - host + type: object + type: object cliOverrides: additionalProperties: type: string @@ -542,6 +614,8 @@ spec: type: object type: object required: + - celeryBrokerUrl + - celeryResultBackend - roleGroups type: object clusterConfig: @@ -766,8 +840,6 @@ spec: oneOf: - required: - postgresql - - required: - - redis - required: - generic properties: @@ -802,22 +874,6 @@ spec: - databaseName - host type: object - redis: - properties: - credentialsSecret: - type: string - host: - type: string - port: - default: 6379 - format: uint16 - maximum: 65535.0 - minimum: 0.0 - type: integer - required: - - credentialsSecret - - host - type: object type: object vectorAggregatorConfigMapName: description: |- @@ -3295,6 +3351,78 @@ spec: The celery executor. Deployed with an explicit number of replicas. properties: + celeryBrokerUrl: + description: Connection information for the celery broker queue. + oneOf: + - required: + - redis + - required: + - generic + properties: + generic: + properties: + uriSecret: + type: string + required: + - uriSecret + type: object + redis: + properties: + credentialsSecret: + type: string + host: + type: string + port: + default: 6379 + format: uint16 + maximum: 65535.0 + minimum: 0.0 + type: integer + required: + - credentialsSecret + - host + type: object + type: object + celeryResultBackend: + description: Connection information for the celery backend database. + oneOf: + - required: + - postgresql + - required: + - generic + properties: + generic: + properties: + uriSecret: + type: string + required: + - uriSecret + type: object + postgresql: + properties: + credentialsSecret: + type: string + databaseName: + type: string + host: + type: string + parameters: + additionalProperties: + type: string + default: {} + type: object + port: + default: 5432 + format: uint16 + maximum: 65535.0 + minimum: 0.0 + type: integer + required: + - credentialsSecret + - databaseName + - host + type: object + type: object cliOverrides: additionalProperties: type: string @@ -3798,6 +3926,8 @@ spec: type: object type: object required: + - celeryBrokerUrl + - celeryResultBackend - roleGroups type: object clusterConfig: @@ -3998,8 +4128,6 @@ spec: oneOf: - required: - postgresql - - required: - - redis - required: - generic properties: @@ -4034,22 +4162,6 @@ spec: - databaseName - host type: object - redis: - properties: - credentialsSecret: - type: string - host: - type: string - port: - default: 6379 - format: uint16 - maximum: 65535.0 - minimum: 0.0 - type: integer - required: - - credentialsSecret - - host - type: object type: object vectorAggregatorConfigMapName: description: |- diff --git a/rust/operator-binary/src/connection.rs b/rust/operator-binary/src/connection.rs new file mode 100644 index 00000000..fa798392 --- /dev/null +++ b/rust/operator-binary/src/connection.rs @@ -0,0 +1,142 @@ +use std::collections::BTreeMap; + +use k8s_openapi::api::core::v1::EnvVar; + +use crate::{ + connections::{database::DbType, queue::QueueType}, + crd::v1alpha2, + util::env_var_from_secret, +}; + +const AIRFLOW_DATABASE_SQL_ALCHEMY_CONN: &str = "AIRFLOW__DATABASE__SQL_ALCHEMY_CONN"; +const AIRFLOW_CELERY_RESULT_BACKEND: &str = "AIRFLOW__CELERY__RESULT_BACKEND"; +const AIRFLOW_CELERY_BROKER_URL: &str = "AIRFLOW__CELERY__BROKER_URL"; + +// metadata credentials +const META_DB_USERNAME: &str = "META_DB_USERNAME"; +const META_DB_PASSWORD: &str = "META_DB_PASSWORD"; +// celery backend credentials +const CELERY_DB_USERNAME: &str = "CELERY_DB_USERNAME"; +const CELERY_DB_PASSWORD: &str = "CELERY_DB_PASSWORD"; +// celery broker credentials +const CELERY_BROKER_USERNAME: &str = "CELERY_BROKER_USERNAME"; +const CELERY_BROKER_PASSWORD: &str = "CELERY_BROKER_PASSWORD"; +// secret fields +const USERNAME_FIELD: &str = "username"; +const PASSWORD_FIELD: &str = "password"; +const URI_FIELD: &str = "uri"; + +pub fn add_metadata_credentials( + airflow: &v1alpha2::AirflowCluster, + env: &mut BTreeMap, +) { + let db_type = &airflow.spec.cluster_config.metadata_database; + let db_secret = db_type.credentials_secret(); + + match db_type { + DbType::Postgresql(db) => { + add_typed_credentials( + env, + &db_secret, + META_DB_USERNAME, + META_DB_PASSWORD, + AIRFLOW_DATABASE_SQL_ALCHEMY_CONN, + db.connection_string_alchemy(META_DB_USERNAME, META_DB_PASSWORD), + ); + } + DbType::Generic(_) => { + add_generic_credentials(env, &db_secret, AIRFLOW_DATABASE_SQL_ALCHEMY_CONN); + } + } +} + +pub fn add_celery_backend_credentials( + celery_result_backend: &DbType, + env: &mut BTreeMap, +) { + let db_secret = celery_result_backend.credentials_secret(); + + match celery_result_backend { + DbType::Postgresql(db) => { + add_typed_credentials( + env, + &db_secret, + CELERY_DB_USERNAME, + CELERY_DB_PASSWORD, + AIRFLOW_CELERY_RESULT_BACKEND, + db.connection_string_celery(CELERY_DB_USERNAME, CELERY_DB_PASSWORD), + ); + } + DbType::Generic(_) => { + add_generic_credentials(env, &db_secret, AIRFLOW_CELERY_RESULT_BACKEND); + } + } +} + +pub fn add_celery_broker_credentials( + celery_broker_url: &QueueType, + env: &mut BTreeMap, +) { + let queue_secret = celery_broker_url.credentials_secret(); + + match celery_broker_url { + QueueType::Redis(queue) => { + add_typed_credentials( + env, + &queue_secret, + CELERY_BROKER_USERNAME, + CELERY_BROKER_PASSWORD, + AIRFLOW_CELERY_BROKER_URL, + queue.connection_string(CELERY_BROKER_USERNAME, CELERY_BROKER_PASSWORD), + ); + } + QueueType::Generic(_) => { + add_generic_credentials(env, &queue_secret, AIRFLOW_CELERY_BROKER_URL); + } + } +} + +fn add_typed_credentials( + env: &mut BTreeMap, + secret: &str, + username_key: &str, + password_key: &str, + connection_key: &str, + connection_string: String, +) { + // Add username and password from secret + add_secret_env_vars( + env, + secret, + &[ + (username_key, USERNAME_FIELD), + (password_key, PASSWORD_FIELD), + ], + ); + + // Build connection string using username/password env vars + env.insert( + connection_key.into(), + EnvVar { + name: connection_key.into(), + value: Some(connection_string), + ..Default::default() + }, + ); +} + +fn add_generic_credentials(env: &mut BTreeMap, secret: &str, connection_key: &str) { + env.insert( + connection_key.into(), + env_var_from_secret(connection_key, secret, URI_FIELD), + ); +} + +fn add_secret_env_vars(env: &mut BTreeMap, secret: &str, vars: &[(&str, &str)]) { + for (env_key, secret_key) in vars { + env.insert( + (*env_key).into(), + env_var_from_secret(env_key, secret, secret_key), + ); + } +} diff --git a/rust/operator-binary/src/connections/database.rs b/rust/operator-binary/src/connections/database.rs index 6e4175f0..60fca6d9 100644 --- a/rust/operator-binary/src/connections/database.rs +++ b/rust/operator-binary/src/connections/database.rs @@ -8,18 +8,10 @@ use stackable_operator::schemars::{self, JsonSchema}; pub enum DbType { #[serde(rename = "postgresql")] Postgresql(PostgresqlDb), - #[serde(rename = "redis")] - Redis(RedisDb), #[serde(rename = "generic")] Generic(GenericDb), } -#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct GenericDb { - pub uri_secret: String, -} - #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct PostgresqlDb { @@ -34,25 +26,21 @@ pub struct PostgresqlDb { #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] -pub struct RedisDb { - pub host: String, - #[serde(default = "default_redis_port")] - pub port: u16, - pub credentials_secret: String, +pub struct GenericDb { + pub uri_secret: String, } impl DbType { - pub fn connection_string(&self, username_env: &str, password_env: &str) -> String { + pub fn credentials_secret(&self) -> String { match self { - DbType::Postgresql(db) => db.connection_string(username_env, password_env), - DbType::Redis(db) => db.connection_string(username_env, password_env), - DbType::Generic(db) => db.connection_string(), + DbType::Postgresql(db) => db.credentials_secret.to_owned(), + DbType::Generic(db) => db.uri_secret.to_owned(), } } } impl PostgresqlDb { - pub fn connection_string(&self, username_env: &str, password_env: &str) -> String { + fn connection_string(&self, prefix: &str, username_env: &str, password_env: &str) -> String { let params = if self.parameters.is_empty() { String::new() } else { @@ -65,24 +53,17 @@ impl PostgresqlDb { }; format!( - "postgresql+psycopg2://${{env:{}}}:${{env:{}}}@{}:{}/{}{}", - username_env, password_env, self.host, self.port, self.database_name, params + "{}://${}:${}@{}:{}/{}{}", + prefix, username_env, password_env, self.host, self.port, self.database_name, params ) } -} -impl RedisDb { - pub fn connection_string(&self, username_env: &str, password_env: &str) -> String { - format!( - "redis://${{env:{}}}:${{env:{}}}@{}:{}/0", - username_env, password_env, self.host, self.port - ) + pub fn connection_string_alchemy(&self, username_env: &str, password_env: &str) -> String { + self.connection_string("postgresql+psycopg2", username_env, password_env) } -} -impl GenericDb { - pub fn connection_string(&self) -> String { - format!("${{secret:{}}}", self.uri_secret) + pub fn connection_string_celery(&self, username_env: &str, password_env: &str) -> String { + self.connection_string("db+postgresql", username_env, password_env) } } @@ -90,46 +71,25 @@ fn default_postgres_port() -> u16 { 5432 } -fn default_redis_port() -> u16 { - 6379 -} - #[cfg(test)] mod tests { use std::collections::BTreeMap; - use crate::connections::database::{ - DbType, PostgresqlDb, RedisDb, default_postgres_port, default_redis_port, - }; + use crate::connections::database::{PostgresqlDb, default_postgres_port}; #[test] - fn test_postgresql_connection() { - let db_type = DbType::Postgresql(PostgresqlDb { + fn test_postgresql_alchemy() { + let db_type = PostgresqlDb { host: "airflow-postgresql".to_string(), database_name: "airflow".to_string(), credentials_secret: "airflow-credentials".to_string(), port: default_postgres_port(), parameters: BTreeMap::new(), - }); - let connection_string = db_type.connection_string("ADMIN_USERNAME", "ADMIN_PASSWORD"); - - assert_eq!( - "postgresql+psycopg2://${env:ADMIN_USERNAME}:${env:ADMIN_PASSWORD}@airflow-postgresql:5432/airflow", - connection_string - ); - } - - #[test] - fn test_redis_connection() { - let db_type = DbType::Redis(RedisDb { - host: "airflow-postgresql".to_string(), - credentials_secret: "airflow-credentials".to_string(), - port: default_redis_port(), - }); - let connection_string = db_type.connection_string("ADMIN_USERNAME", "ADMIN_PASSWORD"); + }; + let connection_string = db_type.connection_string_alchemy("DB_USERNAME", "DB_PASSWORD"); assert_eq!( - "redis://${env:ADMIN_USERNAME}:${env:ADMIN_PASSWORD}@airflow-postgresql:6379/0", + "postgresql+psycopg2://$DB_USERNAME:$DB_PASSWORD@airflow-postgresql:5432/airflow", connection_string ); } diff --git a/rust/operator-binary/src/connections/mod.rs b/rust/operator-binary/src/connections/mod.rs index 8fd0a6be..f7608e18 100644 --- a/rust/operator-binary/src/connections/mod.rs +++ b/rust/operator-binary/src/connections/mod.rs @@ -1 +1,2 @@ pub mod database; +pub mod queue; diff --git a/rust/operator-binary/src/connections/queue.rs b/rust/operator-binary/src/connections/queue.rs new file mode 100644 index 00000000..ddb62537 --- /dev/null +++ b/rust/operator-binary/src/connections/queue.rs @@ -0,0 +1,68 @@ +use serde::{Deserialize, Serialize}; +use stackable_operator::schemars::{self, JsonSchema}; + +#[derive(Clone, Deserialize, Debug, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum QueueType { + #[serde(rename = "redis")] + Redis(RedisQueue), + #[serde(rename = "generic")] + Generic(GenericQueue), +} + +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct RedisQueue { + pub host: String, + #[serde(default = "default_redis_port")] + pub port: u16, + pub credentials_secret: String, +} + +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct GenericQueue { + pub uri_secret: String, +} + +impl QueueType { + pub fn credentials_secret(&self) -> String { + match self { + QueueType::Redis(queue) => queue.credentials_secret.to_owned(), + QueueType::Generic(queue) => queue.uri_secret.to_owned(), + } + } +} + +impl RedisQueue { + pub fn connection_string(&self, username_env: &str, password_env: &str) -> String { + format!( + "redis://${}:${}@{}:{}/0", + username_env, password_env, self.host, self.port + ) + } +} + +fn default_redis_port() -> u16 { + 6379 +} + +#[cfg(test)] +mod tests { + use crate::connections::queue::{RedisQueue, default_redis_port}; + + #[test] + fn test_redis_queue() { + let queue_type = RedisQueue { + host: "airflow-postgresql".to_string(), + credentials_secret: "airflow-credentials".to_string(), + port: default_redis_port(), + }; + let connection_string = queue_type.connection_string("QUEUE_USERNAME", "QUEUE_PASSWORD"); + + assert_eq!( + "redis://$QUEUE_USERNAME:$QUEUE_PASSWORD@airflow-postgresql:6379/0", + connection_string + ); + } +} diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index c3f0edf6..36830b78 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -82,6 +82,15 @@ mod tests { default: replicas: 1 celeryExecutors: + celeryResultBackend: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials + celeryBrokerUrl: + redis: + host: airflow-redis-master + credentialsSecret: redis-credentials roleGroups: default: replicas: 2 diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 457d9e22..d00a503d 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -47,7 +47,7 @@ use stackable_operator::{ use strum::{Display, EnumIter, EnumString, IntoEnumIterator}; use crate::{ - connections::database::DbType, + connections::{database::DbType, queue::QueueType}, crd::{ affinity::{get_affinity, get_executor_affinity}, authentication::{ @@ -386,7 +386,7 @@ impl v1alpha2::AirflowCluster { AirflowRole::DagProcessor => self.spec.dag_processors.to_owned(), AirflowRole::Triggerer => self.spec.triggerers.to_owned(), AirflowRole::Worker => { - if let AirflowExecutor::CeleryExecutor { config } = &self.spec.executor { + if let AirflowExecutor::CeleryExecutor { config, .. } = &self.spec.executor { Some(config.clone()) } else { None @@ -816,7 +816,7 @@ impl AirflowRole { .context(UnknownAirflowRoleSnafu { role, roles })?, ), AirflowRole::Worker => { - if let AirflowExecutor::CeleryExecutor { config } = &airflow.spec.executor { + if let AirflowExecutor::CeleryExecutor { config, .. } = &airflow.spec.executor { config } else { return Err(Error::NoRoleForExecutorFailure); @@ -847,6 +847,7 @@ fn container_debug_command() -> String { } #[derive(Clone, Debug, Deserialize, Display, JsonSchema, PartialEq, Serialize)] +#[allow(clippy::large_enum_variant)] pub enum AirflowExecutor { /// The celery executor. /// Deployed with an explicit number of replicas. @@ -854,6 +855,12 @@ pub enum AirflowExecutor { CeleryExecutor { #[serde(flatten)] config: Role, + /// Connection information for the celery backend database. + #[serde(rename = "celeryResultBackend")] + celery_result_backend: DbType, + /// Connection information for the celery broker queue. + #[serde(rename = "celeryBrokerUrl")] + celery_broker_url: QueueType, }, /// With the Kuberentes executor, executor Pods are created on demand. diff --git a/rust/operator-binary/src/env_vars.rs b/rust/operator-binary/src/env_vars.rs index d67c1c33..856cdffc 100644 --- a/rust/operator-binary/src/env_vars.rs +++ b/rust/operator-binary/src/env_vars.rs @@ -14,6 +14,9 @@ use stackable_operator::{ }; use crate::{ + connection::{ + add_celery_backend_credentials, add_celery_broker_credentials, add_metadata_credentials, + }, crd::{ AirflowExecutor, AirflowRole, ExecutorConfig, LOG_CONFIG_DIR, STACKABLE_LOG_DIR, TEMPLATE_LOCATION, TEMPLATE_NAME, @@ -40,12 +43,9 @@ const AIRFLOW_METRICS_STATSD_ON: &str = "AIRFLOW__METRICS__STATSD_ON"; const AIRFLOW_METRICS_STATSD_HOST: &str = "AIRFLOW__METRICS__STATSD_HOST"; const AIRFLOW_METRICS_STATSD_PORT: &str = "AIRFLOW__METRICS__STATSD_PORT"; const AIRFLOW_WEBSERVER_SECRET_KEY: &str = "AIRFLOW__WEBSERVER__SECRET_KEY"; -const AIRFLOW_CELERY_RESULT_BACKEND: &str = "AIRFLOW__CELERY__RESULT_BACKEND"; -const AIRFLOW_CELERY_BROKER_URL: &str = "AIRFLOW__CELERY__BROKER_URL"; const AIRFLOW_CORE_DAGS_FOLDER: &str = "AIRFLOW__CORE__DAGS_FOLDER"; const AIRFLOW_CORE_LOAD_EXAMPLES: &str = "AIRFLOW__CORE__LOAD_EXAMPLES"; const AIRFLOW_API_AUTH_BACKENDS: &str = "AIRFLOW__API__AUTH_BACKENDS"; -const AIRFLOW_DATABASE_SQL_ALCHEMY_CONN: &str = "AIRFLOW__DATABASE__SQL_ALCHEMY_CONN"; const AIRFLOW_WEBSERVER_EXPOSE_CONFIG: &str = "AIRFLOW__WEBSERVER__EXPOSE_CONFIG"; const AIRFLOW_CORE_EXECUTOR: &str = "AIRFLOW__CORE__EXECUTOR"; @@ -84,7 +84,6 @@ pub fn build_airflow_statefulset_envs( resolved_product_image: &ResolvedProductImage, ) -> Result, Error> { let mut env: BTreeMap = BTreeMap::new(); - let secret = airflow.spec.cluster_config.credentials_secret.as_str(); let internal_secret_name = airflow.shared_internal_secret_secret_name(); env.extend(static_envs(git_sync_resources)); @@ -94,7 +93,8 @@ pub fn build_airflow_statefulset_envs( add_version_specific_env_vars(airflow, airflow_role, resolved_product_image, &mut env); - // N.B. this has been deprecated and replaced with AIRFLOW__API__SECRET_KEY since 3.0.2. Can be removed when 3.0.1 is no longer supported. + // N.B. this has been deprecated and replaced with AIRFLOW__API__SECRET_KEY + // since 3.0.2. Can be removed when 3.0.1 is no longer supported. env.insert( AIRFLOW_WEBSERVER_SECRET_KEY.into(), // The secret key is used to run the webserver flask app and also @@ -124,34 +124,18 @@ pub fn build_airflow_statefulset_envs( ), ); - env.insert( - AIRFLOW_DATABASE_SQL_ALCHEMY_CONN.into(), - env_var_from_secret( - AIRFLOW_DATABASE_SQL_ALCHEMY_CONN, - secret, - "connections.sqlalchemyDatabaseUri", - ), - ); + add_metadata_credentials(airflow, &mut env); // Redis is only needed when celery executors are used // see https://github.com/stackabletech/airflow-operator/issues/424 for details - if matches!(executor, AirflowExecutor::CeleryExecutor { .. }) { - env.insert( - AIRFLOW_CELERY_RESULT_BACKEND.into(), - env_var_from_secret( - AIRFLOW_CELERY_RESULT_BACKEND, - secret, - "connections.celeryResultBackend", - ), - ); - env.insert( - AIRFLOW_CELERY_BROKER_URL.into(), - env_var_from_secret( - AIRFLOW_CELERY_BROKER_URL, - secret, - "connections.celeryBrokerUrl", - ), - ); + if let AirflowExecutor::CeleryExecutor { + celery_result_backend, + celery_broker_url, + .. + } = executor + { + add_celery_backend_credentials(celery_result_backend, &mut env); + add_celery_broker_credentials(celery_broker_url, &mut env); } let dags_folder = get_dags_folder(git_sync_resources); @@ -376,16 +360,8 @@ pub fn build_airflow_template_envs( resolved_product_image: &ResolvedProductImage, ) -> Vec { let mut env: BTreeMap = BTreeMap::new(); - let secret = airflow.spec.cluster_config.credentials_secret.as_str(); - env.insert( - AIRFLOW_DATABASE_SQL_ALCHEMY_CONN.into(), - env_var_from_secret( - AIRFLOW_DATABASE_SQL_ALCHEMY_CONN, - secret, - "connections.sqlalchemyDatabaseUri", - ), - ); + add_metadata_credentials(airflow, &mut env); env.insert( AIRFLOW_CORE_EXECUTOR.into(), diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index cf4c6dbc..71b3045e 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -39,6 +39,7 @@ use crate::{ mod airflow_controller; mod config; +mod connection; mod connections; mod controller_commons; mod crd; diff --git a/tests/templates/kuttl/cluster-operation/08-install-airflow.yaml.j2 b/tests/templates/kuttl/cluster-operation/08-install-airflow.yaml.j2 index 3aa50e6e..e6fe8092 100644 --- a/tests/templates/kuttl/cluster-operation/08-install-airflow.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/08-install-airflow.yaml.j2 @@ -7,7 +7,7 @@ timeout: 480 apiVersion: v1 kind: Secret metadata: - name: test-airflow-credentials + name: admin-user-credentials type: Opaque stringData: adminUser.username: airflow @@ -15,9 +15,22 @@ stringData: adminUser.lastname: Admin adminUser.email: airflow@airflow.com adminUser.password: airflow - connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow - connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql/airflow - connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgresql-credentials +stringData: + username: airflow + password: airflow +--- +apiVersion: v1 +kind: Secret +metadata: + name: redis-credentials +stringData: + username: "" + password: redis --- apiVersion: airflow.stackable.tech/v1alpha1 kind: AirflowCluster @@ -36,7 +49,12 @@ spec: {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} - credentialsSecret: test-airflow-credentials + credentialsSecret: admin-user-credentials + metadataDatabase: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials webservers: roleConfig: listenerClass: external-unstable @@ -47,6 +65,15 @@ spec: default: replicas: 1 celeryExecutors: + celeryResultBackend: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials + celeryBrokerUrl: + redis: + host: airflow-redis-master + credentialsSecret: redis-credentials config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} diff --git a/tests/templates/kuttl/cluster-operation/10-pause-airflow.yaml.j2 b/tests/templates/kuttl/cluster-operation/10-pause-airflow.yaml.j2 index e84c9653..8c91133f 100644 --- a/tests/templates/kuttl/cluster-operation/10-pause-airflow.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/10-pause-airflow.yaml.j2 @@ -24,7 +24,12 @@ spec: {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} - credentialsSecret: test-airflow-credentials + credentialsSecret: admin-user-credentials + metadataDatabase: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials webservers: roleConfig: listenerClass: external-unstable @@ -35,6 +40,15 @@ spec: default: replicas: 1 celeryExecutors: + celeryResultBackend: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials + celeryBrokerUrl: + redis: + host: airflow-redis-master + credentialsSecret: redis-credentials config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} diff --git a/tests/templates/kuttl/cluster-operation/20-stop-airflow.yaml.j2 b/tests/templates/kuttl/cluster-operation/20-stop-airflow.yaml.j2 index 470f8d7d..08b6ec8c 100644 --- a/tests/templates/kuttl/cluster-operation/20-stop-airflow.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/20-stop-airflow.yaml.j2 @@ -24,7 +24,12 @@ spec: {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} - credentialsSecret: test-airflow-credentials + credentialsSecret: admin-user-credentials + metadataDatabase: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials webservers: roleConfig: listenerClass: external-unstable @@ -35,6 +40,15 @@ spec: default: replicas: 1 celeryExecutors: + celeryResultBackend: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials + celeryBrokerUrl: + redis: + host: airflow-redis-master + credentialsSecret: redis-credentials config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} diff --git a/tests/templates/kuttl/cluster-operation/30-restart-airflow.yaml.j2 b/tests/templates/kuttl/cluster-operation/30-restart-airflow.yaml.j2 index 7b90ad6b..a25a97fb 100644 --- a/tests/templates/kuttl/cluster-operation/30-restart-airflow.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/30-restart-airflow.yaml.j2 @@ -24,7 +24,12 @@ spec: {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} - credentialsSecret: test-airflow-credentials + credentialsSecret: admin-user-credentials + metadataDatabase: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials databaseInitialization: enabled: false webservers: @@ -37,6 +42,15 @@ spec: default: replicas: 1 celeryExecutors: + celeryResultBackend: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials + celeryBrokerUrl: + redis: + host: airflow-redis-master + credentialsSecret: redis-credentials config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} diff --git a/tests/templates/kuttl/external-access/install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/external-access/install-airflow-cluster.yaml.j2 index 84f6547a..3549484d 100644 --- a/tests/templates/kuttl/external-access/install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/external-access/install-airflow-cluster.yaml.j2 @@ -2,7 +2,7 @@ apiVersion: v1 kind: Secret metadata: - name: test-airflow-credentials + name: admin-user-credentials type: Opaque stringData: adminUser.username: airflow @@ -10,9 +10,22 @@ stringData: adminUser.lastname: Admin adminUser.email: airflow@airflow.com adminUser.password: airflow - connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow - connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql/airflow - connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgresql-credentials +stringData: + username: airflow + password: airflow +--- +apiVersion: v1 +kind: Secret +metadata: + name: redis-credentials +stringData: + username: "" + password: redis --- apiVersion: airflow.stackable.tech/v1alpha1 kind: AirflowCluster @@ -29,7 +42,12 @@ spec: pullPolicy: IfNotPresent clusterConfig: loadExamples: false - credentialsSecret: test-airflow-credentials + credentialsSecret: admin-user-credentials + metadataDatabase: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials webservers: roleConfig: listenerClass: test-external-stable-$NAMESPACE @@ -45,6 +63,15 @@ spec: replicas: 1 {% if test_scenario['values']['executor'] == 'celery' %} celeryExecutors: + celeryResultBackend: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials + celeryBrokerUrl: + redis: + host: airflow-redis-master + credentialsSecret: redis-credentials roleGroups: default: replicas: 1 diff --git a/tests/templates/kuttl/ldap/60-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/ldap/60-install-airflow-cluster.yaml.j2 index cbba7152..19740ed9 100644 --- a/tests/templates/kuttl/ldap/60-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/ldap/60-install-airflow-cluster.yaml.j2 @@ -10,7 +10,7 @@ commands: apiVersion: v1 kind: Secret metadata: - name: test-airflow-credentials + name: admin-user-credentials type: Opaque stringData: adminUser.username: airflow @@ -18,11 +18,22 @@ commands: adminUser.lastname: Admin adminUser.email: airflow@airflow.com adminUser.password: airflow - connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow -{% if test_scenario['values']['executor'] == 'celery' %} - connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql/airflow - connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 -{% endif %} + --- + apiVersion: v1 + kind: Secret + metadata: + name: postgresql-credentials + stringData: + username: airflow + password: airflow + --- + apiVersion: v1 + kind: Secret + metadata: + name: redis-credentials + stringData: + username: "" + password: redis --- apiVersion: v1 kind: Secret @@ -52,7 +63,12 @@ commands: vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} loadExamples: true - credentialsSecret: test-airflow-credentials + credentialsSecret: admin-user-credentials + metadataDatabase: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials authentication: - authenticationClass: {% if test_scenario['values']['ldap-authentication'] == 'no-tls' -%} no-tls-$NAMESPACE @@ -74,6 +90,15 @@ commands: replicas: 1 {% if test_scenario['values']['executor'] == 'celery' %} celeryExecutors: + celeryResultBackend: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials + celeryBrokerUrl: + redis: + host: airflow-redis-master + credentialsSecret: redis-credentials config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} diff --git a/tests/templates/kuttl/logging/41-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/logging/41-install-airflow-cluster.yaml.j2 index 498f6db0..e5dc25ff 100644 --- a/tests/templates/kuttl/logging/41-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/logging/41-install-airflow-cluster.yaml.j2 @@ -3,11 +3,11 @@ kind: TestStep metadata: name: install-airflow timeout: 480 ---- +---- apiVersion: v1 kind: Secret metadata: - name: test-airflow-credentials + name: admin-user-credentials type: Opaque stringData: adminUser.username: airflow @@ -15,11 +15,22 @@ stringData: adminUser.lastname: Admin adminUser.email: airflow@airflow.com adminUser.password: airflow - connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow -{% if test_scenario['values']['executor'] == 'celery' %} - connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql/airflow - connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 -{% endif %} +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgresql-credentials +stringData: + username: airflow + password: airflow +--- +apiVersion: v1 +kind: Secret +metadata: + name: redis-credentials +stringData: + username: "" + password: redis --- apiVersion: v1 kind: ConfigMap @@ -70,7 +81,12 @@ spec: clusterConfig: vectorAggregatorConfigMapName: airflow-vector-aggregator-discovery loadExamples: true - credentialsSecret: test-airflow-credentials + credentialsSecret: admin-user-credentials + metadataDatabase: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials dagsGitSync: - repo: https://github.com/stackabletech/example-dags gitFolder: dags @@ -137,6 +153,15 @@ spec: configMap: airflow-log-config {% if test_scenario['values']['executor'] == 'celery' %} celeryExecutors: + celeryResultBackend: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials + celeryBrokerUrl: + redis: + host: airflow-redis-master + credentialsSecret: redis-credentials config: resources: cpu: diff --git a/tests/templates/kuttl/mount-dags-configmap/30-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/mount-dags-configmap/30-install-airflow-cluster.yaml.j2 index b3bf36c8..a6f2436e 100644 --- a/tests/templates/kuttl/mount-dags-configmap/30-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/mount-dags-configmap/30-install-airflow-cluster.yaml.j2 @@ -7,7 +7,7 @@ timeout: 480 apiVersion: v1 kind: Secret metadata: - name: test-airflow-credentials + name: admin-user-credentials type: Opaque stringData: adminUser.username: airflow @@ -15,11 +15,22 @@ stringData: adminUser.lastname: Admin adminUser.email: airflow@airflow.com adminUser.password: airflow - connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow -{% if test_scenario['values']['executor'] == 'celery' %} - connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql/airflow - connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 -{% endif %} +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgresql-credentials +stringData: + username: airflow + password: airflow +--- +apiVersion: v1 +kind: Secret +metadata: + name: redis-credentials +stringData: + username: "" + password: redis --- apiVersion: v1 kind: ConfigMap @@ -74,7 +85,12 @@ spec: {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} - credentialsSecret: test-airflow-credentials + credentialsSecret: admin-user-credentials + metadataDatabase: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials volumes: - name: test-cm-dag configMap: @@ -96,6 +112,15 @@ spec: replicas: 1 {% if test_scenario['values']['executor'] == 'celery' %} celeryExecutors: + celeryResultBackend: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials + celeryBrokerUrl: + redis: + host: airflow-redis-master + credentialsSecret: redis-credentials config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} diff --git a/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 index 1e3620dd..6a04fc0b 100644 --- a/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 @@ -34,7 +34,7 @@ data: apiVersion: v1 kind: Secret metadata: - name: test-airflow-credentials + name: admin-user-credentials type: Opaque stringData: adminUser.username: airflow @@ -42,11 +42,22 @@ stringData: adminUser.lastname: Admin adminUser.email: airflow@airflow.com adminUser.password: airflow - connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow -{% if test_scenario['values']['executor'] == 'celery' %} - connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql/airflow - connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 -{% endif %} +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgresql-credentials +stringData: + username: airflow + password: airflow +--- +apiVersion: v1 +kind: Secret +metadata: + name: redis-credentials +stringData: + username: "" + password: redis --- apiVersion: v1 kind: ConfigMap @@ -73,7 +84,12 @@ spec: {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} - credentialsSecret: test-airflow-credentials + credentialsSecret: admin-user-credentials + metadataDatabase: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials dagsGitSync: {% if test_scenario['values']['access'] == 'ssh' %} - repo: ssh://git@github.com/stackable-airflow/dags.git @@ -116,6 +132,15 @@ spec: replicas: 1 {% if test_scenario['values']['executor'] == 'celery' %} celeryExecutors: + celeryResultBackend: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials + celeryBrokerUrl: + redis: + host: airflow-redis-master + credentialsSecret: redis-credentials config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} diff --git a/tests/templates/kuttl/oidc/install-airflow.yaml.j2 b/tests/templates/kuttl/oidc/install-airflow.yaml.j2 index 48826e4b..778ce98a 100644 --- a/tests/templates/kuttl/oidc/install-airflow.yaml.j2 +++ b/tests/templates/kuttl/oidc/install-airflow.yaml.j2 @@ -3,7 +3,7 @@ apiVersion: v1 kind: Secret metadata: - name: airflow-credentials + name: admin-user-credentials type: Opaque stringData: adminUser.username: airflow @@ -11,7 +11,14 @@ stringData: adminUser.lastname: Admin adminUser.email: airflow@airflow.com adminUser.password: airflow - connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgresql-credentials +stringData: + username: airflow + password: airflow --- apiVersion: v1 kind: Secret @@ -52,7 +59,12 @@ spec: oidc: clientCredentialsSecret: airflow-keycloak2-client userRegistrationRole: Admin - credentialsSecret: airflow-credentials + credentialsSecret: admin-user-credentials + metadataDatabase: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/opa/30-install-airflow.yaml.j2 b/tests/templates/kuttl/opa/30-install-airflow.yaml.j2 index 9fe3daa0..c149e217 100644 --- a/tests/templates/kuttl/opa/30-install-airflow.yaml.j2 +++ b/tests/templates/kuttl/opa/30-install-airflow.yaml.j2 @@ -7,7 +7,7 @@ metadata: apiVersion: v1 kind: Secret metadata: - name: airflow-credentials + name: admin-user-credentials type: Opaque stringData: adminUser.username: airflow @@ -15,7 +15,22 @@ stringData: adminUser.lastname: Admin adminUser.email: airflow@airflow.com adminUser.password: airflow - connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgresql-credentials +stringData: + username: airflow + password: airflow +--- +apiVersion: v1 +kind: Secret +metadata: + name: redis-credentials +stringData: + username: "" + password: redis --- apiVersion: airflow.stackable.tech/v1alpha1 kind: AirflowCluster @@ -38,7 +53,12 @@ spec: cache: entryTimeToLive: 5s maxEntries: 10 - credentialsSecret: airflow-credentials + credentialsSecret: admin-user-credentials + metadataDatabase: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials exposeConfig: true loadExamples: true {% if lookup('env', 'VECTOR_AGGREGATOR') %} diff --git a/tests/templates/kuttl/orphaned-resources/30-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/orphaned-resources/30-install-airflow-cluster.yaml.j2 index 3aa50e6e..e6fe8092 100644 --- a/tests/templates/kuttl/orphaned-resources/30-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/orphaned-resources/30-install-airflow-cluster.yaml.j2 @@ -7,7 +7,7 @@ timeout: 480 apiVersion: v1 kind: Secret metadata: - name: test-airflow-credentials + name: admin-user-credentials type: Opaque stringData: adminUser.username: airflow @@ -15,9 +15,22 @@ stringData: adminUser.lastname: Admin adminUser.email: airflow@airflow.com adminUser.password: airflow - connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow - connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql/airflow - connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgresql-credentials +stringData: + username: airflow + password: airflow +--- +apiVersion: v1 +kind: Secret +metadata: + name: redis-credentials +stringData: + username: "" + password: redis --- apiVersion: airflow.stackable.tech/v1alpha1 kind: AirflowCluster @@ -36,7 +49,12 @@ spec: {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} - credentialsSecret: test-airflow-credentials + credentialsSecret: admin-user-credentials + metadataDatabase: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials webservers: roleConfig: listenerClass: external-unstable @@ -47,6 +65,15 @@ spec: default: replicas: 1 celeryExecutors: + celeryResultBackend: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials + celeryBrokerUrl: + redis: + host: airflow-redis-master + credentialsSecret: redis-credentials config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} diff --git a/tests/templates/kuttl/overrides/10-install-airflow.yaml.j2 b/tests/templates/kuttl/overrides/10-install-airflow.yaml.j2 index 7ae53a0c..db7162b0 100644 --- a/tests/templates/kuttl/overrides/10-install-airflow.yaml.j2 +++ b/tests/templates/kuttl/overrides/10-install-airflow.yaml.j2 @@ -2,7 +2,7 @@ apiVersion: v1 kind: Secret metadata: - name: airflow-credentials + name: admin-user-credentials type: Opaque stringData: adminUser.username: airflow @@ -10,9 +10,22 @@ stringData: adminUser.lastname: Admin adminUser.email: airflow@airflow.com adminUser.password: airflow - connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow - connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql/airflow - connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgresql-credentials +stringData: + username: airflow + password: airflow +--- +apiVersion: v1 +kind: Secret +metadata: + name: redis-credentials +stringData: + username: "" + password: redis --- apiVersion: v1 kind: Secret @@ -25,9 +38,6 @@ stringData: adminUser.lastname: Admin adminUser.email: airflow@airflow.com adminUser.password: airflow - connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow - connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql/airflow - connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 --- apiVersion: airflow.stackable.tech/v1alpha1 kind: AirflowCluster @@ -45,7 +55,12 @@ spec: clusterConfig: loadExamples: true exposeConfig: false - credentialsSecret: airflow-credentials + credentialsSecret: admin-user-credentials + metadataDatabase: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials webservers: roleConfig: listenerClass: external-unstable @@ -60,6 +75,15 @@ spec: COMMON_VAR: group-value # overrides role value GROUP_VAR: group-value # only defined here at group level celeryExecutors: + celeryResultBackend: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials + celeryBrokerUrl: + redis: + host: airflow-redis-master + credentialsSecret: redis-credentials envOverrides: COMMON_VAR: role-value # overridden by role group below ROLE_VAR: role-value # only defined here at role level diff --git a/tests/templates/kuttl/overrides/20-install-airflow2.yaml.j2 b/tests/templates/kuttl/overrides/20-install-airflow2.yaml.j2 index 96edce21..80b6c258 100644 --- a/tests/templates/kuttl/overrides/20-install-airflow2.yaml.j2 +++ b/tests/templates/kuttl/overrides/20-install-airflow2.yaml.j2 @@ -15,7 +15,12 @@ spec: clusterConfig: loadExamples: true exposeConfig: false - credentialsSecret: airflow-credentials + credentialsSecret: admin-user-credentials + metadataDatabase: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials webservers: roleConfig: listenerClass: external-unstable diff --git a/tests/templates/kuttl/remote-logging/40-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/remote-logging/40-install-airflow-cluster.yaml.j2 index c0719d48..20395b4c 100644 --- a/tests/templates/kuttl/remote-logging/40-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/remote-logging/40-install-airflow-cluster.yaml.j2 @@ -7,7 +7,7 @@ timeout: 480 apiVersion: v1 kind: Secret metadata: - name: test-airflow-credentials + name: admin-user-credentials type: Opaque stringData: adminUser.username: airflow @@ -15,11 +15,22 @@ stringData: adminUser.lastname: Admin adminUser.email: airflow@airflow.com adminUser.password: airflow - connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow -{% if test_scenario['values']['executor'] == 'celery' %} - connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql/airflow - connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 -{% endif %} +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgresql-credentials +stringData: + username: airflow + password: airflow +--- +apiVersion: v1 +kind: Secret +metadata: + name: redis-credentials +stringData: + username: "" + password: redis --- apiVersion: airflow.stackable.tech/v1alpha1 kind: AirflowCluster @@ -36,7 +47,12 @@ spec: pullPolicy: IfNotPresent clusterConfig: loadExamples: true - credentialsSecret: test-airflow-credentials + credentialsSecret: admin-user-credentials + metadataDatabase: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials webservers: roleConfig: listenerClass: external-unstable @@ -50,6 +66,15 @@ spec: AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID: minio_conn {% if test_scenario['values']['executor'] == 'celery' %} celeryExecutors: + celeryResultBackend: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials + celeryBrokerUrl: + redis: + host: airflow-redis-master + credentialsSecret: redis-credentials roleGroups: default: replicas: 2 diff --git a/tests/templates/kuttl/resources/30-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/resources/30-install-airflow-cluster.yaml.j2 index 5af22cb8..1814b947 100644 --- a/tests/templates/kuttl/resources/30-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/resources/30-install-airflow-cluster.yaml.j2 @@ -7,7 +7,7 @@ timeout: 480 apiVersion: v1 kind: Secret metadata: - name: test-airflow-credentials + name: admin-user-credentials type: Opaque stringData: adminUser.username: airflow @@ -15,9 +15,22 @@ stringData: adminUser.lastname: Admin adminUser.email: airflow@airflow.com adminUser.password: airflow - connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow - connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql/airflow - connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgresql-credentials +stringData: + username: airflow + password: airflow +--- +apiVersion: v1 +kind: Secret +metadata: + name: redis-credentials +stringData: + username: "" + password: redis --- apiVersion: airflow.stackable.tech/v1alpha1 kind: AirflowCluster @@ -36,7 +49,12 @@ spec: {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} - credentialsSecret: test-airflow-credentials + credentialsSecret: admin-user-credentials + metadataDatabase: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials webservers: roleConfig: listenerClass: external-unstable @@ -47,6 +65,15 @@ spec: default: replicas: 1 celeryExecutors: + celeryResultBackend: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials + celeryBrokerUrl: + redis: + host: airflow-redis-master + credentialsSecret: redis-credentials config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} diff --git a/tests/templates/kuttl/smoke/40-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/smoke/40-install-airflow-cluster.yaml.j2 index 5227979c..39fdaaa2 100644 --- a/tests/templates/kuttl/smoke/40-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/smoke/40-install-airflow-cluster.yaml.j2 @@ -7,7 +7,7 @@ timeout: 480 apiVersion: v1 kind: Secret metadata: - name: test-airflow-credentials + name: admin-user-credentials type: Opaque stringData: adminUser.username: airflow @@ -15,11 +15,22 @@ stringData: adminUser.lastname: Admin adminUser.email: airflow@airflow.com adminUser.password: airflow - connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow -{% if test_scenario['values']['executor'] == 'celery' %} - connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql/airflow - connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 -{% endif %} +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgresql-credentials +stringData: + username: airflow + password: airflow +--- +apiVersion: v1 +kind: Secret +metadata: + name: redis-credentials +stringData: + username: "" + password: redis --- apiVersion: airflow.stackable.tech/v1alpha1 kind: AirflowCluster @@ -39,7 +50,12 @@ spec: vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} loadExamples: true - credentialsSecret: test-airflow-credentials + credentialsSecret: admin-user-credentials + metadataDatabase: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials webservers: roleConfig: listenerClass: external-unstable @@ -62,6 +78,15 @@ spec: COMMON_HEADER_VAR = "group-value" {% if test_scenario['values']['executor'] == 'celery' %} celeryExecutors: + celeryResultBackend: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials + celeryBrokerUrl: + redis: + host: airflow-redis-master + credentialsSecret: redis-credentials config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} diff --git a/tests/templates/kuttl/triggerer/30-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/triggerer/30-install-airflow-cluster.yaml.j2 index 2f728798..ddc2da01 100644 --- a/tests/templates/kuttl/triggerer/30-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/triggerer/30-install-airflow-cluster.yaml.j2 @@ -7,7 +7,7 @@ timeout: 480 apiVersion: v1 kind: Secret metadata: - name: test-airflow-credentials + name: admin-user-credentials type: Opaque stringData: adminUser.username: airflow @@ -15,11 +15,22 @@ stringData: adminUser.lastname: Admin adminUser.email: airflow@airflow.com adminUser.password: airflow - connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow -{% if test_scenario['values']['executor'] == 'celery' %} - connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql/airflow - connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 -{% endif %} +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgresql-credentials +stringData: + username: airflow + password: airflow +--- +apiVersion: v1 +kind: Secret +metadata: + name: redis-credentials +stringData: + username: "" + password: redis --- apiVersion: v1 kind: ConfigMap @@ -96,7 +107,12 @@ spec: {% endif %} pullPolicy: IfNotPresent clusterConfig: - credentialsSecret: test-airflow-credentials + credentialsSecret: admin-user-credentials + metadataDatabase: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials volumes: - name: triggerer-dag configMap: @@ -115,6 +131,15 @@ spec: replicas: 1 {% if test_scenario['values']['executor'] == 'celery' %} celeryExecutors: + celeryResultBackend: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials + celeryBrokerUrl: + redis: + host: airflow-redis-master + credentialsSecret: redis-credentials roleGroups: default: envOverrides: *envOverrides diff --git a/tests/templates/kuttl/versioning/30-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/versioning/30-install-airflow-cluster.yaml.j2 index a30cc10e..99203d52 100644 --- a/tests/templates/kuttl/versioning/30-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/versioning/30-install-airflow-cluster.yaml.j2 @@ -19,7 +19,7 @@ data: apiVersion: v1 kind: Secret metadata: - name: test-airflow-credentials + name: admin-user-credentials type: Opaque stringData: adminUser.username: airflow @@ -27,7 +27,22 @@ stringData: adminUser.lastname: Admin adminUser.email: airflow@airflow.com adminUser.password: airflow - connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgresql-credentials +stringData: + username: airflow + password: airflow +--- +apiVersion: v1 +kind: Secret +metadata: + name: redis-credentials +stringData: + username: "" + password: redis --- apiVersion: airflow.stackable.tech/v1alpha1 kind: AirflowCluster @@ -46,7 +61,12 @@ spec: {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} - credentialsSecret: test-airflow-credentials + credentialsSecret: admin-user-credentials + metadataDatabase: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials dagsGitSync: - repo: https://github.com/stackable-airflow/dags # v1alpha1 field which should be converted to an enum From d543b23641d20342c53985c1c5d8dfd941fcb1d8 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 5 Feb 2026 15:11:42 +0100 Subject: [PATCH 04/26] added some comments --- .../src/connections/database.rs | 27 +++++++++++++++++++ rust/operator-binary/src/connections/queue.rs | 16 +++++++++++ 2 files changed, 43 insertions(+) diff --git a/rust/operator-binary/src/connections/database.rs b/rust/operator-binary/src/connections/database.rs index 60fca6d9..c9436ab7 100644 --- a/rust/operator-binary/src/connections/database.rs +++ b/rust/operator-binary/src/connections/database.rs @@ -3,6 +3,22 @@ use std::collections::BTreeMap; use serde::{Deserialize, Serialize}; use stackable_operator::schemars::{self, JsonSchema}; +// Struct defining supported backend database types. The implementation should +// conform to ADR 29. Concrete types will define all fields necessary to +// construct a connection for that database type. Each backend requires a +// secret containing `username` and `password`. Additionally, a generic type +// is defined which will contain a single field pointing to a secret with a +// single `uri` field that contains the entire connection string. A single +// backend type may use multiple drivers that are prefixed to the connection +// string. The operator knows the context of the backend connection (e.g. is it +// for a SqlALchemyURI- or a Celery-connection) and is therefore responsible +// for calling the correct function. The connection string will not contain +// resolved credentials, but rather embedded environment variables that point +// to values that have been set via the resource definitions (created by the +// operator). In this way the operator never has to read the secret itself. +// These embedded values within enviroment variables are resolved when read by +// product containers, as they are read through a process started within a +// shell (which performs variable substitution). #[derive(Clone, Deserialize, Debug, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub enum DbType { @@ -40,6 +56,17 @@ impl DbType { } impl PostgresqlDb { + // At this point the necessary secrets have been added to the product + // statefulset (as env-vars). If a product has been started via a shell, + // then any embedded environment variables will be resolved (i.e. variable + // substitution) automatically. If for some reason the env-var is accessed + // directly via program code (e.g. python), then this will not work, and + // the env-vars will instead need to be replaced with the same information + // written to a relevant configuration file (as env-vars cannot be safely + // templated/re-written from Rust in a multi-threaded environment): in this + // case the resulting connection will be written (where env-vars are in the + // form of ${{env:...}}) to a file which is then templated on container + // start-up. fn connection_string(&self, prefix: &str, username_env: &str, password_env: &str) -> String { let params = if self.parameters.is_empty() { String::new() diff --git a/rust/operator-binary/src/connections/queue.rs b/rust/operator-binary/src/connections/queue.rs index ddb62537..858bfc70 100644 --- a/rust/operator-binary/src/connections/queue.rs +++ b/rust/operator-binary/src/connections/queue.rs @@ -1,6 +1,22 @@ use serde::{Deserialize, Serialize}; use stackable_operator::schemars::{self, JsonSchema}; +// Struct defining supported backend queue/broker types. These are similar to +// backend databases but are kept separate as they are not strictly covered by +// ADR 29. Concrete types will define all fields necessary to construct a +// connection for that queue type. Each queue requires a secret containing +// `username` and `password`. Additionally, a generic type is defined which +// will contain a single field pointing to a secret with a single `uri` field +// that contains the entire connection string. A single queue type may use +// multiple drivers that are prefixed to the connection string. The operator +// knows the context of the queue connection and is therefore responsible for +// calling the correct function. The connection string will not contain +// resolved credentials, but rather embedded environment variables that point +// to values that have been set via the resource definitions (created by the +// operator). In this way the operator never has to read the secret itself. +// These embedded values within enviroment variables are resolved when read by +// product containers, as they are read through a process started within a +// shell (which performs variable substitution). #[derive(Clone, Deserialize, Debug, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub enum QueueType { From 9ea139cc3671873863583be6776c5cd8d53e8e70 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 5 Feb 2026 15:49:39 +0100 Subject: [PATCH 05/26] added test for generic URIs --- extra/crds.yaml | 4 +- rust/operator-binary/src/crd/mod.rs | 2 +- .../kuttl/smoke-generic/00-patch-ns.yaml.j2 | 9 ++ .../kuttl/smoke-generic/00-range-limit.yaml | 11 ++ .../kuttl/smoke-generic/10-assert.yaml | 14 ++ .../smoke-generic/10-install-postgresql.yaml | 12 ++ .../kuttl/smoke-generic/20-assert.yaml.j2 | 22 +++ .../kuttl/smoke-generic/20-install-redis.yaml | 12 ++ .../kuttl/smoke-generic/30-assert.yaml.j2 | 10 ++ ...tor-aggregator-discovery-configmap.yaml.j2 | 9 ++ .../kuttl/smoke-generic/40-assert.yaml | 134 ++++++++++++++++++ .../40-install-airflow-cluster.yaml.j2 | 104 ++++++++++++++ .../kuttl/smoke-generic/50-assert.yaml | 14 ++ .../50-install-airflow-python.yaml | 30 ++++ .../kuttl/smoke-generic/60-assert.yaml.j2 | 12 ++ .../kuttl/smoke-generic/60-health-check.yaml | 7 + .../kuttl/smoke-generic/70-assert.yaml.j2 | 12 ++ .../70-install-metrics-script.yaml | 8 ++ .../kuttl/smoke-generic/80-assert.yaml | 26 ++++ .../helm-bitnami-postgresql-values.yaml.j2 | 43 ++++++ .../helm-bitnami-redis-values.yaml.j2 | 63 ++++++++ tests/test-definition.yaml | 4 + 22 files changed, 559 insertions(+), 3 deletions(-) create mode 100644 tests/templates/kuttl/smoke-generic/00-patch-ns.yaml.j2 create mode 100644 tests/templates/kuttl/smoke-generic/00-range-limit.yaml create mode 100644 tests/templates/kuttl/smoke-generic/10-assert.yaml create mode 100644 tests/templates/kuttl/smoke-generic/10-install-postgresql.yaml create mode 100644 tests/templates/kuttl/smoke-generic/20-assert.yaml.j2 create mode 100644 tests/templates/kuttl/smoke-generic/20-install-redis.yaml create mode 100644 tests/templates/kuttl/smoke-generic/30-assert.yaml.j2 create mode 100644 tests/templates/kuttl/smoke-generic/30-install-vector-aggregator-discovery-configmap.yaml.j2 create mode 100644 tests/templates/kuttl/smoke-generic/40-assert.yaml create mode 100644 tests/templates/kuttl/smoke-generic/40-install-airflow-cluster.yaml.j2 create mode 100644 tests/templates/kuttl/smoke-generic/50-assert.yaml create mode 100644 tests/templates/kuttl/smoke-generic/50-install-airflow-python.yaml create mode 100644 tests/templates/kuttl/smoke-generic/60-assert.yaml.j2 create mode 100644 tests/templates/kuttl/smoke-generic/60-health-check.yaml create mode 100644 tests/templates/kuttl/smoke-generic/70-assert.yaml.j2 create mode 100644 tests/templates/kuttl/smoke-generic/70-install-metrics-script.yaml create mode 100644 tests/templates/kuttl/smoke-generic/80-assert.yaml create mode 100644 tests/templates/kuttl/smoke-generic/helm-bitnami-postgresql-values.yaml.j2 create mode 100644 tests/templates/kuttl/smoke-generic/helm-bitnami-redis-values.yaml.j2 diff --git a/extra/crds.yaml b/extra/crds.yaml index 8e03f633..dfb91664 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -721,7 +721,7 @@ spec: type: object credentialsSecret: description: |- - The name of the Secret object containing the admin user credentials and database connection details. + The name of the Secret object containing the admin user credentials. Read the [getting started guide first steps](https://docs.stackable.tech/home/nightly/airflow/getting_started/first_steps) to find out more. @@ -4033,7 +4033,7 @@ spec: type: object credentialsSecret: description: |- - The name of the Secret object containing the admin user credentials and database connection details. + The name of the Secret object containing the admin user credentials. Read the [getting started guide first steps](https://docs.stackable.tech/home/nightly/airflow/getting_started/first_steps) to find out more. diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 85d715fd..9935a18a 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -245,7 +245,7 @@ pub mod versioned { #[serde(skip_serializing_if = "Option::is_none")] pub authorization: Option, - /// The name of the Secret object containing the admin user credentials and database connection details. + /// The name of the Secret object containing the admin user credentials. /// Read the /// [getting started guide first steps](DOCS_BASE_URL_PLACEHOLDER/airflow/getting_started/first_steps) /// to find out more. diff --git a/tests/templates/kuttl/smoke-generic/00-patch-ns.yaml.j2 b/tests/templates/kuttl/smoke-generic/00-patch-ns.yaml.j2 new file mode 100644 index 00000000..67185acf --- /dev/null +++ b/tests/templates/kuttl/smoke-generic/00-patch-ns.yaml.j2 @@ -0,0 +1,9 @@ +{% if test_scenario['values']['openshift'] == 'true' %} +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + timeout: 120 +{% endif %} diff --git a/tests/templates/kuttl/smoke-generic/00-range-limit.yaml b/tests/templates/kuttl/smoke-generic/00-range-limit.yaml new file mode 100644 index 00000000..8fd02210 --- /dev/null +++ b/tests/templates/kuttl/smoke-generic/00-range-limit.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: v1 +kind: LimitRange +metadata: + name: limit-request-ratio +spec: + limits: + - type: "Container" + maxLimitRequestRatio: + cpu: 5 + memory: 1 diff --git a/tests/templates/kuttl/smoke-generic/10-assert.yaml b/tests/templates/kuttl/smoke-generic/10-assert.yaml new file mode 100644 index 00000000..319e927a --- /dev/null +++ b/tests/templates/kuttl/smoke-generic/10-assert.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-airflow-postgresql +timeout: 480 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: airflow-postgresql +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/smoke-generic/10-install-postgresql.yaml b/tests/templates/kuttl/smoke-generic/10-install-postgresql.yaml new file mode 100644 index 00000000..9e0529d1 --- /dev/null +++ b/tests/templates/kuttl/smoke-generic/10-install-postgresql.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: >- + helm install airflow-postgresql + --namespace $NAMESPACE + --version 16.4.2 + -f helm-bitnami-postgresql-values.yaml + oci://registry-1.docker.io/bitnamicharts/postgresql + --wait + timeout: 600 diff --git a/tests/templates/kuttl/smoke-generic/20-assert.yaml.j2 b/tests/templates/kuttl/smoke-generic/20-assert.yaml.j2 new file mode 100644 index 00000000..f038df42 --- /dev/null +++ b/tests/templates/kuttl/smoke-generic/20-assert.yaml.j2 @@ -0,0 +1,22 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-airflow-redis +timeout: 360 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: airflow-redis-master +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: airflow-redis-replicas +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/smoke-generic/20-install-redis.yaml b/tests/templates/kuttl/smoke-generic/20-install-redis.yaml new file mode 100644 index 00000000..cc1edc53 --- /dev/null +++ b/tests/templates/kuttl/smoke-generic/20-install-redis.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: >- + helm install airflow-redis + --namespace $NAMESPACE + --version 17.11.3 + -f helm-bitnami-redis-values.yaml + --repo https://charts.bitnami.com/bitnami redis + --wait + timeout: 600 diff --git a/tests/templates/kuttl/smoke-generic/30-assert.yaml.j2 b/tests/templates/kuttl/smoke-generic/30-assert.yaml.j2 new file mode 100644 index 00000000..50b1d4c3 --- /dev/null +++ b/tests/templates/kuttl/smoke-generic/30-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/smoke-generic/30-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/smoke-generic/30-install-vector-aggregator-discovery-configmap.yaml.j2 new file mode 100644 index 00000000..2d6a0df5 --- /dev/null +++ b/tests/templates/kuttl/smoke-generic/30-install-vector-aggregator-discovery-configmap.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/smoke-generic/40-assert.yaml b/tests/templates/kuttl/smoke-generic/40-assert.yaml new file mode 100644 index 00000000..6c16b2a7 --- /dev/null +++ b/tests/templates/kuttl/smoke-generic/40-assert.yaml @@ -0,0 +1,134 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-available-condition +timeout: 600 +commands: + - script: kubectl -n $NAMESPACE wait --for=condition=available airflowclusters.airflow.stackable.tech/airflow --timeout 301s +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-airflow-cluster +timeout: 1200 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: airflow-webserver-default + generation: 1 # There should be no unneeded Pod restarts + labels: + restarter.stackable.tech/enabled: "true" +spec: + template: + spec: + terminationGracePeriodSeconds: 120 +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: airflow-worker-default + generation: 1 # There should be no unneeded Pod restarts + labels: + restarter.stackable.tech/enabled: "true" +spec: + template: + spec: + terminationGracePeriodSeconds: 300 +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: airflow-scheduler-default + generation: 1 # There should be no unneeded Pod restarts + labels: + restarter.stackable.tech/enabled: "true" +spec: + template: + spec: + terminationGracePeriodSeconds: 120 +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: airflow-dagprocessor-default + generation: 1 # There should be no unneeded Pod restarts + labels: + restarter.stackable.tech/enabled: "true" +spec: + template: + spec: + terminationGracePeriodSeconds: 120 +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: airflow-triggerer-default + generation: 1 # There should be no unneeded Pod restarts + labels: + restarter.stackable.tech/enabled: "true" +spec: + template: + spec: + terminationGracePeriodSeconds: 120 +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: airflow-webserver +status: + expectedPods: 1 + currentHealthy: 1 + disruptionsAllowed: 1 +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: airflow-worker +status: + expectedPods: 1 + currentHealthy: 1 + disruptionsAllowed: 1 +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: airflow-scheduler +status: + expectedPods: 1 + currentHealthy: 1 + disruptionsAllowed: 1 +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: airflow-dagprocessor +status: + expectedPods: 1 + currentHealthy: 1 + disruptionsAllowed: 1 +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: airflow-triggerer +status: + expectedPods: 1 + currentHealthy: 1 + disruptionsAllowed: 1 diff --git a/tests/templates/kuttl/smoke-generic/40-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/smoke-generic/40-install-airflow-cluster.yaml.j2 new file mode 100644 index 00000000..cd0b4484 --- /dev/null +++ b/tests/templates/kuttl/smoke-generic/40-install-airflow-cluster.yaml.j2 @@ -0,0 +1,104 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +metadata: + name: install-airflow +timeout: 480 +--- +apiVersion: v1 +kind: Secret +metadata: + name: admin-user-credentials +type: Opaque +stringData: + adminUser.username: airflow + adminUser.firstname: Airflow + adminUser.lastname: Admin + adminUser.email: airflow@airflow.com + adminUser.password: airflow +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgresql-metadata +stringData: + uri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgresql-celery +stringData: + uri: db+postgresql://airflow:airflow@airflow-postgresql/airflow +--- +apiVersion: v1 +kind: Secret +metadata: + name: redis-celery +stringData: + uri: redis://:redis@airflow-redis-master:6379/0 +--- +apiVersion: airflow.stackable.tech/v1alpha1 +kind: AirflowCluster +metadata: + name: airflow +spec: + image: +{% if test_scenario['values']['airflow'].find(",") > 0 %} + custom: "{{ test_scenario['values']['airflow'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['airflow'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['airflow'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + loadExamples: true + credentialsSecret: admin-user-credentials + metadataDatabase: + generic: + uriSecret: postgresql-metadata + webservers: + roleConfig: + listenerClass: external-unstable + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 + celeryExecutors: + celeryResultBackend: + generic: + uriSecret: postgresql-celery + celeryBrokerUrl: + generic: + uriSecret: redis-celery + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 + schedulers: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 + dagProcessors: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 + triggerers: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 diff --git a/tests/templates/kuttl/smoke-generic/50-assert.yaml b/tests/templates/kuttl/smoke-generic/50-assert.yaml new file mode 100644 index 00000000..6edaa3c3 --- /dev/null +++ b/tests/templates/kuttl/smoke-generic/50-assert.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-airflow-python +timeout: 240 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-airflow-python +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/smoke-generic/50-install-airflow-python.yaml b/tests/templates/kuttl/smoke-generic/50-install-airflow-python.yaml new file mode 100644 index 00000000..b5ffa00c --- /dev/null +++ b/tests/templates/kuttl/smoke-generic/50-install-airflow-python.yaml @@ -0,0 +1,30 @@ +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-airflow-python + labels: + app: test-airflow-python +spec: + replicas: 1 + selector: + matchLabels: + app: test-airflow-python + template: + metadata: + labels: + app: test-airflow-python + spec: + containers: + - name: test-airflow-python + image: oci.stackable.tech/sdp/testing-tools:0.3.0-stackable0.0.0-dev + imagePullPolicy: IfNotPresent + stdin: true + tty: true + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "128Mi" + cpu: "400m" diff --git a/tests/templates/kuttl/smoke-generic/60-assert.yaml.j2 b/tests/templates/kuttl/smoke-generic/60-assert.yaml.j2 new file mode 100644 index 00000000..8b1f71bf --- /dev/null +++ b/tests/templates/kuttl/smoke-generic/60-assert.yaml.j2 @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-airflow-webserver-health-check +timeout: 480 +commands: +{% if test_scenario['values']['airflow'].find(",") > 0 %} + - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/health.py --airflow-version "{{ test_scenario['values']['airflow'].split(',')[0] }}" +{% else %} + - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/health.py --airflow-version "{{ test_scenario['values']['airflow'] }}" +{% endif %} diff --git a/tests/templates/kuttl/smoke-generic/60-health-check.yaml b/tests/templates/kuttl/smoke-generic/60-health-check.yaml new file mode 100644 index 00000000..c72c4222 --- /dev/null +++ b/tests/templates/kuttl/smoke-generic/60-health-check.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 480 +commands: + - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/health.py test-airflow-python-0:/tmp + timeout: 240 diff --git a/tests/templates/kuttl/smoke-generic/70-assert.yaml.j2 b/tests/templates/kuttl/smoke-generic/70-assert.yaml.j2 new file mode 100644 index 00000000..a811cb8e --- /dev/null +++ b/tests/templates/kuttl/smoke-generic/70-assert.yaml.j2 @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: metrics +timeout: 480 +commands: +{% if test_scenario['values']['airflow'].find(",") > 0 %} + - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/metrics.py --airflow-version "{{ test_scenario['values']['airflow'].split(',')[0] }}" +{% else %} + - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/metrics.py --airflow-version "{{ test_scenario['values']['airflow'] }}" +{% endif %} diff --git a/tests/templates/kuttl/smoke-generic/70-install-metrics-script.yaml b/tests/templates/kuttl/smoke-generic/70-install-metrics-script.yaml new file mode 100644 index 00000000..17136688 --- /dev/null +++ b/tests/templates/kuttl/smoke-generic/70-install-metrics-script.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +metadata: + name: metrics +commands: + - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/metrics.py test-airflow-python-0:/tmp + timeout: 240 diff --git a/tests/templates/kuttl/smoke-generic/80-assert.yaml b/tests/templates/kuttl/smoke-generic/80-assert.yaml new file mode 100644 index 00000000..e54c4c84 --- /dev/null +++ b/tests/templates/kuttl/smoke-generic/80-assert.yaml @@ -0,0 +1,26 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-log-endpoint +timeout: 240 +commands: + - script: | + set -eu + + # Log-Endpoint Test: + # This is executed from the Webserver as JWT keys must be present. + # A small server is started on each worker that serves the logs on its + # 8793 port for the Webserver: we don't use the token as that is an + # internal implementation, but check that the endpoint is reachable, + # indicated by a 403. + # N.B. this behaviour is stricter in 3.1.0, whereby the "/log" suffix + # has to be declared. + # See https://github.com/apache/airflow/pull/52581. + CURL_RESPONSE=$( + kubectl -n $NAMESPACE exec airflow-webserver-default-0 -- sh -c 'CODE=$(curl -s -o /dev/null -w "%{http_code}" http://airflow-worker-default-headless:8793/log 2>/dev/null || true);echo "$CODE"' + ) + + # Log-Endpoint Test Assertion: + echo "The HTTP Code is $CURL_RESPONSE (an internal JWT token is needed for full access)" + [ "$CURL_RESPONSE" -eq 403 ] diff --git a/tests/templates/kuttl/smoke-generic/helm-bitnami-postgresql-values.yaml.j2 b/tests/templates/kuttl/smoke-generic/helm-bitnami-postgresql-values.yaml.j2 new file mode 100644 index 00000000..8067902b --- /dev/null +++ b/tests/templates/kuttl/smoke-generic/helm-bitnami-postgresql-values.yaml.j2 @@ -0,0 +1,43 @@ +--- +global: + security: + allowInsecureImages: true + +image: + repository: bitnamilegacy/postgresql + +volumePermissions: + enabled: false + image: + repository: bitnamilegacy/os-shell + securityContext: + runAsUser: auto + +metrics: + image: + repository: bitnamilegacy/postgres-exporter + +primary: + podSecurityContext: +{% if test_scenario['values']['openshift'] == 'true' %} + enabled: false +{% else %} + enabled: true +{% endif %} + containerSecurityContext: + enabled: false + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "128Mi" + cpu: "400m" +shmVolume: + chmod: + enabled: false + +auth: + username: airflow + password: airflow + database: airflow diff --git a/tests/templates/kuttl/smoke-generic/helm-bitnami-redis-values.yaml.j2 b/tests/templates/kuttl/smoke-generic/helm-bitnami-redis-values.yaml.j2 new file mode 100644 index 00000000..293abaf1 --- /dev/null +++ b/tests/templates/kuttl/smoke-generic/helm-bitnami-redis-values.yaml.j2 @@ -0,0 +1,63 @@ +--- +global: + security: + allowInsecureImages: true # needed starting with Chart version 20.5.0 if modifying images +image: + repository: bitnamilegacy/redis +sentinel: + image: + repository: bitnamilegacy/redis-sentinel +metrics: + image: + repository: bitnamilegacy/redis-exporter +kubectl: + image: + repository: bitnamilegacy/kubectl +sysctl: + image: + repository: bitnamilegacy/os-shell + +volumePermissions: + enabled: false + image: + repository: bitnamilegacy/os-shell + containerSecurityContext: + runAsUser: auto + +master: + podSecurityContext: +{% if test_scenario['values']['openshift'] == 'true' %} + enabled: false +{% else %} + enabled: true +{% endif %} + containerSecurityContext: + enabled: false + resources: + requests: + memory: "128Mi" + cpu: "200m" + limits: + memory: "128Mi" + cpu: "800m" + +replica: + replicaCount: 1 + podSecurityContext: +{% if test_scenario['values']['openshift'] == 'true' %} + enabled: false +{% else %} + enabled: true +{% endif %} + containerSecurityContext: + enabled: false + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "128Mi" + cpu: "400m" + +auth: + password: redis diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index d031431b..e2192f30 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -42,6 +42,10 @@ tests: - airflow - openshift - executor + - name: smoke-generic + dimensions: + - airflow + - openshift - name: mount-dags-configmap dimensions: - airflow-latest From d210148dec2d3bd7ff01997944a365ca6e96a53f Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 5 Feb 2026 16:41:07 +0100 Subject: [PATCH 06/26] typo --- .../templates/kuttl/logging/41-install-airflow-cluster.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/templates/kuttl/logging/41-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/logging/41-install-airflow-cluster.yaml.j2 index e5dc25ff..f40be4c5 100644 --- a/tests/templates/kuttl/logging/41-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/logging/41-install-airflow-cluster.yaml.j2 @@ -3,7 +3,7 @@ kind: TestStep metadata: name: install-airflow timeout: 480 ----- +--- apiVersion: v1 kind: Secret metadata: From 50a2c5c46c7a3d8f0b37f437e5deb3837b0ba14c Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 5 Feb 2026 17:22:17 +0100 Subject: [PATCH 07/26] code style --- .../src/connections/database.rs | 26 +++++++------------ rust/operator-binary/src/connections/queue.rs | 26 +++++++------------ 2 files changed, 20 insertions(+), 32 deletions(-) diff --git a/rust/operator-binary/src/connections/database.rs b/rust/operator-binary/src/connections/database.rs index c9436ab7..2bb861d8 100644 --- a/rust/operator-binary/src/connections/database.rs +++ b/rust/operator-binary/src/connections/database.rs @@ -3,22 +3,16 @@ use std::collections::BTreeMap; use serde::{Deserialize, Serialize}; use stackable_operator::schemars::{self, JsonSchema}; -// Struct defining supported backend database types. The implementation should -// conform to ADR 29. Concrete types will define all fields necessary to -// construct a connection for that database type. Each backend requires a -// secret containing `username` and `password`. Additionally, a generic type -// is defined which will contain a single field pointing to a secret with a -// single `uri` field that contains the entire connection string. A single -// backend type may use multiple drivers that are prefixed to the connection -// string. The operator knows the context of the backend connection (e.g. is it -// for a SqlALchemyURI- or a Celery-connection) and is therefore responsible -// for calling the correct function. The connection string will not contain -// resolved credentials, but rather embedded environment variables that point -// to values that have been set via the resource definitions (created by the -// operator). In this way the operator never has to read the secret itself. -// These embedded values within enviroment variables are resolved when read by -// product containers, as they are read through a process started within a -// shell (which performs variable substitution). +// Struct defining supported backend database types. +// The implementation should conform to ADR 29. +// Concrete types will define all fields necessary to construct a connection for that database type. +// Each backend requires a secret containing `username` and `password`. +// Additionally, a generic type is defined which will contain a single field pointing to a secret with a single `uri` field that contains the entire connection string. +// A single backend type may use multiple drivers that are prefixed to the connection string. +// The operator knows the context of the backend connection (e.g. is it for a SqlALchemyURI- or a Celery-connection) and is therefore responsible for calling the correct function to obtain the connnection string. +// The connection string will not contain resolved credentials, but rather embedded environment variables that point to values that have been set via the resource definitions (created by the operator). +// In this way the operator never has to read the secret itself. +// These embedded values within enviroment variables are resolved when read by product containers, as they are read through a process started within a shell (which performs variable substitution). #[derive(Clone, Deserialize, Debug, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub enum DbType { diff --git a/rust/operator-binary/src/connections/queue.rs b/rust/operator-binary/src/connections/queue.rs index 858bfc70..b39542a4 100644 --- a/rust/operator-binary/src/connections/queue.rs +++ b/rust/operator-binary/src/connections/queue.rs @@ -1,22 +1,16 @@ use serde::{Deserialize, Serialize}; use stackable_operator::schemars::{self, JsonSchema}; -// Struct defining supported backend queue/broker types. These are similar to -// backend databases but are kept separate as they are not strictly covered by -// ADR 29. Concrete types will define all fields necessary to construct a -// connection for that queue type. Each queue requires a secret containing -// `username` and `password`. Additionally, a generic type is defined which -// will contain a single field pointing to a secret with a single `uri` field -// that contains the entire connection string. A single queue type may use -// multiple drivers that are prefixed to the connection string. The operator -// knows the context of the queue connection and is therefore responsible for -// calling the correct function. The connection string will not contain -// resolved credentials, but rather embedded environment variables that point -// to values that have been set via the resource definitions (created by the -// operator). In this way the operator never has to read the secret itself. -// These embedded values within enviroment variables are resolved when read by -// product containers, as they are read through a process started within a -// shell (which performs variable substitution). +// Struct defining supported backend queue/broker types. +// These are similar to backend databases but are kept separate as they are not strictly covered by ADR 29. +// Concrete types will define all fields necessary to construct a connection for that queue type. +// Each queue requires a secret containing `username` and `password`. +// Additionally, a generic type is defined which will contain a single field pointing to a secret with a single `uri` field that contains the entire connection string. +// A single queue type may use multiple drivers that are prefixed to the connection string. +// The operator knows the context of the queue connection and is therefore responsible for calling the correct function. +// The connection string will not contain resolved credentials, but rather embedded environment variables that point to values that have been set via the resource definitions (created by the operator). +// In this way the operator never has to read the secret itself. +// These embedded values within enviroment variables are resolved when read by product containers, as they are read through a process started within a shell (which performs variable substitution). #[derive(Clone, Deserialize, Debug, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub enum QueueType { From 751fb68d183baacd42c8f68a995bca43007ec942 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Fri, 6 Feb 2026 12:56:46 +0100 Subject: [PATCH 08/26] changelog and doc updates --- CHANGELOG.md | 5 + .../example-airflow-dags-configmap.yaml | 23 +-- .../examples/example-airflow-incluster.yaml | 27 ++- .../examples/example-airflow-secret.yaml | 16 -- .../code/airflow-credentials.yaml | 22 ++- .../getting_started/code/airflow.yaml | 16 +- .../airflow/pages/usage-guide/logging.adoc | 1 + .../pages/usage-guide/storage-resources.adoc | 1 + .../using-kubernetes-executors.adoc | 1 + .../simple-airflow-cluster-dags-cmap.yaml | 116 ----------- ...ple-airflow-cluster-ldap-insecure-tls.yaml | 181 ------------------ examples/simple-airflow-cluster-ldap.yaml | 179 ----------------- examples/simple-airflow-cluster.yaml | 42 ---- 13 files changed, 67 insertions(+), 563 deletions(-) delete mode 100644 docs/modules/airflow/examples/example-airflow-secret.yaml delete mode 100644 examples/simple-airflow-cluster-dags-cmap.yaml delete mode 100644 examples/simple-airflow-cluster-ldap-insecure-tls.yaml delete mode 100644 examples/simple-airflow-cluster-ldap.yaml delete mode 100644 examples/simple-airflow-cluster.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index b9fa1b14..51e5a46e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,10 @@ - Gracefully shutdown all concurrent tasks by forwarding the SIGTERM signal ([#741]). - Bump testing-tools to `0.3.0-stackable0.0.0-dev` ([#733]). +- BREAKING: Replace Airflow credentials-secret with database/broker connections ([#743]). + - Existing secret is retained for the admin user alone. + - Database/broker connections can be defined either using structs or as a generic connection string (see ADR 29). + - Removed standalone examples folder (not affecting the documentation). ### Fixed @@ -28,6 +32,7 @@ [#734]: https://github.com/stackabletech/airflow-operator/pull/734 [#741]: https://github.com/stackabletech/airflow-operator/pull/741 [#742]: https://github.com/stackabletech/airflow-operator/pull/742 +[#743]: https://github.com/stackabletech/airflow-operator/pull/743 ## [25.11.0] - 2025-11-07 diff --git a/docs/modules/airflow/examples/example-airflow-dags-configmap.yaml b/docs/modules/airflow/examples/example-airflow-dags-configmap.yaml index afb8f558..bbc96c1a 100644 --- a/docs/modules/airflow/examples/example-airflow-dags-configmap.yaml +++ b/docs/modules/airflow/examples/example-airflow-dags-configmap.yaml @@ -7,9 +7,12 @@ spec: image: productVersion: 3.1.6 clusterConfig: - loadExamples: false - exposeConfig: false - credentialsSecret: simple-airflow-credentials + credentialsSecret: admin-user-credentials + metadataDatabase: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials volumes: - name: cm-dag # <3> configMap: @@ -23,18 +26,8 @@ spec: listenerClass: external-unstable roleGroups: default: - envOverrides: + envOverrides: &envOverrides AIRFLOW__CORE__DAGS_FOLDER: "/dags" # <8> replicas: 1 celeryExecutors: - roleGroups: - default: - envOverrides: - AIRFLOW__CORE__DAGS_FOLDER: "/dags" # <8> - replicas: 2 - schedulers: - roleGroups: - default: - envOverrides: - AIRFLOW__CORE__DAGS_FOLDER: "/dags" # <8> - replicas: 1 + ... diff --git a/docs/modules/airflow/examples/example-airflow-incluster.yaml b/docs/modules/airflow/examples/example-airflow-incluster.yaml index ca437df9..a52a5825 100644 --- a/docs/modules/airflow/examples/example-airflow-incluster.yaml +++ b/docs/modules/airflow/examples/example-airflow-incluster.yaml @@ -9,28 +9,39 @@ spec: clusterConfig: loadExamples: false exposeConfig: false - credentialsSecret: simple-airflow-credentials + credentialsSecret: admin-user-credentials + metadataDatabase: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials webservers: roleConfig: listenerClass: external-unstable roleGroups: default: - envOverrides: + envOverrides: &envOverrides AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" replicas: 1 schedulers: roleGroups: default: - envOverrides: - AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" + envOverrides: *envOverrides replicas: 1 celeryExecutors: + celeryResultBackend: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials + celeryBrokerUrl: + redis: + host: airflow-redis-master + credentialsSecret: redis-credentials roleGroups: default: - envOverrides: - AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" + envOverrides: *envOverrides replicas: 1 # in case of using kubernetesExecutors # kubernetesExecutors: -# envOverrides: -# AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" +# envOverrides: *envOverrides diff --git a/docs/modules/airflow/examples/example-airflow-secret.yaml b/docs/modules/airflow/examples/example-airflow-secret.yaml deleted file mode 100644 index 5e112e91..00000000 --- a/docs/modules/airflow/examples/example-airflow-secret.yaml +++ /dev/null @@ -1,16 +0,0 @@ ---- -apiVersion: v1 -kind: Secret -metadata: - name: simple-airflow-credentials -type: Opaque -stringData: - adminUser.username: airflow - adminUser.firstname: Airflow - adminUser.lastname: Admin - adminUser.email: airflow@airflow.com - adminUser.password: airflow - connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow - # Only needed when using celery workers (instead of Kubernetes executors) - connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow - connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 diff --git a/docs/modules/airflow/examples/getting_started/code/airflow-credentials.yaml b/docs/modules/airflow/examples/getting_started/code/airflow-credentials.yaml index 5e112e91..160aa3ea 100644 --- a/docs/modules/airflow/examples/getting_started/code/airflow-credentials.yaml +++ b/docs/modules/airflow/examples/getting_started/code/airflow-credentials.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Secret metadata: - name: simple-airflow-credentials + name: admin-user-credentials type: Opaque stringData: adminUser.username: airflow @@ -10,7 +10,19 @@ stringData: adminUser.lastname: Admin adminUser.email: airflow@airflow.com adminUser.password: airflow - connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow - # Only needed when using celery workers (instead of Kubernetes executors) - connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow - connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgresql-credentials +stringData: + username: airflow + password: airflow +--- +apiVersion: v1 +kind: Secret +metadata: + name: redis-credentials +stringData: + username: "" + password: redis diff --git a/docs/modules/airflow/examples/getting_started/code/airflow.yaml b/docs/modules/airflow/examples/getting_started/code/airflow.yaml index a2c1b646..cf3bef98 100644 --- a/docs/modules/airflow/examples/getting_started/code/airflow.yaml +++ b/docs/modules/airflow/examples/getting_started/code/airflow.yaml @@ -10,7 +10,12 @@ spec: clusterConfig: loadExamples: true exposeConfig: false - credentialsSecret: simple-airflow-credentials + credentialsSecret: admin-user-credentials + metadataDatabase: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials webservers: roleConfig: listenerClass: external-unstable @@ -18,6 +23,15 @@ spec: default: replicas: 1 celeryExecutors: + celeryResultBackend: + postgresql: + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials + celeryBrokerUrl: + redis: + host: airflow-redis-master + credentialsSecret: redis-credentials roleGroups: default: replicas: 1 diff --git a/docs/modules/airflow/pages/usage-guide/logging.adoc b/docs/modules/airflow/pages/usage-guide/logging.adoc index bd82ee39..6b682690 100644 --- a/docs/modules/airflow/pages/usage-guide/logging.adoc +++ b/docs/modules/airflow/pages/usage-guide/logging.adoc @@ -23,6 +23,7 @@ spec: "flask_appbuilder": level: WARN celeryExecutors: + ... config: logging: enableVectorAgent: true diff --git a/docs/modules/airflow/pages/usage-guide/storage-resources.adoc b/docs/modules/airflow/pages/usage-guide/storage-resources.adoc index 3c399557..69b7772c 100644 --- a/docs/modules/airflow/pages/usage-guide/storage-resources.adoc +++ b/docs/modules/airflow/pages/usage-guide/storage-resources.adoc @@ -27,6 +27,7 @@ spec: default: replicas: 2 celeryExecutors: + ... config: resources: cpu: diff --git a/docs/modules/airflow/pages/usage-guide/using-kubernetes-executors.adoc b/docs/modules/airflow/pages/usage-guide/using-kubernetes-executors.adoc index cdfa0ae5..499872d9 100644 --- a/docs/modules/airflow/pages/usage-guide/using-kubernetes-executors.adoc +++ b/docs/modules/airflow/pages/usage-guide/using-kubernetes-executors.adoc @@ -12,6 +12,7 @@ E.g. you would change the following example ---- spec: celeryExecutors: + ... roleGroups: default: replicas: 2 diff --git a/examples/simple-airflow-cluster-dags-cmap.yaml b/examples/simple-airflow-cluster-dags-cmap.yaml deleted file mode 100644 index d9f38242..00000000 --- a/examples/simple-airflow-cluster-dags-cmap.yaml +++ /dev/null @@ -1,116 +0,0 @@ ---- -apiVersion: v1 -kind: Secret -metadata: - name: simple-airflow-credentials -type: Opaque -stringData: - adminUser.username: airflow - adminUser.firstname: Airflow - adminUser.lastname: Admin - adminUser.email: airflow@airflow.com - adminUser.password: airflow - connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow - # Only needed when using celery workers (instead of Kubernetes executors) - connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow - connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: cm-dag -data: - test_airflow_dag.py: | - from datetime import datetime, timedelta - from airflow import DAG - from airflow.operators.bash import BashOperator - from airflow.operators.dummy import DummyOperator - - with DAG( - dag_id='test_airflow_dag', - schedule='0 0 * * *', - start_date=datetime(2021, 1, 1), - catchup=False, - dagrun_timeout=timedelta(minutes=60), - tags=['example', 'example2'], - params={"example_key": "example_value"}, - ) as dag: - run_this_last = DummyOperator( - task_id='run_this_last', - ) - - # [START howto_operator_bash] - run_this = BashOperator( - task_id='run_after_loop', - bash_command='echo 1', - ) - # [END howto_operator_bash] - - run_this >> run_this_last - - for i in range(3): - task = BashOperator( - task_id='runme_' + str(i), - bash_command='echo "{{ task_instance_key_str }}" && sleep 1', - ) - task >> run_this - - # [START howto_operator_bash_template] - also_run_this = BashOperator( - task_id='also_run_this', - bash_command='echo "run_id={{ run_id }} | dag_run={{ dag_run }}"', - ) - # [END howto_operator_bash_template] - also_run_this >> run_this_last - - # [START howto_operator_bash_skip] - this_will_skip = BashOperator( - task_id='this_will_skip', - bash_command='echo "hello world"; exit 99;', - dag=dag, - ) - # [END howto_operator_bash_skip] - this_will_skip >> run_this_last - - if __name__ == "__main__": - dag.cli() ---- -apiVersion: airflow.stackable.tech/v1alpha1 -kind: AirflowCluster -metadata: - name: airflow-dags-cmap -spec: - image: - productVersion: 3.1.6 - clusterConfig: - loadExamples: false - exposeConfig: false - credentialsSecret: simple-airflow-credentials - volumes: - - name: cm-dag - configMap: - name: cm-dag - volumeMounts: - - name: cm-dag - mountPath: /dags/test_airflow_dag.py - subPath: test_airflow_dag.py - webservers: - roleConfig: - listenerClass: external-unstable - roleGroups: - default: - envOverrides: - AIRFLOW__CORE__DAGS_FOLDER: "/dags" - replicas: 1 - celeryExecutors: - roleGroups: - default: - envOverrides: - AIRFLOW__CORE__DAGS_FOLDER: "/dags" - replicas: 2 - schedulers: - roleGroups: - default: - envOverrides: - AIRFLOW__CORE__DAGS_FOLDER: "/dags" - replicas: 1 diff --git a/examples/simple-airflow-cluster-ldap-insecure-tls.yaml b/examples/simple-airflow-cluster-ldap-insecure-tls.yaml deleted file mode 100644 index d8a96ee2..00000000 --- a/examples/simple-airflow-cluster-ldap-insecure-tls.yaml +++ /dev/null @@ -1,181 +0,0 @@ -# helm install secret-operator oci://oci.stackable.tech/sdp-charts/secret-operator -# helm install commons-operator oci://oci.stackable.tech/sdp-charts/commons-operator -# helm install listener-operator oci://oci.stackable.tech/sdp-charts/listener-operator -# helm install airflow-operator oci://oci.stackable.tech/sdp-charts/airflow-operator -# helm install --repo https://charts.bitnami.com/bitnami --version 12.1.5 --set auth.username=airflow --set auth.password=airflow --set auth.database=airflow --set image.repository=bitnamilegacy/postgresql --set volumePermissions.image.repository=bitnamilegacy/os-shell --set metrics.image.repository=bitnamilegacy/postgres-exporter --set global.security.allowInsecureImages=true airflow-postgresql postgresql -# helm install --repo https://charts.bitnami.com/bitnami --version 17.3.7 --set auth.password=redis --set replica.replicaCount=1 --set global.security.allowInsecureImages=true --set image.repository=bitnamilegacy/redis --set sentinel.image.repository=bitnamilegacy/redis-sentinel --set metrics.image.repository=bitnamilegacy/redis-exporter --set volumePermissions.image.repository=bitnamilegacy/os-shell --set kubectl.image.repository=bitnamilegacy/kubectl --set sysctl.image.repository=bitnamilegacy/os-shell airflow-redis redis -# Log in with user01/user01 or user02/user02 ---- -apiVersion: secrets.stackable.tech/v1alpha1 -kind: SecretClass -metadata: - name: openldap-tls -spec: - backend: - autoTls: - ca: - autoGenerate: true - secret: - name: openldap-tls-ca - namespace: default ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: openldap - labels: - app.kubernetes.io/name: openldap -spec: - selector: - matchLabels: - app.kubernetes.io/name: openldap - serviceName: openldap - replicas: 1 - template: - metadata: - labels: - app.kubernetes.io/name: openldap - spec: - containers: - - name: openldap - image: docker.io/bitnamilegacy/openldap:2.5 - env: - - name: LDAP_ADMIN_USERNAME - value: admin - - name: LDAP_ADMIN_PASSWORD - value: admin - - name: LDAP_USERS - value: user01,user02 - - name: LDAP_PASSWORDS - value: user01,user02 - - name: LDAP_ENABLE_TLS - value: "yes" - - name: LDAP_TLS_CERT_FILE - value: /tls/tls.crt - - name: LDAP_TLS_KEY_FILE - value: /tls/tls.key - - name: LDAP_TLS_CA_FILE - value: /tls/ca.crt - ports: - - name: tls-ldap - containerPort: 1636 - volumeMounts: - - name: tls - mountPath: /tls - volumes: - - name: tls - ephemeral: - volumeClaimTemplate: - metadata: - annotations: - secrets.stackable.tech/class: openldap-tls - secrets.stackable.tech/scope: pod - spec: - storageClassName: secrets.stackable.tech - accessModes: - - ReadWriteOnce - resources: - requests: - storage: "1" ---- -apiVersion: v1 -kind: Service -metadata: - name: openldap - labels: - app.kubernetes.io/name: openldap -spec: - type: ClusterIP - ports: - - name: tls-ldap - port: 1636 - targetPort: tls-ldap - selector: - app.kubernetes.io/name: openldap ---- -apiVersion: authentication.stackable.tech/v1alpha1 -kind: AuthenticationClass -metadata: - name: airflow-with-ldap-insecure-tls-ldap -spec: - provider: - ldap: - hostname: openldap.default.svc.cluster.local - port: 1636 - searchBase: ou=users,dc=example,dc=org - ldapFieldNames: - uid: uid - group: memberof - givenName: givenName - surname: sn - email: mail - bindCredentials: - secretClass: airflow-with-ldap-bind - tls: - verification: - none: {} ---- -apiVersion: secrets.stackable.tech/v1alpha1 -kind: SecretClass -metadata: - name: airflow-with-ldap-bind -spec: - backend: - k8sSearch: - searchNamespace: - pod: {} ---- -apiVersion: v1 -kind: Secret -metadata: - name: airflow-with-ldap-bind - labels: - secrets.stackable.tech/class: airflow-with-ldap-bind -stringData: - user: cn=admin,dc=example,dc=org - password: admin ---- -apiVersion: v1 -kind: Secret -metadata: - name: airflow-with-ldap-server-veri-tls-credentials -type: Opaque -stringData: - adminUser.username: airflow - adminUser.firstname: Airflow - adminUser.lastname: Admin - adminUser.email: airflow@airflow.com - adminUser.password: airflow - connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow - # Only needed when using celery workers (instead of Kubernetes executors) - connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow - connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 ---- -apiVersion: airflow.stackable.tech/v1alpha1 -kind: AirflowCluster -metadata: - name: airflow-insecure-tls -spec: - image: - productVersion: 3.1.6 - clusterConfig: - loadExamples: true - exposeConfig: true - credentialsSecret: airflow-with-ldap-server-veri-tls-credentials - authentication: - - authenticationClass: airflow-with-ldap-insecure-tls-ldap - userRegistrationRole: Admin - webservers: - roleConfig: - listenerClass: external-unstable - roleGroups: - default: - replicas: 1 - celeryExecutors: - roleGroups: - default: - replicas: 1 - schedulers: - roleGroups: - default: - replicas: 1 diff --git a/examples/simple-airflow-cluster-ldap.yaml b/examples/simple-airflow-cluster-ldap.yaml deleted file mode 100644 index 2cf1b515..00000000 --- a/examples/simple-airflow-cluster-ldap.yaml +++ /dev/null @@ -1,179 +0,0 @@ -# helm install secret-operator oci://oci.stackable.tech/sdp-charts/secret-operator -# helm install commons-operator oci://oci.stackable.tech/sdp-charts/commons-operator -# helm install listener-operator oci://oci.stackable.tech/sdp-charts/listener-operator -# helm install airflow-operator oci://oci.stackable.tech/sdp-charts/airflow-operator -# helm install --repo https://charts.bitnami.com/bitnami --version 12.1.5 --set auth.username=airflow --set auth.password=airflow --set auth.database=airflow --set image.repository=bitnamilegacy/postgresql --set volumePermissions.image.repository=bitnamilegacy/os-shell --set metrics.image.repository=bitnamilegacy/postgres-exporter --set global.security.allowInsecureImages=true airflow-postgresql postgresql -# helm install --repo https://charts.bitnami.com/bitnami --version 17.3.7 --set auth.password=redis --set replica.replicaCount=1 --set global.security.allowInsecureImages=true --set image.repository=bitnamilegacy/redis --set sentinel.image.repository=bitnamilegacy/redis-sentinel --set metrics.image.repository=bitnamilegacy/redis-exporter --set volumePermissions.image.repository=bitnamilegacy/os-shell --set kubectl.image.repository=bitnamilegacy/kubectl --set sysctl.image.repository=bitnamilegacy/os-shell airflow-redis redis -# Log in with user01/user01 or user02/user02 ---- -apiVersion: secrets.stackable.tech/v1alpha1 -kind: SecretClass -metadata: - name: openldap-tls -spec: - backend: - autoTls: - ca: - autoGenerate: true - secret: - name: openldap-tls-ca - namespace: default ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: openldap - labels: - app.kubernetes.io/name: openldap -spec: - selector: - matchLabels: - app.kubernetes.io/name: openldap - serviceName: openldap - replicas: 1 - template: - metadata: - labels: - app.kubernetes.io/name: openldap - spec: - containers: - - name: openldap - image: docker.io/bitnamilegacy/openldap:2.5 - env: - - name: LDAP_ADMIN_USERNAME - value: admin - - name: LDAP_ADMIN_PASSWORD - value: admin - - name: LDAP_USERS - value: user01,user02 - - name: LDAP_PASSWORDS - value: user01,user02 - - name: LDAP_ENABLE_TLS - value: "yes" - - name: LDAP_TLS_CERT_FILE - value: /tls/tls.crt - - name: LDAP_TLS_KEY_FILE - value: /tls/tls.key - - name: LDAP_TLS_CA_FILE - value: /tls/ca.crt - ports: - - name: tls-ldap - containerPort: 1636 - volumeMounts: - - name: tls - mountPath: /tls - volumes: - - name: tls - ephemeral: - volumeClaimTemplate: - metadata: - annotations: - secrets.stackable.tech/class: openldap-tls - secrets.stackable.tech/scope: pod - spec: - storageClassName: secrets.stackable.tech - accessModes: - - ReadWriteOnce - resources: - requests: - storage: "1" ---- -apiVersion: v1 -kind: Service -metadata: - name: openldap - labels: - app.kubernetes.io/name: openldap -spec: - type: ClusterIP - ports: - - name: tls-ldap - port: 636 - targetPort: tls-ldap - selector: - app.kubernetes.io/name: openldap ---- -apiVersion: authentication.stackable.tech/v1alpha1 -kind: AuthenticationClass -metadata: - name: airflow-with-ldap-server-veri-tls-ldap -spec: - provider: - ldap: - hostname: openldap.default.svc.cluster.local - port: 636 - searchBase: ou=users,dc=example,dc=org - ldapFieldNames: - uid: uid - bindCredentials: - secretClass: airflow-with-ldap-server-veri-tls-ldap-bind - tls: - verification: - server: - caCert: - secretClass: openldap-tls ---- -apiVersion: secrets.stackable.tech/v1alpha1 -kind: SecretClass -metadata: - name: airflow-with-ldap-server-veri-tls-ldap-bind -spec: - backend: - k8sSearch: - searchNamespace: - pod: {} ---- -apiVersion: v1 -kind: Secret -metadata: - name: airflow-with-ldap-server-veri-tls-ldap-bind - labels: - secrets.stackable.tech/class: airflow-with-ldap-server-veri-tls-ldap-bind -stringData: - user: cn=admin,dc=example,dc=org - password: admin ---- -apiVersion: v1 -kind: Secret -metadata: - name: airflow-with-ldap-server-veri-tls-credentials -type: Opaque -stringData: - adminUser.username: airflow - adminUser.firstname: Airflow - adminUser.lastname: Admin - adminUser.email: airflow@airflow.com - adminUser.password: airflow - connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow - # Only needed when using celery workers (instead of Kubernetes executors) - connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow - connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 ---- -apiVersion: airflow.stackable.tech/v1alpha1 -kind: AirflowCluster -metadata: - name: airflow-with-ldap-server-veri-tls -spec: - image: - productVersion: 3.1.6 - clusterConfig: - loadExamples: true - exposeConfig: true - credentialsSecret: airflow-with-ldap-server-veri-tls-credentials - authentication: - - authenticationClass: airflow-with-ldap-server-veri-tls-ldap - userRegistrationRole: Admin - webservers: - roleConfig: - listenerClass: external-unstable - roleGroups: - default: - replicas: 1 - celeryExecutors: - roleGroups: - default: - replicas: 1 - schedulers: - roleGroups: - default: - replicas: 1 diff --git a/examples/simple-airflow-cluster.yaml b/examples/simple-airflow-cluster.yaml deleted file mode 100644 index 0dd0449e..00000000 --- a/examples/simple-airflow-cluster.yaml +++ /dev/null @@ -1,42 +0,0 @@ ---- -apiVersion: v1 -kind: Secret -metadata: - name: simple-airflow-credentials -type: Opaque -stringData: - adminUser.username: airflow - adminUser.firstname: Airflow - adminUser.lastname: Admin - adminUser.email: airflow@airflow.com - adminUser.password: airflow - connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow - # Only needed when using celery workers (instead of Kubernetes executors) - connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow - connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 ---- -apiVersion: airflow.stackable.tech/v1alpha1 -kind: AirflowCluster -metadata: - name: airflow -spec: - image: - productVersion: 3.1.6 - clusterConfig: - loadExamples: true - exposeConfig: false - credentialsSecret: simple-airflow-credentials - webservers: - roleConfig: - listenerClass: external-unstable - roleGroups: - default: - replicas: 1 - celeryExecutors: - roleGroups: - default: - replicas: 2 - schedulers: - roleGroups: - default: - replicas: 1 From 69ce87835f6ae7c0dcafc7d4a55c782d0f73ed66 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Fri, 6 Feb 2026 15:38:12 +0100 Subject: [PATCH 09/26] apply cluster secrets separately and remove assert on generations --- .../kuttl/smoke-generic/35-assert.yaml | 24 +++++++++++ .../35-install-airflow-secrets.yaml | 40 +++++++++++++++++++ .../kuttl/smoke-generic/40-assert.yaml | 6 +-- .../40-install-airflow-cluster.yaml.j2 | 33 --------------- tests/templates/kuttl/smoke/35-assert.yaml | 19 +++++++++ .../smoke/35-install-airflow-secrets.yaml | 35 ++++++++++++++++ tests/templates/kuttl/smoke/40-assert.yaml.j2 | 6 +-- .../smoke/40-install-airflow-cluster.yaml.j2 | 28 ------------- tests/test-definition.yaml | 11 ++--- 9 files changed, 126 insertions(+), 76 deletions(-) create mode 100644 tests/templates/kuttl/smoke-generic/35-assert.yaml create mode 100644 tests/templates/kuttl/smoke-generic/35-install-airflow-secrets.yaml create mode 100644 tests/templates/kuttl/smoke/35-assert.yaml create mode 100644 tests/templates/kuttl/smoke/35-install-airflow-secrets.yaml diff --git a/tests/templates/kuttl/smoke-generic/35-assert.yaml b/tests/templates/kuttl/smoke-generic/35-assert.yaml new file mode 100644 index 00000000..8a1ed130 --- /dev/null +++ b/tests/templates/kuttl/smoke-generic/35-assert.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +--- +apiVersion: v1 +kind: Secret +metadata: + name: admin-user-credentials +type: Opaque +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgresql-metadata +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgresql-celery +--- +apiVersion: v1 +kind: Secret +metadata: + name: redis-celery diff --git a/tests/templates/kuttl/smoke-generic/35-install-airflow-secrets.yaml b/tests/templates/kuttl/smoke-generic/35-install-airflow-secrets.yaml new file mode 100644 index 00000000..3de36ab3 --- /dev/null +++ b/tests/templates/kuttl/smoke-generic/35-install-airflow-secrets.yaml @@ -0,0 +1,40 @@ +# N.B. secrets should be applied before the cluster to avoid unecessary restarts. +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +metadata: + name: install-airflow +timeout: 480 +--- +apiVersion: v1 +kind: Secret +metadata: + name: admin-user-credentials +type: Opaque +stringData: + adminUser.username: airflow + adminUser.firstname: Airflow + adminUser.lastname: Admin + adminUser.email: airflow@airflow.com + adminUser.password: airflow +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgresql-metadata +stringData: + uri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgresql-celery +stringData: + uri: db+postgresql://airflow:airflow@airflow-postgresql/airflow +--- +apiVersion: v1 +kind: Secret +metadata: + name: redis-celery +stringData: + uri: redis://:redis@airflow-redis-master:6379/0 diff --git a/tests/templates/kuttl/smoke-generic/40-assert.yaml b/tests/templates/kuttl/smoke-generic/40-assert.yaml index 6c16b2a7..2730fbcd 100644 --- a/tests/templates/kuttl/smoke-generic/40-assert.yaml +++ b/tests/templates/kuttl/smoke-generic/40-assert.yaml @@ -17,7 +17,7 @@ apiVersion: apps/v1 kind: StatefulSet metadata: name: airflow-webserver-default - generation: 1 # There should be no unneeded Pod restarts + # Do not include an assert on generation:1 as this is hard to enforce. Same applies for all roles. labels: restarter.stackable.tech/enabled: "true" spec: @@ -32,7 +32,6 @@ apiVersion: apps/v1 kind: StatefulSet metadata: name: airflow-worker-default - generation: 1 # There should be no unneeded Pod restarts labels: restarter.stackable.tech/enabled: "true" spec: @@ -47,7 +46,6 @@ apiVersion: apps/v1 kind: StatefulSet metadata: name: airflow-scheduler-default - generation: 1 # There should be no unneeded Pod restarts labels: restarter.stackable.tech/enabled: "true" spec: @@ -62,7 +60,6 @@ apiVersion: apps/v1 kind: StatefulSet metadata: name: airflow-dagprocessor-default - generation: 1 # There should be no unneeded Pod restarts labels: restarter.stackable.tech/enabled: "true" spec: @@ -77,7 +74,6 @@ apiVersion: apps/v1 kind: StatefulSet metadata: name: airflow-triggerer-default - generation: 1 # There should be no unneeded Pod restarts labels: restarter.stackable.tech/enabled: "true" spec: diff --git a/tests/templates/kuttl/smoke-generic/40-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/smoke-generic/40-install-airflow-cluster.yaml.j2 index cd0b4484..37e34df1 100644 --- a/tests/templates/kuttl/smoke-generic/40-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/smoke-generic/40-install-airflow-cluster.yaml.j2 @@ -4,39 +4,6 @@ metadata: name: install-airflow timeout: 480 --- -apiVersion: v1 -kind: Secret -metadata: - name: admin-user-credentials -type: Opaque -stringData: - adminUser.username: airflow - adminUser.firstname: Airflow - adminUser.lastname: Admin - adminUser.email: airflow@airflow.com - adminUser.password: airflow ---- -apiVersion: v1 -kind: Secret -metadata: - name: postgresql-metadata -stringData: - uri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow ---- -apiVersion: v1 -kind: Secret -metadata: - name: postgresql-celery -stringData: - uri: db+postgresql://airflow:airflow@airflow-postgresql/airflow ---- -apiVersion: v1 -kind: Secret -metadata: - name: redis-celery -stringData: - uri: redis://:redis@airflow-redis-master:6379/0 ---- apiVersion: airflow.stackable.tech/v1alpha1 kind: AirflowCluster metadata: diff --git a/tests/templates/kuttl/smoke/35-assert.yaml b/tests/templates/kuttl/smoke/35-assert.yaml new file mode 100644 index 00000000..372a3e39 --- /dev/null +++ b/tests/templates/kuttl/smoke/35-assert.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +--- +apiVersion: v1 +kind: Secret +metadata: + name: admin-user-credentials +type: Opaque +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgresql-credentials +--- +apiVersion: v1 +kind: Secret +metadata: + name: redis-credentials diff --git a/tests/templates/kuttl/smoke/35-install-airflow-secrets.yaml b/tests/templates/kuttl/smoke/35-install-airflow-secrets.yaml new file mode 100644 index 00000000..7b9c9e70 --- /dev/null +++ b/tests/templates/kuttl/smoke/35-install-airflow-secrets.yaml @@ -0,0 +1,35 @@ +# N.B. secrets should be applied before the cluster to avoid unecessary restarts. +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +metadata: + name: install-airflow +timeout: 480 +--- +apiVersion: v1 +kind: Secret +metadata: + name: admin-user-credentials +type: Opaque +stringData: + adminUser.username: airflow + adminUser.firstname: Airflow + adminUser.lastname: Admin + adminUser.email: airflow@airflow.com + adminUser.password: airflow +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgresql-credentials +stringData: + username: airflow + password: airflow +--- +apiVersion: v1 +kind: Secret +metadata: + name: redis-credentials +stringData: + username: "" + password: redis diff --git a/tests/templates/kuttl/smoke/40-assert.yaml.j2 b/tests/templates/kuttl/smoke/40-assert.yaml.j2 index 054f68fe..dfc62f39 100644 --- a/tests/templates/kuttl/smoke/40-assert.yaml.j2 +++ b/tests/templates/kuttl/smoke/40-assert.yaml.j2 @@ -17,7 +17,7 @@ apiVersion: apps/v1 kind: StatefulSet metadata: name: airflow-webserver-default - generation: 1 # There should be no unneeded Pod restarts + # Do not include an assert on generation:1 as this is hard to enforce. Same applies for all roles. labels: restarter.stackable.tech/enabled: "true" spec: @@ -33,7 +33,6 @@ apiVersion: apps/v1 kind: StatefulSet metadata: name: airflow-worker-default - generation: 1 # There should be no unneeded Pod restarts labels: restarter.stackable.tech/enabled: "true" spec: @@ -49,7 +48,6 @@ apiVersion: apps/v1 kind: StatefulSet metadata: name: airflow-scheduler-default - generation: 1 # There should be no unneeded Pod restarts labels: restarter.stackable.tech/enabled: "true" spec: @@ -64,7 +62,6 @@ apiVersion: apps/v1 kind: StatefulSet metadata: name: airflow-dagprocessor-default - generation: 1 # There should be no unneeded Pod restarts labels: restarter.stackable.tech/enabled: "true" spec: @@ -79,7 +76,6 @@ apiVersion: apps/v1 kind: StatefulSet metadata: name: airflow-triggerer-default - generation: 1 # There should be no unneeded Pod restarts labels: restarter.stackable.tech/enabled: "true" spec: diff --git a/tests/templates/kuttl/smoke/40-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/smoke/40-install-airflow-cluster.yaml.j2 index 39fdaaa2..13cca402 100644 --- a/tests/templates/kuttl/smoke/40-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/smoke/40-install-airflow-cluster.yaml.j2 @@ -4,34 +4,6 @@ metadata: name: install-airflow timeout: 480 --- -apiVersion: v1 -kind: Secret -metadata: - name: admin-user-credentials -type: Opaque -stringData: - adminUser.username: airflow - adminUser.firstname: Airflow - adminUser.lastname: Admin - adminUser.email: airflow@airflow.com - adminUser.password: airflow ---- -apiVersion: v1 -kind: Secret -metadata: - name: postgresql-credentials -stringData: - username: airflow - password: airflow ---- -apiVersion: v1 -kind: Secret -metadata: - name: redis-credentials -stringData: - username: "" - password: redis ---- apiVersion: airflow.stackable.tech/v1alpha1 kind: AirflowCluster metadata: diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index e2192f30..8a9b5337 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -80,11 +80,12 @@ tests: dimensions: - airflow-latest - openshift - - name: logging - dimensions: - - airflow - - openshift - - executor + # TODO revert this before merging! + # - name: logging + # dimensions: + # - airflow + # - openshift + # - executor - name: cluster-operation dimensions: - airflow-latest From 965176e6eb7ec4f0fb0da2160bcbe5f8de511926 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Fri, 6 Feb 2026 16:55:27 +0100 Subject: [PATCH 10/26] docs --- .../pages/getting_started/first_steps.adoc | 21 +++-- .../airflow/pages/usage-guide/db-connect.adoc | 78 +++++++++++++++++++ docs/modules/airflow/partials/nav.adoc | 1 + 3 files changed, 92 insertions(+), 8 deletions(-) create mode 100644 docs/modules/airflow/pages/usage-guide/db-connect.adoc diff --git a/docs/modules/airflow/pages/getting_started/first_steps.adoc b/docs/modules/airflow/pages/getting_started/first_steps.adoc index f25fcd2c..03bfd1b3 100644 --- a/docs/modules/airflow/pages/getting_started/first_steps.adoc +++ b/docs/modules/airflow/pages/getting_started/first_steps.adoc @@ -10,9 +10,10 @@ With the external dependencies required by Airflow (Postgresql and Redis) instal Supported versions for PostgreSQL and Redis can be found in the https://airflow.apache.org/docs/apache-airflow/stable/installation/prerequisites.html#prerequisites[Airflow documentation]. -== Secret with Airflow credentials +== Airflow secrets -Create a Secret with the necessary credentials, this entails database connection credentials as well as an admin account for Airflow itself. +Secrets are required for the mandatory metadata database connection and the airflow admin user. +When using the celery executor it is also required to provide information for the celery database and broker. Create a file called `airflow-credentials.yaml`: [source,yaml] @@ -23,13 +24,12 @@ And apply it: [source,bash] include::example$getting_started/code/getting_started.sh[tag=apply-airflow-credentials] -`connections.sqlalchemyDatabaseUri` must contain the connection string to the SQL database storing the Airflow metadata. +`postgresql-credentials` contains credentials for the SQL database storing the Airflow metadata. +In this example we will use the same database for both the Airflow job metadata as well as the Celery broker metadata. -`connections.celeryResultBackend` must contain the connection string to the SQL database storing the job metadata (the example above uses the same PostgreSQL database for both). +`redis-credentials` contains credentials for the the Redis instance used for queuing the jobs submitted to the airflow executor(s). -`connections.celeryBrokerUrl` must contain the connection string to the Redis instance used for queuing the jobs submitted to the airflow executor(s). - -The `adminUser` fields are used to create an admin user. +`admin-user-credentials`: the `adminUser` fields are used to create an admin user. NOTE: The admin user is disabled if you use a non-default authentication mechanism like LDAP. @@ -61,20 +61,25 @@ include::example$getting_started/code/getting_started.sh[tag=install-airflow] Where: * `metadata.name` contains the name of the Airflow cluster. +* `spec.clusterConfig.metadataDatabase` specifies one of the supported database types (in this case, `postgresql`) along with references to the host, database and the secret containing the connection credentials. * the product version of the Docker image provided by Stackable must be set in `spec.image.productVersion`. * `spec.celeryExecutors`: deploy executors managed by Airflow's Celery engine. Alternatively you can use `kuberenetesExectors` that use Airflow's Kubernetes engine for executor management. For more information see https://airflow.apache.org/docs/apache-airflow/stable/executor/index.html#executor-types). +* `spec.celeryExecutors.celeryResultBackend`: specifies one of the supported database types (in this case, `postgresql`) along with references to the host, database and the secret containing the connection credentials. +* `spec.celeryExecutors.celeryBrokerUrl`: specifies one of the supported queue/broker types (in this case, `redis`) along with references to the host and the secret containing the connection credentials. * the `spec.clusterConfig.loadExamples` key is optional and defaults to `false`. It is set to `true` here as the example DAGs are used when verifying the installation. * the `spec.clusterConfig.exposeConfig` key is optional and defaults to `false`. It is set to `true` only as an aid to verify the configuration and should never be used as such in anything other than test or demo clusters. -* the previously created secret must be referenced in `spec.clusterConfig.credentialsSecret`. +* the secret containing the admin user infomration must be referenced in `spec.clusterConfig.credentialsSecret`. NOTE: The version you need to specify for `spec.image.productVersion` is the desired version of Apache Airflow. You can optionally specify the `spec.image.stackableVersion` to a certain release like `23.11.0` but it is recommended to leave it out and use the default provided by the operator. Check our https://oci.stackable.tech/[image registry,window=_blank] for a list of available versions. Information on how to browse the registry can be found xref:contributor:project-overview.adoc#docker-images[here,window=_blank]. It should generally be safe to simply use the latest version that is available. +NOTE: Refer to xref:usage-guide/db-connect.adoc[] for more information about database/broker connections. + This creates the actual Airflow cluster. After a while, all the Pods in the StatefulSets should be ready: diff --git a/docs/modules/airflow/pages/usage-guide/db-connect.adoc b/docs/modules/airflow/pages/usage-guide/db-connect.adoc new file mode 100644 index 00000000..7ea7872f --- /dev/null +++ b/docs/modules/airflow/pages/usage-guide/db-connect.adoc @@ -0,0 +1,78 @@ += Database connections +:description: Configure Airflow Database connectivity. + +Airflow requires a metadata database for storing e.g. DAG, task and Job data. +The actual connection string is provided by the operator so that the user does not need to remember the exact structure. +The same database can be accessed using different drivers: this is also handled by the operator, since the context is known (e.g. job metadata vs. queued job metadata) when parsing the resource file. + +== Typed connections + +[source,yaml] +---- +--- +spec: + clusterConfig: + metadataDatabase: + postgresql: # <1> + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials # <2> +---- +<1> A reference to one of the supported database backends (e.g. `postgresql`). +<2> A reference to a secret which must contain the two fields `username` and `password`. + +The queue/broker metadata and URL is only needed when running the celery executor. +The `celeryResultBackend` definition uses the same structure as `metadataDatabase` shown above. +The `celeryBrokerUrl` definition is similar but does not require a `databaseName`. + +[source,yaml] +---- +--- +spec: + celeryExecutors: + celeryResultBackend: + postgresql: # <1> + host: airflow-postgresql + databaseName: airflow + credentialsSecret: postgresql-credentials # <2> + celeryBrokerUrl: + redis: # <3> + host: airflow-redis-master + credentialsSecret: redis-credentials # <2> +---- +<1> A reference to one of the supported database backends (e.g. `postgresql`). +<2> A reference to a secret which must contain the two fields `username` and `password`. +<3> A reference to one of the supported queue brokers (e.g. `redis`). + +== Generic connections + +Alternatively, these connections can also be defined in full in a referenced secret: + +[source,yaml] +---- +--- +spec: + clusterConfig: + metadataDatabase: + generic: + uriSecret: postgresql-metadata # <1> +---- + +[source,yaml] +---- +--- +spec: + celeryResultBackend: + generic: + uriSecret: postgresql-celery # <2> + celeryBrokerUrl: + generic: + uriSecret: redis-celery # <3> +---- + +<1> A reference to a secret which must contain the single fields `uri` e.g. +`uri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow` +<2> A reference to a secret which must contain the single fields `uri` e.g. +`uri: db+postgresql://airflow:airflow@airflow-postgresql/airflow` +<3> A reference to a secret which must contain the single fields `uri` e.g. +`uri: redis://:redis@airflow-redis-master:6379/0` diff --git a/docs/modules/airflow/partials/nav.adoc b/docs/modules/airflow/partials/nav.adoc index e1e64483..9c40a3b5 100644 --- a/docs/modules/airflow/partials/nav.adoc +++ b/docs/modules/airflow/partials/nav.adoc @@ -4,6 +4,7 @@ * xref:airflow:required-external-components.adoc[] * xref:airflow:usage-guide/index.adoc[] ** xref:airflow:usage-guide/db-init.adoc[] +** xref:airflow:usage-guide/db-connect.adoc[] ** xref:airflow:usage-guide/mounting-dags.adoc[] ** xref:airflow:usage-guide/applying-custom-resources.adoc[] ** xref:airflow:usage-guide/listenerclass.adoc[] From 93e3a8e4bbf2d1262d8394cdb1b72c022e141b95 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Tue, 10 Feb 2026 09:06:10 +0100 Subject: [PATCH 11/26] Update docs/modules/airflow/examples/getting_started/code/airflow.yaml Co-authored-by: Sebastian Bernauer --- docs/modules/airflow/examples/getting_started/code/airflow.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/airflow/examples/getting_started/code/airflow.yaml b/docs/modules/airflow/examples/getting_started/code/airflow.yaml index cf3bef98..c0437275 100644 --- a/docs/modules/airflow/examples/getting_started/code/airflow.yaml +++ b/docs/modules/airflow/examples/getting_started/code/airflow.yaml @@ -10,7 +10,7 @@ spec: clusterConfig: loadExamples: true exposeConfig: false - credentialsSecret: admin-user-credentials + credentialsSecret: airflow-admin-user-credentials metadataDatabase: postgresql: host: airflow-postgresql From 3c4e66d73bc5bf847d7204a3b0dffb170b92e793 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Tue, 10 Feb 2026 09:06:24 +0100 Subject: [PATCH 12/26] Update docs/modules/airflow/examples/getting_started/code/airflow.yaml Co-authored-by: Sebastian Bernauer --- docs/modules/airflow/examples/getting_started/code/airflow.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/airflow/examples/getting_started/code/airflow.yaml b/docs/modules/airflow/examples/getting_started/code/airflow.yaml index c0437275..0f0386de 100644 --- a/docs/modules/airflow/examples/getting_started/code/airflow.yaml +++ b/docs/modules/airflow/examples/getting_started/code/airflow.yaml @@ -15,7 +15,7 @@ spec: postgresql: host: airflow-postgresql databaseName: airflow - credentialsSecret: postgresql-credentials + credentialsSecret: airflow-postgresql-credentials webservers: roleConfig: listenerClass: external-unstable From 07cca3db3bee25e11e0763a83f3cf0a434f897be Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Tue, 10 Feb 2026 09:06:36 +0100 Subject: [PATCH 13/26] Update docs/modules/airflow/examples/getting_started/code/airflow.yaml Co-authored-by: Sebastian Bernauer --- docs/modules/airflow/examples/getting_started/code/airflow.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/airflow/examples/getting_started/code/airflow.yaml b/docs/modules/airflow/examples/getting_started/code/airflow.yaml index 0f0386de..8e4e953c 100644 --- a/docs/modules/airflow/examples/getting_started/code/airflow.yaml +++ b/docs/modules/airflow/examples/getting_started/code/airflow.yaml @@ -31,7 +31,7 @@ spec: celeryBrokerUrl: redis: host: airflow-redis-master - credentialsSecret: redis-credentials + credentialsSecret: airflow-redis-credentials roleGroups: default: replicas: 1 From f58899451122e6531882b66825e8960cb5dd0599 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Tue, 10 Feb 2026 09:06:51 +0100 Subject: [PATCH 14/26] Update docs/modules/airflow/examples/getting_started/code/airflow.yaml Co-authored-by: Sebastian Bernauer --- docs/modules/airflow/examples/getting_started/code/airflow.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/airflow/examples/getting_started/code/airflow.yaml b/docs/modules/airflow/examples/getting_started/code/airflow.yaml index 8e4e953c..8e976fdd 100644 --- a/docs/modules/airflow/examples/getting_started/code/airflow.yaml +++ b/docs/modules/airflow/examples/getting_started/code/airflow.yaml @@ -27,7 +27,7 @@ spec: postgresql: host: airflow-postgresql databaseName: airflow - credentialsSecret: postgresql-credentials + credentialsSecret: airflow-postgresql-credentials celeryBrokerUrl: redis: host: airflow-redis-master From 784d5c883bd535416327d3048a05871a16203f5a Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Tue, 10 Feb 2026 09:07:15 +0100 Subject: [PATCH 15/26] Update docs/modules/airflow/pages/usage-guide/db-connect.adoc Co-authored-by: Malte Sander --- docs/modules/airflow/pages/usage-guide/db-connect.adoc | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/modules/airflow/pages/usage-guide/db-connect.adoc b/docs/modules/airflow/pages/usage-guide/db-connect.adoc index 7ea7872f..1a72f508 100644 --- a/docs/modules/airflow/pages/usage-guide/db-connect.adoc +++ b/docs/modules/airflow/pages/usage-guide/db-connect.adoc @@ -9,7 +9,6 @@ The same database can be accessed using different drivers: this is also handled [source,yaml] ---- ---- spec: clusterConfig: metadataDatabase: From e9fcbf4511ca9d3712c968404b92831ae19a06d4 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Tue, 10 Feb 2026 09:07:46 +0100 Subject: [PATCH 16/26] Update rust/operator-binary/src/connections/database.rs Co-authored-by: Malte Sander --- rust/operator-binary/src/connections/database.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/rust/operator-binary/src/connections/database.rs b/rust/operator-binary/src/connections/database.rs index 2bb861d8..0290de68 100644 --- a/rust/operator-binary/src/connections/database.rs +++ b/rust/operator-binary/src/connections/database.rs @@ -68,14 +68,16 @@ impl PostgresqlDb { let param_str: Vec = self .parameters .iter() - .map(|(k, v)| format!("{}={}", k, v)) + .map(|(k, v)| format!("{k}={v}")) .collect(); format!("?{}", param_str.join("&")) }; format!( - "{}://${}:${}@{}:{}/{}{}", - prefix, username_env, password_env, self.host, self.port, self.database_name, params + "{prefix}://${username_env}:${password_env}@{host}:{port}/{database_name}{params}", + host = self.host, + port = self.port, + database_name = self.database_name ) } From b4847c046310def560f0fa4194da442b79c67a59 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Tue, 10 Feb 2026 09:08:18 +0100 Subject: [PATCH 17/26] Update CHANGELOG.md Co-authored-by: Malte Sander --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 51e5a46e..3c22bf86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ - Gracefully shutdown all concurrent tasks by forwarding the SIGTERM signal ([#741]). - Bump testing-tools to `0.3.0-stackable0.0.0-dev` ([#733]). - BREAKING: Replace Airflow credentials-secret with database/broker connections ([#743]). - - Existing secret is retained for the admin user alone. + - Existing secret is retained for the airflow admin user alone. - Database/broker connections can be defined either using structs or as a generic connection string (see ADR 29). - Removed standalone examples folder (not affecting the documentation). From ce50743cacc0431b630fe393914a1be3062821b5 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Tue, 10 Feb 2026 09:08:29 +0100 Subject: [PATCH 18/26] Update docs/modules/airflow/pages/getting_started/first_steps.adoc Co-authored-by: Malte Sander --- docs/modules/airflow/pages/getting_started/first_steps.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/airflow/pages/getting_started/first_steps.adoc b/docs/modules/airflow/pages/getting_started/first_steps.adoc index 03bfd1b3..c63e7e74 100644 --- a/docs/modules/airflow/pages/getting_started/first_steps.adoc +++ b/docs/modules/airflow/pages/getting_started/first_steps.adoc @@ -71,7 +71,7 @@ For more information see https://airflow.apache.org/docs/apache-airflow/stable/e * the `spec.clusterConfig.loadExamples` key is optional and defaults to `false`. It is set to `true` here as the example DAGs are used when verifying the installation. * the `spec.clusterConfig.exposeConfig` key is optional and defaults to `false`. It is set to `true` only as an aid to verify the configuration and should never be used as such in anything other than test or demo clusters. -* the secret containing the admin user infomration must be referenced in `spec.clusterConfig.credentialsSecret`. +* the secret containing the admin user information must be referenced in `spec.clusterConfig.credentialsSecret`. NOTE: The version you need to specify for `spec.image.productVersion` is the desired version of Apache Airflow. You can optionally specify the `spec.image.stackableVersion` to a certain release like `23.11.0` but it is recommended to leave it out and use the default provided by the operator. From f690097bf357fecf5c64c9572cbcf394b9c36e55 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Tue, 10 Feb 2026 09:10:07 +0100 Subject: [PATCH 19/26] Update rust/operator-binary/src/connections/queue.rs Co-authored-by: Malte Sander --- rust/operator-binary/src/connections/queue.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rust/operator-binary/src/connections/queue.rs b/rust/operator-binary/src/connections/queue.rs index b39542a4..a0c1d1a6 100644 --- a/rust/operator-binary/src/connections/queue.rs +++ b/rust/operator-binary/src/connections/queue.rs @@ -47,8 +47,9 @@ impl QueueType { impl RedisQueue { pub fn connection_string(&self, username_env: &str, password_env: &str) -> String { format!( - "redis://${}:${}@{}:{}/0", - username_env, password_env, self.host, self.port + "redis://${username_env}:${password_env}@{host}:{port}/0", + host = self.host, + port = self.port ) } } From 652a1bba337e28a051194a65cf6deb843b8fdd55 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Tue, 10 Feb 2026 09:18:05 +0100 Subject: [PATCH 20/26] Update tests/templates/kuttl/ldap/60-install-airflow-cluster.yaml.j2 Co-authored-by: Malte Sander --- tests/templates/kuttl/ldap/60-install-airflow-cluster.yaml.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/templates/kuttl/ldap/60-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/ldap/60-install-airflow-cluster.yaml.j2 index 19740ed9..8750eb12 100644 --- a/tests/templates/kuttl/ldap/60-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/ldap/60-install-airflow-cluster.yaml.j2 @@ -26,6 +26,7 @@ commands: stringData: username: airflow password: airflow +{% if test_scenario['values']['executor'] == 'celery' %} --- apiVersion: v1 kind: Secret @@ -34,6 +35,7 @@ commands: stringData: username: "" password: redis +{% endif %} --- apiVersion: v1 kind: Secret From cf5ab7024b49b22fc5cb77b51c15621eb4dfd65f Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Tue, 10 Feb 2026 09:19:11 +0100 Subject: [PATCH 21/26] Update tests/templates/kuttl/logging/41-install-airflow-cluster.yaml.j2 Co-authored-by: Malte Sander --- .../templates/kuttl/logging/41-install-airflow-cluster.yaml.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/templates/kuttl/logging/41-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/logging/41-install-airflow-cluster.yaml.j2 index f40be4c5..ca13490e 100644 --- a/tests/templates/kuttl/logging/41-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/logging/41-install-airflow-cluster.yaml.j2 @@ -23,6 +23,7 @@ metadata: stringData: username: airflow password: airflow +{% if test_scenario['values']['executor'] == 'celery' %} --- apiVersion: v1 kind: Secret @@ -31,6 +32,7 @@ metadata: stringData: username: "" password: redis +{% endif %} --- apiVersion: v1 kind: ConfigMap From fe3ac91fb22993f8e5648aa9a0625d1cd2753675 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Tue, 10 Feb 2026 09:20:07 +0100 Subject: [PATCH 22/26] Update tests/templates/kuttl/mount-dags-configmap/30-install-airflow-cluster.yaml.j2 Co-authored-by: Malte Sander --- .../mount-dags-configmap/30-install-airflow-cluster.yaml.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/templates/kuttl/mount-dags-configmap/30-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/mount-dags-configmap/30-install-airflow-cluster.yaml.j2 index a6f2436e..3c3500d5 100644 --- a/tests/templates/kuttl/mount-dags-configmap/30-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/mount-dags-configmap/30-install-airflow-cluster.yaml.j2 @@ -23,6 +23,7 @@ metadata: stringData: username: airflow password: airflow +{% if test_scenario['values']['executor'] == 'celery' %} --- apiVersion: v1 kind: Secret @@ -31,6 +32,7 @@ metadata: stringData: username: "" password: redis +{% endif %} --- apiVersion: v1 kind: ConfigMap From 20f12e92871192eade5a165e647fc66405383572 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Tue, 10 Feb 2026 09:21:02 +0100 Subject: [PATCH 23/26] Update tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 Co-authored-by: Malte Sander --- .../kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 index 6a04fc0b..0e3250d8 100644 --- a/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 @@ -50,6 +50,7 @@ metadata: stringData: username: airflow password: airflow +{% if test_scenario['values']['executor'] == 'celery' %} --- apiVersion: v1 kind: Secret @@ -58,6 +59,7 @@ metadata: stringData: username: "" password: redis +{% endif %} --- apiVersion: v1 kind: ConfigMap From 322a7c57967c22fe2cdec5c67c011bc1a06612f3 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Tue, 10 Feb 2026 09:22:08 +0100 Subject: [PATCH 24/26] Update tests/templates/kuttl/versioning/30-install-airflow-cluster.yaml.j2 Co-authored-by: Malte Sander --- .../kuttl/versioning/30-install-airflow-cluster.yaml.j2 | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/templates/kuttl/versioning/30-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/versioning/30-install-airflow-cluster.yaml.j2 index 99203d52..806aa924 100644 --- a/tests/templates/kuttl/versioning/30-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/versioning/30-install-airflow-cluster.yaml.j2 @@ -84,7 +84,6 @@ spec: default: envOverrides: &envOverrides AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: '{"conn_type": "kubernetes", "extra": {"extra__kubernetes__in_cluster": true}}' - SOME_TEST_CONCAT: 'this is my admin password: $ADMIN_PASSWORD' replicas: 1 kubernetesExecutors: envOverrides: *envOverrides From dbe5c941ca9f698569026e7e8c0680728225a383 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Tue, 10 Feb 2026 09:22:31 +0100 Subject: [PATCH 25/26] Update tests/templates/kuttl/remote-logging/40-install-airflow-cluster.yaml.j2 Co-authored-by: Malte Sander --- .../kuttl/remote-logging/40-install-airflow-cluster.yaml.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/templates/kuttl/remote-logging/40-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/remote-logging/40-install-airflow-cluster.yaml.j2 index 20395b4c..fb36aa5a 100644 --- a/tests/templates/kuttl/remote-logging/40-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/remote-logging/40-install-airflow-cluster.yaml.j2 @@ -23,6 +23,7 @@ metadata: stringData: username: airflow password: airflow +{% if test_scenario['values']['executor'] == 'celery' %} --- apiVersion: v1 kind: Secret @@ -31,6 +32,7 @@ metadata: stringData: username: "" password: redis +{% endif %} --- apiVersion: airflow.stackable.tech/v1alpha1 kind: AirflowCluster From 1c04edc7e7d87c8effaa95a491fad18969017e02 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 10 Feb 2026 09:28:36 +0100 Subject: [PATCH 26/26] consistent naming of credentials references, removed unused secret --- .../getting_started/code/airflow-credentials.yaml | 6 +++--- .../airflow/pages/getting_started/first_steps.adoc | 6 +++--- .../kuttl/versioning/30-install-airflow-cluster.yaml.j2 | 8 -------- 3 files changed, 6 insertions(+), 14 deletions(-) diff --git a/docs/modules/airflow/examples/getting_started/code/airflow-credentials.yaml b/docs/modules/airflow/examples/getting_started/code/airflow-credentials.yaml index 160aa3ea..7468cd67 100644 --- a/docs/modules/airflow/examples/getting_started/code/airflow-credentials.yaml +++ b/docs/modules/airflow/examples/getting_started/code/airflow-credentials.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Secret metadata: - name: admin-user-credentials + name: airflow-admin-user-credentials type: Opaque stringData: adminUser.username: airflow @@ -14,7 +14,7 @@ stringData: apiVersion: v1 kind: Secret metadata: - name: postgresql-credentials + name: airflow-postgresql-credentials stringData: username: airflow password: airflow @@ -22,7 +22,7 @@ stringData: apiVersion: v1 kind: Secret metadata: - name: redis-credentials + name: airflow-redis-credentials stringData: username: "" password: redis diff --git a/docs/modules/airflow/pages/getting_started/first_steps.adoc b/docs/modules/airflow/pages/getting_started/first_steps.adoc index c63e7e74..e146a490 100644 --- a/docs/modules/airflow/pages/getting_started/first_steps.adoc +++ b/docs/modules/airflow/pages/getting_started/first_steps.adoc @@ -24,12 +24,12 @@ And apply it: [source,bash] include::example$getting_started/code/getting_started.sh[tag=apply-airflow-credentials] -`postgresql-credentials` contains credentials for the SQL database storing the Airflow metadata. +`airflow-postgresql-credentials` contains credentials for the SQL database storing the Airflow metadata. In this example we will use the same database for both the Airflow job metadata as well as the Celery broker metadata. -`redis-credentials` contains credentials for the the Redis instance used for queuing the jobs submitted to the airflow executor(s). +`airflow-redis-credentials` contains credentials for the the Redis instance used for queuing the jobs submitted to the airflow executor(s). -`admin-user-credentials`: the `adminUser` fields are used to create an admin user. +`airflow-admin-user-credentials`: the `adminUser` fields are used to create an admin user. NOTE: The admin user is disabled if you use a non-default authentication mechanism like LDAP. diff --git a/tests/templates/kuttl/versioning/30-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/versioning/30-install-airflow-cluster.yaml.j2 index 806aa924..15e43d93 100644 --- a/tests/templates/kuttl/versioning/30-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/versioning/30-install-airflow-cluster.yaml.j2 @@ -36,14 +36,6 @@ stringData: username: airflow password: airflow --- -apiVersion: v1 -kind: Secret -metadata: - name: redis-credentials -stringData: - username: "" - password: redis ---- apiVersion: airflow.stackable.tech/v1alpha1 kind: AirflowCluster metadata: