diff --git a/Cargo.lock b/Cargo.lock index 257c8b6..d4d4499 100755 --- a/Cargo.lock +++ b/Cargo.lock @@ -1540,10 +1540,12 @@ dependencies = [ "kube", "rustls", "rustls-pemfile", + "rustls-webpki", "schemars", "serde", "serde_json", "serde_yaml_ng", + "sha2", "shadow-rs", "snafu", "strum", diff --git a/Cargo.toml b/Cargo.toml index 6d0606f..c3700df 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,8 @@ schemars = "1" clap = { version = "4.5.54", features = ["derive"] } rustls = { version = "0.23", default-features = false, features = ["ring"] } rustls-pemfile = "2.2.0" +webpki = { package = "rustls-webpki", version = "0.103" } +sha2 = "0.10" shadow-rs = "1.5.0" snafu = { version = "0.8.9", features = ["futures"] } diff --git a/deploy/k8s-dev/console-rbac.yaml b/deploy/k8s-dev/console-rbac.yaml index 1be1043..cef5b46 100755 --- a/deploy/k8s-dev/console-rbac.yaml +++ b/deploy/k8s-dev/console-rbac.yaml @@ -42,6 +42,9 @@ rules: - apiGroups: ["apps"] resources: ["statefulsets"] verbs: ["get", "list", "watch"] + - apiGroups: ["cert-manager.io"] + resources: ["certificates", "issuers", "clusterissuers"] + verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch"] diff --git a/deploy/k8s-dev/operator-rbac.yaml b/deploy/k8s-dev/operator-rbac.yaml index 9f776d4..c178a52 100755 --- a/deploy/k8s-dev/operator-rbac.yaml +++ b/deploy/k8s-dev/operator-rbac.yaml @@ -36,6 +36,16 @@ rules: - apiGroups: ["apps"] resources: ["statefulsets"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["cert-manager.io"] + resources: ["certificates"] + verbs: ["get", "list", "watch", "create", "patch", "update"] + - apiGroups: ["cert-manager.io"] + resources: ["issuers", "clusterissuers"] + verbs: ["get", "list", "watch"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + resourceNames: ["certificates.cert-manager.io"] + verbs: ["get"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch"] diff --git a/deploy/rustfs-operator/crds/tenant-crd.yaml b/deploy/rustfs-operator/crds/tenant-crd.yaml index 84120e3..b7556d5 100644 --- a/deploy/rustfs-operator/crds/tenant-crd.yaml +++ b/deploy/rustfs-operator/crds/tenant-crd.yaml @@ -1282,6 +1282,138 @@ spec: serviceAccountName: nullable: true type: string + tls: + nullable: true + properties: + certManager: + nullable: true + properties: + caTrust: + nullable: true + properties: + caSecretRef: + nullable: true + properties: + key: + default: ca.crt + type: string + name: + type: string + required: + - name + type: object + clientCaSecretRef: + nullable: true + properties: + key: + default: ca.crt + type: string + name: + type: string + required: + - name + type: object + source: + default: CertificateSecretCa + enum: + - CertificateSecretCa + - SecretRef + - SystemCa + type: string + trustLeafCertificateAsCa: + default: false + type: boolean + trustSystemCa: + default: false + type: boolean + type: object + certificateName: + nullable: true + type: string + commonName: + nullable: true + type: string + dnsNames: + items: + type: string + type: array + duration: + nullable: true + type: string + includeGeneratedDnsNames: + default: true + type: boolean + issuerRef: + nullable: true + properties: + group: + default: cert-manager.io + type: string + kind: + default: Issuer + type: string + name: + type: string + required: + - name + type: object + manageCertificate: + default: false + type: boolean + privateKey: + nullable: true + properties: + algorithm: + nullable: true + type: string + encoding: + nullable: true + type: string + rotationPolicy: + nullable: true + type: string + size: + format: int32 + nullable: true + type: integer + type: object + renewBefore: + nullable: true + type: string + secretName: + nullable: true + type: string + secretType: + nullable: true + type: string + usages: + items: + type: string + type: array + type: object + enableInternodeHttps: + default: false + type: boolean + mode: + default: disabled + enum: + - disabled + - external + - certManager + type: string + mountPath: + default: /var/run/rustfs/tls + type: string + requireSanMatch: + default: true + type: boolean + rotationStrategy: + default: Rollout + enum: + - Rollout + - HotReload + type: string + type: object required: - pools type: object @@ -1291,6 +1423,134 @@ spec: availableReplicas: format: int32 type: integer + certificates: + properties: + tls: + nullable: true + properties: + caSecretRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + type: string + resourceVersion: + nullable: true + type: string + required: + - name + type: object + certificateRef: + nullable: true + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + observedGeneration: + format: int64 + nullable: true + type: integer + ready: + nullable: true + type: boolean + reason: + nullable: true + type: string + required: + - apiVersion + - kind + - name + type: object + clientCaSecretRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + type: string + resourceVersion: + nullable: true + type: string + required: + - name + type: object + dnsNames: + items: + type: string + type: array + expiresInSeconds: + format: int64 + nullable: true + type: integer + ipAddresses: + items: + type: string + type: array + lastErrorMessage: + nullable: true + type: string + lastErrorReason: + nullable: true + type: string + lastRolloutTriggerTime: + nullable: true + type: string + lastValidatedTime: + nullable: true + type: string + managedCertificate: + nullable: true + type: boolean + mode: + type: string + mountPath: + nullable: true + type: string + notAfter: + nullable: true + type: string + notBefore: + nullable: true + type: string + observedHash: + nullable: true + type: string + ready: + type: boolean + rotationStrategy: + nullable: true + type: string + sanMatched: + nullable: true + type: boolean + serverSecretRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + type: string + resourceVersion: + nullable: true + type: string + required: + - name + type: object + trustSource: + nullable: true + type: string + required: + - mode + - ready + type: object + type: object conditions: description: Kubernetes standard conditions items: diff --git a/deploy/rustfs-operator/crds/tenant.yaml b/deploy/rustfs-operator/crds/tenant.yaml index 84120e3..b7556d5 100755 --- a/deploy/rustfs-operator/crds/tenant.yaml +++ b/deploy/rustfs-operator/crds/tenant.yaml @@ -1282,6 +1282,138 @@ spec: serviceAccountName: nullable: true type: string + tls: + nullable: true + properties: + certManager: + nullable: true + properties: + caTrust: + nullable: true + properties: + caSecretRef: + nullable: true + properties: + key: + default: ca.crt + type: string + name: + type: string + required: + - name + type: object + clientCaSecretRef: + nullable: true + properties: + key: + default: ca.crt + type: string + name: + type: string + required: + - name + type: object + source: + default: CertificateSecretCa + enum: + - CertificateSecretCa + - SecretRef + - SystemCa + type: string + trustLeafCertificateAsCa: + default: false + type: boolean + trustSystemCa: + default: false + type: boolean + type: object + certificateName: + nullable: true + type: string + commonName: + nullable: true + type: string + dnsNames: + items: + type: string + type: array + duration: + nullable: true + type: string + includeGeneratedDnsNames: + default: true + type: boolean + issuerRef: + nullable: true + properties: + group: + default: cert-manager.io + type: string + kind: + default: Issuer + type: string + name: + type: string + required: + - name + type: object + manageCertificate: + default: false + type: boolean + privateKey: + nullable: true + properties: + algorithm: + nullable: true + type: string + encoding: + nullable: true + type: string + rotationPolicy: + nullable: true + type: string + size: + format: int32 + nullable: true + type: integer + type: object + renewBefore: + nullable: true + type: string + secretName: + nullable: true + type: string + secretType: + nullable: true + type: string + usages: + items: + type: string + type: array + type: object + enableInternodeHttps: + default: false + type: boolean + mode: + default: disabled + enum: + - disabled + - external + - certManager + type: string + mountPath: + default: /var/run/rustfs/tls + type: string + requireSanMatch: + default: true + type: boolean + rotationStrategy: + default: Rollout + enum: + - Rollout + - HotReload + type: string + type: object required: - pools type: object @@ -1291,6 +1423,134 @@ spec: availableReplicas: format: int32 type: integer + certificates: + properties: + tls: + nullable: true + properties: + caSecretRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + type: string + resourceVersion: + nullable: true + type: string + required: + - name + type: object + certificateRef: + nullable: true + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + observedGeneration: + format: int64 + nullable: true + type: integer + ready: + nullable: true + type: boolean + reason: + nullable: true + type: string + required: + - apiVersion + - kind + - name + type: object + clientCaSecretRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + type: string + resourceVersion: + nullable: true + type: string + required: + - name + type: object + dnsNames: + items: + type: string + type: array + expiresInSeconds: + format: int64 + nullable: true + type: integer + ipAddresses: + items: + type: string + type: array + lastErrorMessage: + nullable: true + type: string + lastErrorReason: + nullable: true + type: string + lastRolloutTriggerTime: + nullable: true + type: string + lastValidatedTime: + nullable: true + type: string + managedCertificate: + nullable: true + type: boolean + mode: + type: string + mountPath: + nullable: true + type: string + notAfter: + nullable: true + type: string + notBefore: + nullable: true + type: string + observedHash: + nullable: true + type: string + ready: + type: boolean + rotationStrategy: + nullable: true + type: string + sanMatched: + nullable: true + type: boolean + serverSecretRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + type: string + resourceVersion: + nullable: true + type: string + required: + - name + type: object + trustSource: + nullable: true + type: string + required: + - mode + - ready + type: object + type: object conditions: description: Kubernetes standard conditions items: diff --git a/deploy/rustfs-operator/templates/clusterrole.yaml b/deploy/rustfs-operator/templates/clusterrole.yaml index b0e900a..02be419 100755 --- a/deploy/rustfs-operator/templates/clusterrole.yaml +++ b/deploy/rustfs-operator/templates/clusterrole.yaml @@ -37,6 +37,18 @@ rules: resources: ["statefulsets"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + # cert-manager Certificate orchestration and readiness watches + - apiGroups: ["cert-manager.io"] + resources: ["certificates"] + verbs: ["get", "list", "watch", "create", "patch", "update"] + - apiGroups: ["cert-manager.io"] + resources: ["issuers", "clusterissuers"] + verbs: ["get", "list", "watch"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + resourceNames: ["certificates.cert-manager.io"] + verbs: ["get"] + # PersistentVolumeClaims - read (tenant-scoped event discovery lists PVC names) - apiGroups: [""] resources: ["persistentvolumeclaims"] diff --git a/deploy/rustfs-operator/templates/console-clusterrole.yaml b/deploy/rustfs-operator/templates/console-clusterrole.yaml index 0f93554..4f70de3 100755 --- a/deploy/rustfs-operator/templates/console-clusterrole.yaml +++ b/deploy/rustfs-operator/templates/console-clusterrole.yaml @@ -43,6 +43,11 @@ rules: resources: ["statefulsets"] verbs: ["get", "list", "watch"] + # cert-manager resources - read only for Console diagnostics + - apiGroups: ["cert-manager.io"] + resources: ["certificates", "issuers", "clusterissuers"] + verbs: ["get", "list", "watch"] + # PersistentVolumeClaims - read only - apiGroups: [""] resources: ["persistentvolumeclaims"] diff --git a/e2e/Cargo.lock b/e2e/Cargo.lock index c8017e1..785b508 100644 --- a/e2e/Cargo.lock +++ b/e2e/Cargo.lock @@ -1670,10 +1670,12 @@ dependencies = [ "kube", "rustls", "rustls-pemfile", + "rustls-webpki", "schemars", "serde", "serde_json", "serde_yaml_ng", + "sha2", "shadow-rs", "snafu", "strum", diff --git a/e2e/src/cases/cert_manager_tls.rs b/e2e/src/cases/cert_manager_tls.rs new file mode 100644 index 0000000..46a5448 --- /dev/null +++ b/e2e/src/cases/cert_manager_tls.rs @@ -0,0 +1,106 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::{CaseSpec, Suite}; + +pub fn cases() -> Vec { + vec![ + CaseSpec::new( + Suite::CertManagerTls, + "cert_manager_managed_certificate_reaches_tls_ready_and_https_wiring", + "Apply a cert-manager managed Certificate Tenant in an isolated namespace/storage layout and verify TLSReady, Certificate Ready, HTTPS services, probes, RUSTFS_VOLUMES, and TLS file mounts.", + "cert-manager/managed-certificate", + "cert-manager-tls", + ), + CaseSpec::new( + Suite::CertManagerTls, + "cert_manager_external_secret_reaches_tls_ready_and_rolls_on_secret_hash", + "Use an isolated namespace/storage layout with a pre-created kubernetes.io/tls Secret plus external CA Secret and verify TLS hash changes trigger rollout wiring.", + "cert-manager/external-secret", + "cert-manager-tls", + ), + CaseSpec::new( + Suite::CertManagerTls, + "cert_manager_rejects_secret_missing_tls_crt", + "Verify an API-admissible configured Opaque Secret without tls.crt reports TLSReady=False with CertificateSecretMissingKey.", + "cert-manager/negative-secret", + "cert-manager-tls", + ), + CaseSpec::new( + Suite::CertManagerTls, + "cert_manager_rejects_secret_missing_tls_key", + "Verify an API-admissible configured Opaque Secret without tls.key reports TLSReady=False with CertificateSecretMissingKey.", + "cert-manager/negative-secret", + "cert-manager-tls", + ), + CaseSpec::new( + Suite::CertManagerTls, + "cert_manager_rejects_secret_missing_ca_for_internode_https", + "Verify enableInternodeHttps without trusted CA material reports TLSReady=False with CaBundleMissing.", + "cert-manager/negative-ca-trust", + "cert-manager-tls", + ), + CaseSpec::new( + Suite::CertManagerTls, + "cert_manager_rejects_missing_issuer_for_managed_certificate", + "Verify manageCertificate=true with a missing Issuer reports TLSReady=False with CertManagerIssuerNotFound.", + "cert-manager/negative-issuer", + "cert-manager-tls", + ), + CaseSpec::new( + Suite::CertManagerTls, + "cert_manager_reports_pending_certificate_not_ready", + "Verify a cert-manager Certificate that remains Pending/NotReady reports TLSReady=False with CertManagerCertificateNotReady.", + "cert-manager/certificate-pending", + "cert-manager-tls", + ), + CaseSpec::new( + Suite::CertManagerTls, + "cert_manager_rejects_hot_reload", + "Verify rotationStrategy=HotReload remains blocked until RustFS supports clean-directory reload.", + "cert-manager/negative-hot-reload", + "cert-manager-tls", + ), + CaseSpec::new( + Suite::CertManagerTls, + "cert_manager_artifacts_do_not_expose_secret_material", + "Ensure generated manifests, command displays, status assertions, and artifact collection paths do not print PEM or Secret payloads.", + "cert-manager/security", + "cert-manager-tls", + ), + ] +} + +#[cfg(test)] +mod tests { + use super::cases; + + #[test] + fn cert_manager_case_inventory_matches_expected_order() { + let names = cases() + .into_iter() + .map(|case| case.name) + .collect::>(); + + assert_eq!(names.len(), 9); + assert_eq!( + names.first().copied(), + Some("cert_manager_managed_certificate_reaches_tls_ready_and_https_wiring") + ); + assert_eq!( + names.last().copied(), + Some("cert_manager_artifacts_do_not_expose_secret_material") + ); + } +} diff --git a/e2e/src/cases/mod.rs b/e2e/src/cases/mod.rs index 02fcef3..d227667 100644 --- a/e2e/src/cases/mod.rs +++ b/e2e/src/cases/mod.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +pub mod cert_manager_tls; pub mod console; pub mod operator; pub mod smoke; @@ -21,6 +22,7 @@ pub enum Suite { Smoke, Operator, Console, + CertManagerTls, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -55,6 +57,7 @@ pub fn all_cases() -> Vec { cases.extend(smoke::cases()); cases.extend(operator::cases()); cases.extend(console::cases()); + cases.extend(cert_manager_tls::cases()); cases } @@ -71,6 +74,7 @@ mod tests { assert!(suites.contains(&Suite::Smoke)); assert!(suites.contains(&Suite::Operator)); assert!(suites.contains(&Suite::Console)); + assert!(suites.contains(&Suite::CertManagerTls)); } #[test] @@ -111,5 +115,12 @@ mod tests { assert_eq!(counts.get(&Suite::Smoke).copied().unwrap_or_default(), 3); assert_eq!(counts.get(&Suite::Operator).copied().unwrap_or_default(), 1); assert_eq!(counts.get(&Suite::Console).copied().unwrap_or_default(), 1); + assert_eq!( + counts + .get(&Suite::CertManagerTls) + .copied() + .unwrap_or_default(), + 9 + ); } } diff --git a/e2e/src/framework/assertions.rs b/e2e/src/framework/assertions.rs index 39023bb..21c700b 100644 --- a/e2e/src/framework/assertions.rs +++ b/e2e/src/framework/assertions.rs @@ -12,8 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -use anyhow::{Result, bail, ensure}; +use anyhow::{Context, Result, bail, ensure}; +use k8s_openapi::api::{apps::v1::StatefulSet, core::v1 as corev1}; +use operator::types::v1alpha1::status::certificate::SecretStatusRef; use operator::types::v1alpha1::tenant::Tenant; +use operator::types::v1alpha1::tls::{ + RUSTFS_CA_FILE, RUSTFS_TLS_CERT_FILE, RUSTFS_TLS_KEY_FILE, TLS_HASH_ANNOTATION, +}; pub fn current_state(tenant: &Tenant) -> Option<&str> { tenant @@ -64,6 +69,281 @@ pub fn require_observed_generation_current(tenant: &Tenant) -> Result<()> { Ok(()) } +pub fn tenant_tls_observed_hash(tenant: &Tenant) -> Result { + tenant + .status + .as_ref() + .and_then(|status| status.certificates.tls.as_ref()) + .and_then(|tls| tls.observed_hash.clone()) + .context("Tenant status.certificates.tls.observedHash is missing") +} + +pub fn require_tls_service_https_wiring(service: &corev1::Service) -> Result<()> { + let ports = service + .spec + .as_ref() + .and_then(|spec| spec.ports.as_ref()) + .context("Service spec.ports is missing")?; + ensure!( + ports + .iter() + .any(|port| port.name.as_deref() == Some("https-rustfs") && port.port == 9000), + "Service {} does not expose the https-rustfs port on 9000", + service + .metadata + .name + .as_deref() + .unwrap_or("") + ); + Ok(()) +} + +pub fn require_tls_statefulset_https_wiring( + statefulset: &StatefulSet, + expected_hash: &str, + expected_secret_name: &str, + expected_ca_secret_ref: &SecretStatusRef, +) -> Result<()> { + let template = &statefulset + .spec + .as_ref() + .context("StatefulSet spec is missing")? + .template; + let annotations = template + .metadata + .as_ref() + .and_then(|metadata| metadata.annotations.as_ref()) + .context("StatefulSet pod template annotations are missing")?; + ensure!( + annotations.get(TLS_HASH_ANNOTATION).map(String::as_str) == Some(expected_hash), + "StatefulSet pod template TLS hash mismatch: expected {expected_hash}, got {:?}", + annotations.get(TLS_HASH_ANNOTATION) + ); + + let pod_spec = template + .spec + .as_ref() + .context("StatefulSet pod template spec is missing")?; + let volumes = pod_spec.volumes.as_deref().unwrap_or_default(); + let server_volume = volumes + .iter() + .find(|volume| { + volume.name == "rustfs-tls-server" + && volume_references_secret(volume, expected_secret_name) + }) + .with_context(|| { + format!( + "StatefulSet pod volumes do not mount expected TLS Secret {expected_secret_name}" + ) + })?; + ensure!( + volume_has_secret_item( + server_volume, + expected_secret_name, + "tls.crt", + RUSTFS_TLS_CERT_FILE, + ), + "StatefulSet rustfs-tls-server volume does not map tls.crt to {RUSTFS_TLS_CERT_FILE}" + ); + ensure!( + volume_has_secret_item( + server_volume, + expected_secret_name, + "tls.key", + RUSTFS_TLS_KEY_FILE, + ), + "StatefulSet rustfs-tls-server volume does not map tls.key to {RUSTFS_TLS_KEY_FILE}" + ); + + let container = pod_spec + .containers + .iter() + .find(|container| container.name == "rustfs") + .context("rustfs container not found")?; + let tls_path = env_value(container, "RUSTFS_TLS_PATH") + .context("rustfs container missing RUSTFS_TLS_PATH")?; + ensure!( + env_value(container, "RUSTFS_VOLUMES").is_some_and(|value| value.contains("https://")), + "rustfs container RUSTFS_VOLUMES does not use https://" + ); + ensure!( + probe_scheme(container.readiness_probe.as_ref()) == Some("HTTPS"), + "rustfs readiness probe is not HTTPS" + ); + ensure!( + probe_scheme(container.liveness_probe.as_ref()) == Some("HTTPS"), + "rustfs liveness probe is not HTTPS" + ); + + let mounts = container.volume_mounts.as_deref().unwrap_or_default(); + for file_name in [RUSTFS_TLS_CERT_FILE, RUSTFS_TLS_KEY_FILE] { + require_read_only_tls_material_mount(mounts, "rustfs-tls-server", tls_path, file_name)?; + } + + let ca_key = expected_ca_secret_ref.key.as_deref().unwrap_or("ca.crt"); + if volume_has_secret_item( + server_volume, + &expected_ca_secret_ref.name, + ca_key, + RUSTFS_CA_FILE, + ) { + require_read_only_tls_material_mount( + mounts, + "rustfs-tls-server", + tls_path, + RUSTFS_CA_FILE, + )?; + } else { + let ca_volume = volumes + .iter() + .find(|volume| { + volume.name == "rustfs-tls-ca" + && secret_volume_name(volume) == Some(expected_ca_secret_ref.name.as_str()) + }) + .with_context(|| { + format!( + "StatefulSet pod volumes do not mount expected CA Secret {}", + expected_ca_secret_ref.name + ) + })?; + ensure!( + secret_volume_has_item(ca_volume, ca_key, RUSTFS_CA_FILE), + "StatefulSet rustfs-tls-ca volume does not map CA key {ca_key} to {RUSTFS_CA_FILE}" + ); + require_read_only_tls_material_mount(mounts, "rustfs-tls-ca", tls_path, RUSTFS_CA_FILE)?; + } + + Ok(()) +} + +pub fn require_no_secret_material(label: &str, content: &str) -> Result<()> { + for forbidden in [ + "-----BEGIN", + "PRIVATE KEY", + "tls.key:", + "tls.crt:", + "accesskey:", + "secretkey:", + ] { + ensure!( + !content.contains(forbidden), + "{label} exposes forbidden secret material marker {forbidden}" + ); + } + Ok(()) +} + +fn env_value<'a>(container: &'a corev1::Container, name: &str) -> Option<&'a str> { + container + .env + .as_ref()? + .iter() + .find(|var| var.name == name)? + .value + .as_deref() +} + +fn secret_volume_name(volume: &corev1::Volume) -> Option<&str> { + volume + .secret + .as_ref() + .and_then(|secret| secret.secret_name.as_deref()) +} + +fn secret_volume_has_item(volume: &corev1::Volume, key: &str, path: &str) -> bool { + volume + .secret + .as_ref() + .and_then(|secret| secret.items.as_ref()) + .is_some_and(|items| { + items + .iter() + .any(|item| item.key == key && item.path == path) + }) +} + +fn volume_references_secret(volume: &corev1::Volume, secret_name: &str) -> bool { + secret_volume_name(volume) == Some(secret_name) + || projected_volume_references_secret(volume, secret_name) +} + +fn volume_has_secret_item( + volume: &corev1::Volume, + secret_name: &str, + key: &str, + path: &str, +) -> bool { + if secret_volume_name(volume) == Some(secret_name) && secret_volume_has_item(volume, key, path) + { + return true; + } + projected_volume_has_secret_item(volume, secret_name, key, path) +} + +fn projected_volume_references_secret(volume: &corev1::Volume, secret_name: &str) -> bool { + volume + .projected + .as_ref() + .and_then(|projected| projected.sources.as_ref()) + .is_some_and(|sources| { + sources.iter().any(|source| { + source + .secret + .as_ref() + .is_some_and(|secret| secret.name == secret_name) + }) + }) +} + +fn projected_volume_has_secret_item( + volume: &corev1::Volume, + secret_name: &str, + key: &str, + path: &str, +) -> bool { + volume + .projected + .as_ref() + .and_then(|projected| projected.sources.as_ref()) + .is_some_and(|sources| { + sources.iter().any(|source| { + source + .secret + .as_ref() + .filter(|secret| secret.name == secret_name) + .and_then(|secret| secret.items.as_ref()) + .is_some_and(|items| { + items + .iter() + .any(|item| item.key == key && item.path == path) + }) + }) + }) +} + +fn require_read_only_tls_material_mount( + mounts: &[corev1::VolumeMount], + volume_name: &str, + tls_path: &str, + file_name: &str, +) -> Result<()> { + ensure!( + mounts.iter().any(|mount| { + mount.name == volume_name + && mount.read_only == Some(true) + && ((mount.mount_path.ends_with(file_name) + && mount.sub_path.as_deref() == Some(file_name)) + || (mount.mount_path == tls_path && mount.sub_path.is_none())) + }), + "rustfs container does not expose {file_name} read-only from volume {volume_name}" + ); + Ok(()) +} + +fn probe_scheme(probe: Option<&corev1::Probe>) -> Option<&str> { + probe?.http_get.as_ref()?.scheme.as_deref() +} + #[cfg(test)] mod tests { use super::{condition_status, current_state, require_condition}; diff --git a/e2e/src/framework/cert_manager_tls.rs b/e2e/src/framework/cert_manager_tls.rs new file mode 100644 index 0000000..39126c6 --- /dev/null +++ b/e2e/src/framework/cert_manager_tls.rs @@ -0,0 +1,1144 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use anyhow::{Context, Result}; +use k8s_openapi::ByteString; +use k8s_openapi::api::{apps::v1::StatefulSet, core::v1 as corev1}; +use k8s_openapi::apimachinery::pkg::apis::meta::v1 as metav1; +use kube::core::{ApiResource, DynamicObject, GroupVersionKind}; +use kube::{Api, Client}; +use operator::types::v1alpha1::tenant::Tenant; +use operator::types::v1alpha1::tls::{ + CaTrustConfig, CaTrustSource, CertManagerIssuerRef, CertManagerTlsConfig, SecretKeyReference, + TlsConfig, TlsMode, TlsPlan, TlsRotationStrategy, +}; +use serde_json::Value; +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::path::Path; +use std::time::Duration; +use tempfile::TempDir; + +use crate::framework::{ + assertions, command::CommandSpec, config::E2eConfig, kubectl::Kubectl, resources, storage, + tenant_factory::TenantTemplate, wait, +}; + +const CERT_MANAGER_GROUP: &str = "cert-manager.io"; +const CERT_MANAGER_VERSION: &str = "v1"; +const CERT_MANAGER_CERTIFICATE_KIND: &str = "Certificate"; +const CERT_MANAGER_CERTIFICATE_PLURAL: &str = "certificates"; +const SELF_SIGNED_ISSUER_NAME: &str = "rustfs-e2e-selfsigned"; +const PENDING_ISSUER_NAME: &str = "rustfs-e2e-pending-issuer"; +const MISSING_ISSUER_NAME: &str = "rustfs-e2e-missing-issuer"; +const KUBERNETES_TLS_SECRET_TYPE: &str = "kubernetes.io/tls"; +const OPAQUE_SECRET_TYPE: &str = "Opaque"; +const REDACTED_FIXTURE_BYTES: &[u8] = b"redacted-test-fixture"; +const MANAGED_CERTIFICATE_CASE_SUFFIX: &str = "cert-manager-managed"; +const EXTERNAL_SECRET_CASE_SUFFIX: &str = "cert-manager-external"; +const RUSTFS_TENANT_LABEL: &str = "rustfs.tenant"; +pub const POSITIVE_CERT_MANAGER_TLS_TIMEOUT: Duration = Duration::from_secs(600); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NegativeTlsCase { + MissingTlsCrt, + MissingTlsKey, + MissingCaForInternodeHttps, + MissingIssuer, + PendingCertificate, + HotReloadUnsupported, +} + +impl NegativeTlsCase { + pub fn case_name(self) -> &'static str { + match self { + Self::MissingTlsCrt => "cert_manager_rejects_secret_missing_tls_crt", + Self::MissingTlsKey => "cert_manager_rejects_secret_missing_tls_key", + Self::MissingCaForInternodeHttps => { + "cert_manager_rejects_secret_missing_ca_for_internode_https" + } + Self::MissingIssuer => "cert_manager_rejects_missing_issuer_for_managed_certificate", + Self::PendingCertificate => "cert_manager_reports_pending_certificate_not_ready", + Self::HotReloadUnsupported => "cert_manager_rejects_hot_reload", + } + } +} + +pub fn managed_certificate_case_config(config: &E2eConfig) -> E2eConfig { + positive_tls_case_config(config, MANAGED_CERTIFICATE_CASE_SUFFIX) +} + +pub fn external_secret_case_config(config: &E2eConfig) -> E2eConfig { + positive_tls_case_config(config, EXTERNAL_SECRET_CASE_SUFFIX) +} + +pub fn positive_cert_manager_tls_timeout(config: &E2eConfig) -> Duration { + std::cmp::max(config.timeout, POSITIVE_CERT_MANAGER_TLS_TIMEOUT) +} + +fn tenant_watch_labels(config: &E2eConfig) -> BTreeMap { + BTreeMap::from([(RUSTFS_TENANT_LABEL.to_string(), config.tenant_name.clone())]) +} + +pub fn managed_certificate_storage_layout(config: &E2eConfig) -> storage::LocalStorageLayout { + positive_tls_storage_layout(config, MANAGED_CERTIFICATE_CASE_SUFFIX) +} + +pub fn external_secret_storage_layout(config: &E2eConfig) -> storage::LocalStorageLayout { + positive_tls_storage_layout(config, EXTERNAL_SECRET_CASE_SUFFIX) +} + +pub fn managed_secret_name(config: &E2eConfig) -> String { + format!("{}-managed-tls", config.tenant_name) +} + +pub fn managed_certificate_name(config: &E2eConfig) -> String { + format!("{}-managed-cert", config.tenant_name) +} + +pub fn external_secret_name(config: &E2eConfig) -> String { + format!("{}-external-tls", config.tenant_name) +} + +pub fn external_ca_secret_name(config: &E2eConfig) -> String { + format!("{}-external-ca", config.tenant_name) +} + +pub fn external_tls_subject_alt_name(config: &E2eConfig) -> String { + subject_alt_name(&external_tls_certificate_dns_names(config)) +} + +pub fn external_tls_rotation_subject_alt_name(config: &E2eConfig) -> String { + external_tls_subject_alt_name(config) +} + +pub fn external_tls_certificate_dns_names(config: &E2eConfig) -> Vec { + tls_certificate_dns_names(config, &external_secret_tenant(config)) +} + +fn tls_certificate_dns_names(config: &E2eConfig, tenant: &Tenant) -> Vec { + let mut names = BTreeSet::from([config.tenant_name.clone(), "localhost".to_string()]); + + if let Some(cert_manager) = tenant + .spec + .tls + .as_ref() + .and_then(|tls| tls.cert_manager.as_ref()) + { + if let Some(common_name) = cert_manager + .common_name + .as_deref() + .filter(|name| !name.is_empty()) + { + names.insert(common_name.to_string()); + } + names.extend( + cert_manager + .dns_names + .iter() + .filter(|name| !name.is_empty()) + .cloned(), + ); + + if cert_manager.include_generated_dns_names { + let tenant_name = &config.tenant_name; + let namespace = &config.test_namespace; + let io_service = format!("{tenant_name}-io"); + let headless_service = format!("{tenant_name}-hl"); + names.insert(format!("{io_service}.{namespace}.svc")); + names.insert(format!("{io_service}.{namespace}.svc.cluster.local")); + names.insert(format!("{headless_service}.{namespace}.svc")); + names.insert(format!("{headless_service}.{namespace}.svc.cluster.local")); + for pool in &tenant.spec.pools { + for ordinal in 0..pool.servers.max(0) { + names.insert(format!( + "{tenant_name}-{}-{ordinal}.{headless_service}.{namespace}.svc.cluster.local", + pool.name + )); + } + } + } + } + + names.into_iter().collect() +} + +pub fn managed_certificate_tenant(config: &E2eConfig) -> Tenant { + let secret_name = managed_secret_name(config); + let certificate_name = managed_certificate_name(config); + positive_tenant_with_tls( + config, + cert_manager_tls_config( + true, + secret_name, + Some(certificate_name), + Some(issuer_ref(SELF_SIGNED_ISSUER_NAME)), + true, + TlsRotationStrategy::Rollout, + CaTrustConfig { + source: CaTrustSource::CertificateSecretCa, + ..Default::default() + }, + ), + ) +} + +pub fn managed_certificate_tenant_manifest(config: &E2eConfig) -> Result { + tenant_manifest(&managed_certificate_tenant(config)) +} + +pub fn external_secret_tenant(config: &E2eConfig) -> Tenant { + positive_tenant_with_tls( + config, + cert_manager_tls_config( + false, + external_secret_name(config), + None, + None, + true, + TlsRotationStrategy::Rollout, + CaTrustConfig { + source: CaTrustSource::SecretRef, + ca_secret_ref: Some(SecretKeyReference { + name: external_ca_secret_name(config), + key: "ca.crt".to_string(), + }), + ..Default::default() + }, + ), + ) +} + +pub fn external_secret_tenant_manifest(config: &E2eConfig) -> Result { + tenant_manifest(&external_secret_tenant(config)) +} + +fn positive_tls_case_config(config: &E2eConfig, suffix: &str) -> E2eConfig { + let mut isolated = config.clone(); + isolated.test_namespace = format!("{}-{suffix}", config.test_namespace_prefix); + isolated.tenant_name = format!("{}-{suffix}", config.tenant_name); + isolated.storage_class = format!("{}-{suffix}", config.storage_class); + isolated.pv_count = topology_safe_pv_count_for_tls_tenant(&isolated); + isolated +} + +fn topology_safe_pv_count_for_tls_tenant(config: &E2eConfig) -> usize { + let template = positive_tls_tenant_template(config); + let servers = usize::try_from(template.servers) + .expect("TLS e2e Tenant template server count must be non-negative"); + let volumes_per_server = usize::try_from(template.volumes_per_server) + .expect("TLS e2e Tenant template volumes_per_server must be non-negative"); + servers * volumes_per_server +} + +fn positive_tls_tenant_template(config: &E2eConfig) -> TenantTemplate { + let mut template = tls_tenant_template(config); + template.volumes_per_server = 1; + template +} + +fn tls_tenant_template(config: &E2eConfig) -> TenantTemplate { + TenantTemplate::kind_local( + &config.test_namespace, + &config.tenant_name, + &config.rustfs_image, + &config.storage_class, + resources::credential_secret_name(config), + ) +} + +fn positive_tls_storage_layout(config: &E2eConfig, suffix: &str) -> storage::LocalStorageLayout { + storage::LocalStorageLayout::new( + config.storage_class.clone(), + format!("{}-{suffix}-pv", config.cluster_name), + format!("/mnt/data/{suffix}"), + config.pv_count, + ) +} + +pub fn negative_case_tenant_manifest(config: &E2eConfig, case: NegativeTlsCase) -> Result { + tenant_manifest(&negative_case_tenant(config, case)) +} + +pub fn negative_tls_secret_manifest(config: &E2eConfig, case: NegativeTlsCase) -> Result { + let (secret_type, data) = match case { + NegativeTlsCase::MissingCaForInternodeHttps => { + let material = generate_missing_ca_tls_material(config)?; + ( + KUBERNETES_TLS_SECRET_TYPE, + secret_data(Some(&material.cert), Some(&material.key), None), + ) + } + _ => negative_tls_secret_fixture(case).with_context(|| { + format!( + "case {} does not apply a TLS Secret fixture", + case.case_name() + ) + })?, + }; + tls_secret_manifest( + &config.test_namespace, + &negative_secret_name(config, case), + secret_type, + data, + ) +} + +pub fn sample_tls_plan(hash: &str, server_secret_name: String) -> TlsPlan { + TlsPlan::rollout( + "/var/run/rustfs/tls".to_string(), + hash.to_string(), + server_secret_name, + Some("ca.crt".to_string()), + None, + None, + true, + false, + false, + None, + ) +} + +pub fn external_tls_secret_apply_command( + config: &E2eConfig, + secret_name: String, +) -> Result { + Ok( + Kubectl::new(config).apply_yaml_command(external_tls_secret_manifest( + config, + &secret_name, + KUBERNETES_TLS_SECRET_TYPE, + secret_data( + Some(REDACTED_FIXTURE_BYTES), + Some(REDACTED_FIXTURE_BYTES), + Some(REDACTED_FIXTURE_BYTES), + ), + )?), + ) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ExternalTlsSecretManifests { + pub tls_secret_manifest: String, + pub ca_secret_manifest: String, +} + +pub fn external_tls_secret_manifests(config: &E2eConfig) -> Result { + let material = generate_external_tls_material(config)?; + external_tls_secret_manifests_from_material(config, &material) +} + +fn external_tls_secret_manifests_from_material( + config: &E2eConfig, + material: &GeneratedTlsMaterial, +) -> Result { + Ok(ExternalTlsSecretManifests { + tls_secret_manifest: external_tls_secret_manifest( + config, + &external_secret_name(config), + KUBERNETES_TLS_SECRET_TYPE, + secret_data( + Some(&material.cert), + Some(&material.key), + Some(&material.ca), + ), + )?, + ca_secret_manifest: external_tls_secret_manifest( + config, + &external_ca_secret_name(config), + OPAQUE_SECRET_TYPE, + secret_data(None, None, Some(&material.ca)), + )?, + }) +} + +pub fn apply_managed_certificate_case_resources(config: &E2eConfig) -> Result<()> { + apply_positive_case_base_resources(config, managed_certificate_storage_layout(config))?; + apply_yaml(config, self_signed_issuer_manifest(config))?; + apply_yaml(config, managed_certificate_tenant_manifest(config)?)?; + Ok(()) +} + +pub fn apply_external_secret_case_resources(config: &E2eConfig) -> Result<()> { + apply_positive_case_base_resources(config, external_secret_storage_layout(config))?; + let material = generate_external_tls_material(config)?; + apply_external_tls_material(config, &material)?; + apply_yaml(config, external_secret_tenant_manifest(config)?)?; + Ok(()) +} + +pub fn rotate_external_tls_secret(config: &E2eConfig) -> Result<()> { + let material = generate_external_tls_material(config)?; + apply_external_tls_material(config, &material)?; + Ok(()) +} + +pub fn apply_negative_case_resources(config: &E2eConfig, case: NegativeTlsCase) -> Result<()> { + apply_base_resources(config)?; + + match case { + NegativeTlsCase::MissingTlsCrt + | NegativeTlsCase::MissingTlsKey + | NegativeTlsCase::MissingCaForInternodeHttps => { + apply_yaml(config, negative_tls_secret_manifest(config, case)?)?; + } + NegativeTlsCase::PendingCertificate => { + apply_yaml(config, pending_ca_issuer_manifest(config))?; + } + NegativeTlsCase::MissingIssuer | NegativeTlsCase::HotReloadUnsupported => {} + } + + apply_yaml( + config, + tenant_manifest(&negative_case_tenant(config, case))?, + )?; + Ok(()) +} + +pub async fn wait_for_tenant_tls_ready( + client: Client, + namespace: &str, + name: &str, + timeout: Duration, +) -> Result { + let tenants: Api = Api::namespaced(client, namespace); + let name = name.to_string(); + wait::wait_until( + &format!("Tenant {name} TLSReady=True and Ready=True"), + timeout, + Duration::from_secs(5), + move || { + let tenants = tenants.clone(); + let name = name.clone(); + async move { + let tenant = tenants.get(&name).await?; + let tls_ready = assertions::condition_status(&tenant, "TlsReady") == Some("True") + || assertions::condition_status(&tenant, "TLSReady") == Some("True"); + let tenant_ready = assertions::current_state(&tenant) == Some("Ready") + && assertions::condition_status(&tenant, "Ready") == Some("True") + && assertions::condition_status(&tenant, "Degraded") == Some("False"); + if tls_ready && tenant_ready { + Ok(Some(tenant)) + } else { + Ok(None) + } + } + }, + ) + .await +} + +pub async fn wait_for_tenant_tls_hash_change( + client: Client, + namespace: &str, + name: &str, + previous_hash: &str, + timeout: Duration, +) -> Result { + let tenants: Api = Api::namespaced(client, namespace); + let name = name.to_string(); + let previous_hash = previous_hash.to_string(); + wait::wait_until( + &format!("Tenant {name} TLS hash to change from {previous_hash}"), + timeout, + Duration::from_secs(5), + move || { + let tenants = tenants.clone(); + let name = name.clone(); + let previous_hash = previous_hash.clone(); + async move { + let tenant = tenants.get(&name).await?; + match assertions::tenant_tls_observed_hash(&tenant) { + Ok(hash) if hash != previous_hash => Ok(Some(tenant)), + _ => Ok(None), + } + } + }, + ) + .await +} + +pub async fn wait_for_tenant_tls_reason( + client: Client, + namespace: &str, + name: &str, + reason: &str, + timeout: Duration, +) -> Result { + let tenants: Api = Api::namespaced(client, namespace); + let name = name.to_string(); + let reason = reason.to_string(); + wait::wait_until( + &format!("Tenant {name} TLSReady reason {reason}"), + timeout, + Duration::from_secs(5), + move || { + let tenants = tenants.clone(); + let name = name.clone(); + let reason = reason.clone(); + async move { + let tenant = tenants.get(&name).await?; + if tenant_tls_reason(&tenant).as_deref() == Some(reason.as_str()) { + Ok(Some(tenant)) + } else { + Ok(None) + } + } + }, + ) + .await +} + +pub async fn wait_for_certificate_ready( + client: Client, + namespace: &str, + name: &str, + timeout: Duration, +) -> Result { + let certificates: Api = Api::namespaced_with( + client, + namespace, + &ApiResource::from_gvk_with_plural( + &GroupVersionKind::gvk( + CERT_MANAGER_GROUP, + CERT_MANAGER_VERSION, + CERT_MANAGER_CERTIFICATE_KIND, + ), + CERT_MANAGER_CERTIFICATE_PLURAL, + ), + ); + let name = name.to_string(); + wait::wait_until( + &format!("cert-manager Certificate {name} Ready=True"), + timeout, + Duration::from_secs(5), + move || { + let certificates = certificates.clone(); + let name = name.clone(); + async move { + let certificate = certificates.get(&name).await?; + if dynamic_certificate_ready(&certificate) { + Ok(Some(certificate)) + } else { + Ok(None) + } + } + }, + ) + .await +} + +pub async fn assert_live_workload_tls_wiring( + client: Client, + config: &E2eConfig, + tenant: &Tenant, +) -> Result<()> { + let tls_status = tenant + .status + .as_ref() + .and_then(|status| status.certificates.tls.as_ref()) + .context("Tenant status.certificates.tls is missing")?; + let hash = tls_status + .observed_hash + .as_deref() + .context("Tenant TLS status missing observedHash")?; + let secret_name = tls_status + .server_secret_ref + .as_ref() + .map(|secret| secret.name.as_str()) + .context("Tenant TLS status missing serverSecretRef")?; + let ca_secret_ref = tls_status + .ca_secret_ref + .as_ref() + .context("Tenant TLS status missing caSecretRef")?; + + let statefulsets: Api = Api::namespaced(client.clone(), &config.test_namespace); + let services: Api = Api::namespaced(client, &config.test_namespace); + let pool = tenant + .spec + .pools + .first() + .context("Tenant fixture must include at least one pool")?; + let statefulset = statefulsets + .get(&format!("{}-{}", config.tenant_name, pool.name)) + .await?; + let io_service = services.get(&format!("{}-io", config.tenant_name)).await?; + let headless_service = services.get(&format!("{}-hl", config.tenant_name)).await?; + + assertions::require_tls_statefulset_https_wiring( + &statefulset, + hash, + secret_name, + ca_secret_ref, + )?; + assertions::require_tls_service_https_wiring(&io_service)?; + assertions::require_tls_service_https_wiring(&headless_service)?; + Ok(()) +} + +fn positive_tenant_with_tls(config: &E2eConfig, tls: TlsConfig) -> Tenant { + let mut tenant = positive_tls_tenant_template(config).build(); + tenant.spec.tls = Some(tls); + tenant +} + +fn tenant_with_tls(config: &E2eConfig, tls: TlsConfig) -> Tenant { + let mut tenant = tls_tenant_template(config).build(); + tenant.spec.tls = Some(tls); + tenant +} + +#[allow(clippy::too_many_arguments)] +fn cert_manager_tls_config( + manage_certificate: bool, + secret_name: String, + certificate_name: Option, + issuer_ref: Option, + enable_internode_https: bool, + rotation_strategy: TlsRotationStrategy, + ca_trust: CaTrustConfig, +) -> TlsConfig { + TlsConfig { + mode: TlsMode::CertManager, + rotation_strategy, + enable_internode_https, + cert_manager: Some(CertManagerTlsConfig { + manage_certificate, + secret_name: Some(secret_name), + secret_type: Some(KUBERNETES_TLS_SECRET_TYPE.to_string()), + certificate_name, + issuer_ref, + common_name: Some("rustfs-e2e.local".to_string()), + dns_names: vec!["rustfs-e2e.local".to_string()], + ca_trust: Some(ca_trust), + ..CertManagerTlsConfig::default() + }), + ..TlsConfig::default() + } +} + +fn issuer_ref(name: &str) -> CertManagerIssuerRef { + CertManagerIssuerRef { + group: CERT_MANAGER_GROUP.to_string(), + kind: "Issuer".to_string(), + name: name.to_string(), + } +} + +fn tenant_manifest(tenant: &Tenant) -> Result { + Ok(serde_yaml_ng::to_string(tenant)?) +} + +fn apply_base_resources(config: &E2eConfig) -> Result<()> { + storage::prepare_local_storage(config)?; + apply_shared_namespace_resources(config) +} + +fn apply_positive_case_base_resources( + config: &E2eConfig, + layout: storage::LocalStorageLayout, +) -> Result<()> { + storage::prepare_local_storage_with_layout(config, &layout)?; + apply_shared_namespace_resources(config) +} + +fn apply_shared_namespace_resources(config: &E2eConfig) -> Result<()> { + apply_yaml( + config, + resources::namespace_manifest(&config.test_namespace), + )?; + apply_yaml(config, resources::credential_secret_manifest(config))?; + Ok(()) +} + +fn apply_yaml(config: &E2eConfig, yaml: String) -> Result<()> { + Kubectl::new(config) + .apply_yaml_command(yaml) + .run_checked()?; + Ok(()) +} + +fn self_signed_issuer_manifest(config: &E2eConfig) -> String { + format!( + r#"apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {SELF_SIGNED_ISSUER_NAME} + namespace: {namespace} +spec: + selfSigned: {{}} +"#, + namespace = config.test_namespace + ) +} + +fn pending_ca_issuer_manifest(config: &E2eConfig) -> String { + format!( + r#"apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {PENDING_ISSUER_NAME} + namespace: {namespace} +spec: + ca: + secretName: rustfs-e2e-missing-ca-secret +"#, + namespace = config.test_namespace + ) +} + +fn negative_case_tenant(config: &E2eConfig, case: NegativeTlsCase) -> Tenant { + match case { + NegativeTlsCase::MissingTlsCrt => tenant_with_expected_secret_type( + tenant_with_tls( + config, + cert_manager_tls_config( + false, + negative_secret_name(config, case), + None, + None, + false, + TlsRotationStrategy::Rollout, + CaTrustConfig::default(), + ), + ), + OPAQUE_SECRET_TYPE, + ), + NegativeTlsCase::MissingTlsKey => tenant_with_expected_secret_type( + tenant_with_tls( + config, + cert_manager_tls_config( + false, + negative_secret_name(config, case), + None, + None, + false, + TlsRotationStrategy::Rollout, + CaTrustConfig::default(), + ), + ), + OPAQUE_SECRET_TYPE, + ), + NegativeTlsCase::MissingCaForInternodeHttps => tenant_with_tls( + config, + cert_manager_tls_config( + false, + negative_secret_name(config, case), + None, + None, + true, + TlsRotationStrategy::Rollout, + CaTrustConfig::default(), + ), + ), + NegativeTlsCase::MissingIssuer => tenant_with_tls( + config, + cert_manager_tls_config( + true, + negative_secret_name(config, case), + Some(format!("{}-missing-issuer-cert", config.tenant_name)), + Some(issuer_ref(MISSING_ISSUER_NAME)), + true, + TlsRotationStrategy::Rollout, + CaTrustConfig::default(), + ), + ), + NegativeTlsCase::PendingCertificate => tenant_with_tls( + config, + cert_manager_tls_config( + true, + negative_secret_name(config, case), + Some(format!("{}-pending-cert", config.tenant_name)), + Some(issuer_ref(PENDING_ISSUER_NAME)), + true, + TlsRotationStrategy::Rollout, + CaTrustConfig::default(), + ), + ), + NegativeTlsCase::HotReloadUnsupported => tenant_with_tls( + config, + cert_manager_tls_config( + false, + negative_secret_name(config, case), + None, + None, + false, + TlsRotationStrategy::HotReload, + CaTrustConfig::default(), + ), + ), + } +} + +fn tenant_with_expected_secret_type(mut tenant: Tenant, secret_type: &str) -> Tenant { + let cert_manager = tenant + .spec + .tls + .as_mut() + .and_then(|tls| tls.cert_manager.as_mut()) + .expect("negative TLS fixture should configure cert-manager TLS"); + cert_manager.secret_type = Some(secret_type.to_string()); + tenant +} + +fn negative_secret_name(config: &E2eConfig, case: NegativeTlsCase) -> String { + let suffix = match case { + NegativeTlsCase::MissingTlsCrt => "missing-crt", + NegativeTlsCase::MissingTlsKey => "missing-key", + NegativeTlsCase::MissingCaForInternodeHttps => "missing-ca", + NegativeTlsCase::MissingIssuer => "missing-issuer", + NegativeTlsCase::PendingCertificate => "pending", + NegativeTlsCase::HotReloadUnsupported => "hot-reload", + }; + format!("{}-{suffix}-tls", config.tenant_name) +} + +fn negative_tls_secret_fixture( + case: NegativeTlsCase, +) -> Option<(&'static str, BTreeMap)> { + match case { + NegativeTlsCase::MissingTlsCrt => Some(( + OPAQUE_SECRET_TYPE, + secret_data( + None, + Some(REDACTED_FIXTURE_BYTES), + Some(REDACTED_FIXTURE_BYTES), + ), + )), + NegativeTlsCase::MissingTlsKey => Some(( + OPAQUE_SECRET_TYPE, + secret_data( + Some(REDACTED_FIXTURE_BYTES), + None, + Some(REDACTED_FIXTURE_BYTES), + ), + )), + NegativeTlsCase::MissingCaForInternodeHttps + | NegativeTlsCase::MissingIssuer + | NegativeTlsCase::PendingCertificate + | NegativeTlsCase::HotReloadUnsupported => None, + } +} + +struct GeneratedTlsMaterial { + cert: Vec, + key: Vec, + ca: Vec, + _dir: TempDir, +} + +fn generate_external_tls_material(config: &E2eConfig) -> Result { + let dns_names = external_tls_certificate_dns_names(config); + generate_ca_signed_tls_material(&config.tenant_name, &dns_names) +} + +fn generate_missing_ca_tls_material(config: &E2eConfig) -> Result { + let tenant = negative_case_tenant(config, NegativeTlsCase::MissingCaForInternodeHttps); + let dns_names = tls_certificate_dns_names(config, &tenant); + generate_self_signed_tls_material(&config.tenant_name, &dns_names) +} + +fn generate_ca_signed_tls_material( + common_name: &str, + dns_names: &[String], +) -> Result { + let dir = tempfile::tempdir()?; + let ca_key_path = dir.path().join("ca.key"); + let ca_cert_path = dir.path().join("ca.crt"); + let key_path = dir.path().join("tls.key"); + let csr_path = dir.path().join("tls.csr"); + let cert_path = dir.path().join("tls.crt"); + let leaf_ext_path = dir.path().join("leaf.ext"); + let san = subject_alt_name(dns_names); + + openssl_ca_certificate_command(dir.path(), &ca_key_path, &ca_cert_path, common_name) + .run_checked()?; + openssl_leaf_csr_command(dir.path(), &key_path, &csr_path, common_name).run_checked()?; + fs::write( + &leaf_ext_path, + format!( + "basicConstraints=critical,CA:FALSE\n\ + keyUsage=critical,digitalSignature,keyEncipherment\n\ + extendedKeyUsage=serverAuth,clientAuth\n\ + {san}\n" + ), + ) + .with_context(|| format!("write {}", leaf_ext_path.display()))?; + openssl_sign_leaf_command( + dir.path(), + &csr_path, + &ca_cert_path, + &ca_key_path, + &cert_path, + &leaf_ext_path, + ) + .run_checked()?; + + let cert = fs::read(&cert_path).with_context(|| format!("read {}", cert_path.display()))?; + let key = fs::read(&key_path).with_context(|| format!("read {}", key_path.display()))?; + let ca = fs::read(&ca_cert_path).with_context(|| format!("read {}", ca_cert_path.display()))?; + Ok(GeneratedTlsMaterial { + cert, + key, + ca, + _dir: dir, + }) +} + +fn generate_self_signed_tls_material( + common_name: &str, + dns_names: &[String], +) -> Result { + let dir = tempfile::tempdir()?; + let key_path = dir.path().join("tls.key"); + let cert_path = dir.path().join("tls.crt"); + let san = subject_alt_name(dns_names); + + openssl_self_signed_command(dir.path(), &key_path, &cert_path, common_name, &san) + .run_checked()?; + + let cert = fs::read(&cert_path).with_context(|| format!("read {}", cert_path.display()))?; + let key = fs::read(&key_path).with_context(|| format!("read {}", key_path.display()))?; + Ok(GeneratedTlsMaterial { + ca: cert.clone(), + cert, + key, + _dir: dir, + }) +} + +fn subject_alt_name(dns_names: &[String]) -> String { + format!( + "subjectAltName={}", + dns_names + .iter() + .map(|name| format!("DNS:{name}")) + .collect::>() + .join(",") + ) +} + +fn openssl_ca_certificate_command( + cwd: &Path, + key_path: &Path, + cert_path: &Path, + common_name: &str, +) -> CommandSpec { + CommandSpec::new("openssl") + .args(["req", "-x509", "-newkey", "rsa:2048", "-nodes", "-keyout"]) + .arg(key_path.display().to_string()) + .args(["-out"]) + .arg(cert_path.display().to_string()) + .args(["-days", "1", "-sha256", "-subj"]) + .arg(format!("/CN={common_name}-ca")) + .args(["-addext", "basicConstraints=critical,CA:TRUE"]) + .args(["-addext", "keyUsage=critical,keyCertSign,cRLSign"]) + .cwd(cwd) +} + +fn openssl_leaf_csr_command( + cwd: &Path, + key_path: &Path, + csr_path: &Path, + common_name: &str, +) -> CommandSpec { + CommandSpec::new("openssl") + .args(["req", "-newkey", "rsa:2048", "-nodes", "-keyout"]) + .arg(key_path.display().to_string()) + .args(["-out"]) + .arg(csr_path.display().to_string()) + .args(["-subj"]) + .arg(format!("/CN={common_name}")) + .cwd(cwd) +} + +fn openssl_sign_leaf_command( + cwd: &Path, + csr_path: &Path, + ca_cert_path: &Path, + ca_key_path: &Path, + cert_path: &Path, + leaf_ext_path: &Path, +) -> CommandSpec { + CommandSpec::new("openssl") + .args(["x509", "-req", "-in"]) + .arg(csr_path.display().to_string()) + .args(["-CA"]) + .arg(ca_cert_path.display().to_string()) + .args(["-CAkey"]) + .arg(ca_key_path.display().to_string()) + .args(["-CAcreateserial", "-out"]) + .arg(cert_path.display().to_string()) + .args(["-days", "1", "-sha256", "-extfile"]) + .arg(leaf_ext_path.display().to_string()) + .cwd(cwd) +} + +fn openssl_self_signed_command( + cwd: &Path, + key_path: &Path, + cert_path: &Path, + common_name: &str, + san: &str, +) -> CommandSpec { + CommandSpec::new("openssl") + .args(["req", "-x509", "-newkey", "rsa:2048", "-nodes", "-keyout"]) + .arg(key_path.display().to_string()) + .args(["-out"]) + .arg(cert_path.display().to_string()) + .args(["-days", "1", "-subj"]) + .arg(format!("/CN={common_name}")) + .args(["-addext"]) + .arg(san.to_string()) + .cwd(cwd) +} + +fn apply_external_tls_material(config: &E2eConfig, material: &GeneratedTlsMaterial) -> Result<()> { + let manifests = external_tls_secret_manifests_from_material(config, material)?; + apply_yaml(config, manifests.tls_secret_manifest)?; + apply_yaml(config, manifests.ca_secret_manifest)?; + Ok(()) +} + +fn external_tls_secret_manifest( + config: &E2eConfig, + name: &str, + secret_type: &str, + data: BTreeMap, +) -> Result { + tls_secret_manifest_with_labels( + &config.test_namespace, + name, + secret_type, + data, + tenant_watch_labels(config), + ) +} + +fn tls_secret_manifest( + namespace: &str, + name: &str, + secret_type: &str, + data: BTreeMap, +) -> Result { + tls_secret_manifest_with_labels(namespace, name, secret_type, data, BTreeMap::new()) +} + +fn tls_secret_manifest_with_labels( + namespace: &str, + name: &str, + secret_type: &str, + data: BTreeMap, + labels: BTreeMap, +) -> Result { + let labels = (!labels.is_empty()).then_some(labels); + let secret = corev1::Secret { + metadata: metav1::ObjectMeta { + name: Some(name.to_string()), + namespace: Some(namespace.to_string()), + labels, + ..Default::default() + }, + type_: Some(secret_type.to_string()), + data: Some(data), + ..Default::default() + }; + Ok(serde_yaml_ng::to_string(&secret)?) +} + +fn secret_data( + cert: Option<&[u8]>, + key: Option<&[u8]>, + ca: Option<&[u8]>, +) -> BTreeMap { + let mut data = BTreeMap::new(); + if let Some(cert) = cert { + data.insert("tls.crt".to_string(), ByteString(cert.to_vec())); + } + if let Some(key) = key { + data.insert("tls.key".to_string(), ByteString(key.to_vec())); + } + if let Some(ca) = ca { + data.insert("ca.crt".to_string(), ByteString(ca.to_vec())); + } + data +} + +fn dynamic_certificate_ready(certificate: &DynamicObject) -> bool { + status_conditions(&certificate.data) + .iter() + .any(|condition| { + condition.get("type").and_then(Value::as_str) == Some("Ready") + && condition.get("status").and_then(Value::as_str) == Some("True") + }) +} + +fn status_conditions(data: &Value) -> Vec<&Value> { + data.pointer("/status/conditions") + .and_then(Value::as_array) + .map(|conditions| conditions.iter().collect()) + .unwrap_or_default() +} + +fn tenant_tls_reason(tenant: &Tenant) -> Option { + tenant + .status + .as_ref() + .and_then(|status| { + status + .conditions + .iter() + .find(|condition| condition.type_ == "TlsReady" || condition.type_ == "TLSReady") + .map(|condition| condition.reason.clone()) + }) + .or_else(|| { + tenant + .status + .as_ref() + .and_then(|status| status.certificates.tls.as_ref()) + .and_then(|tls| tls.last_error_reason.clone()) + }) +} + +#[cfg(test)] +mod tests { + use super::{ + external_secret_name, external_secret_tenant_manifest, managed_certificate_tenant_manifest, + }; + use crate::framework::assertions; + use crate::framework::config::E2eConfig; + + #[test] + fn tenant_manifests_do_not_embed_pem_material() { + let config = E2eConfig::defaults(); + + assertions::require_no_secret_material( + "managed manifest", + &managed_certificate_tenant_manifest(&config).expect("managed manifest"), + ) + .expect("managed manifest should not expose secrets"); + assertions::require_no_secret_material( + "external manifest", + &external_secret_tenant_manifest(&config).expect("external manifest"), + ) + .expect("external manifest should not expose secrets"); + } + + #[test] + fn external_secret_name_is_stable() { + let config = E2eConfig::defaults(); + + assert_eq!(external_secret_name(&config), "e2e-tenant-external-tls"); + } +} diff --git a/e2e/src/framework/mod.rs b/e2e/src/framework/mod.rs index fe443bf..d7d557d 100644 --- a/e2e/src/framework/mod.rs +++ b/e2e/src/framework/mod.rs @@ -14,6 +14,7 @@ pub mod artifacts; pub mod assertions; +pub mod cert_manager_tls; pub mod command; pub mod config; pub mod console_client; diff --git a/e2e/src/framework/storage.rs b/e2e/src/framework/storage.rs index 8bd5e89..8af1269 100644 --- a/e2e/src/framework/storage.rs +++ b/e2e/src/framework/storage.rs @@ -22,6 +22,50 @@ use crate::framework::{ pub const RUSTFS_RUN_AS_UID: u32 = 10001; +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LocalStorageLayout { + pub storage_class: String, + pub pv_name_prefix: String, + pub volume_path_prefix: String, + pub pv_count: usize, +} + +impl LocalStorageLayout { + pub fn from_config(config: &E2eConfig) -> Self { + Self { + storage_class: config.storage_class.clone(), + pv_name_prefix: format!("{}-pv", config.cluster_name), + volume_path_prefix: "/mnt/data".to_string(), + pv_count: config.pv_count, + } + } + + pub fn new( + storage_class: impl Into, + pv_name_prefix: impl Into, + volume_path_prefix: impl Into, + pv_count: usize, + ) -> Self { + Self { + storage_class: storage_class.into(), + pv_name_prefix: pv_name_prefix.into(), + volume_path_prefix: volume_path_prefix.into(), + pv_count, + } + } + + fn pv_name(&self, index: usize) -> String { + format!("{}-{index}", self.pv_name_prefix) + } + + fn volume_path(&self, index: usize) -> String { + format!( + "{}/vol{index}", + self.volume_path_prefix.trim_end_matches('/') + ) + } +} + pub fn worker_node_names(config: &E2eConfig) -> Vec { (1..=KIND_WORKER_COUNT) .map(|index| match index { @@ -36,10 +80,17 @@ pub fn volume_path(index: usize) -> String { } pub fn volume_directory_commands(config: &E2eConfig) -> Vec { + volume_directory_commands_for_layout(config, &LocalStorageLayout::from_config(config)) +} + +pub fn volume_directory_commands_for_layout( + config: &E2eConfig, + layout: &LocalStorageLayout, +) -> Vec { let mut commands = Vec::new(); for node in worker_node_names(config) { - for index in 1..=config.pv_count { - let path = volume_path(index); + for index in 1..=layout.pv_count { + let path = layout.volume_path(index); commands.push(CommandSpec::new("docker").args([ "exec".to_string(), node.clone(), @@ -61,6 +112,13 @@ pub fn volume_directory_commands(config: &E2eConfig) -> Vec { } pub fn local_storage_manifest(config: &E2eConfig) -> String { + local_storage_manifest_for_layout(config, &LocalStorageLayout::from_config(config)) +} + +pub fn local_storage_manifest_for_layout( + _config: &E2eConfig, + layout: &LocalStorageLayout, +) -> String { let mut manifest = format!( r#"--- apiVersion: storage.k8s.io/v1 @@ -70,17 +128,17 @@ metadata: provisioner: kubernetes.io/no-provisioner volumeBindingMode: WaitForFirstConsumer "#, - storage_class = config.storage_class + storage_class = layout.storage_class ); - for index in 1..=config.pv_count { + for index in 1..=layout.pv_count { let worker_group = ((index - 1) % KIND_WORKER_COUNT) + 1; manifest.push_str(&format!( r#"--- apiVersion: v1 kind: PersistentVolume metadata: - name: {cluster}-pv-{index} + name: {pv_name} spec: capacity: storage: 10Gi @@ -100,10 +158,9 @@ spec: values: - storage-{worker_group} "#, - cluster = config.cluster_name, - index = index, - storage_class = config.storage_class, - path = volume_path(index), + pv_name = layout.pv_name(index), + storage_class = layout.storage_class, + path = layout.volume_path(index), worker_group = worker_group )); } @@ -112,12 +169,19 @@ spec: } pub fn prepare_local_storage(config: &E2eConfig) -> Result<()> { - for command in volume_directory_commands(config) { + prepare_local_storage_with_layout(config, &LocalStorageLayout::from_config(config)) +} + +pub fn prepare_local_storage_with_layout( + config: &E2eConfig, + layout: &LocalStorageLayout, +) -> Result<()> { + for command in volume_directory_commands_for_layout(config, layout) { command.run_checked()?; } Kubectl::new(config) - .apply_yaml_command(local_storage_manifest(config)) + .apply_yaml_command(local_storage_manifest_for_layout(config, layout)) .run_checked()?; Ok(()) } diff --git a/e2e/tests/cert_manager_tls.rs b/e2e/tests/cert_manager_tls.rs new file mode 100644 index 0000000..9d47a9b --- /dev/null +++ b/e2e/tests/cert_manager_tls.rs @@ -0,0 +1,1526 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use anyhow::{Context, Result, ensure}; +use k8s_openapi::api::{apps::v1::StatefulSet, core::v1 as corev1}; +use operator::types::v1alpha1::{ + status::certificate::SecretStatusRef, + tenant::Tenant, + tls::{ + CaTrustSource, DEFAULT_TLS_MOUNT_PATH, RUSTFS_CA_FILE, RUSTFS_TLS_CERT_FILE, + RUSTFS_TLS_KEY_FILE, SecretKeyReference, TlsPlan, + }, +}; +use rustfs_operator_e2e::{ + cases::cert_manager_tls, + framework::{ + artifacts::ArtifactCollector, + assertions, cert_manager_tls as tls_e2e, + command::CommandSpec, + config::{E2eConfig, KIND_WORKER_COUNT}, + kube_client, live, + }, +}; +use std::collections::BTreeSet; +use std::time::Duration; +use tempfile::TempDir; + +#[test] +fn cert_manager_case_inventory_matches_executable_tests() { + let names = cert_manager_tls::cases() + .into_iter() + .map(|case| case.name) + .collect::>(); + + assert_eq!( + names, + vec![ + "cert_manager_managed_certificate_reaches_tls_ready_and_https_wiring", + "cert_manager_external_secret_reaches_tls_ready_and_rolls_on_secret_hash", + "cert_manager_rejects_secret_missing_tls_crt", + "cert_manager_rejects_secret_missing_tls_key", + "cert_manager_rejects_secret_missing_ca_for_internode_https", + "cert_manager_rejects_missing_issuer_for_managed_certificate", + "cert_manager_reports_pending_certificate_not_ready", + "cert_manager_rejects_hot_reload", + "cert_manager_artifacts_do_not_expose_secret_material", + ] + ); +} + +#[test] +fn managed_certificate_tenant_manifest_wires_ca_trust_without_secret_material() -> Result<()> { + let config = E2eConfig::defaults(); + let manifest = tls_e2e::managed_certificate_tenant_manifest(&config)?; + + ensure!(manifest.contains("mode: certManager")); + ensure!(manifest.contains("manageCertificate: true")); + ensure!(manifest.contains("issuerRef:")); + ensure!(manifest.contains("source: CertificateSecretCa")); + ensure!(manifest.contains("enableInternodeHttps: true")); + assertions::require_no_secret_material("managed cert-manager Tenant manifest", &manifest)?; + + Ok(()) +} + +#[test] +fn external_secret_tenant_manifest_uses_shared_secret_and_rollout_strategy() -> Result<()> { + let config = E2eConfig::defaults(); + let manifest = tls_e2e::external_secret_tenant_manifest(&config)?; + + ensure!(manifest.contains("mode: certManager")); + ensure!(manifest.contains("manageCertificate: false")); + ensure!(manifest.contains("secretName: e2e-tenant-external-tls")); + ensure!(manifest.contains("source: SecretRef")); + ensure!(manifest.contains("caSecretRef:")); + ensure!(manifest.contains("rotationStrategy: Rollout")); + assertions::require_no_secret_material("external Secret Tenant manifest", &manifest)?; + + Ok(()) +} + +#[test] +fn external_secret_manifests_carry_tenant_watch_label_for_initial_create_and_rotation() -> Result<()> +{ + let config = tls_e2e::external_secret_case_config(&E2eConfig::defaults()); + + let initial = tls_e2e::external_tls_secret_manifests(&config)?; + assert_secret_manifest_tenant_watch_label( + "initial external TLS Secret", + &initial.tls_secret_manifest, + &config.tenant_name, + )?; + assert_secret_manifest_tenant_watch_label( + "initial external CA Secret", + &initial.ca_secret_manifest, + &config.tenant_name, + )?; + + let rotated = tls_e2e::external_tls_secret_manifests(&config)?; + assert_secret_manifest_tenant_watch_label( + "rotated external TLS Secret", + &rotated.tls_secret_manifest, + &config.tenant_name, + )?; + assert_secret_manifest_tenant_watch_label( + "rotated external CA Secret", + &rotated.ca_secret_manifest, + &config.tenant_name, + )?; + + Ok(()) +} + +#[test] +fn positive_cert_manager_tls_timeout_uses_named_longer_readiness_window() -> Result<()> { + let default_config = E2eConfig::defaults(); + ensure!(default_config.timeout < Duration::from_secs(600)); + ensure!( + tls_e2e::positive_cert_manager_tls_timeout(&default_config) >= Duration::from_secs(600), + "positive cert-manager TLS waits should use at least a 600s readiness window" + ); + + let overridden = E2eConfig::from_env_with(|name| match name { + "RUSTFS_E2E_TIMEOUT_SECONDS" => Some("900".to_string()), + _ => None, + }); + ensure!( + tls_e2e::positive_cert_manager_tls_timeout(&overridden) == Duration::from_secs(900), + "positive cert-manager TLS waits should preserve explicit longer timeouts" + ); + + Ok(()) +} + +#[test] +fn positive_tls_live_case_configs_are_isolated_from_smoke_namespace_tenant_and_storage() +-> Result<()> { + let smoke = E2eConfig::defaults(); + let managed = tls_e2e::managed_certificate_case_config(&smoke); + let external = tls_e2e::external_secret_case_config(&smoke); + + for (case_name, config, manifest) in [ + ( + "managed cert-manager Tenant", + &managed, + tls_e2e::managed_certificate_tenant_manifest(&managed)?, + ), + ( + "external Secret Tenant", + &external, + tls_e2e::external_secret_tenant_manifest(&external)?, + ), + ] { + ensure!( + config.test_namespace != smoke.test_namespace, + "{case_name} should not reuse the smoke namespace {}", + smoke.test_namespace + ); + ensure!( + config.tenant_name != smoke.tenant_name, + "{case_name} should not mutate the smoke Tenant {}", + smoke.tenant_name + ); + ensure!( + config.storage_class != smoke.storage_class, + "{case_name} should not bind the smoke storage class {}", + smoke.storage_class + ); + ensure!( + config.pv_count > 0, + "{case_name} should request positive isolated PV capacity, got {}", + config.pv_count + ); + ensure!( + manifest.contains(&format!("namespace: {}", config.test_namespace)), + "{case_name} manifest should use isolated namespace {}, got:\n{manifest}", + config.test_namespace + ); + ensure!( + manifest.contains(&format!("name: {}", config.tenant_name)), + "{case_name} manifest should use isolated Tenant {}, got:\n{manifest}", + config.tenant_name + ); + ensure!( + manifest.contains(&format!("storageClassName: {}", config.storage_class)), + "{case_name} manifest should use isolated StorageClass {}, got:\n{manifest}", + config.storage_class + ); + } + + ensure!(managed.test_namespace != external.test_namespace); + ensure!(managed.tenant_name != external.tenant_name); + ensure!(managed.storage_class != external.storage_class); + + Ok(()) +} + +#[test] +fn positive_tls_storage_layouts_use_dedicated_pv_names_and_paths() -> Result<()> { + let smoke = E2eConfig::defaults(); + let managed_config = tls_e2e::managed_certificate_case_config(&smoke); + let managed_layout = tls_e2e::managed_certificate_storage_layout(&managed_config); + let external_config = tls_e2e::external_secret_case_config(&smoke); + let external_layout = tls_e2e::external_secret_storage_layout(&external_config); + + for (case_name, config, layout, tenant, path_fragment, pv_name_fragment) in [ + ( + "managed cert-manager Tenant", + &managed_config, + managed_layout, + tls_e2e::managed_certificate_tenant(&managed_config), + "/mnt/data/cert-manager-managed/vol1", + "name: rustfs-e2e-cert-manager-managed-pv-1", + ), + ( + "external Secret Tenant", + &external_config, + external_layout, + tls_e2e::external_secret_tenant(&external_config), + "/mnt/data/cert-manager-external/vol1", + "name: rustfs-e2e-cert-manager-external-pv-1", + ), + ] { + let manifest = rustfs_operator_e2e::framework::storage::local_storage_manifest_for_layout( + &smoke, &layout, + ); + let first_command = + rustfs_operator_e2e::framework::storage::volume_directory_commands_for_layout( + &smoke, &layout, + ) + .into_iter() + .next() + .expect("layout should create volume directory commands") + .display(); + let required_pvc_count = tenant + .spec + .pools + .iter() + .map(|pool| { + assert!( + pool.servers > 0, + "{case_name} pool server count should be positive" + ); + assert!( + pool.persistence.volumes_per_server > 0, + "{case_name} volumes_per_server should be positive" + ); + pool.servers as usize * pool.persistence.volumes_per_server as usize + }) + .sum::(); + let rendered_pv_count = manifest.matches("kind: PersistentVolume").count(); + + ensure!( + layout.storage_class == config.storage_class, + "{case_name} layout StorageClass should match Tenant config" + ); + ensure!( + layout.pv_count == config.pv_count, + "{case_name} layout PV count should match isolated config PV count" + ); + ensure!( + config.pv_count >= required_pvc_count, + "{case_name} isolated config should create at least one PV per Tenant PVC: pv_count={} required_pvc_count={required_pvc_count}", + config.pv_count + ); + ensure!( + rendered_pv_count >= required_pvc_count, + "{case_name} rendered PV manifest should cover Tenant PVCs: rendered_pv_count={rendered_pv_count} required_pvc_count={required_pvc_count}" + ); + ensure!( + rendered_pv_count == layout.pv_count, + "{case_name} rendered PV count should match layout PV count" + ); + + let per_worker_pv_counts = (1..=KIND_WORKER_COUNT) + .map(|worker_group| { + manifest + .matches(&format!(" - storage-{worker_group}\n")) + .count() + }) + .collect::>(); + ensure!( + per_worker_pv_counts.iter().sum::() == rendered_pv_count, + "{case_name} should assign every PV to a storage worker: per_worker_pv_counts={per_worker_pv_counts:?} rendered_pv_count={rendered_pv_count}" + ); + for pool in &tenant.spec.pools { + let servers = pool.servers as usize; + let volumes_per_server = pool.persistence.volumes_per_server as usize; + let schedulable_pods = per_worker_pv_counts + .iter() + .map(|pv_count| pv_count / volumes_per_server) + .sum::(); + + ensure!( + per_worker_pv_counts + .iter() + .all(|pv_count| *pv_count >= volumes_per_server), + "{case_name} should allocate at least {volumes_per_server} PVs on every storage worker: per_worker_pv_counts={per_worker_pv_counts:?}" + ); + ensure!( + schedulable_pods >= servers, + "{case_name} PV topology should schedule {servers} pods with {volumes_per_server} PVCs each across {KIND_WORKER_COUNT} workers: per_worker_pv_counts={per_worker_pv_counts:?} schedulable_pods={schedulable_pods}" + ); + } + + ensure!( + manifest.contains(&format!("storageClassName: {}", config.storage_class)), + "{case_name} PV manifest should use isolated StorageClass {}, got:\n{manifest}", + config.storage_class + ); + ensure!( + manifest.contains(path_fragment), + "{case_name} PV manifest should use isolated local path {path_fragment}, got:\n{manifest}" + ); + ensure!( + manifest.contains(pv_name_fragment), + "{case_name} PV manifest should use isolated PV name {pv_name_fragment}, got:\n{manifest}" + ); + ensure!( + !manifest.contains("path: /mnt/data/vol1"), + "{case_name} PV manifest should not reuse smoke local volume paths, got:\n{manifest}" + ); + ensure!( + !manifest.contains("name: rustfs-e2e-pv-1"), + "{case_name} PV manifest should not reuse smoke PV names, got:\n{manifest}" + ); + ensure!( + first_command.contains(path_fragment), + "{case_name} docker directory setup should prepare isolated path {path_fragment}, got {first_command}" + ); + } + + Ok(()) +} + +#[test] +fn positive_tls_fixtures_use_minimal_four_volume_https_erasure_sets() -> Result<()> { + let smoke = E2eConfig::defaults(); + + let managed_config = tls_e2e::managed_certificate_case_config(&smoke); + let managed_tenant = tls_e2e::managed_certificate_tenant(&managed_config); + let managed_tls_plan = tls_e2e::sample_tls_plan( + "sha256:e2e-test", + tls_e2e::managed_secret_name(&managed_config), + ); + assert_positive_tls_fixture_uses_minimal_four_volume_https_erasure_set( + "managed cert-manager Tenant", + &managed_config, + &managed_tenant, + &managed_tls_plan, + )?; + + let external_config = tls_e2e::external_secret_case_config(&smoke); + let external_tenant = tls_e2e::external_secret_tenant(&external_config); + let external_tls_plan = TlsPlan::rollout( + DEFAULT_TLS_MOUNT_PATH.to_string(), + "sha256:e2e-test".to_string(), + tls_e2e::external_secret_name(&external_config), + None, + Some(SecretKeyReference { + name: tls_e2e::external_ca_secret_name(&external_config), + key: "ca.crt".to_string(), + }), + None, + true, + false, + false, + None, + ); + assert_positive_tls_fixture_uses_minimal_four_volume_https_erasure_set( + "external Secret Tenant", + &external_config, + &external_tenant, + &external_tls_plan, + )?; + + Ok(()) +} + +#[test] +fn cert_manager_tenant_fixtures_use_default_tls_mount_path() -> Result<()> { + let config = E2eConfig::defaults(); + + for (case_name, tenant, manifest) in [ + ( + "managed cert-manager Tenant", + tls_e2e::managed_certificate_tenant(&config), + tls_e2e::managed_certificate_tenant_manifest(&config)?, + ), + ( + "external Secret Tenant", + tls_e2e::external_secret_tenant(&config), + tls_e2e::external_secret_tenant_manifest(&config)?, + ), + ] { + let tls = tenant + .spec + .tls + .as_ref() + .expect("cert-manager fixture should enable TLS"); + + ensure!( + tls.mount_path == DEFAULT_TLS_MOUNT_PATH, + "{case_name} should use default TLS mount path {DEFAULT_TLS_MOUNT_PATH}, got {:?}", + tls.mount_path + ); + ensure!( + manifest.contains(&format!("mountPath: {DEFAULT_TLS_MOUNT_PATH}")), + "{case_name} manifest should contain the default TLS mount path, got:\n{manifest}" + ); + ensure!( + !manifest.contains("mountPath: ''") && !manifest.contains("mountPath: \"\""), + "{case_name} manifest should not contain an empty TLS mountPath, got:\n{manifest}" + ); + } + + Ok(()) +} + +#[test] +fn generated_tls_workload_assertions_cover_https_mounts_services_and_rollout_hash() -> Result<()> { + let config = E2eConfig::defaults(); + let tenant = tls_e2e::managed_certificate_tenant(&config); + let pool = tenant + .spec + .pools + .first() + .expect("managed certificate fixture should have a pool"); + let tls_plan = + tls_e2e::sample_tls_plan("sha256:e2e-test", tls_e2e::managed_secret_name(&config)); + let managed_ca_secret_ref = SecretStatusRef { + name: tls_e2e::managed_secret_name(&config), + key: Some("ca.crt".to_string()), + resource_version: None, + }; + + let statefulset = tenant.new_statefulset_with_tls_plan(pool, &tls_plan)?; + let io_service = tenant.new_io_service_with_tls_plan(&tls_plan); + let headless_service = tenant.new_headless_service_with_tls_plan(&tls_plan); + + assertions::require_tls_statefulset_https_wiring( + &statefulset, + "sha256:e2e-test", + &tls_e2e::managed_secret_name(&config), + &managed_ca_secret_ref, + )?; + assertions::require_tls_service_https_wiring(&io_service)?; + assertions::require_tls_service_https_wiring(&headless_service)?; + + Ok(()) +} + +#[test] +fn generated_tls_workload_assertions_accept_explicit_ca_secret_ref() -> Result<()> { + let config = E2eConfig::defaults(); + let tenant = tls_e2e::external_secret_tenant(&config); + let pool = tenant + .spec + .pools + .first() + .expect("external Secret fixture should have a pool"); + let ca_secret_ref = SecretStatusRef { + name: tls_e2e::external_ca_secret_name(&config), + key: Some("ca.crt".to_string()), + resource_version: None, + }; + let tls_plan = TlsPlan::rollout( + DEFAULT_TLS_MOUNT_PATH.to_string(), + "sha256:e2e-test".to_string(), + tls_e2e::external_secret_name(&config), + None, + Some(SecretKeyReference { + name: ca_secret_ref.name.clone(), + key: ca_secret_ref + .key + .clone() + .expect("test CA status ref should include a key"), + }), + None, + true, + false, + false, + None, + ); + + let statefulset = tenant.new_statefulset_with_tls_plan(pool, &tls_plan)?; + + assertions::require_tls_statefulset_https_wiring( + &statefulset, + "sha256:e2e-test", + &tls_e2e::external_secret_name(&config), + &ca_secret_ref, + )?; + + Ok(()) +} + +#[test] +fn external_secret_tls_plan_projects_server_and_explicit_ca_into_single_tls_directory() -> Result<()> +{ + let config = E2eConfig::defaults(); + let tenant = tls_e2e::external_secret_tenant(&config); + let pool = tenant + .spec + .pools + .first() + .expect("external Secret fixture should have a pool"); + let tls_plan = TlsPlan::rollout( + DEFAULT_TLS_MOUNT_PATH.to_string(), + "sha256:e2e-test".to_string(), + tls_e2e::external_secret_name(&config), + None, + Some(SecretKeyReference { + name: tls_e2e::external_ca_secret_name(&config), + key: "ca.crt".to_string(), + }), + None, + true, + false, + false, + None, + ); + + let statefulset = tenant.new_statefulset_with_tls_plan(pool, &tls_plan)?; + let pod_spec = statefulset + .spec + .as_ref() + .context("StatefulSet should have spec")? + .template + .spec + .as_ref() + .context("StatefulSet pod template should have spec")?; + let tls_volume = pod_spec + .volumes + .as_deref() + .unwrap_or_default() + .iter() + .find(|volume| volume.name == "rustfs-tls-server") + .context("TLS material should use the rustfs-tls-server volume")?; + let rustfs_container = pod_spec + .containers + .iter() + .find(|container| container.name == "rustfs") + .context("rustfs container should exist")?; + let tls_mount = rustfs_container + .volume_mounts + .as_deref() + .unwrap_or_default() + .iter() + .find(|mount| mount.name == "rustfs-tls-server") + .context("rustfs container should mount the projected TLS material volume")?; + + ensure!( + projected_secret_item( + tls_volume, + &tls_e2e::external_secret_name(&config), + "tls.crt", + RUSTFS_TLS_CERT_FILE, + ), + "projected TLS material volume should include server tls.crt" + ); + ensure!( + projected_secret_item( + tls_volume, + &tls_e2e::external_secret_name(&config), + "tls.key", + RUSTFS_TLS_KEY_FILE, + ), + "projected TLS material volume should include server tls.key" + ); + ensure!( + projected_secret_item( + tls_volume, + &tls_e2e::external_ca_secret_name(&config), + "ca.crt", + RUSTFS_CA_FILE, + ), + "projected TLS material volume should include explicit CA SecretRef ca.crt" + ); + ensure!( + tls_mount.mount_path == DEFAULT_TLS_MOUNT_PATH, + "projected TLS material should mount the whole TLS directory, got {}", + tls_mount.mount_path + ); + ensure!( + tls_mount.sub_path.is_none(), + "projected TLS material should not rely on subPath file mounts" + ); + ensure!(tls_mount.read_only == Some(true)); + ensure!( + !pod_spec + .volumes + .as_deref() + .unwrap_or_default() + .iter() + .any(|volume| volume.name == "rustfs-tls-ca"), + "explicit CA SecretRef should be projected into the TLS directory instead of mounted as a separate subPath volume" + ); + + Ok(()) +} + +#[test] +fn external_secret_tls_sans_cover_rendered_https_peer_hosts() -> Result<()> { + let smoke = E2eConfig::defaults(); + let config = tls_e2e::external_secret_case_config(&smoke); + let subject_alt_name = tls_e2e::external_tls_subject_alt_name(&config); + let rotated_subject_alt_name = tls_e2e::external_tls_rotation_subject_alt_name(&config); + let san_names = dns_names_from_subject_alt_name(&subject_alt_name)?; + let rendered_peer_hosts = rendered_rustfs_volume_hosts(&config)?; + + ensure!( + subject_alt_name == rotated_subject_alt_name, + "external TLS Secret rotation should preserve the same SAN profile: initial={subject_alt_name} rotated={rotated_subject_alt_name}" + ); + ensure!( + san_names.contains(&config.tenant_name), + "external TLS SANs should retain the Tenant DNS name {}, got {san_names:?}", + config.tenant_name + ); + ensure!( + san_names.contains("localhost"), + "external TLS SANs should retain localhost compatibility, got {san_names:?}" + ); + ensure!( + san_names.contains("rustfs-e2e.local"), + "external TLS SANs should retain the configured service DNS rustfs-e2e.local, got {san_names:?}" + ); + ensure!( + !san_names + .iter() + .any(|name| name.contains("cert-manager-external-rotated")), + "external TLS rotation must not switch to a rotated-only DNS identity, got {san_names:?}" + ); + + for host in &rendered_peer_hosts { + ensure!( + san_names.contains(host), + "external TLS SANs should cover rendered RUSTFS_VOLUMES host {host}; sans={san_names:?} rendered_peer_hosts={rendered_peer_hosts:?}" + ); + } + + Ok(()) +} + +#[test] +fn external_secret_tls_material_uses_real_ca_chain_and_preserves_tls_guards() -> Result<()> { + let smoke = E2eConfig::defaults(); + let config = tls_e2e::external_secret_case_config(&smoke); + let manifests = tls_e2e::external_tls_secret_manifests(&config)?; + let tls_secret: corev1::Secret = serde_yaml_ng::from_str(&manifests.tls_secret_manifest)?; + let ca_secret: corev1::Secret = serde_yaml_ng::from_str(&manifests.ca_secret_manifest)?; + let tls_data = tls_secret + .data + .as_ref() + .context("external TLS Secret should contain data")?; + let ca_data = ca_secret + .data + .as_ref() + .context("external CA Secret should contain data")?; + let leaf_cert = secret_data_value(&tls_secret, "tls.crt")?; + let leaf_key = secret_data_value(&tls_secret, "tls.key")?; + let bundled_ca = secret_data_value(&tls_secret, "ca.crt")?; + let external_ca = secret_data_value(&ca_secret, "ca.crt")?; + + ensure!(tls_secret.type_.as_deref() == Some("kubernetes.io/tls")); + ensure!(ca_secret.type_.as_deref() == Some("Opaque")); + ensure!(tls_data.contains_key("tls.crt")); + ensure!(tls_data.contains_key("tls.key")); + ensure!(tls_data.contains_key("ca.crt")); + ensure!(ca_data.contains_key("ca.crt")); + ensure!( + !leaf_key.is_empty(), + "external TLS Secret should include a private key" + ); + ensure!( + leaf_cert != external_ca, + "external positive TLS fixture should not reuse the leaf certificate as ca.crt" + ); + ensure!( + bundled_ca == external_ca, + "server TLS Secret ca.crt and explicit external CA Secret ca.crt should carry the same CA bundle" + ); + + require_certificate_is_ca(external_ca)?; + require_certificate_verifies_with_ca(leaf_cert, external_ca)?; + + let san_names = certificate_dns_sans(leaf_cert)?; + for host in rendered_rustfs_volume_hosts(&config)? { + ensure!( + san_names.contains(&host), + "external TLS leaf SANs should cover rendered RUSTFS_VOLUMES peer host {host}; sans={san_names:?}" + ); + } + + let tenant = tls_e2e::external_secret_tenant(&config); + let tls = tenant + .spec + .tls + .as_ref() + .context("external Secret Tenant should enable TLS")?; + let cert_manager = tls + .cert_manager + .as_ref() + .context("external Secret Tenant should use cert-manager TLS mode")?; + let ca_trust = cert_manager + .ca_trust + .as_ref() + .context("external Secret Tenant should configure CA trust")?; + + ensure!( + !cert_manager.manage_certificate, + "external Secret fixture should keep using user-provided TLS Secrets" + ); + ensure!( + tls.enable_internode_https, + "external Secret fixture should keep internode HTTPS enabled" + ); + ensure!( + tls.require_san_match, + "external Secret fixture should keep SAN matching enabled" + ); + ensure!( + ca_trust.source == CaTrustSource::SecretRef, + "external Secret fixture should trust the explicit CA SecretRef, got {:?}", + ca_trust.source + ); + ensure!( + ca_trust + .ca_secret_ref + .as_ref() + .map(|reference| (reference.name.as_str(), reference.key.as_str())) + == Some((tls_e2e::external_ca_secret_name(&config).as_str(), "ca.crt")), + "external Secret fixture should point CA trust at the external CA Secret ca.crt" + ); + ensure!( + !ca_trust.trust_leaf_certificate_as_ca, + "external Secret fixture should not bypass CA-chain validation with trustLeafCertificateAsCa" + ); + + Ok(()) +} + +#[test] +fn negative_secret_fixtures_are_api_admissible_and_still_test_missing_tls_keys() -> Result<()> { + let config = E2eConfig::defaults(); + + for (case, present_key, missing_key) in [ + ( + tls_e2e::NegativeTlsCase::MissingTlsCrt, + "tls.key", + "tls.crt", + ), + ( + tls_e2e::NegativeTlsCase::MissingTlsKey, + "tls.crt", + "tls.key", + ), + ] { + let secret_manifest = tls_e2e::negative_tls_secret_manifest(&config, case)?; + let tenant_manifest = tls_e2e::negative_case_tenant_manifest(&config, case)?; + + ensure!( + secret_manifest.contains("type: Opaque"), + "negative Secret fixture should use an API-admissible Opaque Secret, got:\n{secret_manifest}" + ); + ensure!( + secret_manifest.contains(present_key), + "negative Secret fixture should include {present_key}, got:\n{secret_manifest}" + ); + ensure!( + !secret_manifest.contains(missing_key), + "negative Secret fixture should omit {missing_key}, got:\n{secret_manifest}" + ); + ensure!( + tenant_manifest.contains("secretType: Opaque"), + "negative Tenant fixture should explicitly expect Opaque so Operator validation reaches missing key checks, got:\n{tenant_manifest}" + ); + ensure!( + !secret_manifest.contains("redacted-test-fixture") + && !secret_manifest.contains("-----BEGIN") + && !secret_manifest.contains("PRIVATE KEY"), + "negative Secret fixture should not contain raw secret material, got:\n{secret_manifest}" + ); + } + + Ok(()) +} + +#[test] +fn missing_ca_fixture_uses_valid_server_cert_key_and_omits_ca_bundle() -> Result<()> { + let config = E2eConfig::defaults(); + let secret_manifest = tls_e2e::negative_tls_secret_manifest( + &config, + tls_e2e::NegativeTlsCase::MissingCaForInternodeHttps, + )?; + let tenant_manifest = tls_e2e::negative_case_tenant_manifest( + &config, + tls_e2e::NegativeTlsCase::MissingCaForInternodeHttps, + )?; + let secret: corev1::Secret = serde_yaml_ng::from_str(&secret_manifest)?; + let data = secret + .data + .as_ref() + .context("missing-ca Secret fixture should include data")?; + let cert = data + .get("tls.crt") + .context("missing-ca Secret fixture should include tls.crt")? + .0 + .as_slice(); + let key = data + .get("tls.key") + .context("missing-ca Secret fixture should include tls.key")? + .0 + .as_slice(); + + ensure!(secret.type_.as_deref() == Some("kubernetes.io/tls")); + ensure!( + pem_contains_label(cert, "CERTIFICATE"), + "missing-ca Secret fixture should use a parseable-looking PEM certificate so Operator validation reaches CaBundleMissing" + ); + ensure!( + pem_contains_label(key, "PRIVATE KEY"), + "missing-ca Secret fixture should use a parseable-looking PEM private key so Operator validation reaches CaBundleMissing" + ); + ensure!( + !data.contains_key("ca.crt"), + "missing-ca Secret fixture should omit ca.crt to exercise CaBundleMissing" + ); + ensure!( + tenant_manifest.contains("enableInternodeHttps: true"), + "missing-ca Tenant fixture should keep internode HTTPS enabled, got:\n{tenant_manifest}" + ); + ensure!( + tenant_manifest.contains("source: CertificateSecretCa"), + "missing-ca Tenant fixture should use the server Secret CA trust source, got:\n{tenant_manifest}" + ); + ensure!( + !secret_manifest.contains("redacted-test-fixture") + && !secret_manifest.contains("-----BEGIN") + && !secret_manifest.contains("PRIVATE KEY"), + "missing-ca Secret fixture manifest should not print raw secret material, got:\n{secret_manifest}" + ); + + Ok(()) +} + +#[test] +fn cert_manager_artifacts_do_not_expose_secret_material() -> Result<()> { + let command = tls_e2e::external_tls_secret_apply_command( + &E2eConfig::defaults(), + tls_e2e::external_secret_name(&E2eConfig::defaults()), + )?; + + assertions::require_no_secret_material( + "external TLS Secret apply command", + &command.display(), + )?; + ensure!( + !command.display().contains("tls.crt") && !command.display().contains("tls.key"), + "kubectl apply display should hide Secret stdin payload" + ); + + Ok(()) +} + +#[tokio::test] +#[ignore = "requires cert-manager installed in the dedicated Kind cluster; run after `make e2e-live-run`"] +async fn cert_manager_managed_certificate_reaches_tls_ready_and_https_wiring() -> Result<()> { + let base_config = E2eConfig::from_env(); + live::require_live_enabled(&base_config)?; + live::ensure_dedicated_context(&base_config)?; + let config = tls_e2e::managed_certificate_case_config(&base_config); + let positive_timeout = tls_e2e::positive_cert_manager_tls_timeout(&config); + + let result = async { + tls_e2e::apply_managed_certificate_case_resources(&config)?; + let client = kube_client::default_client().await?; + let tenant = tls_e2e::wait_for_tenant_tls_ready( + client.clone(), + &config.test_namespace, + &config.tenant_name, + positive_timeout, + ) + .await?; + tls_e2e::wait_for_certificate_ready( + client.clone(), + &config.test_namespace, + &tls_e2e::managed_certificate_name(&config), + positive_timeout, + ) + .await?; + tls_e2e::assert_live_workload_tls_wiring(client, &config, &tenant).await?; + Ok(()) + } + .await; + + collect_tls_artifacts_on_error( + &config, + "cert_manager_managed_certificate_reaches_tls_ready_and_https_wiring", + &result, + ); + result +} + +#[tokio::test] +#[ignore = "creates an external TLS Secret and waits for rollout; run after `make e2e-live-run`"] +async fn cert_manager_external_secret_reaches_tls_ready_and_rolls_on_secret_hash() -> Result<()> { + let base_config = E2eConfig::from_env(); + live::require_live_enabled(&base_config)?; + live::ensure_dedicated_context(&base_config)?; + let config = tls_e2e::external_secret_case_config(&base_config); + let positive_timeout = tls_e2e::positive_cert_manager_tls_timeout(&config); + + let result = async { + tls_e2e::apply_external_secret_case_resources(&config)?; + let client = kube_client::default_client().await?; + let tenant = tls_e2e::wait_for_tenant_tls_ready( + client.clone(), + &config.test_namespace, + &config.tenant_name, + positive_timeout, + ) + .await?; + let initial_hash = assertions::tenant_tls_observed_hash(&tenant)?; + tls_e2e::rotate_external_tls_secret(&config)?; + let rotated = tls_e2e::wait_for_tenant_tls_hash_change( + client.clone(), + &config.test_namespace, + &config.tenant_name, + &initial_hash, + positive_timeout, + ) + .await?; + tls_e2e::assert_live_workload_tls_wiring(client, &config, &rotated).await?; + Ok(()) + } + .await; + + collect_tls_artifacts_on_error( + &config, + "cert_manager_external_secret_reaches_tls_ready_and_rolls_on_secret_hash", + &result, + ); + result +} + +#[tokio::test] +#[ignore = "mutates live Tenant fixtures; run after `make e2e-live-run`"] +async fn cert_manager_rejects_secret_missing_tls_crt() -> Result<()> { + assert_negative_case_tls_reason( + tls_e2e::NegativeTlsCase::MissingTlsCrt, + "CertificateSecretMissingKey", + ) + .await +} + +#[tokio::test] +#[ignore = "mutates live Tenant fixtures; run after `make e2e-live-run`"] +async fn cert_manager_rejects_secret_missing_tls_key() -> Result<()> { + assert_negative_case_tls_reason( + tls_e2e::NegativeTlsCase::MissingTlsKey, + "CertificateSecretMissingKey", + ) + .await +} + +#[tokio::test] +#[ignore = "mutates live Tenant fixtures; run after `make e2e-live-run`"] +async fn cert_manager_rejects_secret_missing_ca_for_internode_https() -> Result<()> { + assert_negative_case_tls_reason( + tls_e2e::NegativeTlsCase::MissingCaForInternodeHttps, + "CaBundleMissing", + ) + .await +} + +#[tokio::test] +#[ignore = "requires cert-manager API and mutates live Tenant fixtures; run after `make e2e-live-run`"] +async fn cert_manager_rejects_missing_issuer_for_managed_certificate() -> Result<()> { + assert_negative_case_tls_reason( + tls_e2e::NegativeTlsCase::MissingIssuer, + "CertManagerIssuerNotFound", + ) + .await +} + +#[tokio::test] +#[ignore = "requires cert-manager API and mutates live Tenant fixtures; run after `make e2e-live-run`"] +async fn cert_manager_reports_pending_certificate_not_ready() -> Result<()> { + assert_negative_case_tls_reason( + tls_e2e::NegativeTlsCase::PendingCertificate, + "CertManagerCertificateNotReady", + ) + .await +} + +#[tokio::test] +#[ignore = "mutates live Tenant fixtures; run after `make e2e-live-run`"] +async fn cert_manager_rejects_hot_reload() -> Result<()> { + assert_negative_case_tls_reason( + tls_e2e::NegativeTlsCase::HotReloadUnsupported, + "TlsHotReloadUnsupported", + ) + .await +} + +async fn assert_negative_case_tls_reason( + case: tls_e2e::NegativeTlsCase, + reason: &str, +) -> Result<()> { + let config = E2eConfig::from_env(); + live::require_live_enabled(&config)?; + live::ensure_dedicated_context(&config)?; + + let result = async { + tls_e2e::apply_negative_case_resources(&config, case)?; + let client = kube_client::default_client().await?; + let tenant = tls_e2e::wait_for_tenant_tls_reason( + client, + &config.test_namespace, + &config.tenant_name, + reason, + config.timeout, + ) + .await?; + assertions::require_no_secret_material( + "Tenant TLS status", + &format!("{:?}", tenant.status), + )?; + Ok(()) + } + .await; + + collect_tls_artifacts_on_error(&config, case.case_name(), &result); + result +} + +fn collect_tls_artifacts_on_error(config: &E2eConfig, case_name: &str, result: &Result<()>) { + if let Err(error) = result { + let collector = ArtifactCollector::new(&config.artifacts_dir); + if let Err(artifact_error) = collector.collect_kubernetes_snapshot(case_name, config) { + eprintln!("failed to collect e2e artifacts after {error}: {artifact_error}"); + } + } +} + +fn assert_secret_manifest_tenant_watch_label( + context: &str, + manifest: &str, + expected_tenant: &str, +) -> Result<()> { + let secret: corev1::Secret = serde_yaml_ng::from_str(manifest) + .with_context(|| format!("parse {context} Secret manifest"))?; + let labels = secret + .metadata + .labels + .as_ref() + .with_context(|| format!("{context} missing labels"))?; + ensure!( + labels.get("rustfs.tenant").map(String::as_str) == Some(expected_tenant), + "{context} should carry rustfs.tenant label for Tenant {expected_tenant}" + ); + Ok(()) +} + +fn assert_positive_tls_fixture_uses_minimal_four_volume_https_erasure_set( + case_name: &str, + config: &E2eConfig, + tenant: &Tenant, + tls_plan: &TlsPlan, +) -> Result<()> { + let pool = tenant + .spec + .pools + .first() + .with_context(|| format!("{case_name} should include one pool"))?; + ensure!( + pool.servers == 4, + "{case_name} should keep the 4-server erasure set, got {}", + pool.servers + ); + ensure!( + pool.persistence.volumes_per_server == 1, + "{case_name} should use one volume per server for a 4-volume erasure set, got {}", + pool.persistence.volumes_per_server + ); + let required_pvc_count = usize::try_from(pool.servers) + .context("positive TLS pool server count should be non-negative")? + * usize::try_from(pool.persistence.volumes_per_server) + .context("positive TLS pool volumes_per_server should be non-negative")?; + ensure!( + required_pvc_count == 4, + "{case_name} should render exactly four RustFS data PVCs, got {required_pvc_count}" + ); + ensure!( + config.pv_count == required_pvc_count, + "{case_name} isolated PV count should match its 4x1 PVC count: pv_count={} required_pvc_count={required_pvc_count}", + config.pv_count + ); + + let statefulset = tenant.new_statefulset_with_tls_plan(pool, tls_plan)?; + let statefulset_spec = statefulset + .spec + .as_ref() + .context("StatefulSet should have spec")?; + let expected_headless_service = format!("{}-hl", config.tenant_name); + ensure!( + statefulset_spec.service_name.as_deref() == Some(expected_headless_service.as_str()), + "{case_name} StatefulSet should target the headless Service {expected_headless_service}, got {:?}", + statefulset_spec.service_name + ); + ensure!( + statefulset_spec + .volume_claim_templates + .as_ref() + .map(Vec::len) + == Some(1), + "{case_name} StatefulSet should render exactly one PVC template for the single-volume fixture" + ); + + let headless_service = tenant.new_headless_service_with_tls_plan(tls_plan); + let headless_service_spec = headless_service + .spec + .as_ref() + .context("headless Service should have spec")?; + ensure!( + headless_service.metadata.name.as_deref() == Some(expected_headless_service.as_str()), + "{case_name} headless Service should be named {expected_headless_service}, got {:?}", + headless_service.metadata.name + ); + ensure!( + headless_service_spec.publish_not_ready_addresses == Some(true), + "{case_name} headless Service should publish not-ready pod DNS records for TLS bootstrap" + ); + + let pod_spec = statefulset_pod_spec(&statefulset)?; + let rustfs_container = rustfs_container(pod_spec)?; + let rustfs_volumes = env_value(rustfs_container, "RUSTFS_VOLUMES")?; + require_single_volume_https_rustfs_volumes(case_name, rustfs_volumes)?; + + let expected_sans = expected_tls_dns_names(config, tenant); + for host in expand_rustfs_volume_hosts(rustfs_volumes)? { + ensure!( + expected_sans.contains(&host), + "{case_name} expected SANs should cover rendered RUSTFS_VOLUMES host {host}; expected_sans={expected_sans:?} rustfs_volumes={rustfs_volumes}" + ); + } + + ensure!( + env_value(rustfs_container, "RUSTFS_TLS_PATH")? == DEFAULT_TLS_MOUNT_PATH, + "{case_name} should set RUSTFS_TLS_PATH={DEFAULT_TLS_MOUNT_PATH}" + ); + require_tls_material_files_share_runtime_dir(case_name, pod_spec, rustfs_container)?; + + Ok(()) +} + +fn statefulset_pod_spec(statefulset: &StatefulSet) -> Result<&corev1::PodSpec> { + statefulset + .spec + .as_ref() + .context("StatefulSet should have spec")? + .template + .spec + .as_ref() + .context("StatefulSet pod template should have spec") +} + +fn rustfs_container(pod_spec: &corev1::PodSpec) -> Result<&corev1::Container> { + pod_spec + .containers + .iter() + .find(|container| container.name == "rustfs") + .context("rustfs container should exist") +} + +fn env_value<'a>(container: &'a corev1::Container, name: &str) -> Result<&'a str> { + container + .env + .as_ref() + .and_then(|vars| vars.iter().find(|var| var.name == name)) + .and_then(|var| var.value.as_deref()) + .with_context(|| format!("rustfs container should set {name}")) +} + +fn require_single_volume_https_rustfs_volumes(case_name: &str, volumes_value: &str) -> Result<()> { + let specs = volumes_value.split_whitespace().collect::>(); + ensure!( + specs.len() == 1, + "{case_name} should render one RUSTFS_VOLUMES entry for the single positive TLS pool, got {volumes_value}" + ); + + for spec in specs { + let (host_pattern, path_expression) = spec + .strip_prefix("https://") + .with_context(|| { + format!("{case_name} RUSTFS_VOLUMES entry should use https://: {spec}") + })? + .split_once(":9000") + .with_context(|| { + format!("{case_name} RUSTFS_VOLUMES entry should include :9000: {spec}") + })?; + ensure!( + host_pattern.contains(".svc.cluster.local"), + "{case_name} RUSTFS_VOLUMES should use peer FQDNs, got host pattern {host_pattern}" + ); + ensure!( + path_expression == "/data/rustfs{0...0}" || path_expression == "/data/rustfs0", + "{case_name} RUSTFS_VOLUMES should use a single-volume path expression, got {path_expression} in {volumes_value}" + ); + } + + Ok(()) +} + +fn expected_tls_dns_names(config: &E2eConfig, tenant: &Tenant) -> BTreeSet { + let mut names = BTreeSet::from([config.tenant_name.clone(), "localhost".to_string()]); + + if let Some(cert_manager) = tenant + .spec + .tls + .as_ref() + .and_then(|tls| tls.cert_manager.as_ref()) + { + if let Some(common_name) = cert_manager + .common_name + .as_deref() + .filter(|name| !name.is_empty()) + { + names.insert(common_name.to_string()); + } + names.extend( + cert_manager + .dns_names + .iter() + .filter(|name| !name.is_empty()) + .cloned(), + ); + + if cert_manager.include_generated_dns_names { + let tenant_name = &config.tenant_name; + let namespace = &config.test_namespace; + let io_service = format!("{tenant_name}-io"); + let headless_service = format!("{tenant_name}-hl"); + names.insert(format!("{io_service}.{namespace}.svc")); + names.insert(format!("{io_service}.{namespace}.svc.cluster.local")); + names.insert(format!("{headless_service}.{namespace}.svc")); + names.insert(format!("{headless_service}.{namespace}.svc.cluster.local")); + for pool in &tenant.spec.pools { + for ordinal in 0..pool.servers.max(0) { + names.insert(format!( + "{tenant_name}-{}-{ordinal}.{headless_service}.{namespace}.svc.cluster.local", + pool.name + )); + } + } + } + } + + names +} + +fn require_tls_material_files_share_runtime_dir( + case_name: &str, + pod_spec: &corev1::PodSpec, + rustfs_container: &corev1::Container, +) -> Result<()> { + let tls_volume = pod_spec + .volumes + .as_deref() + .unwrap_or_default() + .iter() + .find(|volume| volume.name == "rustfs-tls-server") + .with_context(|| format!("{case_name} should render rustfs-tls-server volume"))?; + for file in [RUSTFS_TLS_CERT_FILE, RUSTFS_TLS_KEY_FILE, RUSTFS_CA_FILE] { + ensure!( + tls_volume_has_item_path(tls_volume, file), + "{case_name} TLS volume should render {file} into the runtime TLS directory" + ); + } + + let mounts = rustfs_container + .volume_mounts + .as_deref() + .unwrap_or_default() + .iter() + .filter(|mount| mount.name == "rustfs-tls-server") + .collect::>(); + ensure!( + !mounts.is_empty(), + "{case_name} rustfs container should mount rustfs-tls-server" + ); + + let has_directory_mount = mounts + .iter() + .any(|mount| mount.mount_path == DEFAULT_TLS_MOUNT_PATH && mount.sub_path.is_none()); + if !has_directory_mount { + for file in [RUSTFS_TLS_CERT_FILE, RUSTFS_TLS_KEY_FILE, RUSTFS_CA_FILE] { + let expected_mount_path = format!("{DEFAULT_TLS_MOUNT_PATH}/{file}"); + ensure!( + mounts + .iter() + .any(|mount| mount.mount_path == expected_mount_path + && mount.sub_path.as_deref() == Some(file)), + "{case_name} should mount {file} under {DEFAULT_TLS_MOUNT_PATH}, mounts={mounts:?}" + ); + } + } + + Ok(()) +} + +fn tls_volume_has_item_path(volume: &corev1::Volume, path: &str) -> bool { + volume + .secret + .as_ref() + .and_then(|secret| secret.items.as_ref()) + .is_some_and(|items| items.iter().any(|item| item.path == path)) + || volume + .projected + .as_ref() + .and_then(|projected| projected.sources.as_ref()) + .is_some_and(|sources| { + sources.iter().any(|source| { + source + .secret + .as_ref() + .and_then(|secret| secret.items.as_ref()) + .is_some_and(|items| items.iter().any(|item| item.path == path)) + }) + }) +} + +fn dns_names_from_subject_alt_name(subject_alt_name: &str) -> Result> { + let value = subject_alt_name + .strip_prefix("subjectAltName=") + .context("subjectAltName addext should start with subjectAltName=")?; + let names = value + .split(',') + .map(|entry| { + entry + .strip_prefix("DNS:") + .with_context(|| format!("subjectAltName entry should be a DNS name: {entry}")) + .map(ToString::to_string) + }) + .collect::>>()?; + ensure!(!names.is_empty(), "subjectAltName should contain DNS SANs"); + Ok(names) +} + +fn rendered_rustfs_volume_hosts(config: &E2eConfig) -> Result> { + let tenant = tls_e2e::external_secret_tenant(config); + let pool = tenant + .spec + .pools + .first() + .context("external Secret fixture should have a pool")?; + let tls_plan = + tls_e2e::sample_tls_plan("sha256:e2e-test", tls_e2e::external_secret_name(config)); + let statefulset = tenant.new_statefulset_with_tls_plan(pool, &tls_plan)?; + let pod_spec = statefulset + .spec + .as_ref() + .context("StatefulSet should have spec")? + .template + .spec + .as_ref() + .context("StatefulSet pod template should have spec")?; + let rustfs_container = pod_spec + .containers + .iter() + .find(|container| container.name == "rustfs") + .context("rustfs container should exist")?; + let volumes_value = rustfs_container + .env + .as_ref() + .and_then(|vars| vars.iter().find(|var| var.name == "RUSTFS_VOLUMES")) + .and_then(|var| var.value.as_deref()) + .context("rustfs container should render RUSTFS_VOLUMES")?; + + expand_rustfs_volume_hosts(volumes_value) +} + +fn expand_rustfs_volume_hosts(volumes_value: &str) -> Result> { + let mut hosts = BTreeSet::new(); + for spec in volumes_value.split_whitespace() { + let host_pattern = spec + .strip_prefix("https://") + .with_context(|| format!("RUSTFS_VOLUMES entry should use https://: {spec}"))? + .split_once(":9000") + .with_context(|| format!("RUSTFS_VOLUMES entry should include :9000: {spec}"))? + .0; + if let Some(range_start) = host_pattern.find("{0...") { + let range_end = host_pattern[range_start..] + .find('}') + .map(|offset| range_start + offset) + .with_context(|| format!("host range should close with }}: {host_pattern}"))?; + let last_ordinal = host_pattern[range_start + "{0...".len()..range_end] + .parse::() + .with_context(|| { + format!("host range should end with an ordinal: {host_pattern}") + })?; + let prefix = &host_pattern[..range_start]; + let suffix = &host_pattern[range_end + 1..]; + for ordinal in 0..=last_ordinal { + hosts.insert(format!("{prefix}{ordinal}{suffix}")); + } + } else { + hosts.insert(host_pattern.to_string()); + } + } + ensure!( + !hosts.is_empty(), + "RUSTFS_VOLUMES should render at least one peer host" + ); + Ok(hosts) +} + +fn secret_data_value<'a>(secret: &'a corev1::Secret, key: &str) -> Result<&'a [u8]> { + secret + .data + .as_ref() + .with_context(|| format!("Secret {:?} should contain data", secret.metadata.name))? + .get(key) + .with_context(|| format!("Secret {:?} should contain {key}", secret.metadata.name)) + .map(|value| value.0.as_slice()) +} + +fn require_certificate_is_ca(cert: &[u8]) -> Result<()> { + let text = openssl_x509_text(cert)?; + ensure!( + text.contains("CA:TRUE"), + "certificate should have BasicConstraints CA:TRUE, got:\n{text}" + ); + Ok(()) +} + +fn require_certificate_verifies_with_ca(cert: &[u8], ca: &[u8]) -> Result<()> { + let dir = TempDir::new()?; + let cert_path = dir.path().join("tls.crt"); + let ca_path = dir.path().join("ca.crt"); + std::fs::write(&cert_path, cert)?; + std::fs::write(&ca_path, ca)?; + + CommandSpec::new("openssl") + .args(["verify", "-CAfile"]) + .arg(ca_path.display().to_string()) + .arg(cert_path.display().to_string()) + .run_checked()?; + Ok(()) +} + +fn certificate_dns_sans(cert: &[u8]) -> Result> { + let text = openssl_x509_text(cert)?; + let names = text + .split([',', '\n']) + .filter_map(|entry| entry.trim().strip_prefix("DNS:")) + .map(ToString::to_string) + .collect::>(); + ensure!( + !names.is_empty(), + "certificate should contain DNS subjectAltName entries, got:\n{text}" + ); + Ok(names) +} + +fn openssl_x509_text(cert: &[u8]) -> Result { + let dir = TempDir::new()?; + let cert_path = dir.path().join("cert.pem"); + std::fs::write(&cert_path, cert)?; + let output = CommandSpec::new("openssl") + .args(["x509", "-in"]) + .arg(cert_path.display().to_string()) + .args(["-noout", "-text"]) + .run_checked()?; + Ok(output.stdout) +} + +fn projected_secret_item( + volume: &corev1::Volume, + secret_name: &str, + key: &str, + path: &str, +) -> bool { + volume + .projected + .as_ref() + .and_then(|projected| projected.sources.as_ref()) + .map(|sources| { + sources.iter().any(|source| { + source + .secret + .as_ref() + .filter(|secret| secret.name == secret_name) + .and_then(|secret| secret.items.as_ref()) + .map(|items| { + items + .iter() + .any(|item| item.key == key && item.path == path) + }) + .unwrap_or(false) + }) + }) + .unwrap_or(false) +} + +fn pem_contains_label(bytes: &[u8], label: &str) -> bool { + std::str::from_utf8(bytes) + .map(|pem| { + pem.contains(&format!("-----BEGIN {label}-----")) + && pem.contains(&format!("-----END {label}-----")) + }) + .unwrap_or(false) +} diff --git a/examples/cert-manager-ca-trust-tenant.yaml b/examples/cert-manager-ca-trust-tenant.yaml new file mode 100644 index 0000000..5fa32c0 --- /dev/null +++ b/examples/cert-manager-ca-trust-tenant.yaml @@ -0,0 +1,69 @@ +# Minimal RustFS Tenant example for cert-manager managed TLS and CA trust. +# +# This manifest is intentionally static: it does not install cert-manager, create an +# Issuer, create any Secret, or require a live cluster. Before applying it in a +# real cluster, create the referenced Issuer in the same namespace, or change +# issuerRef.kind to ClusterIssuer and provide an existing ClusterIssuer. +# +# Current implementation notes: +# - spec.tls.mode must be certManager for this flow. +# - rotationStrategy=HotReload is currently rejected; use Rollout. +# - enableInternodeHttps=true requires a CA source. CertificateSecretCa expects +# cert-manager to write ca.crt into the TLS Secret. Use SecretRef or SystemCa +# only when that matches your trust model. +# - Existing cert-manager Secrets used with manageCertificate=false should carry +# rustfs.tenant= if you want Secret changes to enqueue the Tenant. +apiVersion: rustfs.com/v1alpha1 +kind: Tenant +metadata: + name: cert-manager-ca-trust + namespace: rustfs-system +spec: + image: rustfs/rustfs:latest + + tls: + mode: certManager + mountPath: /var/run/rustfs/tls + rotationStrategy: Rollout + enableInternodeHttps: true + requireSanMatch: true + certManager: + manageCertificate: true + certificateName: cert-manager-ca-trust-server + secretName: cert-manager-ca-trust-server-tls + issuerRef: + group: cert-manager.io + kind: Issuer + name: rustfs-issuer + includeGeneratedDnsNames: true + dnsNames: + - cert-manager-ca-trust-io.rustfs-system.svc + - cert-manager-ca-trust-io.rustfs-system.svc.cluster.local + duration: 2160h + renewBefore: 360h + usages: + - server auth + caTrust: + source: CertificateSecretCa + trustSystemCa: false + trustLeafCertificateAsCa: false + # Alternative for a separate CA Secret: + # source: SecretRef + # caSecretRef: + # name: cert-manager-ca-trust-ca + # key: ca.crt + # clientCaSecretRef: + # name: cert-manager-ca-trust-client-ca + # key: ca.crt + + pools: + - name: primary + servers: 1 + persistence: + volumesPerServer: 4 + volumeClaimTemplate: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/scripts/test/script-test.sh b/scripts/test/script-test.sh index 465732c..d262b29 100755 --- a/scripts/test/script-test.sh +++ b/scripts/test/script-test.sh @@ -91,5 +91,34 @@ else exit 1 fi +assert_cert_manager_crd_discovery_rbac() { + local manifest="$1" + python3 - "$manifest" <<'PY' +import pathlib +import re +import sys + +manifest = pathlib.Path(sys.argv[1]) +pattern = re.compile( + r'^ - apiGroups: \["apiextensions\.k8s\.io"\]\n' + r' resources: \["customresourcedefinitions"\]\n' + r' resourceNames: \["certificates\.cert-manager\.io"\]\n' + r' verbs: \["get"\]$', + re.MULTILINE, +) +if not pattern.search(manifest.read_text()): + sys.exit(1) +PY +} + +echo "11. Checking Operator RBAC can discover the cert-manager Certificate CRD..." +for manifest in \ + deploy/k8s-dev/operator-rbac.yaml \ + deploy/rustfs-operator/templates/clusterrole.yaml; do + assert_cert_manager_crd_discovery_rbac "$manifest" \ + || { echo " ✗ $manifest must grant get on customresourcedefinitions/certificates.cert-manager.io"; exit 1; } +done +echo " ✓ Operator RBAC grants scoped cert-manager Certificate CRD discovery" + echo "" echo "All script checks passed! ✅" diff --git a/src/console/handlers/tenants.rs b/src/console/handlers/tenants.rs index 261d129..a5e4c55 100755 --- a/src/console/handlers/tenants.rs +++ b/src/console/handlers/tenants.rs @@ -163,6 +163,7 @@ pub async fn get_tenant_details( let status_summary = tenant_status_summary(&tenant); let conditions = tenant_conditions(&tenant); let next_actions = status_summary.next_actions.clone(); + let certificates = tenant_certificates(&tenant); Ok(Json(TenantDetailsResponse { name: tenant.name_any(), @@ -181,6 +182,7 @@ pub async fn get_tenant_details( status_summary, conditions, next_actions, + certificates, image: tenant.spec.image.clone(), mount_path: tenant.spec.mount_path.clone(), created_at: tenant diff --git a/src/console/models/tenant.rs b/src/console/models/tenant.rs index 16d017d..0412c3a 100755 --- a/src/console/models/tenant.rs +++ b/src/console/models/tenant.rs @@ -15,7 +15,8 @@ use crate::types::v1alpha1::{ status::{ ConditionStatus, ConditionType, CurrentState, Reason, Status, canonical_filter_state, - canonical_state, next_actions_for_reason, primary_condition, summarize_current_state, + canonical_state, certificate, next_actions_for_reason, primary_condition, + summarize_current_state, }, tenant::Tenant, }; @@ -80,6 +81,8 @@ pub struct TenantDetailsResponse { pub status_summary: TenantStatusSummary, pub conditions: Vec, pub next_actions: Vec, + #[serde(skip_serializing_if = "certificate::Status::is_empty")] + pub certificates: certificate::Status, pub image: Option, pub mount_path: Option, pub created_at: Option, @@ -316,6 +319,14 @@ pub fn tenant_conditions(tenant: &Tenant) -> Vec { .unwrap_or_default() } +pub fn tenant_certificates(tenant: &Tenant) -> certificate::Status { + tenant + .status + .as_ref() + .map(|status| status.certificates.clone()) + .unwrap_or_default() +} + pub fn tenant_to_list_item(tenant: Tenant) -> TenantListItem { let summary = tenant_status_summary(&tenant); TenantListItem { @@ -404,6 +415,7 @@ mod tests { condition("CredentialsReady", "False", "CredentialSecretNotFound"), condition("Degraded", "True", "CredentialSecretNotFound"), ], + ..Default::default() }); let summary = tenant_status_summary(&tenant); @@ -430,6 +442,7 @@ mod tests { condition("Ready", "True", "ReconcileSucceeded"), condition("Degraded", "False", "ReconcileSucceeded"), ], + ..Default::default() }); let summary = tenant_status_summary(&tenant); @@ -455,6 +468,7 @@ mod tests { pools: Vec::new(), observed_generation: None, conditions: vec![condition("Ready", "True", "ReconcileSucceeded")], + ..Default::default() }); let summary = tenant_status_summary(&tenant); @@ -536,6 +550,36 @@ mod tests { ); } + #[test] + fn tenant_certificates_exposes_tls_status_for_details_api() { + let mut tenant = crate::tests::create_test_tenant(None, None); + tenant.status = Some(Status { + certificates: crate::types::v1alpha1::status::certificate::Status { + tls: Some( + crate::types::v1alpha1::status::certificate::TlsCertificateStatus { + mode: "certManager".to_string(), + ready: false, + last_error_reason: Some("CertManagerCertificateNotReady".to_string()), + ..Default::default() + }, + ), + }, + ..Default::default() + }); + + let certificates = tenant_certificates(&tenant); + + let Some(tls) = certificates.tls.as_ref() else { + panic!("tls status should be exposed"); + }; + assert_eq!(tls.mode, "certManager"); + assert!(!tls.ready); + assert_eq!( + tls.last_error_reason.as_deref(), + Some("CertManagerCertificateNotReady") + ); + } + fn condition(type_: &str, status: &str, reason: &str) -> Condition { Condition { type_: type_.to_string(), diff --git a/src/lib.rs b/src/lib.rs index 79bd2f1..650b719 100755 --- a/src/lib.rs +++ b/src/lib.rs @@ -20,14 +20,23 @@ use crate::types::v1alpha1::tenant::Tenant; use futures::StreamExt; use k8s_openapi::api::apps::v1 as appsv1; use k8s_openapi::api::core::v1 as corev1; -use kube::CustomResourceExt; +use k8s_openapi::apimachinery::pkg::apis::meta::v1 as metav1; +use kube::core::{ApiResource, DynamicObject, GroupVersionKind}; +use kube::runtime::reflector::ObjectRef; use kube::runtime::{Controller, watcher}; -use kube::{Api, Client}; +use kube::{Api, Client, CustomResourceExt, Resource}; +use std::collections::BTreeMap; use std::pin::Pin; use std::sync::Arc; use tokio::io::{AsyncWrite, AsyncWriteExt}; use tracing::{info, warn}; +const RUSTFS_TENANT_LABEL: &str = "rustfs.tenant"; +const CERT_MANAGER_GROUP: &str = "cert-manager.io"; +const CERT_MANAGER_VERSION: &str = "v1"; +const CERT_MANAGER_CERTIFICATE_KIND: &str = "Certificate"; +const CERT_MANAGER_CERTIFICATE_PLURAL: &str = "certificates"; + mod context; pub mod reconcile; mod status; @@ -53,14 +62,15 @@ pub async fn run() -> Result<(), Box> { let tenant_client = Api::::all(client.clone()); let context = Context::new(client.clone()); - Controller::new(tenant_client, watcher::Config::default()) + let controller = Controller::new(tenant_client, watcher::Config::default()) .owns( Api::::all(client.clone()), watcher::Config::default(), ) - .owns( + .watches( Api::::all(client.clone()), watcher::Config::default(), + tenant_refs_for_secret, ) .owns( Api::::all(client.clone()), @@ -73,7 +83,29 @@ pub async fn run() -> Result<(), Box> { .owns( Api::::all(client.clone()), watcher::Config::default(), - ) + ); + + let certificate_gvk = cert_manager_certificate_gvk(); + let controller = match kube::discovery::pinned_kind(&client, &certificate_gvk).await { + Ok((_resource, _capabilities)) => { + let resource = cert_manager_certificate_api_resource(); + controller.watches_with( + Api::::all_with(client.clone(), &resource), + resource, + watcher::Config::default(), + tenant_refs_for_cert_manager_certificate, + ) + } + Err(error) => { + warn!( + %error, + "cert-manager Certificate API not discovered; skipping Certificate watch" + ); + controller + } + }; + + controller .run(reconcile_rustfs, error_policy, Arc::new(context)) .for_each(|res| async move { match res { @@ -86,6 +118,94 @@ pub async fn run() -> Result<(), Box> { Ok(()) } +fn cert_manager_certificate_gvk() -> GroupVersionKind { + GroupVersionKind::gvk( + CERT_MANAGER_GROUP, + CERT_MANAGER_VERSION, + CERT_MANAGER_CERTIFICATE_KIND, + ) +} + +fn cert_manager_certificate_api_resource() -> ApiResource { + ApiResource::from_gvk_with_plural( + &cert_manager_certificate_gvk(), + CERT_MANAGER_CERTIFICATE_PLURAL, + ) +} + +fn tenant_refs_for_secret(secret: corev1::Secret) -> Vec> { + tenant_refs_from_metadata( + secret.metadata.namespace.as_deref(), + secret.metadata.owner_references.as_deref(), + secret.metadata.labels.as_ref(), + ) +} + +fn tenant_refs_for_cert_manager_certificate(certificate: DynamicObject) -> Vec> { + tenant_refs_from_metadata( + certificate.metadata.namespace.as_deref(), + certificate.metadata.owner_references.as_deref(), + certificate.metadata.labels.as_ref(), + ) +} + +fn tenant_refs_from_metadata( + namespace: Option<&str>, + owner_references: Option<&[metav1::OwnerReference]>, + labels: Option<&BTreeMap>, +) -> Vec> { + let mut refs = Vec::new(); + + if let Some(owner_references) = owner_references { + for owner in owner_references { + if let Some(tenant_ref) = tenant_ref_from_owner_reference(namespace, owner) { + push_unique_tenant_ref(&mut refs, tenant_ref); + } + } + } + + if let Some(labels) = labels + && let Some(tenant_ref) = tenant_ref_from_labels(namespace, labels) + { + push_unique_tenant_ref(&mut refs, tenant_ref); + } + + refs +} + +fn tenant_ref_from_owner_reference( + namespace: Option<&str>, + owner: &metav1::OwnerReference, +) -> Option> { + if namespace.is_none() + || owner.api_version != Tenant::api_version(&()) + || owner.kind != Tenant::kind(&()) + || owner.name.is_empty() + { + return None; + } + + Some(ObjectRef::new(&owner.name).within(namespace?)) +} + +fn tenant_ref_from_labels( + namespace: Option<&str>, + labels: &BTreeMap, +) -> Option> { + let name = labels + .get(RUSTFS_TENANT_LABEL) + .map(String::as_str) + .filter(|name| !name.is_empty())?; + + Some(ObjectRef::new(name).within(namespace?)) +} + +fn push_unique_tenant_ref(refs: &mut Vec>, tenant_ref: ObjectRef) { + if !refs.iter().any(|existing| existing == &tenant_ref) { + refs.push(tenant_ref); + } +} + pub async fn crd(file: Option) -> Result<(), Box> { let mut writer: Pin> = if let Some(file) = file { Box::pin( @@ -106,3 +226,97 @@ pub async fn crd(file: Option) -> Result<(), Box> Ok(()) } + +#[cfg(test)] +mod controller_watch_tests { + use super::*; + use k8s_openapi::apimachinery::pkg::apis::meta::v1 as metav1; + use std::collections::BTreeMap; + + #[test] + fn cert_manager_certificate_api_resource_is_stable() { + let resource = cert_manager_certificate_api_resource(); + + assert_eq!(resource.group, "cert-manager.io"); + assert_eq!(resource.version, "v1"); + assert_eq!(resource.api_version, "cert-manager.io/v1"); + assert_eq!(resource.kind, "Certificate"); + assert_eq!(resource.plural, "certificates"); + } + + #[test] + fn secret_mapper_uses_tenant_owner_reference() { + let secret = corev1::Secret { + metadata: metav1::ObjectMeta { + name: Some("server-tls".to_string()), + namespace: Some("storage".to_string()), + owner_references: Some(vec![tenant_owner_ref("tenant-a")]), + ..Default::default() + }, + ..Default::default() + }; + + let refs = tenant_refs_for_secret(secret); + + assert_single_ref(&refs, "tenant-a", "storage"); + } + + #[test] + fn secret_mapper_uses_rustfs_tenant_label_for_cert_manager_output_secret() { + let secret = corev1::Secret { + metadata: metav1::ObjectMeta { + name: Some("server-tls".to_string()), + namespace: Some("storage".to_string()), + labels: Some(BTreeMap::from([ + ( + "app.kubernetes.io/managed-by".to_string(), + "rustfs-operator".to_string(), + ), + ("rustfs.tenant".to_string(), "tenant-b".to_string()), + ])), + ..Default::default() + }, + ..Default::default() + }; + + let refs = tenant_refs_for_secret(secret); + + assert_single_ref(&refs, "tenant-b", "storage"); + } + + #[test] + fn cert_manager_certificate_mapper_uses_owner_reference_or_label() { + let resource = cert_manager_certificate_api_resource(); + let mut owned = DynamicObject::new("tenant-c-cert", &resource).within("storage"); + owned.metadata.owner_references = Some(vec![tenant_owner_ref("tenant-c")]); + + let refs = tenant_refs_for_cert_manager_certificate(owned); + assert_single_ref(&refs, "tenant-c", "storage"); + + let mut labeled = DynamicObject::new("tenant-d-cert", &resource).within("storage"); + labeled.metadata.labels = Some(BTreeMap::from([( + "rustfs.tenant".to_string(), + "tenant-d".to_string(), + )])); + + let refs = tenant_refs_for_cert_manager_certificate(labeled); + assert_single_ref(&refs, "tenant-d", "storage"); + } + + fn tenant_owner_ref(name: &str) -> metav1::OwnerReference { + metav1::OwnerReference { + api_version: "rustfs.com/v1alpha1".to_string(), + kind: "Tenant".to_string(), + name: name.to_string(), + uid: format!("{name}-uid"), + controller: Some(true), + block_owner_deletion: Some(true), + } + } + + fn assert_single_ref(refs: &[ObjectRef], name: &str, namespace: &str) { + assert_eq!(refs.len(), 1); + assert_eq!(refs[0].name, name); + assert_eq!(refs[0].namespace.as_deref(), Some(namespace)); + } +} diff --git a/src/reconcile.rs b/src/reconcile.rs index 61d44ac..8d206a7 100755 --- a/src/reconcile.rs +++ b/src/reconcile.rs @@ -28,6 +28,7 @@ use std::time::Duration; use tracing::{debug, error, info, warn}; mod phases; +mod tls; use phases::{ finalize_tenant_status, maybe_cleanup_terminating_pods, reconcile_pool_statefulsets, @@ -42,6 +43,12 @@ pub enum Error { #[snafu(transparent)] Types { source: types::error::Error }, + + #[snafu(display("TLS reconciliation blocked ({reason}): {message}"))] + TlsBlocked { reason: String, message: String }, + + #[snafu(display("TLS reconciliation pending ({reason}): {message}"))] + TlsPending { reason: String, message: String }, } pub async fn reconcile_rustfs(tenant: Arc, ctx: Arc) -> Result { @@ -62,17 +69,18 @@ pub async fn reconcile_rustfs(tenant: Arc, ctx: Arc) -> Result< } validate_tenant_prerequisites(&ctx, &latest_tenant).await?; + let tls_plan = tls::reconcile_tls(&ctx, &latest_tenant, &ns).await?; maybe_cleanup_terminating_pods(&ctx, &latest_tenant, &ns).await?; reconcile_rbac_resources(&ctx, &latest_tenant, &ns).await?; - reconcile_services(&ctx, &latest_tenant, &ns).await?; + reconcile_services(&ctx, &latest_tenant, &ns, &tls_plan).await?; validate_no_pool_rename(&ctx, &latest_tenant, &ns).await?; - let summary = reconcile_pool_statefulsets(&ctx, &latest_tenant, &ns).await?; - finalize_tenant_status(&ctx, &latest_tenant, summary).await + let summary = reconcile_pool_statefulsets(&ctx, &latest_tenant, &ns, &tls_plan).await?; + finalize_tenant_status(&ctx, &latest_tenant, summary, tls_plan).await } #[cfg(test)] @@ -527,6 +535,9 @@ pub fn error_policy(_object: Arc, error: &Error, _ctx: Arc) -> // Other type errors - use moderate requeue _ => Action::requeue(Duration::from_secs(15)), }, + + Error::TlsBlocked { .. } => Action::requeue(Duration::from_secs(60)), + Error::TlsPending { .. } => Action::requeue(Duration::from_secs(20)), } } diff --git a/src/reconcile/phases.rs b/src/reconcile/phases.rs index 22f59d0..a051f0a 100644 --- a/src/reconcile/phases.rs +++ b/src/reconcile/phases.rs @@ -21,6 +21,7 @@ use crate::status::{StatusBuilder, StatusError}; use crate::types; use crate::types::v1alpha1::status::{ConditionType, Reason}; use crate::types::v1alpha1::tenant::Tenant; +use crate::types::v1alpha1::tls::TlsPlan; use kube::ResourceExt; use kube::api::ListParams; use kube::runtime::controller::Action; @@ -158,9 +159,11 @@ pub(super) async fn reconcile_services( ctx: &Context, tenant: &Tenant, namespace: &str, + tls_plan: &TlsPlan, ) -> Result<(), Error> { context_result( - ctx.apply(&tenant.new_io_service(), namespace).await, + ctx.apply(&tenant.new_io_service_with_tls_plan(tls_plan), namespace) + .await, ctx, tenant, ) @@ -172,7 +175,11 @@ pub(super) async fn reconcile_services( ) .await?; context_result( - ctx.apply(&tenant.new_headless_service(), namespace).await, + ctx.apply( + &tenant.new_headless_service_with_tls_plan(tls_plan), + namespace, + ) + .await, ctx, tenant, ) @@ -267,6 +274,7 @@ pub(super) async fn reconcile_pool_statefulsets( ctx: &Context, tenant: &Tenant, namespace: &str, + tls_plan: &TlsPlan, ) -> Result { let mut summary = PoolReconcileSummary::default(); @@ -282,8 +290,8 @@ pub(super) async fn reconcile_pool_statefulsets( tenant, namespace, pool, - &ss_name, existing_ss, + tls_plan, &mut summary, ) .await?; @@ -295,6 +303,7 @@ pub(super) async fn reconcile_pool_statefulsets( namespace, pool, &ss_name, + tls_plan, &mut summary, ) .await?; @@ -325,22 +334,23 @@ async fn reconcile_existing_pool_statefulset( tenant: &Tenant, namespace: &str, pool: &crate::types::v1alpha1::pool::Pool, - ss_name: &str, existing_ss: k8s_openapi::api::apps::v1::StatefulSet, + tls_plan: &TlsPlan, summary: &mut PoolReconcileSummary, ) -> Result<(), Error> { + let ss_name = existing_ss.name_any(); debug!("StatefulSet {} exists, checking if update needed", ss_name); - if let Err(e) = tenant.validate_statefulset_update(&existing_ss, pool) { + if let Err(e) = tenant.validate_statefulset_update_with_tls_plan(&existing_ss, pool, tls_plan) { error!("StatefulSet {} update validation failed: {}", ss_name, e); - let status_error = StatusError::statefulset_update_validation_failed(ss_name); + let status_error = StatusError::statefulset_update_validation_failed(&ss_name); patch_status_error(ctx, tenant, &status_error).await; return Err(e.into()); } if types_result( - tenant.statefulset_needs_update(&existing_ss, pool), + tenant.statefulset_needs_update_with_tls_plan(&existing_ss, pool, tls_plan), ctx, tenant, ) @@ -357,9 +367,14 @@ async fn reconcile_existing_pool_statefulset( ) .await; - let desired = types_result(tenant.new_statefulset(pool), ctx, tenant).await?; + let desired = types_result( + tenant.new_statefulset_with_tls_plan(pool, tls_plan), + ctx, + tenant, + ) + .await?; if let Err(e) = ctx.apply(&desired, namespace).await { - let status_error = StatusError::statefulset_apply_failed(ss_name); + let status_error = StatusError::statefulset_apply_failed(&ss_name); patch_status_error(ctx, tenant, &status_error).await; return Err(e.into()); } @@ -370,7 +385,7 @@ async fn reconcile_existing_pool_statefulset( } let ss = context_result( - ctx.get::(ss_name, namespace) + ctx.get::(&ss_name, namespace) .await, ctx, tenant, @@ -388,6 +403,7 @@ async fn reconcile_missing_pool_statefulset( namespace: &str, pool: &crate::types::v1alpha1::pool::Pool, ss_name: &str, + tls_plan: &TlsPlan, summary: &mut PoolReconcileSummary, ) -> Result<(), Error> { debug!("StatefulSet {} not found, creating", ss_name); @@ -401,7 +417,12 @@ async fn reconcile_missing_pool_statefulset( ) .await; - let desired = types_result(tenant.new_statefulset(pool), ctx, tenant).await?; + let desired = types_result( + tenant.new_statefulset_with_tls_plan(pool, tls_plan), + ctx, + tenant, + ) + .await?; if let Err(e) = ctx.apply(&desired, namespace).await { let status_error = StatusError::statefulset_apply_failed(ss_name); patch_status_error(ctx, tenant, &status_error).await; @@ -450,9 +471,13 @@ pub(super) async fn finalize_tenant_status( ctx: &Context, tenant: &Tenant, summary: PoolReconcileSummary, + tls_plan: TlsPlan, ) -> Result { let mut builder = StatusBuilder::from_tenant(tenant); builder.set_pool_statuses(summary.pool_statuses); + if let Some(tls_status) = tls_plan.status { + builder.set_tls_status(tls_status); + } let (event_condition, event_reason, event_type, event_message) = if summary.any_degraded { builder.finish_degraded( diff --git a/src/reconcile/tls.rs b/src/reconcile/tls.rs new file mode 100644 index 0000000..47e984b --- /dev/null +++ b/src/reconcile/tls.rs @@ -0,0 +1,2086 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::{Error, patch_status_and_record, patch_status_error}; +use crate::context::{self, Context}; +use crate::status::{StatusBuilder, StatusError}; +use crate::types::v1alpha1::status::Reason; +use crate::types::v1alpha1::status::certificate::{ + CertificateObjectRef, SecretStatusRef, TlsCertificateStatus, +}; +use crate::types::v1alpha1::tenant::Tenant; +use crate::types::v1alpha1::tls::{ + CaTrustSource, CertManagerIssuerRef, CertManagerTlsConfig, SecretKeyReference, TlsConfig, + TlsMode, TlsPlan, TlsRotationStrategy, +}; +use k8s_openapi::api::core::v1::Secret; +use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition; +use kube::api::{Api, Patch, PatchParams}; +use kube::core::{ApiResource, DynamicObject, GroupVersionKind}; +use rustls::pki_types::{CertificateDer, ServerName}; +use serde_json::{Map, Value, json}; +use sha2::{Digest, Sha256}; +use std::collections::BTreeSet; +use std::io::Cursor; + +const TLS_CERT_KEY: &str = "tls.crt"; +const TLS_KEY_KEY: &str = "tls.key"; +const CA_CERT_KEY: &str = "ca.crt"; +const KUBERNETES_TLS_SECRET_TYPE: &str = "kubernetes.io/tls"; +const CERT_MANAGER_V1_SECRET_TYPE: &str = "cert-manager.io/v1"; +const CERT_MANAGER_V1ALPHA2_SECRET_TYPE: &str = "cert-manager.io/v1alpha2"; +const CERT_MANAGER_GROUP: &str = "cert-manager.io"; +const CERT_MANAGER_VERSION: &str = "v1"; +const CERT_MANAGER_CERTIFICATE_KIND: &str = "Certificate"; +const CERT_MANAGER_CERTIFICATE_PLURAL: &str = "certificates"; +const CERT_MANAGER_CERTIFICATE_CRD: &str = "certificates.cert-manager.io"; +const CERT_MANAGER_ISSUER_KIND: &str = "Issuer"; +const CERT_MANAGER_ISSUER_PLURAL: &str = "issuers"; +const CERT_MANAGER_CLUSTER_ISSUER_KIND: &str = "ClusterIssuer"; +const CERT_MANAGER_CLUSTER_ISSUER_PLURAL: &str = "clusterissuers"; +const STATUS_MESSAGE_LIMIT: usize = 256; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum CertManagerPrerequisite { + CertificateCrd, + Issuer, + ClusterIssuer, +} + +#[derive(Debug, PartialEq)] +struct TlsValidationFailure { + reason: Reason, + message: String, +} + +#[derive(Debug, PartialEq)] +struct ServerCaMaterial { + key: String, + bytes: Vec, +} + +#[derive(Clone, Debug, PartialEq)] +struct CertManagerCertificateObservation { + name: String, + observed_generation: Option, + ready: bool, + reason: Option, + message: Option, +} + +impl CertManagerCertificateObservation { + fn status_ref(&self) -> CertificateObjectRef { + CertificateObjectRef { + api_version: format!("{CERT_MANAGER_GROUP}/{CERT_MANAGER_VERSION}"), + kind: CERT_MANAGER_CERTIFICATE_KIND.to_string(), + name: self.name.clone(), + observed_generation: self.observed_generation, + ready: Some(self.ready), + reason: self.reason.clone(), + } + } +} + +pub(super) async fn reconcile_tls( + ctx: &Context, + tenant: &Tenant, + namespace: &str, +) -> Result { + let Some(config) = tenant.spec.tls.as_ref() else { + return Ok(TlsPlan::disabled()); + }; + if !config.is_enabled() { + return Ok(TlsPlan::disabled()); + } + + if !config.mount_path.starts_with('/') { + return tls_blocked( + ctx, + tenant, + config, + Reason::CertificateInvalid, + format!( + "spec.tls.mountPath must be an absolute path (got '{}')", + config.mount_path + ), + ) + .await; + } + + if config.rotation_strategy == TlsRotationStrategy::HotReload { + return tls_blocked( + ctx, + tenant, + config, + Reason::TlsHotReloadUnsupported, + "TLS hot reload is not supported until RustFS clean-directory reload support is implemented; use rotationStrategy=Rollout".to_string(), + ) + .await; + } + + match config.mode { + TlsMode::Disabled => Ok(TlsPlan::disabled()), + TlsMode::External => { + tls_blocked( + ctx, + tenant, + config, + Reason::CertificateSecretNotFound, + "spec.tls.mode=external is reserved for the external TLS Secret API and is not wired in this phase".to_string(), + ) + .await + } + TlsMode::CertManager => reconcile_cert_manager_tls(ctx, tenant, namespace, config).await, + } +} + +async fn reconcile_cert_manager_tls( + ctx: &Context, + tenant: &Tenant, + namespace: &str, + config: &TlsConfig, +) -> Result { + let Some(cert_manager) = config.cert_manager.as_ref() else { + return tls_blocked( + ctx, + tenant, + config, + Reason::CertificateSecretNotFound, + "spec.tls.certManager.secretName is required for certManager TLS mode".to_string(), + ) + .await; + }; + + let Some(secret_name) = cert_manager + .secret_name + .as_deref() + .filter(|name| !name.is_empty()) + else { + return tls_blocked( + ctx, + tenant, + config, + Reason::CertificateSecretNotFound, + "spec.tls.certManager.secretName is required for certManager TLS mode".to_string(), + ) + .await; + }; + + let mut certificate_ref = None; + if cert_manager.manage_certificate { + let Some(issuer_ref) = cert_manager.issuer_ref.as_ref() else { + return tls_blocked( + ctx, + tenant, + config, + Reason::CertManagerIssuerNotFound, + "spec.tls.certManager.issuerRef is required when manageCertificate=true" + .to_string(), + ) + .await; + }; + let certificate_name = certificate_name(tenant, cert_manager); + + if let Err(error) = ensure_cert_manager_certificate_crd(ctx).await { + return cert_manager_prerequisite_failed( + ctx, + tenant, + config, + CertManagerPrerequisite::CertificateCrd, + error, + format!( + "cert-manager Certificate CRD '{}' is not installed", + CERT_MANAGER_CERTIFICATE_CRD + ), + ) + .await; + } + + if let Err(error) = ensure_cert_manager_issuer(ctx, namespace, issuer_ref).await { + return cert_manager_prerequisite_failed( + ctx, + tenant, + config, + issuer_prerequisite(issuer_ref), + error, + format!( + "cert-manager {} '{}' was not found", + issuer_ref.kind, issuer_ref.name + ), + ) + .await; + } + + let desired_certificate = build_cert_manager_certificate( + tenant, + namespace, + config, + cert_manager, + secret_name, + &certificate_name, + ); + let observed_certificate = match apply_cert_manager_certificate( + ctx, + namespace, + &certificate_name, + &desired_certificate, + ) + .await + { + Ok(certificate) => certificate, + Err(error) if context::is_kube_not_found(&error) => { + return tls_blocked( + ctx, + tenant, + config, + Reason::CertManagerCrdMissing, + format!( + "cert-manager Certificate API was not found while applying '{}'", + certificate_name + ), + ) + .await; + } + Err(error) => { + return tls_blocked( + ctx, + tenant, + config, + Reason::CertManagerCertificateApplyFailed, + format!( + "failed to apply cert-manager Certificate '{}': {}", + certificate_name, + sanitize_status_message(&error.to_string()) + ), + ) + .await; + } + }; + let observation = observe_cert_manager_certificate(&observed_certificate); + certificate_ref = Some(observation.status_ref()); + if !observation.ready { + return tls_pending_with_certificate_ref( + ctx, + tenant, + config, + Reason::CertManagerCertificateNotReady, + certificate_not_ready_message(&certificate_name, &observation), + certificate_ref.clone(), + ) + .await; + } + } + + let secret = get_server_secret_or_tls_error( + ctx, + tenant, + config, + namespace, + secret_name, + cert_manager.manage_certificate, + certificate_ref.clone(), + ) + .await?; + + if let Err(failure) = validate_tls_secret_type( + &secret, + secret_name, + cert_manager + .secret_type + .as_deref() + .filter(|secret_type| !secret_type.is_empty()), + ) { + return tls_validation_blocked(ctx, tenant, config, failure).await; + } + + let cert_bytes = require_secret_key( + ctx, + tenant, + config, + &secret, + secret_name, + TLS_CERT_KEY, + Reason::CertificateSecretMissingKey, + ) + .await?; + require_secret_key( + ctx, + tenant, + config, + &secret, + secret_name, + TLS_KEY_KEY, + Reason::CertificateSecretMissingKey, + ) + .await?; + + if config.require_san_match && config.enable_internode_https { + let expected_dns_names = certificate_dns_names(tenant, namespace, cert_manager); + if let Err(failure) = + validate_tls_secret_san_match(secret_name, &cert_bytes, &expected_dns_names) + { + return tls_validation_blocked(ctx, tenant, config, failure).await; + } + } + + let ca_trust = config.ca_trust(); + let trust_system_ca = ca_trust.trust_system_ca || ca_trust.source == CaTrustSource::SystemCa; + let mut server_ca_key = None; + let mut explicit_ca = None; + let mut explicit_ca_secret = None; + let mut explicit_ca_bytes: Option> = None; + + match ca_trust.source { + CaTrustSource::CertificateSecretCa => match certificate_secret_ca_material( + &secret, + secret_name, + config.enable_internode_https, + trust_system_ca, + ) { + Ok(Some(material)) => { + server_ca_key = Some(material.key); + explicit_ca_bytes = Some(material.bytes); + } + Ok(None) => {} + Err(failure) => return tls_validation_blocked(ctx, tenant, config, failure).await, + }, + CaTrustSource::SecretRef => { + let Some(ca_secret_ref) = ca_trust.ca_secret_ref.clone() else { + return tls_blocked( + ctx, + tenant, + config, + Reason::CaBundleMissing, + "spec.tls.certManager.caTrust.caSecretRef is required when caTrust.source=SecretRef".to_string(), + ) + .await; + }; + let ca_secret = get_secret_or_tls_blocked( + ctx, + tenant, + config, + namespace, + &ca_secret_ref.name, + Reason::CaBundleMissing, + format!("CA Secret '{}' was not found", ca_secret_ref.name), + ) + .await?; + let ca_bytes = require_secret_key( + ctx, + tenant, + config, + &ca_secret, + &ca_secret_ref.name, + &ca_secret_ref.key, + Reason::CaBundleMissing, + ) + .await?; + if let Err(failure) = validate_ca_bundle_bytes( + &ca_secret_ref.name, + &ca_secret_ref.key, + ca_bytes.as_slice(), + ) { + return tls_validation_blocked(ctx, tenant, config, failure).await; + } + explicit_ca_bytes = Some(ca_bytes); + explicit_ca = Some(ca_secret_ref); + explicit_ca_secret = Some(ca_secret); + } + CaTrustSource::SystemCa => {} + } + + let mut client_ca = None; + let mut client_ca_secret = None; + let mut client_ca_bytes: Option> = None; + if let Some(client_ca_secret_ref) = ca_trust.client_ca_secret_ref.clone() { + let ca_secret = get_secret_or_tls_blocked( + ctx, + tenant, + config, + namespace, + &client_ca_secret_ref.name, + Reason::CaBundleMissing, + format!( + "Client CA Secret '{}' was not found", + client_ca_secret_ref.name + ), + ) + .await?; + client_ca_bytes = Some( + require_secret_key( + ctx, + tenant, + config, + &ca_secret, + &client_ca_secret_ref.name, + &client_ca_secret_ref.key, + Reason::CaBundleMissing, + ) + .await?, + ); + if let Err(failure) = validate_ca_bundle_bytes( + &client_ca_secret_ref.name, + &client_ca_secret_ref.key, + client_ca_bytes.as_deref().unwrap_or_default(), + ) { + return tls_validation_blocked(ctx, tenant, config, failure).await; + } + client_ca = Some(client_ca_secret_ref); + client_ca_secret = Some(ca_secret); + } + + let hash = tls_hash( + config, + &secret, + explicit_ca.as_ref(), + explicit_ca_bytes.as_deref(), + client_ca.as_ref(), + client_ca_bytes.as_deref(), + trust_system_ca, + ); + let status = cert_manager_tls_status( + config, + secret_name, + &secret, + explicit_ca.as_ref().zip(explicit_ca_secret.as_ref()), + client_ca.as_ref().zip(client_ca_secret.as_ref()), + &hash, + certificate_ref, + ); + + Ok(TlsPlan::rollout( + config.mount_path.clone(), + hash, + secret_name.to_string(), + server_ca_key, + explicit_ca, + client_ca, + config.enable_internode_https, + trust_system_ca, + ca_trust.trust_leaf_certificate_as_ca, + Some(status), + )) +} + +async fn get_server_secret_or_tls_error( + ctx: &Context, + tenant: &Tenant, + config: &TlsConfig, + namespace: &str, + secret_name: &str, + managed_certificate: bool, + certificate_ref: Option, +) -> Result { + match ctx.get::(secret_name, namespace).await { + Ok(secret) => Ok(secret), + Err(error) if context::is_kube_not_found(&error) => { + let reason = secret_missing_reason(managed_certificate); + let message = if managed_certificate { + format!( + "TLS Secret '{}' has not been created by cert-manager yet", + secret_name + ) + } else { + format!("TLS Secret '{}' was not found", secret_name) + }; + if managed_certificate { + tls_pending_with_certificate_ref( + ctx, + tenant, + config, + reason, + message, + certificate_ref, + ) + .await + } else { + tls_blocked(ctx, tenant, config, reason, message).await + } + } + Err(error) => { + let status_error = StatusError::from_context_error(&error); + patch_status_error(ctx, tenant, &status_error).await; + Err(error.into()) + } + } +} + +async fn ensure_cert_manager_certificate_crd(ctx: &Context) -> Result<(), context::Error> { + let api: Api = Api::all(ctx.client.clone()); + api.get(CERT_MANAGER_CERTIFICATE_CRD) + .await + .map(|_| ()) + .map_err(|source| context::Error::Kube { source }) +} + +async fn ensure_cert_manager_issuer( + ctx: &Context, + namespace: &str, + issuer_ref: &CertManagerIssuerRef, +) -> Result<(), context::Error> { + let resource = issuer_api_resource(issuer_ref); + if issuer_is_cluster_scoped(issuer_ref) { + let api: Api = Api::all_with(ctx.client.clone(), &resource); + api.get(&issuer_ref.name) + .await + .map(|_| ()) + .map_err(|source| context::Error::Kube { source }) + } else { + let api: Api = + Api::namespaced_with(ctx.client.clone(), namespace, &resource); + api.get(&issuer_ref.name) + .await + .map(|_| ()) + .map_err(|source| context::Error::Kube { source }) + } +} + +async fn apply_cert_manager_certificate( + ctx: &Context, + namespace: &str, + certificate_name: &str, + certificate: &DynamicObject, +) -> Result { + let resource = certificate_api_resource(); + let api: Api = Api::namespaced_with(ctx.client.clone(), namespace, &resource); + api.patch( + certificate_name, + &PatchParams::apply("rustfs-operator"), + &Patch::Apply(certificate), + ) + .await + .map_err(|source| context::Error::Kube { source }) +} + +async fn cert_manager_prerequisite_failed( + ctx: &Context, + tenant: &Tenant, + config: &TlsConfig, + prerequisite: CertManagerPrerequisite, + error: context::Error, + missing_message: String, +) -> Result { + if context::is_kube_not_found(&error) { + return tls_blocked( + ctx, + tenant, + config, + missing_cert_manager_prerequisite_reason(prerequisite), + missing_message, + ) + .await; + } + + let status_error = StatusError::from_context_error(&error); + patch_status_error(ctx, tenant, &status_error).await; + Err(error.into()) +} + +fn build_cert_manager_certificate( + tenant: &Tenant, + namespace: &str, + _config: &TlsConfig, + cert_manager: &CertManagerTlsConfig, + secret_name: &str, + certificate_name: &str, +) -> DynamicObject { + let mut spec = Map::new(); + spec.insert("secretName".to_string(), json!(secret_name)); + if let Some(issuer_ref) = cert_manager.issuer_ref.as_ref() { + spec.insert("issuerRef".to_string(), issuer_ref_value(issuer_ref)); + } + if let Some(common_name) = cert_manager + .common_name + .as_deref() + .filter(|common_name| !common_name.is_empty()) + { + spec.insert("commonName".to_string(), json!(common_name)); + } + spec.insert( + "dnsNames".to_string(), + json!(certificate_dns_names(tenant, namespace, cert_manager)), + ); + spec.insert( + "usages".to_string(), + json!(certificate_usages(cert_manager)), + ); + if let Some(duration) = cert_manager + .duration + .as_deref() + .filter(|duration| !duration.is_empty()) + { + spec.insert("duration".to_string(), json!(duration)); + } + if let Some(renew_before) = cert_manager + .renew_before + .as_deref() + .filter(|renew_before| !renew_before.is_empty()) + { + spec.insert("renewBefore".to_string(), json!(renew_before)); + } + if let Some(private_key) = cert_manager.private_key.as_ref() { + spec.insert("privateKey".to_string(), json!(private_key)); + } + spec.insert( + "secretTemplate".to_string(), + json!({ "labels": tenant.common_labels() }), + ); + + let resource = certificate_api_resource(); + let mut certificate = DynamicObject::new(certificate_name, &resource) + .within(namespace) + .data(json!({ "spec": Value::Object(spec) })); + certificate.metadata.labels = Some(tenant.common_labels()); + certificate.metadata.owner_references = Some(vec![tenant.new_owner_ref()]); + certificate +} + +fn observe_cert_manager_certificate( + certificate: &DynamicObject, +) -> CertManagerCertificateObservation { + let ready_condition = certificate + .data + .pointer("/status/conditions") + .and_then(Value::as_array) + .and_then(|conditions| { + conditions.iter().find(|condition| { + condition + .get("type") + .and_then(Value::as_str) + .is_some_and(|type_| type_ == "Ready") + }) + }); + let observed_generation = ready_condition + .and_then(|condition| condition.get("observedGeneration")) + .and_then(Value::as_i64) + .or_else(|| { + certificate + .data + .pointer("/status/observedGeneration") + .and_then(Value::as_i64) + }); + let condition_ready = ready_condition + .and_then(|condition| condition.get("status")) + .and_then(Value::as_str) + .is_some_and(|status| status == "True"); + let generation_current = + observed_generation_matches(certificate.metadata.generation, observed_generation); + let ready = condition_ready && generation_current; + let reason = if condition_ready && !generation_current { + Some(Reason::ObservedGenerationStale.as_str().to_string()) + } else { + ready_condition + .and_then(|condition| condition.get("reason")) + .and_then(Value::as_str) + .map(sanitize_status_message) + }; + let message = if condition_ready && !generation_current { + Some(format!( + "Certificate observedGeneration {} is older than metadata.generation {}", + observed_generation + .map(|generation| generation.to_string()) + .unwrap_or_else(|| "".to_string()), + certificate + .metadata + .generation + .map(|generation| generation.to_string()) + .unwrap_or_else(|| "".to_string()) + )) + } else { + ready_condition + .and_then(|condition| condition.get("message")) + .and_then(Value::as_str) + .map(sanitize_status_message) + }; + + CertManagerCertificateObservation { + name: certificate + .metadata + .name + .clone() + .unwrap_or_else(|| "".to_string()), + observed_generation, + ready, + reason, + message, + } +} + +fn certificate_not_ready_message( + certificate_name: &str, + observation: &CertManagerCertificateObservation, +) -> String { + let detail = observation + .message + .as_deref() + .or(observation.reason.as_deref()) + .unwrap_or("Ready condition is not True"); + format!( + "cert-manager Certificate '{}' is not Ready: {}", + certificate_name, detail + ) +} + +fn observed_generation_matches(generation: Option, observed_generation: Option) -> bool { + match (generation, observed_generation) { + (Some(generation), Some(observed_generation)) => observed_generation >= generation, + (Some(_), None) => false, + _ => true, + } +} + +#[cfg(test)] +fn tls_reason_for_certificate_observation( + observation: &CertManagerCertificateObservation, +) -> Reason { + if observation.ready { + Reason::TlsConfigured + } else { + Reason::CertManagerCertificateNotReady + } +} + +fn secret_missing_reason(managed_certificate: bool) -> Reason { + if managed_certificate { + Reason::CertificateSecretPending + } else { + Reason::CertificateSecretNotFound + } +} + +fn missing_cert_manager_prerequisite_reason(prerequisite: CertManagerPrerequisite) -> Reason { + match prerequisite { + CertManagerPrerequisite::CertificateCrd => Reason::CertManagerCrdMissing, + CertManagerPrerequisite::Issuer | CertManagerPrerequisite::ClusterIssuer => { + Reason::CertManagerIssuerNotFound + } + } +} + +fn certificate_name(tenant: &Tenant, cert_manager: &CertManagerTlsConfig) -> String { + cert_manager + .certificate_name + .as_deref() + .filter(|name| !name.is_empty()) + .map(ToString::to_string) + .unwrap_or_else(|| format!("{}-server-tls", tenant.name())) +} + +fn issuer_ref_value(issuer_ref: &CertManagerIssuerRef) -> Value { + json!({ + "group": if issuer_ref.group.is_empty() { CERT_MANAGER_GROUP } else { issuer_ref.group.as_str() }, + "kind": if issuer_ref.kind.is_empty() { CERT_MANAGER_ISSUER_KIND } else { issuer_ref.kind.as_str() }, + "name": issuer_ref.name, + }) +} + +fn certificate_dns_names( + tenant: &Tenant, + namespace: &str, + cert_manager: &CertManagerTlsConfig, +) -> Vec { + let mut names = BTreeSet::new(); + names.extend( + cert_manager + .dns_names + .iter() + .filter(|name| !name.is_empty()) + .cloned(), + ); + if cert_manager.include_generated_dns_names { + let tenant_name = tenant.name(); + let io_service = format!("{tenant_name}-io"); + let headless_service = tenant.headless_service_name(); + names.insert(format!("{io_service}.{namespace}.svc")); + names.insert(format!("{io_service}.{namespace}.svc.cluster.local")); + names.insert(format!("{headless_service}.{namespace}.svc")); + names.insert(format!("{headless_service}.{namespace}.svc.cluster.local")); + for pool in &tenant.spec.pools { + for ordinal in 0..pool.servers.max(0) { + names.insert(format!( + "{tenant_name}-{}-{ordinal}.{headless_service}.{namespace}.svc.cluster.local", + pool.name + )); + } + } + } + names.into_iter().collect() +} + +fn certificate_usages(cert_manager: &CertManagerTlsConfig) -> Vec { + if cert_manager.usages.is_empty() { + vec!["server auth".to_string()] + } else { + cert_manager.usages.clone() + } +} + +fn issuer_prerequisite(issuer_ref: &CertManagerIssuerRef) -> CertManagerPrerequisite { + if issuer_is_cluster_scoped(issuer_ref) { + CertManagerPrerequisite::ClusterIssuer + } else { + CertManagerPrerequisite::Issuer + } +} + +fn issuer_is_cluster_scoped(issuer_ref: &CertManagerIssuerRef) -> bool { + issuer_ref.kind == CERT_MANAGER_CLUSTER_ISSUER_KIND +} + +fn certificate_api_resource() -> ApiResource { + ApiResource::from_gvk_with_plural( + &GroupVersionKind::gvk( + CERT_MANAGER_GROUP, + CERT_MANAGER_VERSION, + CERT_MANAGER_CERTIFICATE_KIND, + ), + CERT_MANAGER_CERTIFICATE_PLURAL, + ) +} + +fn issuer_api_resource(issuer_ref: &CertManagerIssuerRef) -> ApiResource { + if issuer_is_cluster_scoped(issuer_ref) { + ApiResource::from_gvk_with_plural( + &GroupVersionKind::gvk( + CERT_MANAGER_GROUP, + CERT_MANAGER_VERSION, + CERT_MANAGER_CLUSTER_ISSUER_KIND, + ), + CERT_MANAGER_CLUSTER_ISSUER_PLURAL, + ) + } else { + ApiResource::from_gvk_with_plural( + &GroupVersionKind::gvk( + CERT_MANAGER_GROUP, + CERT_MANAGER_VERSION, + CERT_MANAGER_ISSUER_KIND, + ), + CERT_MANAGER_ISSUER_PLURAL, + ) + } +} + +fn sanitize_status_message(message: &str) -> String { + let collapsed = message.split_whitespace().collect::>().join(" "); + let mut chars = collapsed.chars(); + let truncated = chars + .by_ref() + .take(STATUS_MESSAGE_LIMIT) + .collect::(); + if chars.next().is_some() { + format!("{}...", truncated) + } else { + truncated + } +} + +async fn get_secret_or_tls_blocked( + ctx: &Context, + tenant: &Tenant, + config: &TlsConfig, + namespace: &str, + secret_name: &str, + missing_reason: Reason, + missing_message: String, +) -> Result { + match ctx.get::(secret_name, namespace).await { + Ok(secret) => Ok(secret), + Err(error) if context::is_kube_not_found(&error) => { + tls_blocked(ctx, tenant, config, missing_reason, missing_message).await + } + Err(error) => { + let status_error = StatusError::from_context_error(&error); + patch_status_error(ctx, tenant, &status_error).await; + Err(error.into()) + } + } +} + +async fn require_secret_key( + ctx: &Context, + tenant: &Tenant, + config: &TlsConfig, + secret: &Secret, + secret_name: &str, + key: &str, + missing_reason: Reason, +) -> Result, Error> { + match require_secret_key_bytes(secret, secret_name, key, missing_reason) { + Ok(bytes) => Ok(bytes.to_vec()), + Err(failure) => tls_validation_blocked(ctx, tenant, config, failure).await, + } +} + +fn require_secret_key_bytes<'a>( + secret: &'a Secret, + secret_name: &str, + key: &str, + missing_reason: Reason, +) -> Result<&'a [u8], TlsValidationFailure> { + secret_bytes(secret, key).ok_or_else(|| TlsValidationFailure { + reason: missing_reason, + message: format!( + "TLS Secret '{}' is missing required key '{}'", + secret_name, key + ), + }) +} + +fn validate_tls_secret_type( + secret: &Secret, + secret_name: &str, + expected_type: Option<&str>, +) -> Result<(), TlsValidationFailure> { + let actual_type = secret.type_.as_deref().unwrap_or(""); + if let Some(expected_type) = expected_type { + if actual_type == expected_type { + return Ok(()); + } + return Err(TlsValidationFailure { + reason: Reason::CertificateSecretInvalidType, + message: format!( + "TLS Secret '{}' has type '{}', expected '{}'", + secret_name, actual_type, expected_type + ), + }); + } + + if supported_tls_secret_type(actual_type) { + return Ok(()); + } + + Err(TlsValidationFailure { + reason: Reason::CertificateSecretInvalidType, + message: format!( + "TLS Secret '{}' has type '{}', expected one of: {}, {}, {}", + secret_name, + actual_type, + KUBERNETES_TLS_SECRET_TYPE, + CERT_MANAGER_V1_SECRET_TYPE, + CERT_MANAGER_V1ALPHA2_SECRET_TYPE + ), + }) +} + +fn supported_tls_secret_type(secret_type: &str) -> bool { + matches!( + secret_type, + KUBERNETES_TLS_SECRET_TYPE + | CERT_MANAGER_V1_SECRET_TYPE + | CERT_MANAGER_V1ALPHA2_SECRET_TYPE + ) +} + +fn validate_tls_secret_san_match( + secret_name: &str, + cert_bytes: &[u8], + expected_dns_names: &[String], +) -> Result<(), TlsValidationFailure> { + if expected_dns_names.is_empty() { + return Ok(()); + } + + let certs = rustls_pemfile::certs(&mut Cursor::new(cert_bytes)) + .collect::>, _>>() + .map_err(|_| TlsValidationFailure { + reason: Reason::CertificateInvalid, + message: format!( + "TLS certificate in Secret '{}' key '{}' must contain a valid PEM certificate", + secret_name, TLS_CERT_KEY + ), + })?; + let cert_der = certs.first().ok_or_else(|| TlsValidationFailure { + reason: Reason::CertificateInvalid, + message: format!( + "TLS certificate in Secret '{}' key '{}' must contain at least one valid PEM certificate", + secret_name, TLS_CERT_KEY + ), + })?; + let cert = webpki::EndEntityCert::try_from(cert_der).map_err(|_| TlsValidationFailure { + reason: Reason::CertificateInvalid, + message: format!( + "TLS certificate in Secret '{}' key '{}' must be a valid X.509 end-entity certificate", + secret_name, TLS_CERT_KEY + ), + })?; + + let mut missing = Vec::new(); + for dns_name in expected_dns_names { + let server_name = + ServerName::try_from(dns_name.as_str()).map_err(|_| TlsValidationFailure { + reason: Reason::CertificateSanMismatch, + message: format!("required TLS DNS name '{dns_name}' is invalid"), + })?; + if cert.verify_is_valid_for_subject_name(&server_name).is_err() { + missing.push(dns_name.clone()); + } + } + + if missing.is_empty() { + Ok(()) + } else { + Err(TlsValidationFailure { + reason: Reason::CertificateSanMismatch, + message: format!( + "TLS certificate in Secret '{}' key '{}' does not cover required DNS names: {}", + secret_name, + TLS_CERT_KEY, + missing.join(", ") + ), + }) + } +} + +fn certificate_secret_ca_material( + secret: &Secret, + secret_name: &str, + enable_internode_https: bool, + trust_system_ca: bool, +) -> Result, TlsValidationFailure> { + if let Some(ca_bytes) = secret_bytes(secret, CA_CERT_KEY) { + validate_ca_bundle_bytes(secret_name, CA_CERT_KEY, ca_bytes)?; + return Ok(Some(ServerCaMaterial { + key: CA_CERT_KEY.to_string(), + bytes: ca_bytes.to_vec(), + })); + } + + if enable_internode_https && !trust_system_ca { + return Err(TlsValidationFailure { + reason: Reason::CaBundleMissing, + message: format!( + "TLS Secret '{}' is missing '{}' while spec.tls.enableInternodeHttps=true and trustSystemCa is false", + secret_name, CA_CERT_KEY + ), + }); + } + + Ok(None) +} + +fn validate_ca_bundle_bytes( + secret_name: &str, + key: &str, + bytes: &[u8], +) -> Result<(), TlsValidationFailure> { + let parsed = rustls_pemfile::certs(&mut Cursor::new(bytes)).collect::, _>>(); + match parsed { + Ok(certs) if !certs.is_empty() => Ok(()), + Ok(_) | Err(_) => Err(TlsValidationFailure { + reason: Reason::CaBundleInvalid, + message: format!( + "CA bundle in Secret '{}' key '{}' must contain at least one valid PEM certificate", + secret_name, key + ), + }), + } +} + +async fn tls_validation_blocked( + ctx: &Context, + tenant: &Tenant, + config: &TlsConfig, + failure: TlsValidationFailure, +) -> Result { + tls_blocked(ctx, tenant, config, failure.reason, failure.message).await +} + +async fn tls_blocked( + ctx: &Context, + tenant: &Tenant, + config: &TlsConfig, + reason: Reason, + message: String, +) -> Result { + patch_tls_error(ctx, tenant, config, reason, &message, true).await?; + Err(Error::TlsBlocked { + reason: reason.as_str().to_string(), + message, + }) +} + +async fn tls_pending_with_certificate_ref( + ctx: &Context, + tenant: &Tenant, + config: &TlsConfig, + reason: Reason, + message: String, + certificate_ref: Option, +) -> Result { + patch_tls_error_with_certificate_ref( + ctx, + tenant, + config, + reason, + &message, + false, + certificate_ref, + ) + .await?; + Err(Error::TlsPending { + reason: reason.as_str().to_string(), + message, + }) +} + +async fn patch_tls_error( + ctx: &Context, + tenant: &Tenant, + config: &TlsConfig, + reason: Reason, + message: &str, + blocked: bool, +) -> Result<(), Error> { + patch_tls_error_with_certificate_ref(ctx, tenant, config, reason, message, blocked, None).await +} + +async fn patch_tls_error_with_certificate_ref( + ctx: &Context, + tenant: &Tenant, + config: &TlsConfig, + reason: Reason, + message: &str, + blocked: bool, + certificate_ref: Option, +) -> Result<(), Error> { + let status_error = if blocked { + StatusError::tls_blocked(reason, message.to_string()) + } else { + StatusError::tls_reconciling(reason, message.to_string()) + }; + let mut builder = StatusBuilder::from_tenant(tenant); + builder.set_tls_status(error_tls_status_with_certificate_ref( + config, + reason, + message, + certificate_ref, + )); + builder.mark_error(&status_error); + let status = builder.build(); + patch_status_and_record( + ctx, + tenant, + status, + status_error.condition_type, + status_error.reason, + status_error.event_type, + &status_error.safe_message, + ) + .await +} + +fn cert_manager_tls_status( + config: &TlsConfig, + secret_name: &str, + secret: &Secret, + explicit_ca: Option<(&SecretKeyReference, &Secret)>, + client_ca: Option<(&SecretKeyReference, &Secret)>, + hash: &str, + certificate_ref: Option, +) -> TlsCertificateStatus { + let ca_trust = config.ca_trust(); + TlsCertificateStatus { + mode: tls_mode_name(config.mode).to_string(), + ready: true, + managed_certificate: config + .cert_manager + .as_ref() + .map(|cert_manager| cert_manager.manage_certificate), + rotation_strategy: Some(rotation_strategy_name(config.rotation_strategy).to_string()), + mount_path: Some(config.mount_path.clone()), + certificate_ref, + server_secret_ref: Some(SecretStatusRef { + name: secret_name.to_string(), + key: None, + resource_version: secret.metadata.resource_version.clone(), + }), + ca_secret_ref: ca_status_ref(secret_name, secret, explicit_ca), + client_ca_secret_ref: client_ca.map(|(secret_ref, ca_secret)| SecretStatusRef { + name: secret_ref.name.clone(), + key: Some(secret_ref.key.clone()), + resource_version: ca_secret.metadata.resource_version.clone(), + }), + observed_hash: Some(hash.to_string()), + dns_names: config + .cert_manager + .as_ref() + .map(|cert_manager| cert_manager.dns_names.clone()) + .unwrap_or_default(), + trust_source: Some(ca_trust_source_name(ca_trust.source).to_string()), + last_validated_time: Some( + chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true), + ), + ..Default::default() + } +} + +fn ca_status_ref( + secret_name: &str, + secret: &Secret, + explicit_ca: Option<(&SecretKeyReference, &Secret)>, +) -> Option { + if let Some((secret_ref, ca_secret)) = explicit_ca { + return Some(SecretStatusRef { + name: secret_ref.name.clone(), + key: Some(secret_ref.key.clone()), + resource_version: ca_secret.metadata.resource_version.clone(), + }); + } + secret_bytes(secret, CA_CERT_KEY).map(|_| SecretStatusRef { + name: secret_name.to_string(), + key: Some(CA_CERT_KEY.to_string()), + resource_version: secret.metadata.resource_version.clone(), + }) +} + +#[cfg(test)] +fn error_tls_status(config: &TlsConfig, reason: Reason, message: &str) -> TlsCertificateStatus { + error_tls_status_with_certificate_ref(config, reason, message, None) +} + +fn error_tls_status_with_certificate_ref( + config: &TlsConfig, + reason: Reason, + message: &str, + certificate_ref: Option, +) -> TlsCertificateStatus { + TlsCertificateStatus { + mode: tls_mode_name(config.mode).to_string(), + ready: false, + managed_certificate: config + .cert_manager + .as_ref() + .map(|cert_manager| cert_manager.manage_certificate), + rotation_strategy: Some(rotation_strategy_name(config.rotation_strategy).to_string()), + mount_path: Some(config.mount_path.clone()), + certificate_ref, + trust_source: Some(ca_trust_source_name(config.ca_trust().source).to_string()), + last_error_reason: Some(reason.as_str().to_string()), + last_error_message: Some(message.to_string()), + ..Default::default() + } +} + +fn tls_hash( + config: &TlsConfig, + secret: &Secret, + explicit_ca: Option<&SecretKeyReference>, + explicit_ca_bytes: Option<&[u8]>, + client_ca: Option<&SecretKeyReference>, + client_ca_bytes: Option<&[u8]>, + trust_system_ca: bool, +) -> String { + let mut hasher = Sha256::new(); + hash_str(&mut hasher, "mountPath", &config.mount_path); + hash_str( + &mut hasher, + "rotationStrategy", + rotation_strategy_name(config.rotation_strategy), + ); + hash_str( + &mut hasher, + "enableInternodeHttps", + if config.enable_internode_https { + "true" + } else { + "false" + }, + ); + hash_str( + &mut hasher, + "trustSystemCa", + if trust_system_ca { "true" } else { "false" }, + ); + hash_str( + &mut hasher, + "serverSecret.resourceVersion", + secret.metadata.resource_version.as_deref().unwrap_or(""), + ); + hash_bytes(&mut hasher, "tls.crt", secret_bytes(secret, TLS_CERT_KEY)); + hash_bytes( + &mut hasher, + "secret.ca.crt", + secret_bytes(secret, CA_CERT_KEY), + ); + if let Some(secret_ref) = explicit_ca { + hash_str(&mut hasher, "explicitCa.name", &secret_ref.name); + hash_str(&mut hasher, "explicitCa.key", &secret_ref.key); + } + hash_bytes(&mut hasher, "explicitCa.bytes", explicit_ca_bytes); + if let Some(secret_ref) = client_ca { + hash_str(&mut hasher, "clientCa.name", &secret_ref.name); + hash_str(&mut hasher, "clientCa.key", &secret_ref.key); + } + hash_bytes(&mut hasher, "clientCa.bytes", client_ca_bytes); + format!("sha256:{:x}", hasher.finalize()) +} + +fn hash_str(hasher: &mut Sha256, label: &str, value: &str) { + hasher.update(label.as_bytes()); + hasher.update([0]); + hasher.update(value.len().to_le_bytes()); + hasher.update(value.as_bytes()); + hasher.update([0]); +} + +fn hash_bytes(hasher: &mut Sha256, label: &str, value: Option<&[u8]>) { + hasher.update(label.as_bytes()); + hasher.update([0]); + match value { + Some(bytes) => { + hasher.update(bytes.len().to_le_bytes()); + hasher.update(bytes); + } + None => hasher.update(0usize.to_le_bytes()), + } + hasher.update([0]); +} + +fn secret_bytes<'a>(secret: &'a Secret, key: &str) -> Option<&'a [u8]> { + secret + .data + .as_ref()? + .get(key) + .map(|bytes| bytes.0.as_slice()) +} + +const fn tls_mode_name(mode: TlsMode) -> &'static str { + match mode { + TlsMode::Disabled => "Disabled", + TlsMode::External => "External", + TlsMode::CertManager => "CertManager", + } +} + +const fn rotation_strategy_name(strategy: TlsRotationStrategy) -> &'static str { + match strategy { + TlsRotationStrategy::Rollout => "Rollout", + TlsRotationStrategy::HotReload => "HotReload", + } +} + +const fn ca_trust_source_name(source: CaTrustSource) -> &'static str { + match source { + CaTrustSource::CertificateSecretCa => "CertificateSecretCa", + CaTrustSource::SecretRef => "SecretRef", + CaTrustSource::SystemCa => "SystemCa", + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::v1alpha1::tls::{ + CaTrustConfig, CertManagerPrivateKeyConfig, CertManagerTlsConfig, + }; + use k8s_openapi::ByteString; + use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; + use kube::CustomResourceExt; + use std::collections::BTreeMap; + + const PUBLIC_CERT_PEM: &[u8] = b"-----BEGIN CERTIFICATE-----\nMIIDCTCCAfGgAwIBAgIUD4D7ObFcJ5PEZwq2t/cmrTbzcU0wDQYJKoZIhvcNAQEL\nBQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTI1MTExMDA3NDQwNVoXDTI2MTEx\nMDA3NDQwNVowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF\nAAOCAQ8AMIIBCgKCAQEAsnrreaQGztdaTppY7p1ExoDU7FpYjk8MalWs9xIioHTe\ndpDlZmEWak0Q80qTvc+x6GT8VD/pLYqg6B2mot8I+Uv44GUmpPD/+WDxVbjvwL2b\nfvcNGEniqKJUOy2za98WcmI8EoILwbmYy7cZslf6b3D0xuDsmovYJgtjNeziV6ie\nLQfbWWXhAipYhUwaBAdUSQS+BWPPdYFG4LEE/8+BqmYdGU7ujIFlqSU89ZMfpZS4\npVRoEy16fs5O0UkbP1l63Q0qBLrLXjWw874dV8wC2p9iuVwofpDZRGhfYFaviZHb\nMHdUBRUughU4vvTknAGwMzbrIH+eTp7aKrGKWb7ozQIDAQABo1MwUTAdBgNVHQ4E\nFgQUGSE2L3XLbuxlA1Q0iX65aVGKzl4wHwYDVR0jBBgwFoAUGSE2L3XLbuxlA1Q0\niX65aVGKzl4wDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAGHwM\nSYFN1/9ZlriVaJEpSvGlfeDvN5ipXqf0s1Ykux9rsTYchn7tcA6zhWqZUimwy/jO\nI7jLfBNa3r5HT1uX3/RlMs6dMIO4h3vkSWjQ3QaGiuXh6U+erbkaeETtrw9b40ta\nDsj2rruE3Z11JV0y5fGcvXjXMFV7XsFQjNXF5TlXu4OUvfMeo9h4IbPmNQtq+g+t\nnx0ZBloqo+punQVjHjovoQUWlrOOL5ZRZl1vLqqhHfw54a9weCXY8XJNnxWN0l0C\nKzht0TgbidDlWKBsk/CMTY8zpYrfVyPhnjNCeFGFG0DzrsehCgpEiEZ6vlylei7c\nRfKUdp4DXmUZBDzeQw==\n-----END CERTIFICATE-----\n"; + const CERT_WITH_PEER_SANS_PEM: &[u8] = br#"-----BEGIN CERTIFICATE----- +MIIDoDCCAoigAwIBAgIUeB45TQucDL0Dm5Jn7CyeIWTRkQUwDQYJKoZIhvcNAQEL +BQAwEzERMA8GA1UEAwwIdGVuYW50LWEwHhcNMjYwNTEzMDkyODA4WhcNMjYwNTE0 +MDkyODA4WjATMREwDwYDVQQDDAh0ZW5hbnQtYTCCASIwDQYJKoZIhvcNAQEBBQAD +ggEPADCCAQoCggEBAKPvXLnfHwjzz1EsnINmuJfBGcUf6dFgw+seTNXbBDEfQ/+R +tpmTa1TO5Eqo9utDk7TZx9GTGr1vFArOP8MBEJ0qdx5YCvoWVoexVc1FhsFSe9Mv ++EGpV9RGniIfMmkVj8BHR+rmTopRoHEYnsDL9wm9D47GNbSuHuMHG4qlkLY4270a +QDMaTGgaH0iLN63ISl6mf/ca55kWqrcCmERNvpfA7EywYm8wwyPf8fURjfg+nKGL +CJ2roZrpXJnUhQmAMF0RDx+Q02RAgkJXClO59Qk9vm7QpnIKwglUPuYK/LJ3bSA7 +4COHbYZxDatBedyBZFDlUZw0kNnQo5+JJSPfKAUCAwEAAaOB6zCB6DAdBgNVHQ4E +FgQUUGaN5hB6CJ9Ds0s9zlG1R/YhiQ4wHwYDVR0jBBgwFoAUUGaN5hB6CJ9Ds0s9 +zlG1R/YhiQ4wDwYDVR0TAQH/BAUwAwEB/zCBlAYDVR0RBIGMMIGJggh0ZW5hbnQt +YYIJbG9jYWxob3N0gjh0ZW5hbnQtYS1wcmltYXJ5LTAudGVuYW50LWEtaGwuc3Rv +cmFnZS5zdmMuY2x1c3Rlci5sb2NhbII4dGVuYW50LWEtcHJpbWFyeS0xLnRlbmFu +dC1hLWhsLnN0b3JhZ2Uuc3ZjLmNsdXN0ZXIubG9jYWwwDQYJKoZIhvcNAQELBQAD +ggEBAAqx762x484bIVcdQXE1dO6GhFPS8OoZWBxFAURnfep8H9lwVgcoXLgglpjM +dfD9EaPNjXpixDX/SK6nI/rCVnbHXFk1nEBpWBHC+XBPIj/J3nUeuhEGJPjif0KX +wjIUfC3RADGlA7AdgLeFJ21FOwtmjdxUsD2aZ1gqOm3flsyBxuIFozZEi1ZTlBes +90l8P6bqksl/3t9ssTdIF5O/mtKJqy8fBXsE2yazKO6dl1Mt7Zn4Lw6OQraaxNWT +S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj +84SYe6DQkgSslVKrEZIaVd/q8t8= +-----END CERTIFICATE----- +"#; + + #[test] + fn tenant_crd_schema_types_cert_manager_private_key() { + let crd = serde_json::to_value(Tenant::crd()).expect("tenant CRD serializes to JSON"); + let private_key_schema = crd + .pointer("/spec/versions/0/schema/openAPIV3Schema/properties/spec/properties/tls/properties/certManager/properties/privateKey") + .expect("spec.tls.certManager.privateKey schema exists"); + let properties = private_key_schema + .pointer("/properties") + .and_then(Value::as_object) + .expect("privateKey schema has typed properties"); + + assert_eq!( + private_key_schema.pointer("/type").and_then(Value::as_str), + Some("object") + ); + assert_eq!( + properties + .get("algorithm") + .and_then(|schema| schema.pointer("/type")) + .and_then(Value::as_str), + Some("string") + ); + assert_eq!( + properties + .get("encoding") + .and_then(|schema| schema.pointer("/type")) + .and_then(Value::as_str), + Some("string") + ); + assert_eq!( + properties + .get("rotationPolicy") + .and_then(|schema| schema.pointer("/type")) + .and_then(Value::as_str), + Some("string") + ); + assert_eq!( + properties + .get("size") + .and_then(|schema| schema.pointer("/type")) + .and_then(Value::as_str), + Some("integer") + ); + } + + #[test] + fn default_secret_type_rejects_unconventional_server_secret() { + let secret = tls_secret("server-tls", "7", Some("Opaque"), true, true, None); + + let error = validate_tls_secret_type(&secret, "server-tls", None); + + assert_validation_reason(error, Reason::CertificateSecretInvalidType); + } + + #[test] + fn default_secret_type_accepts_kubernetes_and_cert_manager_tls_types() { + for secret_type in [ + "kubernetes.io/tls", + "cert-manager.io/v1", + "cert-manager.io/v1alpha2", + ] { + let secret = tls_secret("server-tls", "7", Some(secret_type), true, true, None); + + assert!(validate_tls_secret_type(&secret, "server-tls", None).is_ok()); + } + } + + #[test] + fn explicit_secret_type_rejects_supported_but_wrong_type() { + let secret = tls_secret( + "server-tls", + "7", + Some("kubernetes.io/tls"), + true, + true, + None, + ); + + let error = validate_tls_secret_type(&secret, "server-tls", Some("cert-manager.io/v1")); + + assert_validation_reason(error, Reason::CertificateSecretInvalidType); + } + + #[test] + fn missing_server_tls_key_maps_to_certificate_secret_missing_key() { + let secret = tls_secret( + "server-tls", + "7", + Some("kubernetes.io/tls"), + false, + true, + None, + ); + + let error = require_secret_key_bytes( + &secret, + "server-tls", + TLS_KEY_KEY, + Reason::CertificateSecretMissingKey, + ); + + assert_validation_reason(error, Reason::CertificateSecretMissingKey); + } + + #[test] + fn missing_ca_key_maps_to_ca_bundle_missing() { + let secret = tls_secret("server-ca", "7", Some("Opaque"), false, true, None); + + let error = + require_secret_key_bytes(&secret, "server-ca", CA_CERT_KEY, Reason::CaBundleMissing); + + assert_validation_reason(error, Reason::CaBundleMissing); + } + + #[test] + fn internode_https_allows_system_ca_without_secret_ca_bundle() { + let secret = tls_secret( + "server-tls", + "7", + Some("kubernetes.io/tls"), + true, + true, + None, + ); + + let ca = certificate_secret_ca_material(&secret, "server-tls", true, true); + + assert!(ca.is_ok()); + assert!(matches!(ca, Ok(None))); + } + + #[test] + fn internode_https_requires_ca_bundle_without_system_ca() { + let secret = tls_secret( + "server-tls", + "7", + Some("kubernetes.io/tls"), + true, + true, + None, + ); + + let error = certificate_secret_ca_material(&secret, "server-tls", true, false); + + assert_validation_reason(error, Reason::CaBundleMissing); + } + + #[test] + fn invalid_ca_bundle_maps_to_ca_bundle_invalid() { + let secret = tls_secret( + "server-tls", + "7", + Some("kubernetes.io/tls"), + true, + true, + Some(b"not a pem certificate"), + ); + + let error = certificate_secret_ca_material(&secret, "server-tls", false, false); + + assert_validation_reason(error, Reason::CaBundleInvalid); + } + + #[test] + fn hot_reload_remains_explicitly_unsupported_in_tls_status() { + let config = TlsConfig { + mode: TlsMode::CertManager, + rotation_strategy: TlsRotationStrategy::HotReload, + ..Default::default() + }; + + let status = error_tls_status( + &config, + Reason::TlsHotReloadUnsupported, + "hot reload unsupported", + ); + + assert!(!status.ready); + assert_eq!( + status.last_error_reason.as_deref(), + Some("TlsHotReloadUnsupported") + ); + assert_eq!(status.rotation_strategy.as_deref(), Some("HotReload")); + } + + #[test] + fn tls_hash_uses_resource_version_public_cert_and_ca_not_private_key() { + let config = TlsConfig::default(); + let first = tls_secret( + "server-tls", + "7", + Some("kubernetes.io/tls"), + true, + true, + Some(PUBLIC_CERT_PEM), + ); + let changed_private_key = tls_secret( + "server-tls", + "7", + Some("kubernetes.io/tls"), + true, + false, + Some(PUBLIC_CERT_PEM), + ); + let changed_resource_version = tls_secret( + "server-tls", + "8", + Some("kubernetes.io/tls"), + true, + false, + Some(PUBLIC_CERT_PEM), + ); + let changed_ca = tls_secret( + "server-tls", + "7", + Some("kubernetes.io/tls"), + true, + true, + Some(b"-----BEGIN CERTIFICATE-----\ninvalid\n-----END CERTIFICATE-----\n"), + ); + + let baseline = tls_hash(&config, &first, None, None, None, None, false); + let private_key_changed = + tls_hash(&config, &changed_private_key, None, None, None, None, false); + let resource_version_changed = tls_hash( + &config, + &changed_resource_version, + None, + None, + None, + None, + false, + ); + let ca_changed = tls_hash(&config, &changed_ca, None, None, None, None, false); + + assert_eq!(baseline, private_key_changed); + assert_ne!(baseline, resource_version_changed); + assert_ne!(baseline, ca_changed); + } + + #[test] + fn require_san_match_accepts_certificate_covering_required_peer_dns_names() { + let expected_dns_names = vec![ + "tenant-a-primary-0.tenant-a-hl.storage.svc.cluster.local".to_string(), + "tenant-a-primary-1.tenant-a-hl.storage.svc.cluster.local".to_string(), + "localhost".to_string(), + ]; + + assert_eq!( + validate_tls_secret_san_match( + "server-tls", + CERT_WITH_PEER_SANS_PEM, + &expected_dns_names + ), + Ok(()) + ); + } + + #[test] + fn require_san_match_rejects_certificate_missing_required_peer_dns_names() { + let expected_dns_names = vec![ + "tenant-a-primary-0.tenant-a-hl.storage.svc.cluster.local".to_string(), + "tenant-a-primary-2.tenant-a-hl.storage.svc.cluster.local".to_string(), + ]; + + let failure = validate_tls_secret_san_match( + "server-tls", + CERT_WITH_PEER_SANS_PEM, + &expected_dns_names, + ) + .expect_err("missing peer DNS should fail SAN validation"); + + assert_eq!(failure.reason, Reason::CertificateSanMismatch); + assert!( + failure + .message + .contains("tenant-a-primary-2.tenant-a-hl.storage.svc.cluster.local"), + "message should name missing peer DNS: {}", + failure.message + ); + assert!( + !failure.message.contains("BEGIN CERTIFICATE"), + "SAN mismatch message must not expose certificate material: {}", + failure.message + ); + } + + #[test] + fn tls_status_records_explicit_ca_and_client_ca_resource_versions() { + let config = TlsConfig { + mode: TlsMode::CertManager, + cert_manager: Some(CertManagerTlsConfig { + secret_name: Some("server-tls".to_string()), + ca_trust: Some(CaTrustConfig { + source: CaTrustSource::SecretRef, + ca_secret_ref: Some(secret_ref("server-ca", "ca.crt")), + client_ca_secret_ref: Some(secret_ref("client-ca", "client_ca.crt")), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + let server = tls_secret( + "server-tls", + "7", + Some("kubernetes.io/tls"), + true, + true, + None, + ); + let ca = tls_secret( + "server-ca", + "11", + Some("Opaque"), + false, + false, + Some(PUBLIC_CERT_PEM), + ); + let client_ca = tls_secret( + "client-ca", + "13", + Some("Opaque"), + false, + false, + Some(PUBLIC_CERT_PEM), + ); + + let status = cert_manager_tls_status( + &config, + "server-tls", + &server, + Some((&secret_ref("server-ca", "ca.crt"), &ca)), + Some((&secret_ref("client-ca", "client_ca.crt"), &client_ca)), + "sha256:test", + None, + ); + + assert_eq!( + status + .ca_secret_ref + .as_ref() + .and_then(|secret| secret.resource_version.as_deref()), + Some("11") + ); + assert_eq!( + status + .client_ca_secret_ref + .as_ref() + .and_then(|secret| secret.resource_version.as_deref()), + Some("13") + ); + } + + #[test] + fn managed_certificate_manifest_renders_spec_owner_and_generated_dns() { + let mut tenant = crate::tests::create_test_tenant(None, None); + tenant.metadata.name = Some("tenant-a".to_string()); + tenant.metadata.namespace = Some("storage".to_string()); + tenant.spec.pools[0].servers = 2; + let config = TlsConfig { + mode: TlsMode::CertManager, + cert_manager: Some(CertManagerTlsConfig { + manage_certificate: true, + certificate_name: Some("tenant-a-server".to_string()), + secret_name: Some("tenant-a-server-tls".to_string()), + issuer_ref: Some(CertManagerIssuerRef { + group: "cert-manager.io".to_string(), + kind: "Issuer".to_string(), + name: "rustfs-issuer".to_string(), + }), + common_name: Some("tenant-a-io.storage.svc".to_string()), + dns_names: vec!["custom.storage.svc".to_string()], + duration: Some("2160h".to_string()), + renew_before: Some("360h".to_string()), + private_key: Some(CertManagerPrivateKeyConfig { + algorithm: Some("RSA".to_string()), + size: Some(2048), + ..Default::default() + }), + usages: vec!["server auth".to_string(), "client auth".to_string()], + ..Default::default() + }), + ..Default::default() + }; + let Some(cert_manager) = config.cert_manager.as_ref() else { + panic!("test config must include cert-manager settings"); + }; + + let certificate = build_cert_manager_certificate( + &tenant, + "storage", + &config, + cert_manager, + "tenant-a-server-tls", + "tenant-a-server", + ); + let dns_names = certificate + .data + .pointer("/spec/dnsNames") + .and_then(serde_json::Value::as_array) + .map(|items| { + items + .iter() + .filter_map(serde_json::Value::as_str) + .map(ToString::to_string) + .collect::>() + }); + + assert_eq!( + certificate.metadata.name.as_deref(), + Some("tenant-a-server") + ); + assert_eq!(certificate.metadata.namespace.as_deref(), Some("storage")); + assert_eq!( + certificate + .metadata + .owner_references + .as_ref() + .and_then(|owners| owners.first()) + .map(|owner| (owner.kind.as_str(), owner.name.as_str(), owner.controller)), + Some(("Tenant", "tenant-a", Some(true))) + ); + assert_eq!( + certificate.data.pointer("/spec/secretName"), + Some(&serde_json::json!("tenant-a-server-tls")) + ); + assert_eq!( + certificate + .data + .pointer("/spec/secretTemplate/labels/rustfs.tenant"), + Some(&serde_json::json!("tenant-a")) + ); + assert_eq!( + certificate + .data + .pointer("/spec/secretTemplate/labels/app.kubernetes.io~1managed-by"), + Some(&serde_json::json!("rustfs-operator")) + ); + assert_eq!( + certificate.data.pointer("/spec/issuerRef/name"), + Some(&serde_json::json!("rustfs-issuer")) + ); + assert_eq!( + certificate.data.pointer("/spec/duration"), + Some(&serde_json::json!("2160h")) + ); + assert_eq!( + certificate.data.pointer("/spec/renewBefore"), + Some(&serde_json::json!("360h")) + ); + assert_eq!( + certificate.data.pointer("/spec/privateKey/algorithm"), + Some(&serde_json::json!("RSA")) + ); + assert_eq!( + dns_names, + Some(vec![ + "custom.storage.svc".to_string(), + "tenant-a-hl.storage.svc".to_string(), + "tenant-a-hl.storage.svc.cluster.local".to_string(), + "tenant-a-io.storage.svc".to_string(), + "tenant-a-io.storage.svc.cluster.local".to_string(), + "tenant-a-pool-0-0.tenant-a-hl.storage.svc.cluster.local".to_string(), + "tenant-a-pool-0-1.tenant-a-hl.storage.svc.cluster.local".to_string(), + ]) + ); + } + + #[test] + fn certificate_observation_requires_ready_condition_for_current_generation() { + let ready = certificate_object( + "tenant-a-server", + Some(3), + serde_json::json!({ + "status": { + "observedGeneration": 3, + "conditions": [{"type": "Ready", "status": "True", "reason": "Ready", "message": "Certificate is up to date"}] + } + }), + ); + let stale = certificate_object( + "tenant-a-server", + Some(4), + serde_json::json!({ + "status": { + "observedGeneration": 3, + "conditions": [{"type": "Ready", "status": "True", "reason": "Ready", "message": "Old revision"}] + } + }), + ); + + let ready_observation = observe_cert_manager_certificate(&ready); + let stale_observation = observe_cert_manager_certificate(&stale); + + assert!(ready_observation.ready); + assert_eq!(ready_observation.observed_generation, Some(3)); + assert_eq!(ready_observation.reason.as_deref(), Some("Ready")); + assert!(!stale_observation.ready); + assert_eq!(stale_observation.observed_generation, Some(3)); + assert_eq!( + stale_observation.reason.as_deref(), + Some("ObservedGenerationStale") + ); + } + + #[test] + fn certificate_observation_uses_ready_condition_observed_generation() { + let ready = certificate_object( + "tenant-a-server", + Some(3), + serde_json::json!({ + "status": { + "conditions": [{ + "type": "Ready", + "status": "True", + "observedGeneration": 3, + "reason": "Ready", + "message": "Certificate is up to date" + }] + } + }), + ); + + let observation = observe_cert_manager_certificate(&ready); + + assert!(observation.ready); + assert_eq!(observation.observed_generation, Some(3)); + assert_eq!(observation.reason.as_deref(), Some("Ready")); + } + + #[test] + fn certificate_observation_marks_stale_ready_condition_observed_generation() { + let stale = certificate_object( + "tenant-a-server", + Some(4), + serde_json::json!({ + "status": { + "conditions": [{ + "type": "Ready", + "status": "True", + "observedGeneration": 3, + "reason": "Ready", + "message": "Old revision" + }] + } + }), + ); + + let observation = observe_cert_manager_certificate(&stale); + + assert!(!observation.ready); + assert_eq!(observation.observed_generation, Some(3)); + assert_eq!( + observation.reason.as_deref(), + Some("ObservedGenerationStale") + ); + } + + #[test] + fn pending_certificate_and_managed_secret_missing_map_to_reconciling_reasons() { + let pending = certificate_object( + "tenant-a-server", + Some(3), + serde_json::json!({ + "status": { + "observedGeneration": 3, + "conditions": [{"type": "Ready", "status": "False", "reason": "DoesNotExist", "message": "Secret is not available\nretrying"}] + } + }), + ); + + let observation = observe_cert_manager_certificate(&pending); + + assert!(!observation.ready); + assert_eq!( + tls_reason_for_certificate_observation(&observation), + Reason::CertManagerCertificateNotReady + ); + assert_eq!( + observation.message.as_deref(), + Some("Secret is not available retrying") + ); + assert_eq!( + secret_missing_reason(true), + Reason::CertificateSecretPending + ); + assert_eq!( + secret_missing_reason(false), + Reason::CertificateSecretNotFound + ); + } + + #[test] + fn cert_manager_prerequisite_missing_resources_map_to_stable_reasons() { + assert_eq!( + missing_cert_manager_prerequisite_reason(CertManagerPrerequisite::CertificateCrd), + Reason::CertManagerCrdMissing + ); + assert_eq!( + missing_cert_manager_prerequisite_reason(CertManagerPrerequisite::Issuer), + Reason::CertManagerIssuerNotFound + ); + assert_eq!( + missing_cert_manager_prerequisite_reason(CertManagerPrerequisite::ClusterIssuer), + Reason::CertManagerIssuerNotFound + ); + } + + fn assert_validation_reason(result: Result, reason: Reason) { + assert!( + matches!(result, Err(TlsValidationFailure { reason: actual, .. }) if actual == reason) + ); + } + + fn secret_ref(name: &str, key: &str) -> SecretKeyReference { + SecretKeyReference { + name: name.to_string(), + key: key.to_string(), + } + } + + fn certificate_object(name: &str, generation: Option, data: Value) -> DynamicObject { + let mut object = DynamicObject::new(name, &certificate_api_resource()).data(data); + object.metadata.generation = generation; + object + } + + fn tls_secret( + name: &str, + resource_version: &str, + type_: Option<&str>, + include_tls_keys: bool, + first_key: bool, + ca: Option<&[u8]>, + ) -> Secret { + let mut data = BTreeMap::new(); + if include_tls_keys { + data.insert( + TLS_CERT_KEY.to_string(), + ByteString(PUBLIC_CERT_PEM.to_vec()), + ); + data.insert( + TLS_KEY_KEY.to_string(), + ByteString( + if first_key { + b"private-a" + } else { + b"private-b" + } + .to_vec(), + ), + ); + } + if let Some(ca) = ca { + data.insert(CA_CERT_KEY.to_string(), ByteString(ca.to_vec())); + } + + Secret { + metadata: ObjectMeta { + name: Some(name.to_string()), + resource_version: Some(resource_version.to_string()), + ..Default::default() + }, + type_: type_.map(ToString::to_string), + data: Some(data), + ..Default::default() + } + } +} diff --git a/src/status.rs b/src/status.rs index 5f1ac2d..9696b49 100644 --- a/src/status.rs +++ b/src/status.rs @@ -15,8 +15,8 @@ use crate::context; use crate::types; use crate::types::v1alpha1::status::{ - ConditionInput, ConditionStatus, ConditionType, Reason, Status, is_blocked_reason, pool, - summarize_current_state, + ConditionInput, ConditionStatus, ConditionType, Reason, Status, certificate, is_blocked_reason, + pool, summarize_current_state, }; use crate::types::v1alpha1::tenant::Tenant; use kube::runtime::events::EventType; @@ -172,6 +172,14 @@ impl StatusError { ) } + pub fn tls_blocked(reason: Reason, safe_message: String) -> Self { + Self::blocked(reason, ConditionType::TlsReady, safe_message) + } + + pub fn tls_reconciling(reason: Reason, safe_message: String) -> Self { + Self::transient(reason, ConditionType::TlsReady, safe_message) + } + fn blocked(reason: Reason, condition_type: ConditionType, safe_message: String) -> Self { Self { reason, @@ -223,6 +231,19 @@ impl StatusBuilder { self.next.pools = pools; } + pub fn set_tls_status(&mut self, tls: certificate::TlsCertificateStatus) { + let ready = tls.ready; + self.next.certificates.tls = Some(tls); + if ready { + self.set_condition( + ConditionType::TlsReady, + ConditionStatus::True, + Reason::TlsConfigured, + "TLS is configured for RustFS workloads".to_string(), + ); + } + } + pub fn mark_started(&mut self) { self.set_condition( ConditionType::Ready, @@ -721,6 +742,7 @@ mod tests { "False", "CredentialSecretNotFound", )], + ..Default::default() }); let status_error = StatusError { reason: Reason::KubernetesApiError, diff --git a/src/types/v1alpha1.rs b/src/types/v1alpha1.rs index 20ec5d0..13f73f2 100755 --- a/src/types/v1alpha1.rs +++ b/src/types/v1alpha1.rs @@ -19,6 +19,7 @@ pub mod persistence; pub mod pool; pub mod status; pub mod tenant; +pub mod tls; // Re-export commonly used types pub use pool::{SchedulingConfig, validate_pool_total_volumes}; diff --git a/src/types/v1alpha1/status.rs b/src/types/v1alpha1/status.rs index 10ca661..7b5a77d 100755 --- a/src/types/v1alpha1/status.rs +++ b/src/types/v1alpha1/status.rs @@ -26,6 +26,7 @@ pub enum ConditionType { SpecValid, CredentialsReady, KmsReady, + TlsReady, PoolsReady, WorkloadsReady, } @@ -39,6 +40,7 @@ impl ConditionType { Self::SpecValid => "SpecValid", Self::CredentialsReady => "CredentialsReady", Self::KmsReady => "KmsReady", + Self::TlsReady => "TlsReady", Self::PoolsReady => "PoolsReady", Self::WorkloadsReady => "WorkloadsReady", } @@ -52,6 +54,7 @@ impl ConditionType { Self::SpecValid, Self::CredentialsReady, Self::KmsReady, + Self::TlsReady, Self::PoolsReady, Self::WorkloadsReady, ] @@ -113,6 +116,24 @@ pub enum Reason { KmsSecretNotFound, KmsSecretMissingKey, KmsConfigInvalid, + TlsDisabled, + TlsConfigured, + CertManagerCrdMissing, + CertManagerIssuerNotFound, + CertManagerCertificateApplyFailed, + CertManagerCertificateNotReady, + CertificateSecretPending, + CertificateSecretNotFound, + CertificateSecretInvalidType, + CertificateSecretMissingKey, + CertificateKeyPairMismatch, + CertificateInvalid, + CertificateExpired, + CertificateSanMismatch, + CaBundleMissing, + CaBundleInvalid, + TlsHotReloadUnsupported, + CertificateExpiring, PoolDeleteBlocked, StatefulSetApplyFailed, StatefulSetUpdateValidationFailed, @@ -138,6 +159,24 @@ impl Reason { Self::KmsSecretNotFound => "KmsSecretNotFound", Self::KmsSecretMissingKey => "KmsSecretMissingKey", Self::KmsConfigInvalid => "KmsConfigInvalid", + Self::TlsDisabled => "TlsDisabled", + Self::TlsConfigured => "TlsConfigured", + Self::CertManagerCrdMissing => "CertManagerCrdMissing", + Self::CertManagerIssuerNotFound => "CertManagerIssuerNotFound", + Self::CertManagerCertificateApplyFailed => "CertManagerCertificateApplyFailed", + Self::CertManagerCertificateNotReady => "CertManagerCertificateNotReady", + Self::CertificateSecretPending => "CertificateSecretPending", + Self::CertificateSecretNotFound => "CertificateSecretNotFound", + Self::CertificateSecretInvalidType => "CertificateSecretInvalidType", + Self::CertificateSecretMissingKey => "CertificateSecretMissingKey", + Self::CertificateKeyPairMismatch => "CertificateKeyPairMismatch", + Self::CertificateInvalid => "CertificateInvalid", + Self::CertificateExpired => "CertificateExpired", + Self::CertificateSanMismatch => "CertificateSanMismatch", + Self::CaBundleMissing => "CaBundleMissing", + Self::CaBundleInvalid => "CaBundleInvalid", + Self::TlsHotReloadUnsupported => "TlsHotReloadUnsupported", + Self::CertificateExpiring => "CertificateExpiring", Self::PoolDeleteBlocked => "PoolDeleteBlocked", Self::StatefulSetApplyFailed => "StatefulSetApplyFailed", Self::StatefulSetUpdateValidationFailed => "StatefulSetUpdateValidationFailed", @@ -202,7 +241,9 @@ pub struct Status { /// Kubernetes standard conditions #[serde(default, skip_serializing_if = "Vec::is_empty")] pub conditions: Vec, - // pub certificates: certificate::Status, + + #[serde(default, skip_serializing_if = "certificate::Status::is_empty")] + pub certificates: certificate::Status, } impl Status { @@ -367,6 +408,19 @@ pub fn is_blocked_reason(reason: &str) -> bool { | "KmsSecretNotFound" | "KmsSecretMissingKey" | "KmsConfigInvalid" + | "CertManagerCrdMissing" + | "CertManagerIssuerNotFound" + | "CertManagerCertificateApplyFailed" + | "CertificateSecretNotFound" + | "CertificateSecretInvalidType" + | "CertificateSecretMissingKey" + | "CertificateKeyPairMismatch" + | "CertificateInvalid" + | "CertificateExpired" + | "CertificateSanMismatch" + | "CaBundleMissing" + | "CaBundleInvalid" + | "TlsHotReloadUnsupported" | "PoolDeleteBlocked" | "StatefulSetUpdateValidationFailed" ) @@ -398,6 +452,25 @@ pub fn next_actions_for_reason(reason: &str) -> Vec<&'static str> { "KmsSecretNotFound" => vec!["createKmsSecret"], "KmsSecretMissingKey" => vec!["addRequiredKmsSecretKey"], "KmsConfigInvalid" => vec!["fixKmsConfig"], + "CertManagerCrdMissing" => vec!["installCertManager", "switchToExternalSecret"], + "CertManagerIssuerNotFound" => vec!["createIssuer", "fixIssuerRef"], + "CertManagerCertificateApplyFailed" => vec!["fixCertificateSpec", "inspectOperatorLogs"], + "CertManagerCertificateNotReady" => vec![ + "inspectCertificate", + "inspectIssuer", + "inspectCertManagerLogs", + ], + "CertificateSecretPending" => vec!["waitForCertificateSecret", "inspectCertificate"], + "CertificateSecretNotFound" => vec!["createCertificateSecret", "fixSecretName"], + "CertificateSecretInvalidType" => vec!["replaceCertificateSecret", "fixSecretType"], + "CertificateSecretMissingKey" => vec!["addRequiredCertificateKey"], + "CertificateKeyPairMismatch" => vec!["replaceCertificateSecret"], + "CertificateInvalid" => vec!["replaceCertificateSecret"], + "CertificateExpired" => vec!["renewCertificate", "inspectCertManager"], + "CertificateSanMismatch" => vec!["addRequiredDnsNames", "reissueCertificate"], + "CaBundleMissing" => vec!["configureCaSecretRef", "enableSystemCaIfAppropriate"], + "CaBundleInvalid" => vec!["replaceCaBundle"], + "TlsHotReloadUnsupported" => vec!["useRolloutRotation", "enableCleanTlsDirectory"], "InvalidTenantName" => vec!["renameTenant"], "ImmutableFieldModified" => vec!["restoreImmutableField"], "PoolDeleteBlocked" => vec!["restorePoolSpec", "startDecommissionAfterRestore"], @@ -473,6 +546,34 @@ mod tests { next_actions_for_reason("DecommissionRequired"), vec!["startDecommission", "inspectPoolStatus"] ); + assert_eq!( + next_actions_for_reason("CertManagerCertificateApplyFailed"), + vec!["fixCertificateSpec", "inspectOperatorLogs"] + ); + } + + #[test] + fn tls_blocked_reasons_are_primary_and_actionable() { + let status = Status { + current_state: "Blocked".to_string(), + observed_generation: Some(4), + conditions: vec![ + condition("Ready", "False", "CaBundleMissing", Some(4)), + condition("TlsReady", "False", "CaBundleMissing", Some(4)), + condition("Degraded", "True", "CaBundleMissing", Some(4)), + ], + ..Default::default() + }; + + assert_eq!(summarize_current_state(&status), "Blocked"); + assert_eq!( + primary_condition(&status).map(|condition| condition.reason.as_str()), + Some("CaBundleMissing") + ); + assert_eq!( + next_actions_for_reason("CaBundleMissing"), + vec!["configureCaSecretRef", "enableSystemCaIfAppropriate"] + ); } fn condition( diff --git a/src/types/v1alpha1/status/certificate.rs b/src/types/v1alpha1/status/certificate.rs index 993159e..eb62f18 100755 --- a/src/types/v1alpha1/status/certificate.rs +++ b/src/types/v1alpha1/status/certificate.rs @@ -1,49 +1,123 @@ -// Copyright 2025 RustFS Team +// Copyright 2025 RustFS Team // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at // -// http:www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; -#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema, Default)] +#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema, ToSchema, Default, PartialEq)] #[serde(rename_all = "camelCase")] pub struct Status { #[serde(skip_serializing_if = "Option::is_none")] - pub auto_cert_enabled: Option, + pub tls: Option, +} - #[serde(skip_serializing_if = "Vec::is_empty")] - pub custom_certificates: Vec, +impl Status { + pub fn is_empty(&self) -> bool { + self.tls.is_none() + } } -#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema, Default)] +#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema, ToSchema, Default, PartialEq)] #[serde(rename_all = "camelCase")] -pub struct CustomCertificates {} +pub struct TlsCertificateStatus { + pub mode: String, + pub ready: bool, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub managed_certificate: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rotation_strategy: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mount_path: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub certificate_ref: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub server_secret_ref: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ca_secret_ref: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub client_ca_secret_ref: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observed_hash: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub not_before: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub not_after: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub expires_in_seconds: Option, + + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub dns_names: Vec, + + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub ip_addresses: Vec, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub san_matched: Option, -#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema, Default)] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub trust_source: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_validated_time: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_rollout_trigger_time: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_error_reason: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_error_message: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema, ToSchema, Default, PartialEq)] #[serde(rename_all = "camelCase")] -pub struct CustomCertificateConfig { - #[serde(skip_serializing_if = "Option::is_none")] - cert_name: Option, +pub struct CertificateObjectRef { + pub api_version: String, + pub kind: String, + pub name: String, - #[serde(skip_serializing_if = "Vec::is_empty")] - domains: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observed_generation: Option, - #[serde(skip_serializing_if = "Option::is_none")] - expiry: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ready: Option, - #[serde(skip_serializing_if = "Option::is_none")] - expires_in: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reason: Option, +} - #[serde(skip_serializing_if = "Option::is_none")] - serial_no: Option, +#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema, ToSchema, Default, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct SecretStatusRef { + pub name: String, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub key: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub resource_version: Option, } diff --git a/src/types/v1alpha1/tenant.rs b/src/types/v1alpha1/tenant.rs index 9dd9356..0968729 100755 --- a/src/types/v1alpha1/tenant.rs +++ b/src/types/v1alpha1/tenant.rs @@ -16,6 +16,7 @@ use crate::types::v1alpha1::encryption::{EncryptionConfig, PodSecurityContextOve use crate::types::v1alpha1::k8s; use crate::types::v1alpha1::logging::LoggingConfig; use crate::types::v1alpha1::pool::Pool; +use crate::types::v1alpha1::tls::TlsConfig; use crate::types::{self, error::NoNamespaceSnafu}; use k8s_openapi::api::core::v1 as corev1; use k8s_openapi::apimachinery::pkg::apis::meta::v1 as metav1; @@ -82,6 +83,9 @@ pub struct TenantSpec { #[serde(default, skip_serializing_if = "Vec::is_empty")] pub env: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tls: Option, + // #[serde(default, skip_serializing_if = "Option::is_none")] // pub request_auto_cert: Option, // diff --git a/src/types/v1alpha1/tenant/services.rs b/src/types/v1alpha1/tenant/services.rs index 9b2d1ae..21d5487 100755 --- a/src/types/v1alpha1/tenant/services.rs +++ b/src/types/v1alpha1/tenant/services.rs @@ -13,6 +13,7 @@ // limitations under the License. use super::Tenant; +use crate::types::v1alpha1::tls::TlsPlan; use k8s_openapi::api::core::v1 as corev1; use k8s_openapi::apimachinery::pkg::apis::meta::v1 as metav1; use k8s_openapi::apimachinery::pkg::util::intstr; @@ -28,6 +29,10 @@ fn console_service_name(tenant: &Tenant) -> String { impl Tenant { /// a new io Service for tenant pub fn new_io_service(&self) -> corev1::Service { + self.new_io_service_with_tls_plan(&TlsPlan::disabled()) + } + + pub fn new_io_service_with_tls_plan(&self, tls_plan: &TlsPlan) -> corev1::Service { corev1::Service { metadata: metav1::ObjectMeta { name: Some(io_service_name(self)), @@ -42,7 +47,7 @@ impl Tenant { ports: Some(vec![corev1::ServicePort { port: 9000, target_port: Some(intstr::IntOrString::Int(9000)), - name: Some("http-rustfs".to_owned()), + name: Some(rustfs_service_port_name(tls_plan).to_owned()), ..Default::default() }]), ..Default::default() @@ -78,6 +83,10 @@ impl Tenant { /// a new headless Service for tenant pub fn new_headless_service(&self) -> corev1::Service { + self.new_headless_service_with_tls_plan(&TlsPlan::disabled()) + } + + pub fn new_headless_service_with_tls_plan(&self, tls_plan: &TlsPlan) -> corev1::Service { corev1::Service { metadata: metav1::ObjectMeta { name: Some(self.headless_service_name()), @@ -93,7 +102,7 @@ impl Tenant { selector: Some(self.selector_labels()), ports: Some(vec![corev1::ServicePort { port: 9000, - name: Some("http-rustfs".to_owned()), + name: Some(rustfs_service_port_name(tls_plan).to_owned()), ..Default::default() }]), ..Default::default() @@ -102,3 +111,57 @@ impl Tenant { } } } + +fn rustfs_service_port_name(tls_plan: &TlsPlan) -> &'static str { + if tls_plan.enabled { + "https-rustfs" + } else { + "http-rustfs" + } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use crate::types::v1alpha1::tls::TlsPlan; + + fn first_port_name(service: &k8s_openapi::api::core::v1::Service) -> Option<&str> { + service + .spec + .as_ref()? + .ports + .as_ref()? + .first()? + .name + .as_deref() + } + + #[test] + fn disabled_tls_keeps_rustfs_services_on_http_port_name() { + let tenant = crate::tests::create_test_tenant(None, None); + + assert_eq!( + first_port_name(&tenant.new_io_service()), + Some("http-rustfs") + ); + assert_eq!( + first_port_name(&tenant.new_headless_service()), + Some("http-rustfs") + ); + } + + #[test] + fn enabled_tls_switches_rustfs_services_to_https_port_name() { + let tenant = crate::tests::create_test_tenant(None, None); + let tls_plan = TlsPlan::for_test("server-tls", "sha256:test"); + + assert_eq!( + first_port_name(&tenant.new_io_service_with_tls_plan(&tls_plan)), + Some("https-rustfs") + ); + assert_eq!( + first_port_name(&tenant.new_headless_service_with_tls_plan(&tls_plan)), + Some("https-rustfs") + ); + } +} diff --git a/src/types/v1alpha1/tenant/workloads.rs b/src/types/v1alpha1/tenant/workloads.rs index 6fbf6a2..8165384 100755 --- a/src/types/v1alpha1/tenant/workloads.rs +++ b/src/types/v1alpha1/tenant/workloads.rs @@ -16,6 +16,7 @@ use super::Tenant; use crate::types; use crate::types::v1alpha1::encryption::KmsBackendType; use crate::types::v1alpha1::pool::Pool; +use crate::types::v1alpha1::tls::{TlsPlan, http_probe}; use k8s_openapi::api::apps::v1; use k8s_openapi::api::core::v1 as corev1; use k8s_openapi::apimachinery::pkg::apis::meta::v1 as metav1; @@ -25,6 +26,18 @@ const DEFAULT_RUN_AS_USER: i64 = 10001; const DEFAULT_RUN_AS_GROUP: i64 = 10001; const DEFAULT_FS_GROUP: i64 = 10001; +const TLS_OPERATOR_MANAGED_ENV_VARS: &[&str] = &[ + "RUSTFS_VOLUMES", + "RUSTFS_TLS_PATH", + "RUSTFS_TRUST_SYSTEM_CA", + "RUSTFS_TRUST_LEAF_CERT_AS_CA", + "RUSTFS_SERVER_MTLS_ENABLE", +]; + +fn is_tls_operator_managed_env_var(name: &str) -> bool { + TLS_OPERATOR_MANAGED_ENV_VARS.contains(&name) +} + fn volume_claim_template_name(shard: i32) -> String { format!("{VOLUME_CLAIM_TEMPLATE_PREFIX}-{shard}") } @@ -38,7 +51,7 @@ impl Tenant { /// Format: http://{tenant}-{pool}-{0...servers-1}.{service}.{namespace}.svc.cluster.local:9000{path}/rustfs{0...volumes-1} /// All pools are combined into a space-separated string for a unified cluster /// Follows RustFS convention: /data/rustfs0, /data/rustfs1, etc. - fn rustfs_volumes_env_value(&self) -> Result { + fn rustfs_volumes_env_value(&self, scheme: &str) -> Result { let namespace = self.namespace()?; let tenant_name = self.name(); let headless_service = self.headless_service_name(); @@ -54,7 +67,7 @@ impl Tenant { // Construct volume specification with range notation // Follows RustFS convention: /data/rustfs{0...N} format!( - "http://{tenant_name}-{pool_name}-{{0...{}}}.{headless_service}.{namespace}.svc.cluster.local:9000{}/rustfs{{0...{}}}", + "{scheme}://{tenant_name}-{pool_name}-{{0...{}}}.{headless_service}.{namespace}.svc.cluster.local:9000{}/rustfs{{0...{}}}", pool.servers - 1, base_path.trim_end_matches('/'), pool.persistence.volumes_per_server - 1 @@ -323,6 +336,14 @@ impl Tenant { } pub fn new_statefulset(&self, pool: &Pool) -> Result { + self.new_statefulset_with_tls_plan(pool, &TlsPlan::disabled()) + } + + pub fn new_statefulset_with_tls_plan( + &self, + pool: &Pool, + tls_plan: &TlsPlan, + ) -> Result { let labels = self.pool_labels(pool); let selector_labels = self.pool_selector_labels(pool); @@ -346,12 +367,13 @@ impl Tenant { let mut env_vars = Vec::new(); // Add RUSTFS_VOLUMES environment variable for multi-node communication - let rustfs_volumes = self.rustfs_volumes_env_value()?; + let rustfs_volumes = self.rustfs_volumes_env_value(tls_plan.internode_scheme)?; env_vars.push(corev1::EnvVar { name: "RUSTFS_VOLUMES".to_owned(), value: Some(rustfs_volumes), ..Default::default() }); + env_vars.extend(tls_plan.env.clone()); // Add required RustFS environment variables env_vars.push(corev1::EnvVar { @@ -403,10 +425,14 @@ impl Tenant { }); } - // Merge with user-provided environment variables - // User-provided vars can override operator-managed ones + // Merge with user-provided environment variables. + // Preserve the legacy override behavior except for TLS runtime values that + // must stay aligned with the rendered TLS mounts, probes, status, and hash. for user_env in &self.spec.env { - // Remove any existing var with the same name to allow override + if tls_plan.enabled && is_tls_operator_managed_env_var(&user_env.name) { + continue; + } + // Remove any existing var with the same name to allow non-reserved overrides. env_vars.retain(|e| e.name != user_env.name); env_vars.push(user_env.clone()); } @@ -423,6 +449,8 @@ impl Tenant { env_vars.extend(kms_env); pod_volumes.append(&mut kms_volumes); volume_mounts.append(&mut kms_mounts); + pod_volumes.extend(tls_plan.volumes.clone()); + volume_mounts.extend(tls_plan.volume_mounts.clone()); // Enforce non-root execution and make mounted volumes writable by RustFS user. // If spec.securityContext overrides are set, use those values instead. @@ -476,6 +504,15 @@ impl Tenant { .image_pull_policy .as_ref() .map(ToString::to_string), + liveness_probe: tls_plan + .enabled + .then(|| http_probe("/health", tls_plan.probe_scheme)), + readiness_probe: tls_plan + .enabled + .then(|| http_probe("/health/ready", tls_plan.probe_scheme)), + startup_probe: tls_plan + .enabled + .then(|| http_probe("/health", tls_plan.probe_scheme)), ..Default::default() }; @@ -505,6 +542,8 @@ impl Tenant { template: corev1::PodTemplateSpec { metadata: Some(metav1::ObjectMeta { labels: Some(labels), + annotations: (!tls_plan.pod_template_annotations.is_empty()) + .then(|| tls_plan.pod_template_annotations.clone()), ..Default::default() }), spec: Some(corev1::PodSpec { @@ -553,7 +592,16 @@ impl Tenant { existing: &v1::StatefulSet, pool: &Pool, ) -> Result { - let desired = self.new_statefulset(pool)?; + self.statefulset_needs_update_with_tls_plan(existing, pool, &TlsPlan::disabled()) + } + + pub fn statefulset_needs_update_with_tls_plan( + &self, + existing: &v1::StatefulSet, + pool: &Pool, + tls_plan: &TlsPlan, + ) -> Result { + let desired = self.new_statefulset_with_tls_plan(pool, tls_plan)?; // Compare key spec fields that should trigger updates let existing_spec = existing @@ -597,6 +645,19 @@ impl Tenant { return Ok(true); } + // Check if pod template annotations changed (TLS hash rollout lives here). + if existing_template + .metadata + .as_ref() + .and_then(|m| m.annotations.as_ref()) + != desired_template + .metadata + .as_ref() + .and_then(|m| m.annotations.as_ref()) + { + return Ok(true); + } + let existing_pod_spec = existing_template .spec @@ -633,6 +694,13 @@ impl Tenant { return Ok(true); } + // Check pod volumes (TLS Secret/CA mounts live here). + if serde_json::to_value(&existing_pod_spec.volumes)? + != serde_json::to_value(&desired_pod_spec.volumes)? + { + return Ok(true); + } + // Check node selector if existing_pod_spec.node_selector != desired_pod_spec.node_selector { return Ok(true); @@ -734,7 +802,16 @@ impl Tenant { existing: &v1::StatefulSet, pool: &Pool, ) -> Result<(), types::error::Error> { - let desired = self.new_statefulset(pool)?; + self.validate_statefulset_update_with_tls_plan(existing, pool, &TlsPlan::disabled()) + } + + pub fn validate_statefulset_update_with_tls_plan( + &self, + existing: &v1::StatefulSet, + pool: &Pool, + tls_plan: &TlsPlan, + ) -> Result<(), types::error::Error> { + let desired = self.new_statefulset_with_tls_plan(pool, tls_plan)?; let existing_spec = existing .spec @@ -850,6 +927,7 @@ impl Tenant { mod tests { use super::{DEFAULT_FS_GROUP, DEFAULT_RUN_AS_GROUP, DEFAULT_RUN_AS_USER}; use crate::types::v1alpha1::logging::{LoggingConfig, LoggingMode}; + use crate::types::v1alpha1::tls::{SecretKeyReference, TlsPlan}; use k8s_openapi::api::core::v1 as corev1; fn image_pull_secret(name: &str) -> corev1::LocalObjectReference { @@ -858,6 +936,242 @@ mod tests { } } + fn tls_plan(hash: &str) -> TlsPlan { + TlsPlan::for_test("server-tls", hash) + } + + fn env_value<'a>(container: &'a corev1::Container, name: &str) -> Option<&'a str> { + container + .env + .as_ref()? + .iter() + .find(|var| var.name == name)? + .value + .as_deref() + } + + #[test] + fn disabled_tls_statefulset_keeps_http_and_has_no_tls_wiring() { + let tenant = crate::tests::create_test_tenant(None, None); + let pool = &tenant.spec.pools[0]; + + let statefulset = tenant + .new_statefulset(pool) + .expect("Should create StatefulSet without TLS"); + + let template = statefulset.spec.unwrap().template; + assert!( + template + .metadata + .as_ref() + .and_then(|metadata| metadata.annotations.as_ref()) + .is_none_or(|annotations| !annotations.contains_key("operator.rustfs.com/tls-hash")) + ); + + let pod_spec = template.spec.unwrap(); + assert!(pod_spec.volumes.as_ref().is_none_or(|volumes| { + !volumes + .iter() + .any(|volume| volume.name.starts_with("rustfs-tls")) + })); + + let container = &pod_spec.containers[0]; + assert!( + env_value(container, "RUSTFS_VOLUMES") + .is_some_and(|value| value.starts_with("http://")) + ); + assert!(env_value(container, "RUSTFS_TLS_PATH").is_none()); + assert!(container.liveness_probe.is_none()); + assert!(container.readiness_probe.is_none()); + assert!(container.startup_probe.is_none()); + assert!(container.volume_mounts.as_ref().is_none_or(|mounts| { + !mounts + .iter() + .any(|mount| mount.name.starts_with("rustfs-tls")) + })); + } + + #[test] + fn cert_manager_tls_statefulset_maps_secret_to_rustfs_tls_files() { + let tenant = crate::tests::create_test_tenant(None, None); + let pool = &tenant.spec.pools[0]; + + let statefulset = tenant + .new_statefulset_with_tls_plan(pool, &tls_plan("sha256:test")) + .expect("Should create StatefulSet with TLS"); + + let template = statefulset.spec.unwrap().template; + let annotations = template.metadata.unwrap().annotations.unwrap(); + assert_eq!( + annotations.get("operator.rustfs.com/tls-hash"), + Some(&"sha256:test".to_string()) + ); + + let pod_spec = template.spec.unwrap(); + let volumes = pod_spec.volumes.unwrap_or_default(); + assert!( + volumes + .iter() + .any(|volume| volume.name == "rustfs-tls-server") + ); + + let container = &pod_spec.containers[0]; + let env = container.env.as_ref().expect("TLS env should be present"); + assert!(env.iter().any(|var| { + var.name == "RUSTFS_TLS_PATH" && var.value.as_deref() == Some("/var/run/rustfs/tls") + })); + assert!(env.iter().any(|var| { + var.name == "RUSTFS_VOLUMES" + && var + .value + .as_deref() + .is_some_and(|value| value.starts_with("https://")) + })); + + let mounts = container + .volume_mounts + .as_ref() + .expect("TLS volume mounts should be present"); + assert!(mounts.iter().any(|mount| { + mount.name == "rustfs-tls-server" + && mount.mount_path == "/var/run/rustfs/tls/rustfs_cert.pem" + && mount.sub_path.as_deref() == Some("rustfs_cert.pem") + })); + assert!(mounts.iter().any(|mount| { + mount.name == "rustfs-tls-server" + && mount.mount_path == "/var/run/rustfs/tls/rustfs_key.pem" + && mount.sub_path.as_deref() == Some("rustfs_key.pem") + })); + assert!(mounts.iter().any(|mount| { + mount.name == "rustfs-tls-server" + && mount.mount_path == "/var/run/rustfs/tls/ca.crt" + && mount.sub_path.as_deref() == Some("ca.crt") + })); + + assert_eq!( + container + .readiness_probe + .as_ref() + .and_then(|probe| probe.http_get.as_ref()) + .and_then(|http_get| http_get.scheme.as_deref()), + Some("HTTPS") + ); + } + + #[test] + fn tls_statefulset_keeps_operator_managed_env_when_spec_env_conflicts() { + let mut tenant = crate::tests::create_test_tenant(None, None); + tenant.spec.env = vec![ + corev1::EnvVar { + name: "RUSTFS_TLS_PATH".to_string(), + value: Some("/wrong/tls".to_string()), + ..Default::default() + }, + corev1::EnvVar { + name: "RUSTFS_VOLUMES".to_string(), + value: Some("http://wrong.example/rustfs0".to_string()), + ..Default::default() + }, + corev1::EnvVar { + name: "RUSTFS_TRUST_SYSTEM_CA".to_string(), + value: Some("false".to_string()), + ..Default::default() + }, + corev1::EnvVar { + name: "RUSTFS_TRUST_LEAF_CERT_AS_CA".to_string(), + value: Some("false".to_string()), + ..Default::default() + }, + corev1::EnvVar { + name: "RUSTFS_SERVER_MTLS_ENABLE".to_string(), + value: Some("false".to_string()), + ..Default::default() + }, + corev1::EnvVar { + name: "CUSTOM_USER_ENV".to_string(), + value: Some("kept".to_string()), + ..Default::default() + }, + ]; + let pool = &tenant.spec.pools[0]; + let plan = TlsPlan::rollout( + "/var/run/rustfs/tls".to_string(), + "sha256:reserved-env".to_string(), + "server-tls".to_string(), + Some("ca.crt".to_string()), + None, + Some(SecretKeyReference { + name: "client-ca".to_string(), + key: "ca.crt".to_string(), + }), + true, + true, + true, + None, + ); + + let statefulset = tenant + .new_statefulset_with_tls_plan(pool, &plan) + .expect("Should create StatefulSet with TLS"); + + let container = &statefulset + .spec + .as_ref() + .expect("StatefulSet should have spec") + .template + .spec + .as_ref() + .expect("Pod template should have spec") + .containers[0]; + let env = container.env.as_ref().expect("TLS env should be present"); + for name in [ + "RUSTFS_TLS_PATH", + "RUSTFS_VOLUMES", + "RUSTFS_TRUST_SYSTEM_CA", + "RUSTFS_TRUST_LEAF_CERT_AS_CA", + "RUSTFS_SERVER_MTLS_ENABLE", + ] { + assert_eq!( + env.iter().filter(|var| var.name == name).count(), + 1, + "reserved env var {name} should appear exactly once" + ); + } + assert_eq!( + env_value(container, "RUSTFS_TLS_PATH"), + Some("/var/run/rustfs/tls") + ); + assert!( + env_value(container, "RUSTFS_VOLUMES") + .is_some_and(|value| value.starts_with("https://") && !value.contains("wrong")) + ); + assert_eq!(env_value(container, "RUSTFS_TRUST_SYSTEM_CA"), Some("true")); + assert_eq!( + env_value(container, "RUSTFS_TRUST_LEAF_CERT_AS_CA"), + Some("true") + ); + assert_eq!( + env_value(container, "RUSTFS_SERVER_MTLS_ENABLE"), + Some("true") + ); + assert_eq!(env_value(container, "CUSTOM_USER_ENV"), Some("kept")); + } + + #[test] + fn tls_hash_annotation_change_triggers_statefulset_update() { + let tenant = crate::tests::create_test_tenant(None, None); + let pool = &tenant.spec.pools[0]; + let statefulset = tenant + .new_statefulset_with_tls_plan(pool, &tls_plan("sha256:old")) + .expect("Should create StatefulSet with TLS"); + + let needs_update = tenant + .statefulset_needs_update_with_tls_plan(&statefulset, pool, &tls_plan("sha256:new")) + .expect("Should compare StatefulSet"); + + assert!(needs_update, "TLS hash change should roll the pod template"); + } + // Test: Pod runs as non-root with proper security context #[test] fn test_statefulset_sets_security_context() { diff --git a/src/types/v1alpha1/tls.rs b/src/types/v1alpha1/tls.rs new file mode 100644 index 0000000..d8c603d --- /dev/null +++ b/src/types/v1alpha1/tls.rs @@ -0,0 +1,497 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use k8s_openapi::api::core::v1 as corev1; +use k8s_openapi::apimachinery::pkg::util::intstr::IntOrString; +use k8s_openapi::schemars::JsonSchema; +use kube::KubeSchema; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +pub const DEFAULT_TLS_MOUNT_PATH: &str = "/var/run/rustfs/tls"; +pub const TLS_HASH_ANNOTATION: &str = "operator.rustfs.com/tls-hash"; +pub const RUSTFS_TLS_CERT_FILE: &str = "rustfs_cert.pem"; +pub const RUSTFS_TLS_KEY_FILE: &str = "rustfs_key.pem"; +pub const RUSTFS_CA_FILE: &str = "ca.crt"; +pub const RUSTFS_CLIENT_CA_FILE: &str = "client_ca.crt"; + +#[derive(Deserialize, Serialize, Clone, Copy, Debug, JsonSchema, Default, PartialEq)] +#[serde(rename_all = "camelCase")] +#[schemars(rename_all = "camelCase")] +pub enum TlsMode { + #[default] + Disabled, + External, + CertManager, +} + +#[derive(Deserialize, Serialize, Clone, Copy, Debug, JsonSchema, Default, PartialEq)] +#[serde(rename_all = "PascalCase")] +#[schemars(rename_all = "PascalCase")] +pub enum TlsRotationStrategy { + #[default] + Rollout, + HotReload, +} + +#[derive(Deserialize, Serialize, Clone, Copy, Debug, JsonSchema, Default, PartialEq)] +#[serde(rename_all = "PascalCase")] +#[schemars(rename_all = "PascalCase")] +pub enum CaTrustSource { + #[default] + CertificateSecretCa, + SecretRef, + SystemCa, +} + +#[derive(Deserialize, Serialize, Clone, Debug, KubeSchema, Default, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct SecretKeyReference { + pub name: String, + + #[serde(default = "default_ca_key")] + pub key: String, +} + +fn default_ca_key() -> String { + "ca.crt".to_string() +} + +#[derive(Deserialize, Serialize, Clone, Debug, KubeSchema, Default, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct CaTrustConfig { + #[serde(default)] + pub source: CaTrustSource, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ca_secret_ref: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub client_ca_secret_ref: Option, + + #[serde(default)] + pub trust_system_ca: bool, + + #[serde(default)] + pub trust_leaf_certificate_as_ca: bool, +} + +#[derive(Deserialize, Serialize, Clone, Debug, KubeSchema, Default, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct CertManagerIssuerRef { + #[serde(default = "default_cert_manager_group")] + pub group: String, + + #[serde(default = "default_cert_manager_issuer_kind")] + pub kind: String, + + pub name: String, +} + +fn default_cert_manager_group() -> String { + "cert-manager.io".to_string() +} + +fn default_cert_manager_issuer_kind() -> String { + "Issuer".to_string() +} + +#[derive(Deserialize, Serialize, Clone, Debug, KubeSchema, Default, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct CertManagerPrivateKeyConfig { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub algorithm: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub encoding: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub size: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rotation_policy: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, KubeSchema, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct CertManagerTlsConfig { + #[serde(default)] + pub manage_certificate: bool, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub secret_name: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub secret_type: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub certificate_name: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub issuer_ref: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub common_name: Option, + + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub dns_names: Vec, + + #[serde(default = "default_include_generated_dns_names")] + pub include_generated_dns_names: bool, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub duration: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub renew_before: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub private_key: Option, + + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub usages: Vec, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ca_trust: Option, +} + +fn default_include_generated_dns_names() -> bool { + true +} + +impl Default for CertManagerTlsConfig { + fn default() -> Self { + Self { + manage_certificate: false, + secret_name: None, + secret_type: None, + certificate_name: None, + issuer_ref: None, + common_name: None, + dns_names: Vec::new(), + include_generated_dns_names: default_include_generated_dns_names(), + duration: None, + renew_before: None, + private_key: None, + usages: Vec::new(), + ca_trust: None, + } + } +} + +#[derive(Deserialize, Serialize, Clone, Debug, KubeSchema, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct TlsConfig { + #[serde(default)] + pub mode: TlsMode, + + #[serde(default = "default_tls_mount_path")] + pub mount_path: String, + + #[serde(default)] + pub rotation_strategy: TlsRotationStrategy, + + #[serde(default)] + pub enable_internode_https: bool, + + #[serde(default = "default_require_san_match")] + pub require_san_match: bool, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cert_manager: Option, +} + +fn default_tls_mount_path() -> String { + DEFAULT_TLS_MOUNT_PATH.to_string() +} + +fn default_require_san_match() -> bool { + true +} + +impl Default for TlsConfig { + fn default() -> Self { + Self { + mode: TlsMode::default(), + mount_path: default_tls_mount_path(), + rotation_strategy: TlsRotationStrategy::default(), + enable_internode_https: false, + require_san_match: default_require_san_match(), + cert_manager: None, + } + } +} + +impl TlsConfig { + pub fn is_enabled(&self) -> bool { + self.mode != TlsMode::Disabled + } + + pub fn ca_trust(&self) -> CaTrustConfig { + self.cert_manager + .as_ref() + .and_then(|cert_manager| cert_manager.ca_trust.clone()) + .unwrap_or_default() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tls_config_default_matches_serde_defaults() { + let config = TlsConfig::default(); + + assert_eq!(config.mode, TlsMode::Disabled); + assert_eq!(config.mount_path, DEFAULT_TLS_MOUNT_PATH); + assert_eq!(config.rotation_strategy, TlsRotationStrategy::Rollout); + assert!(!config.enable_internode_https); + assert!(config.require_san_match); + assert!(config.cert_manager.is_none()); + } +} + +#[derive(Clone, Debug, Default)] +pub struct TlsPlan { + pub enabled: bool, + pub mount_path: String, + pub internode_scheme: &'static str, + pub probe_scheme: &'static str, + pub pod_template_annotations: BTreeMap, + pub env: Vec, + pub volumes: Vec, + pub volume_mounts: Vec, + pub status: Option, +} + +impl TlsPlan { + pub fn disabled() -> Self { + Self { + enabled: false, + mount_path: DEFAULT_TLS_MOUNT_PATH.to_string(), + internode_scheme: "http", + probe_scheme: "HTTP", + pod_template_annotations: BTreeMap::new(), + env: Vec::new(), + volumes: Vec::new(), + volume_mounts: Vec::new(), + status: None, + } + } + + #[allow(clippy::too_many_arguments)] + pub fn rollout( + mount_path: String, + hash: String, + server_secret_name: String, + server_ca_key: Option, + explicit_ca: Option, + client_ca: Option, + enable_internode_https: bool, + trust_system_ca: bool, + trust_leaf_certificate_as_ca: bool, + status: Option, + ) -> Self { + let mut annotations = BTreeMap::new(); + annotations.insert(TLS_HASH_ANNOTATION.to_string(), hash); + + let mut env = vec![corev1::EnvVar { + name: "RUSTFS_TLS_PATH".to_string(), + value: Some(mount_path.clone()), + ..Default::default() + }]; + if trust_system_ca { + env.push(corev1::EnvVar { + name: "RUSTFS_TRUST_SYSTEM_CA".to_string(), + value: Some("true".to_string()), + ..Default::default() + }); + } + if trust_leaf_certificate_as_ca { + env.push(corev1::EnvVar { + name: "RUSTFS_TRUST_LEAF_CERT_AS_CA".to_string(), + value: Some("true".to_string()), + ..Default::default() + }); + } + if client_ca.is_some() { + env.push(corev1::EnvVar { + name: "RUSTFS_SERVER_MTLS_ENABLE".to_string(), + value: Some("true".to_string()), + ..Default::default() + }); + } + + let has_server_ca = server_ca_key.is_some(); + let mut server_items = vec![ + key_to_path("tls.crt", RUSTFS_TLS_CERT_FILE), + key_to_path("tls.key", RUSTFS_TLS_KEY_FILE), + ]; + if let Some(ca_key) = server_ca_key.as_deref() { + server_items.push(key_to_path(ca_key, RUSTFS_CA_FILE)); + } + + let (mut volumes, mut volume_mounts) = if let Some(explicit_ca) = &explicit_ca { + ( + vec![projected_volume( + "rustfs-tls-server", + vec![ + secret_projection(&server_secret_name, server_items), + secret_projection( + &explicit_ca.name, + vec![key_to_path(&explicit_ca.key, RUSTFS_CA_FILE)], + ), + ], + )], + vec![directory_mount("rustfs-tls-server", &mount_path)], + ) + } else { + let mut volume_mounts = vec![ + file_mount("rustfs-tls-server", &mount_path, RUSTFS_TLS_CERT_FILE), + file_mount("rustfs-tls-server", &mount_path, RUSTFS_TLS_KEY_FILE), + ]; + if has_server_ca { + volume_mounts.push(file_mount("rustfs-tls-server", &mount_path, RUSTFS_CA_FILE)); + } + ( + vec![secret_volume( + "rustfs-tls-server", + &server_secret_name, + server_items, + )], + volume_mounts, + ) + }; + + if let Some(client_ca) = &client_ca { + volumes.push(secret_volume( + "rustfs-tls-client-ca", + &client_ca.name, + vec![key_to_path(&client_ca.key, RUSTFS_CLIENT_CA_FILE)], + )); + volume_mounts.push(file_mount( + "rustfs-tls-client-ca", + &mount_path, + RUSTFS_CLIENT_CA_FILE, + )); + } + + Self { + enabled: true, + mount_path, + internode_scheme: if enable_internode_https { + "https" + } else { + "http" + }, + probe_scheme: "HTTPS", + pod_template_annotations: annotations, + env, + volumes, + volume_mounts, + status, + } + } + + #[cfg(test)] + pub(crate) fn for_test(server_secret_name: &str, hash: &str) -> Self { + Self::rollout( + DEFAULT_TLS_MOUNT_PATH.to_string(), + hash.to_string(), + server_secret_name.to_string(), + Some("ca.crt".to_string()), + None, + None, + true, + false, + false, + None, + ) + } +} + +fn secret_volume(name: &str, secret_name: &str, items: Vec) -> corev1::Volume { + corev1::Volume { + name: name.to_string(), + secret: Some(corev1::SecretVolumeSource { + secret_name: Some(secret_name.to_string()), + items: Some(items), + optional: Some(false), + ..Default::default() + }), + ..Default::default() + } +} + +fn projected_volume(name: &str, sources: Vec) -> corev1::Volume { + corev1::Volume { + name: name.to_string(), + projected: Some(corev1::ProjectedVolumeSource { + sources: Some(sources), + ..Default::default() + }), + ..Default::default() + } +} + +fn secret_projection(secret_name: &str, items: Vec) -> corev1::VolumeProjection { + corev1::VolumeProjection { + secret: Some(corev1::SecretProjection { + name: secret_name.to_string(), + items: Some(items), + optional: Some(false), + }), + ..Default::default() + } +} + +fn key_to_path(key: &str, path: &str) -> corev1::KeyToPath { + corev1::KeyToPath { + key: key.to_string(), + path: path.to_string(), + ..Default::default() + } +} + +fn file_mount(volume_name: &str, mount_path: &str, file_name: &str) -> corev1::VolumeMount { + corev1::VolumeMount { + name: volume_name.to_string(), + mount_path: format!("{}/{}", mount_path.trim_end_matches('/'), file_name), + sub_path: Some(file_name.to_string()), + read_only: Some(true), + ..Default::default() + } +} + +fn directory_mount(volume_name: &str, mount_path: &str) -> corev1::VolumeMount { + corev1::VolumeMount { + name: volume_name.to_string(), + mount_path: mount_path.to_string(), + read_only: Some(true), + ..Default::default() + } +} + +pub fn http_probe(path: &str, scheme: &'static str) -> corev1::Probe { + corev1::Probe { + http_get: Some(corev1::HTTPGetAction { + path: Some(path.to_string()), + port: IntOrString::Int(9000), + scheme: Some(scheme.to_string()), + ..Default::default() + }), + ..Default::default() + } +}