diff --git a/antora.yml b/antora.yml index 5798a3217a..2d617a50c6 100644 --- a/antora.yml +++ b/antora.yml @@ -1,13 +1,13 @@ name: ROOT title: Self-Managed -version: 25.3 +version: 26.1 start_page: home:index.adoc nav: - modules/ROOT/nav.adoc asciidoc: attributes: # Date of release in the format YYYY-MM-DD - page-release-date: 2025-11-19 + page-release-date: 2026-03-31 # Only used in the main branch (latest version) page-header-data: order: 2 @@ -17,16 +17,16 @@ asciidoc: # Fallback versions # We try to fetch the latest versions from GitHub at build time # -- - full-version: 25.3.11 - latest-redpanda-tag: 'v25.3.11' + full-version: 26.1.1 + latest-redpanda-tag: 'v26.1.1' latest-console-tag: 'v3.3.1' - latest-release-commit: '6aa5af28b020b66e5caa966094882b7260497a53' + latest-release-commit: '35a825c9c1880ebeedf4c18bb8c6cceaa63566c1' latest-operator-version: 'v2.3.8-24.3.6' operator-beta-tag: '' helm-beta-tag: '' latest-redpanda-helm-chart-version: '' - redpanda-beta-version: '25.3.1-rc4' - redpanda-beta-tag: 'v25.3.1-rc4' + redpanda-beta-version: '' + redpanda-beta-tag: '' console-beta-version: '' console-beta-tag: '' # -- diff --git a/docs-data/property-overrides.json b/docs-data/property-overrides.json index 913ff8476f..370f4ce092 100644 --- a/docs-data/property-overrides.json +++ b/docs-data/property-overrides.json @@ -405,6 +405,9 @@ "cloud_storage_min_chunks_per_segment_threshold": { "description": "The minimum number of chunks per segment for trimming to be enabled. If the number of chunks in a segment is below this threshold, the segment is small enough that all chunks in it can be hydrated at any given time." }, + "cloud_storage_prefetch_segments_max": { + "version": "v26.1.1" + }, "cloud_storage_readreplica_manifest_sync_timeout_ms": { "description": "Timeout to check if new data is available for partitions in object storage for read replicas." }, @@ -472,12 +475,32 @@ "description": "Configure the addressing style that controls how Redpanda formats bucket URLs for S3-compatible object storage.\n\nLeave this property unset (`null`) to use automatic configuration:\n\n* For AWS S3: Redpanda attempts `virtual_host` addressing first, then falls back to `path` style if needed\n* For MinIO: Redpanda automatically uses `path` style regardless of `MINIO_DOMAIN` configuration\n\nSet this property explicitly to override automatic configuration, ensure consistent behavior across deployments, or when using S3-compatible storage that requires a specific URL format.\n\nCAUTION: AWS requires virtual-hosted addressing for buckets created after September 30, 2020. If you use AWS S3 with buckets created after this date, use `virtual_host` addressing.\n\nNOTE: For MinIO deployments, Redpanda defaults to `path` style when this property is unset. To use `virtual_host` addressing with a configured `MINIO_DOMAIN`, set this property explicitly to `virtual_host`. For other S3-compatible storage backends, consult your provider's documentation to determine the required URL style.", "config_scope": "cluster" }, + "cloud_topics_allow_materialization_failure": { + "version": "v26.1.1" + }, + "cloud_topics_compaction_interval_ms": { + "version": "v26.1.1" + }, + "cloud_topics_compaction_key_map_memory": { + "version": "v26.1.1" + }, + "cloud_topics_compaction_max_object_size": { + "version": "v26.1.1" + }, + "cloud_topics_disable_level_zero_gc_for_tests": { + "version": "v26.1.1" + }, + "cloud_topics_disable_metastore_flush_loop_for_tests": { + "version": "v26.1.1" + }, "cloud_topics_disable_reconciliation_loop": { - "exclude_from_docs": true, "config_scope": "cluster" }, "cloud_topics_enabled": { - "exclude_from_docs": true, + "description": "Enable Cloud Topics for the cluster. Cloud Topics are optimized for high-throughput, cost-sensitive workloads that can tolerate higher latencies compared to standard Kafka topics.", + "related_topics": [ + "xref:develop:manage-topics/cloud-topics.adoc[Cloud Topics]" + ], "config_scope": "cluster" }, "cloud_topics_epoch_service_epoch_increment_interval": { @@ -488,26 +511,81 @@ "description": "The duration, in milliseconds, for which a cluster-wide epoch is cached locally on each broker.\n\nCaching the epoch locally reduces the need for frequent coordination with the controller. This property controls how long each broker can use a cached epoch value before fetching the latest value.\n\nIncrease this value to reduce coordination overhead in clusters with stable workloads. Decrease it if you need brokers to react more quickly to epoch changes in Tiered Storage.", "version": "v25.3.3" }, + "cloud_topics_epoch_service_max_same_epoch_duration": { + "version": "v26.1.1" + }, + "cloud_topics_fetch_debounce_enabled": { + "version": "v26.1.1" + }, + "cloud_topics_gc_health_check_interval": { + "version": "v26.1.1" + }, + "cloud_topics_l1_indexing_interval": { + "version": "v26.1.1" + }, + "cloud_topics_long_term_file_deletion_delay": { + "version": "v26.1.1" + }, + "cloud_topics_long_term_flush_interval": { + "version": "v26.1.1" + }, "cloud_topics_long_term_garbage_collection_interval": { - "exclude_from_docs": true, "config_scope": "cluster" }, + "cloud_topics_metastore_lsm_apply_timeout_ms": { + "version": "v26.1.1" + }, + "cloud_topics_metastore_replication_timeout_ms": { + "version": "v26.1.1" + }, + "cloud_topics_num_metastore_partitions": { + "version": "v26.1.1" + }, + "cloud_topics_parallel_fetch_enabled": { + "version": "v26.1.1" + }, + "cloud_topics_preregistered_object_ttl": { + "version": "v26.1.1" + }, "cloud_topics_produce_batching_size_threshold": { - "exclude_from_docs": true, "config_scope": "cluster" }, "cloud_topics_produce_cardinality_threshold": { - "exclude_from_docs": true, "config_scope": "cluster" }, + "cloud_topics_produce_no_pid_concurrency": { + "version": "v26.1.1" + }, "cloud_topics_produce_upload_interval": { - "exclude_from_docs": true, "config_scope": "cluster" }, + "cloud_topics_produce_write_inflight_limit": { + "version": "v26.1.1" + }, "cloud_topics_reconciliation_interval": { - "exclude_from_docs": true, "config_scope": "cluster" }, + "cloud_topics_reconciliation_max_interval": { + "version": "v26.1.1" + }, + "cloud_topics_reconciliation_max_object_size": { + "version": "v26.1.1" + }, + "cloud_topics_reconciliation_min_interval": { + "version": "v26.1.1" + }, + "cloud_topics_reconciliation_parallelism": { + "version": "v26.1.1" + }, + "cloud_topics_reconciliation_slowdown_blend": { + "version": "v26.1.1" + }, + "cloud_topics_reconciliation_speedup_blend": { + "version": "v26.1.1" + }, + "cloud_topics_reconciliation_target_fill_ratio": { + "version": "v26.1.1" + }, "cloud_topics_short_term_gc_backoff_interval": { "description": "The interval, in milliseconds, between invocations of the L0 garbage collection work loop when no progress is being made or errors are occurring.\n\nL0 (level-zero) objects are short-term data objects in Tiered Storage that are periodically garbage collected. When GC encounters errors or cannot make progress (for example, if there are no objects eligible for deletion), this backoff interval prevents excessive retries.\n\nIncrease this value to reduce system load when GC cannot make progress. Decrease it if you need faster retry attempts after transient errors.", "version": "v25.3.3" @@ -520,6 +598,9 @@ "description": "The minimum age, in milliseconds, of an L0 (level-zero) object before it becomes eligible for garbage collection.\n\nThis grace period delays deletion of L0 objects even after they become eligible based on epoch. The delay provides a safety buffer that can support recovery in cases involving accidental deletion or other operational issues.\n\nIncrease this value to extend the retention window for L0 objects, providing more time for recovery from operational errors. Decrease it to free up object storage space more quickly, but with less protection against accidental deletion.", "version": "v25.3.3" }, + "cloud_topics_upload_part_size": { + "version": "v26.1.1" + }, "cluster_id": { "description": "Cluster identifier.", "config_scope": "cluster" @@ -693,6 +774,15 @@ ], "config_scope": "cluster" }, + "default_redpanda_storage_mode": { + "description": "Set the default storage mode for new topics. This value applies to any topic created without an explicit <> setting (that is, when the topic's `redpanda.storage.mode` is `unset`).\n\nAccepted values:\n\n* `unset`: Defer to the legacy <> and <> topic properties for Tiered Storage configuration.\n* `local`: Store data only on local disks, with no object storage involvement.\n* `tiered`: Store data on local disks and replicate it to object storage using Tiered Storage. Equivalent to setting `redpanda.remote.read` and `redpanda.remote.write` to `true`.\n* `cloud`: Store data primarily in object storage using Cloud Topics.", + "related_topics": [ + "self-managed-only: xref:manage:tiered-storage.adoc[Tiered Storage]", + "self-managed-only: xref:develop:manage-topics/cloud-topics.adoc[Manage Cloud Topics]" + ], + "config_scope": "cluster", + "version": "v26.1.1" + }, "delete.retention.ms": { "description": "The retention time for tombstone records in a compacted topic. Redpanda removes tombstone records after the retention limit is exceeded.\n\nIf you have enabled Tiered Storage and set <> or <> for the topic, you cannot enable tombstone removal.\n\nIf both `delete.retention.ms` and the cluster property config_ref:tombstone_retention_ms,true,properties/cluster-properties[] are set, `delete.retention.ms` overrides the cluster level tombstone retention for an individual topic.\n\nThis property supports three states:\n\n* Positive value: Sets the milliseconds to retain tombstone records before removal.\n* 0: Tombstone records are immediately eligible for removal.\n* Negative value: Disables tombstone removal entirely for this topic.", "related_topics": [ @@ -701,6 +791,9 @@ ], "config_scope": "topic" }, + "delete_topic_enable": { + "version": "v26.1.1" + }, "developer_mode": { "description": "CAUTION: Enabling `developer_mode` isn't recommended for production use.\n\nEnable developer mode, which skips most of the checks performed at startup.", "config_scope": "broker", @@ -778,7 +871,7 @@ "related_topics": [ "xref:manage:schema-reg/schema-id-validation.adoc[Server-Side Schema ID Validation]" ], - "description": "Controls whether Redpanda validates schema IDs in records and which topic properties are enforced.\n\nValues:\n\n* `none`: Schema validation is disabled (no schema ID checks are done). Associated topic properties cannot be modified.\n* `redpanda`: Schema validation is enabled. Only Redpanda topic properties are accepted.\n* `compat`: Schema validation is enabled. Both Redpanda and compatible topic properties are accepted.\n\nNOTE: Enabling this property will trigger decompression of message batches for topics on which validation is configured, which may lead to a modest increase in CPU load. Redpanda recommends monitoring CPU utilization after topics are configured.", + "description": "Controls whether Redpanda validates schema IDs in records and which topic properties are enforced.\n\nValues:\n\n* `none`: Schema validation is disabled (no schema ID checks are done). Associated topic properties cannot be modified.\n* `redpanda`: Schema validation is enabled. Only Redpanda topic properties are accepted.\n* `compat`: Schema validation is enabled. Both Redpanda and compatible topic properties are accepted.", "config_scope": "cluster" }, "enable_shadow_linking": { @@ -833,7 +926,7 @@ "config_scope": "cluster" }, "iceberg_default_catalog_namespace": { - "description": "The default Iceberg catalog namespace where Redpanda creates tables. Supports nested namespaces as an array of strings.\n\nIMPORTANT: This value must be configured before enabling Iceberg and must not be changed afterward. Changing it will cause Redpanda to lose track of existing tables.", + "description": "The default namespace (database name) for Iceberg tables. All tables created by Redpanda will be placed in this namespace within the Iceberg catalog. Supports nested namespaces as an array of strings.\n\nIMPORTANT: This value must be configured before enabling Iceberg and must not be changed afterward. Changing it will cause Redpanda to lose track of existing tables.", "version": "v25.3.5" }, "iceberg_default_partition_spec": { @@ -1012,6 +1105,9 @@ ], "config_scope": "cluster" }, + "internal_rpc_request_timeout_ms": { + "version": "v26.1.1" + }, "internal_topic_replication_factor": { "description": "Target replication factor for internal topics.\n\n*Unit*: number of replicas per topic.", "config_scope": "cluster" @@ -1136,7 +1232,6 @@ "config_scope": "cluster" }, "kafka_fetch_request_timeout_ms": { - "description": "Target duration for a single fetch request. The broker tries to complete each fetch within this duration, even if fewer bytes are available than requested.", "version": "v25.3.7" }, "kafka_max_message_size_upper_limit_bytes": { @@ -1222,6 +1317,7 @@ }, "log_compaction_max_priority_wait_ms": { "description": "Maximum time a priority partition (for example, `__consumer_offsets`) can wait for compaction before preempting regular compaction.", + "config_scope": "cluster", "version": "v25.3.8" }, "log_compaction_merge_max_ranges": { @@ -1387,6 +1483,9 @@ "config_scope": "broker", "category": "schema-registry" }, + "nested_group_behavior": { + "version": "v26.1.1" + }, "node_id": { "config_scope": "broker", "category": "redpanda", @@ -1431,6 +1530,9 @@ "description": "The URL pointing to the well-known discovery endpoint for the OIDC provider.", "config_scope": "cluster" }, + "oidc_group_claim_path": { + "version": "v26.1.1" + }, "oidc_principal_mapping": { "description": "Rule for mapping JWT payload claim to a Redpanda user principal.", "related_topics": [ @@ -1503,6 +1605,9 @@ "description": "Controls when and how Redpanda automatically rebalances partition replicas across brokers. For more information, see xref:manage:cluster-maintenance/cluster-balancing.adoc[partition balancing].\n\nValues:\n\n* `continuous`: Partition balancing happens automatically to maintain optimal performance and availability, based on continuous monitoring for node changes (same as `node_add`) and also high disk usage. This option requires an enterprise license, and it is customized by xref:reference:properties/cluster-properties.adoc#partition_autobalancing_node_availability_timeout_sec[`partition_autobalancing_node_availability_timeout_sec`] and xref:reference:properties/cluster-properties.adoc#partition_autobalancing_max_disk_usage_percent[`partition_autobalancing_max_disk_usage_percent`] properties.\n* `node_add`: Partition balancing happens when a node is added.\n* `off`: Partition balancing is disabled. This option is not recommended for production clusters.", "config_scope": "cluster" }, + "partition_autobalancing_node_autodecommission_timeout_sec": { + "version": "v26.1.1" + }, "partition_autobalancing_node_availability_timeout_sec": { "related_topics": [ "xref:manage:cluster-maintenance/continuous-data-balancing.adoc[Configure Continuous Data Balancing]" @@ -1582,9 +1687,13 @@ "category": "redpanda" }, "redpanda.cloud_topic.enabled": { + "description": "Enable Cloud Topic storage mode for this topic. When enabled, topic data is stored primarily in object storage with local storage used only as a write buffer.\n\nTIP: To configure storage modes with more flexibility, use <> which supports `local`, `tiered`, `cloud`, and `unset` modes.", + "related_topics": [ + "self-managed-only: xref:develop:manage-topics/cloud-topics.adoc[Cloud Topics]", + "self-managed-only: xref:manage:tiered-storage.adoc[Tiered Storage]" + ], "config_scope": "topic", - "category": "tiered-storage", - "exclude_from_docs": true + "category": "tiered-storage" }, "redpanda.iceberg.delete": { "description": "Whether the corresponding Iceberg table is deleted upon deleting the topic.", @@ -1685,6 +1794,16 @@ ], "config_scope": "topic" }, + "redpanda.storage.mode": { + "description": "The storage mode for a topic. Determines how topic data is stored and whether it is eligible for upload to object storage.\n\nAccepted values:\n\n* `local`: Topic data is stored only on the broker's local disk. Object storage upload is disabled for the topic, regardless of cluster-level Tiered Storage settings.\n* `tiered`: Topic data is stored on local disk and also uploaded to object storage. Enables Tiered Storage for the topic.\n* `cloud`: Topic data is stored in object storage using the Cloud Topics architecture. Local storage is used only as a write buffer.\n* `unset`: Specifies that the topic's storage mode is unset, regardless of the cluster default. The topic may still have Tiered Storage enabled through the legacy properties `redpanda.remote.read` and `redpanda.remote.write`.\n\nThis property overrides the cluster-wide config_ref:default_redpanda_storage_mode,true,properties/cluster-properties[] setting for individual topics.", + "related_topics": [ + "self-managed-only: xref:manage:tiered-storage.adoc[Tiered Storage]", + "self-managed-only: xref:develop:manage-topics/cloud-topics.adoc[Manage Cloud Topics]" + ], + "config_scope": "topic", + "category": "tiered-storage", + "version": "v26.1.1" + }, "redpanda.value.schema.id.validation": { "description": "Enable validation of the schema ID for values on a record. When enabled, Redpanda validates that the schema ID encoded in the record's value is registered in the Schema Registry according to the configured subject name strategy.", "related_topics": [ @@ -1718,8 +1837,8 @@ "xref:reference:rpk/rpk-topic/rpk-topic-describe.adoc[`rpk topic describe`]", "xref:reference:rpk/rpk-topic/rpk-topic-alter-config.adoc[`rpk topic alter-config`]", "xref:reference:properties/cluster-properties.adoc#default_topic_replication[`default_topic_replication`]", - "xref:develop:config-topics.adoc#choose-the-replication-factor[Choose the replication factor]", - "xref:develop:config-topics.adoc#change-the-replication-factor[Change the replication factor]", + "xref:develop:manage-topics/config-topics.adoc#choose-the-replication-factor[Choose the replication factor]", + "xref:develop:manage-topics/config-topics.adoc#change-the-replication-factor[Change the replication factor]", "xref:reference:properties/cluster-properties.adoc#default_topic_replication[default_topic_replication]" ], "config_scope": "topic" @@ -1829,10 +1948,6 @@ "config_scope": "broker", "category": "redpanda" }, - "rpk_path": { - "description": "Path to RPK binary.", - "config_scope": "cluster" - }, "rps_limit_acls_and_users_operations": { "description": "Rate limit, in requests per second, for ACL and user operations on the controller. The controller processes cluster management requests (such as creating, deleting, and updating ACLs and users) through the controller log (raft0).", "config_scope": "cluster" @@ -1853,6 +1968,10 @@ "description": "Rate limit, in requests per second, for topic operations on the controller. The controller processes cluster management requests (such as creating, deleting, and updating topics and partition counts) through the controller log (raft0).", "config_scope": "cluster" }, + "rpk_path": { + "description": "Path to RPK binary.", + "config_scope": "cluster" + }, "sasl_mechanism": { "description": "The SASL mechanism to use when the HTTP Proxy client connects to the Kafka API. These credentials are used when the HTTP Proxy API listener has <>: `none` but the cluster requires authenticated access to the Kafka API.\n\nThis property specifies which individual SASL mechanism the HTTP Proxy client should use, while the cluster-wide available mechanisms are configured using the xref:reference:properties/cluster-properties.adoc#sasl_mechanisms[`sasl_mechanisms`] cluster property.\n\ninclude::shared:partial$http-proxy-ephemeral-credentials-breaking-change.adoc[]\n\nNOTE: While the cluster-wide xref:reference:properties/cluster-properties.adoc#sasl_mechanisms[`sasl_mechanisms`] property may support additional mechanisms (PLAIN, GSSAPI, OAUTHBEARER), HTTP Proxy client connections only support SCRAM mechanisms.", "related_topics": [ @@ -1901,6 +2020,9 @@ "related_topics": [], "config_scope": "cluster" }, + "schema_registry_enable_qualified_subjects": { + "version": "v26.1.1" + }, "schema_registry_replication_factor": { "description": "Replication factor for internal `_schemas` topic. If unset, defaults to the xref:../cluster-properties.adoc#default_topic_replication[`default_topic_replication`] cluster property.", "related_topics": [ diff --git a/docs-data/redpanda-property-changes-v25.3.1-to-v25.3.3.json b/docs-data/redpanda-property-changes-v25.3.1-to-v25.3.3.json deleted file mode 100644 index b7ff9a7422..0000000000 --- a/docs-data/redpanda-property-changes-v25.3.1-to-v25.3.3.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "comparison": { - "oldVersion": "v25.3.1", - "newVersion": "v25.3.3", - "timestamp": "2025-12-21T10:45:35.556Z" - }, - "summary": { - "newProperties": 6, - "changedDefaults": 0, - "changedDescriptions": 0, - "changedTypes": 0, - "deprecatedProperties": 0, - "removedProperties": 0, - "emptyDescriptions": 3 - }, - "details": { - "newProperties": [ - { - "name": "cloud_topics_epoch_service_epoch_increment_interval", - "type": "integer", - "default": 600000, - "description": "The interval at which the cluster epoch is incremented." - }, - { - "name": "cloud_topics_epoch_service_local_epoch_cache_duration", - "type": "integer", - "default": 60000, - "description": "The local cache duration of a cluster wide epoch." - }, - { - "name": "cloud_topics_short_term_gc_backoff_interval", - "type": "integer", - "default": 60000, - "description": "The interval between invocations of the L0 garbage collection work loop when no progress is being made or errors are occurring." - }, - { - "name": "cloud_topics_short_term_gc_interval", - "type": "integer", - "default": 10000, - "description": "The interval between invocations of the L0 garbage collection work loop when progress is being made." - }, - { - "name": "cloud_topics_short_term_gc_minimum_object_age", - "type": "integer", - "default": 43200000, - "description": "The minimum age of an L0 object before it becomes eligible for garbage collection." - }, - { - "name": "fetch_max_read_concurrency", - "type": "integer", - "default": 1, - "description": "The maximum number of concurrent partition reads per fetch request on each shard. Setting this higher than the default can lead to partition starvation and unneeded memory usage." - } - ], - "changedDefaults": [], - "changedDescriptions": [], - "changedTypes": [], - "deprecatedProperties": [], - "removedProperties": [], - "emptyDescriptions": [ - { - "name": "redpanda.cloud_topic.enabled", - "type": "string" - }, - { - "name": "redpanda.remote.allowgaps", - "type": "boolean" - }, - { - "name": "redpanda.virtual.cluster.id", - "type": "string" - } - ] - } -} \ No newline at end of file diff --git a/docs-data/redpanda-property-changes-v25.3.10-to-v25.3.11.json b/docs-data/redpanda-property-changes-v25.3.10-to-v25.3.11.json deleted file mode 100644 index f1d6676785..0000000000 --- a/docs-data/redpanda-property-changes-v25.3.10-to-v25.3.11.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "comparison": { - "oldVersion": "v25.3.10", - "newVersion": "v25.3.11", - "timestamp": "2026-03-26T03:19:25.076Z" - }, - "summary": { - "newProperties": 1, - "changedDefaults": 0, - "changedDescriptions": 0, - "changedTypes": 0, - "deprecatedProperties": 0, - "removedDeprecatedProperties": 0, - "removedProperties": 0, - "emptyDescriptions": 3 - }, - "details": { - "newProperties": [ - { - "name": "cloud_storage_gc_max_segments_per_run", - "type": "integer", - "default": 300, - "description": "Maximum number of segments to delete per housekeeping run. Each segment maps to up to three object keys (data, index, tx manifest), so a value of 300 produces 600 to 900 deletes plus one per spillover manifest." - } - ], - "changedDefaults": [], - "changedDescriptions": [], - "changedTypes": [], - "deprecatedProperties": [], - "removedDeprecatedProperties": [], - "removedProperties": [], - "emptyDescriptions": [ - { - "name": "redpanda.cloud_topic.enabled", - "type": "string" - }, - { - "name": "redpanda.remote.allowgaps", - "type": "boolean" - }, - { - "name": "redpanda.virtual.cluster.id", - "type": "string" - } - ] - } -} \ No newline at end of file diff --git a/docs-data/redpanda-property-changes-v25.3.7-to-v26.1.1-rc5.json b/docs-data/redpanda-property-changes-v25.3.7-to-v26.1.1-rc5.json new file mode 100644 index 0000000000..afe94c1e50 --- /dev/null +++ b/docs-data/redpanda-property-changes-v25.3.7-to-v26.1.1-rc5.json @@ -0,0 +1,853 @@ +{ + "comparison": { + "oldVersion": "v25.3.7", + "newVersion": "v26.1.1-rc5", + "timestamp": "2026-03-30T15:12:26.249Z" + }, + "summary": { + "newProperties": 5, + "changedDefaults": 38, + "changedDescriptions": 36, + "changedTypes": 36, + "deprecatedProperties": 0, + "removedDeprecatedProperties": 47, + "removedProperties": 0, + "emptyDescriptions": 2 + }, + "details": { + "newProperties": [ + { + "name": "cloud_storage_gc_max_segments_per_run", + "type": "integer", + "default": 300, + "description": "Maximum number of segments to delete per housekeeping run. Each segment maps to up to three object keys (data, index, tx manifest), so a value of 300 produces 600 to 900 deletes plus one per spillover manifest." + }, + { + "name": "cloud_topics_disable_level_zero_gc_for_tests", + "type": "boolean", + "default": false, + "description": "Disables the level-zero garbage collector in cloud topics. This property exists to simplify testing and shouldn't be set in production." + }, + { + "name": "cloud_topics_disable_metastore_flush_loop_for_tests", + "type": "boolean", + "default": false, + "description": "Disables the metastore flush loop in cloud topics. The property exists to simplify testing of read replicas and shouldn't be set in production." + }, + { + "name": "cloud_topics_epoch_service_max_same_epoch_duration", + "type": "integer", + "default": 86400000, + "description": "The duration of time that a node can use the exact same epoch." + }, + { + "name": "schema_registry_avro_use_named_references", + "type": "object", + "description": "No description" + } + ], + "changedDefaults": [ + { + "name": "cloud_storage_prefetch_segments_max", + "oldDefault": null, + "newDefault": 3 + }, + { + "name": "cloud_topics_allow_materialization_failure", + "oldDefault": null, + "newDefault": false + }, + { + "name": "cloud_topics_compaction_interval_ms", + "oldDefault": null, + "newDefault": 30000 + }, + { + "name": "cloud_topics_compaction_key_map_memory", + "oldDefault": null, + "newDefault": 134217728 + }, + { + "name": "cloud_topics_compaction_max_object_size", + "oldDefault": null, + "newDefault": 134217728 + }, + { + "name": "cloud_topics_fetch_debounce_enabled", + "oldDefault": null, + "newDefault": true + }, + { + "name": "cloud_topics_gc_health_check_interval", + "oldDefault": null, + "newDefault": 10000 + }, + { + "name": "cloud_topics_l1_indexing_interval", + "oldDefault": null, + "newDefault": 4194304 + }, + { + "name": "cloud_topics_long_term_file_deletion_delay", + "oldDefault": null, + "newDefault": 3600000 + }, + { + "name": "cloud_topics_long_term_flush_interval", + "oldDefault": null, + "newDefault": 600000 + }, + { + "name": "cloud_topics_metastore_lsm_apply_timeout_ms", + "oldDefault": null, + "newDefault": 300000 + }, + { + "name": "cloud_topics_metastore_replication_timeout_ms", + "oldDefault": null, + "newDefault": 30000 + }, + { + "name": "cloud_topics_num_metastore_partitions", + "oldDefault": null, + "newDefault": 3 + }, + { + "name": "cloud_topics_parallel_fetch_enabled", + "oldDefault": null, + "newDefault": true + }, + { + "name": "cloud_topics_preregistered_object_ttl", + "oldDefault": null, + "newDefault": 3600000 + }, + { + "name": "cloud_topics_produce_no_pid_concurrency", + "oldDefault": null, + "newDefault": 32 + }, + { + "name": "cloud_topics_produce_write_inflight_limit", + "oldDefault": null, + "newDefault": 1024 + }, + { + "name": "cloud_topics_reconciliation_interval", + "oldDefault": 10000, + "newDefault": null + }, + { + "name": "cloud_topics_reconciliation_max_interval", + "oldDefault": null, + "newDefault": 10000 + }, + { + "name": "cloud_topics_reconciliation_max_object_size", + "oldDefault": null, + "newDefault": 83886080 + }, + { + "name": "cloud_topics_reconciliation_min_interval", + "oldDefault": null, + "newDefault": 250 + }, + { + "name": "cloud_topics_reconciliation_parallelism", + "oldDefault": null, + "newDefault": 8 + }, + { + "name": "cloud_topics_reconciliation_slowdown_blend", + "oldDefault": null, + "newDefault": 0.4 + }, + { + "name": "cloud_topics_reconciliation_speedup_blend", + "oldDefault": null, + "newDefault": 0.9 + }, + { + "name": "cloud_topics_reconciliation_target_fill_ratio", + "oldDefault": null, + "newDefault": 0.8 + }, + { + "name": "cloud_topics_upload_part_size", + "oldDefault": null, + "newDefault": 16777216 + }, + { + "name": "default_redpanda_storage_mode", + "oldDefault": null, + "newDefault": "unset" + }, + { + "name": "delete_topic_enable", + "oldDefault": null, + "newDefault": true + }, + { + "name": "internal_rpc_request_timeout_ms", + "oldDefault": null, + "newDefault": 10000 + }, + { + "name": "leader_balancer_mode", + "newDefault": "calibrated" + }, + { + "name": "log_compaction_disable_tx_batch_removal", + "newDefault": null + }, + { + "name": "log_compaction_max_priority_wait_ms", + "oldDefault": null, + "newDefault": 3600000 + }, + { + "name": "log_compaction_tx_batch_removal_enabled", + "oldDefault": false, + "newDefault": true + }, + { + "name": "nested_group_behavior", + "oldDefault": null, + "newDefault": "none" + }, + { + "name": "oidc_group_claim_path", + "oldDefault": null, + "newDefault": "$.groups" + }, + { + "name": "redpanda.storage.mode", + "oldDefault": null, + "newDefault": "unset" + }, + { + "name": "schema_registry_enable_qualified_subjects", + "oldDefault": null, + "newDefault": false + }, + { + "name": "tls_v1_2_cipher_suites", + "oldDefault": "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:AES256-GCM-SHA384:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:AES128-SHA:AES128-CCM:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:AES256-SHA:AES256-CCM", + "newDefault": "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256:TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256:TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384:TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384:TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256:TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256" + } + ], + "changedDescriptions": [ + { + "name": "cloud_storage_prefetch_segments_max", + "oldDescription": "Configuration property: cloud_storage_prefetch_segments_max", + "newDescription": "Maximum number of small segments (size <= chunk size) to prefetch ahead during sequential reads. Set to 0 to disable cross-segment prefetch." + }, + { + "name": "cloud_topics_allow_materialization_failure", + "oldDescription": "Configuration property: cloud_topics_allow_materialization_failure", + "newDescription": "When enabled, the reconciler tolerates missing L0 extent objects (404 errors) during materialization. Failed extents are skipped, producing L1 state with empty offset ranges where deleted data was. Use this to recover partitions after accidental deletion of live extent objects." + }, + { + "name": "cloud_topics_compaction_interval_ms", + "oldDescription": "Configuration property: cloud_topics_compaction_interval_ms", + "newDescription": "How often to trigger background compaction for cloud topics." + }, + { + "name": "cloud_topics_compaction_key_map_memory", + "oldDescription": "Configuration property: cloud_topics_compaction_key_map_memory", + "newDescription": "Maximum number of bytes that may be used on each shard by cloud topics compaction key-offset maps." + }, + { + "name": "cloud_topics_compaction_max_object_size", + "oldDescription": "Configuration property: cloud_topics_compaction_max_object_size", + "newDescription": "Maximum size in bytes for L1 objects produced by cloud topics compaction." + }, + { + "name": "cloud_topics_fetch_debounce_enabled", + "oldDescription": "Configuration property: cloud_topics_fetch_debounce_enabled", + "newDescription": "Enables fetch debouncing in cloud topics. This mechanism guarantees that the broker fetches every object only once improving the performance and lowering the cost." + }, + { + "name": "cloud_topics_gc_health_check_interval", + "oldDescription": "Configuration property: cloud_topics_gc_health_check_interval", + "newDescription": "The interval at which the L0 garbage collector checks cluster health. GC will not proceed while the cluster is unhealthy." + }, + { + "name": "cloud_topics_l1_indexing_interval", + "oldDescription": "Configuration property: cloud_topics_l1_indexing_interval", + "newDescription": "The byte interval at which index entries are created within long term storage objects for cloud topics. Index entries are stored in the object metadata and enable efficient seeking by offset or timestamp within a partition. Lower values produce more index entries (better seek granularity) at the cost of a larger footer." + }, + { + "name": "cloud_topics_long_term_file_deletion_delay", + "oldDescription": "Configuration property: cloud_topics_long_term_file_deletion_delay", + "newDescription": "Delay before deleting stale long term files, allowing concurrent readers (e.g. read replica topics) to finish reading them before removal." + }, + { + "name": "cloud_topics_long_term_flush_interval", + "oldDescription": "Configuration property: cloud_topics_long_term_flush_interval", + "newDescription": "Time interval at which long term storage metadata is flushed to object storage." + }, + { + "name": "cloud_topics_metastore_lsm_apply_timeout_ms", + "oldDescription": "Configuration property: cloud_topics_metastore_lsm_apply_timeout_ms", + "newDescription": "Timeout for applying a replicated write batch to the local LSM database. This may take longer than usual when L0 compaction is behind and writes are being throttled." + }, + { + "name": "cloud_topics_metastore_replication_timeout_ms", + "oldDescription": "Configuration property: cloud_topics_metastore_replication_timeout_ms", + "newDescription": "Timeout for L1 metastore Raft replication and waiting for the STM to apply the replicated write batch." + }, + { + "name": "cloud_topics_num_metastore_partitions", + "oldDescription": "Configuration property: cloud_topics_num_metastore_partitions", + "newDescription": "Number of partitions for the cloud topics metastore topic, used to spread metastore load across the cluster. Higher values allow more parallel metadata operations but reduce the amount of work each partition can batch together. Only takes effect when the metastore topic is first created." + }, + { + "name": "cloud_topics_parallel_fetch_enabled", + "oldDescription": "Configuration property: cloud_topics_parallel_fetch_enabled", + "newDescription": "Enable parallel fetching in cloud topics. This mechanism improves the throughput by allowing the broker to download data needed by the fetch request using multiple shards." + }, + { + "name": "cloud_topics_preregistered_object_ttl", + "oldDescription": "Configuration property: cloud_topics_preregistered_object_ttl", + "newDescription": "Time-to-live for pre-registered L1 objects before they are expired." + }, + { + "name": "cloud_topics_produce_no_pid_concurrency", + "oldDescription": "Configuration property: cloud_topics_produce_no_pid_concurrency", + "newDescription": "Maximum number of concurrent raft replication requests for producers without a producer ID (idempotency disabled). Limits how many no-PID writes can proceed past the producer queue into raft simultaneously." + }, + { + "name": "cloud_topics_produce_write_inflight_limit", + "oldDescription": "Configuration property: cloud_topics_produce_write_inflight_limit", + "newDescription": "Maximum number of in-flight write requests per shard in the cloud topics write pipeline. Requests that exceed this limit are queued until a slot becomes available." + }, + { + "name": "cloud_topics_reconciliation_interval", + "oldDescription": "Time interval after which data is moved from short term storage to long term storage.", + "newDescription": "Configuration property: cloud_topics_reconciliation_interval" + }, + { + "name": "cloud_topics_reconciliation_max_interval", + "oldDescription": "Configuration property: cloud_topics_reconciliation_max_interval", + "newDescription": "Maximum reconciliation interval for adaptive scheduling." + }, + { + "name": "cloud_topics_reconciliation_max_object_size", + "oldDescription": "Configuration property: cloud_topics_reconciliation_max_object_size", + "newDescription": "Maximum size in bytes for L1 objects produced by the reconciler. With the default target fill ratio of 0.8, this gives an effective target object size of 64 MiB." + }, + { + "name": "cloud_topics_reconciliation_min_interval", + "oldDescription": "Configuration property: cloud_topics_reconciliation_min_interval", + "newDescription": "Minimum reconciliation interval for adaptive scheduling. The reconciler will not run more frequently than this." + }, + { + "name": "cloud_topics_reconciliation_parallelism", + "oldDescription": "Configuration property: cloud_topics_reconciliation_parallelism", + "newDescription": "Maximum number, per shard, of concurrent objects built by reconciliation" + }, + { + "name": "cloud_topics_reconciliation_slowdown_blend", + "oldDescription": "Configuration property: cloud_topics_reconciliation_slowdown_blend", + "newDescription": "Blend factor for slowing down reconciliation (0.0 to 1.0). Higher values mean reconciliation lowers its frequency faster when trying to find a frequency that produces well-sized objects. Generally this should be lower than the speedup blend, because reconciliation has less opportunities to adapt its frequency when it runs less frequently." + }, + { + "name": "cloud_topics_reconciliation_speedup_blend", + "oldDescription": "Configuration property: cloud_topics_reconciliation_speedup_blend", + "newDescription": "Blend factor for speeding up reconciliation (0.0 to 1.0). Higher values mean reconciliation increases its frequency faster when trying to find a frequency that produces well-sized objects." + }, + { + "name": "cloud_topics_reconciliation_target_fill_ratio", + "oldDescription": "Configuration property: cloud_topics_reconciliation_target_fill_ratio", + "newDescription": "Target fill ratio for L1 objects. The reconciler adapts its interval to produce objects at approximately this fill level (0.0 to 1.0)." + }, + { + "name": "cloud_topics_upload_part_size", + "oldDescription": "Configuration property: cloud_topics_upload_part_size", + "newDescription": "The part size in bytes used for multipart uploads. The minimum of 5 MiB is the smallest non-terminal part size allowed by cloud object storage providers." + }, + { + "name": "delete_topic_enable", + "oldDescription": "Configuration property: delete_topic_enable", + "newDescription": "Enable or disable topic deletion via the Kafka DeleteTopics API. When set to false, all topic deletion requests are rejected with error code 73 (TOPIC_DELETION_DISABLED). This is a cluster-wide safety setting that cannot be overridden by superusers. Topics in kafka_nodelete_topics are always protected regardless of this setting." + }, + { + "name": "internal_rpc_request_timeout_ms", + "oldDescription": "Configuration property: internal_rpc_request_timeout_ms", + "newDescription": "Default timeout for RPC requests between Redpanda nodes." + }, + { + "name": "leader_balancer_mode", + "oldDescription": "No description", + "newDescription": "Mode of the leader balancer optimization strategy. `calibrated` uses a heuristic that balances leaders based on replica counts per shard. `random` randomly moves leaders to reduce load on heavily-loaded shards. Legacy values `greedy_balanced_shards` and `random_hill_climbing` are treated as `calibrated`." + }, + { + "name": "log_compaction_max_priority_wait_ms", + "oldDescription": "Configuration property: log_compaction_max_priority_wait_ms", + "newDescription": "Maximum time a priority partition (for example, __consumer_offsets) can wait for compaction before preempting regular compaction." + }, + { + "name": "nested_group_behavior", + "oldDescription": "Configuration property: nested_group_behavior", + "newDescription": "Behavior for handling nested groups when extracting groups from authentication tokens. Two options are available - none and suffix. With none, the group is left alone (e.g. '/group/child/grandchild'). Suffix will extract the final component from the nested group (e.g. '/group' -> 'group' and '/group/child/grandchild' -> 'grandchild')." + }, + { + "name": "oidc_group_claim_path", + "oldDescription": "Configuration property: oidc_group_claim_path", + "newDescription": "JSON path to extract groups from the JWT payload." + }, + { + "name": "partition_autobalancing_node_autodecommission_timeout_sec", + "oldDescription": "Configuration property: partition_autobalancing_node_autodecommission_timeout_sec", + "newDescription": "When a node is unavailable for at least this timeout duration, it triggers Redpanda to decommission the node. This property applies only when `partition_autobalancing_mode` is set to `continuous`." + }, + { + "name": "partition_autobalancing_node_availability_timeout_sec", + "oldDescription": "When a node is unavailable for at least this timeout duration, it triggers Redpanda to move partitions off of the node. This property applies only when `partition_autobalancing_mode` is set to `continuous`. ", + "newDescription": "When a node is unavailable for at least this timeout duration, it triggers Redpanda to move partitions off of the node. This property applies only when `partition_autobalancing_mode` is set to `continuous`." + }, + { + "name": "redpanda.cloud_topic.enabled", + "oldDescription": "No description", + "newDescription": "Configuration property: redpanda.cloud_topic.enabled" + }, + { + "name": "schema_registry_enable_qualified_subjects", + "oldDescription": "Configuration property: schema_registry_enable_qualified_subjects", + "newDescription": "Enable parsing of qualified subject syntax (:.context:subject). When false, subjects are treated literally, as subjects in the default context. When true, qualified syntax is parsed to extract context and subject." + } + ], + "changedTypes": [ + { + "name": "cloud_storage_prefetch_segments_max", + "oldType": "string", + "newType": "integer" + }, + { + "name": "cloud_topics_allow_materialization_failure", + "oldType": "string", + "newType": "boolean" + }, + { + "name": "cloud_topics_compaction_interval_ms", + "oldType": "string", + "newType": "integer" + }, + { + "name": "cloud_topics_compaction_key_map_memory", + "oldType": "string", + "newType": "integer" + }, + { + "name": "cloud_topics_compaction_max_object_size", + "oldType": "string", + "newType": "integer" + }, + { + "name": "cloud_topics_fetch_debounce_enabled", + "oldType": "string", + "newType": "boolean" + }, + { + "name": "cloud_topics_gc_health_check_interval", + "oldType": "string", + "newType": "integer" + }, + { + "name": "cloud_topics_l1_indexing_interval", + "oldType": "string", + "newType": "integer" + }, + { + "name": "cloud_topics_long_term_file_deletion_delay", + "oldType": "string", + "newType": "integer" + }, + { + "name": "cloud_topics_long_term_flush_interval", + "oldType": "string", + "newType": "integer" + }, + { + "name": "cloud_topics_metastore_lsm_apply_timeout_ms", + "oldType": "string", + "newType": "integer" + }, + { + "name": "cloud_topics_metastore_replication_timeout_ms", + "oldType": "string", + "newType": "integer" + }, + { + "name": "cloud_topics_num_metastore_partitions", + "oldType": "string", + "newType": "integer" + }, + { + "name": "cloud_topics_parallel_fetch_enabled", + "oldType": "string", + "newType": "boolean" + }, + { + "name": "cloud_topics_preregistered_object_ttl", + "oldType": "string", + "newType": "integer" + }, + { + "name": "cloud_topics_produce_no_pid_concurrency", + "oldType": "string", + "newType": "integer" + }, + { + "name": "cloud_topics_produce_write_inflight_limit", + "oldType": "string", + "newType": "integer" + }, + { + "name": "cloud_topics_reconciliation_interval", + "oldType": "integer", + "newType": "string" + }, + { + "name": "cloud_topics_reconciliation_max_interval", + "oldType": "string", + "newType": "integer" + }, + { + "name": "cloud_topics_reconciliation_max_object_size", + "oldType": "string", + "newType": "integer" + }, + { + "name": "cloud_topics_reconciliation_min_interval", + "oldType": "string", + "newType": "integer" + }, + { + "name": "cloud_topics_reconciliation_parallelism", + "oldType": "string", + "newType": "integer" + }, + { + "name": "cloud_topics_reconciliation_slowdown_blend", + "oldType": "string", + "newType": "number" + }, + { + "name": "cloud_topics_reconciliation_speedup_blend", + "oldType": "string", + "newType": "number" + }, + { + "name": "cloud_topics_reconciliation_target_fill_ratio", + "oldType": "string", + "newType": "number" + }, + { + "name": "cloud_topics_upload_part_size", + "oldType": "string", + "newType": "integer" + }, + { + "name": "coproc_supervisor_server", + "oldType": "deprecated_property", + "newType": "object" + }, + { + "name": "dashboard_dir", + "oldType": "deprecated_property", + "newType": "object" + }, + { + "name": "delete_topic_enable", + "oldType": "string", + "newType": "boolean" + }, + { + "name": "enable_central_config", + "oldType": "deprecated_property", + "newType": "object" + }, + { + "name": "internal_rpc_request_timeout_ms", + "oldType": "string", + "newType": "integer" + }, + { + "name": "leader_balancer_mode", + "oldType": "deprecated_property", + "newType": "string" + }, + { + "name": "log_compaction_disable_tx_batch_removal", + "oldType": "deprecated_property", + "newType": "string" + }, + { + "name": "log_compaction_max_priority_wait_ms", + "oldType": "string", + "newType": "integer" + }, + { + "name": "partition_autobalancing_node_autodecommission_timeout_sec", + "oldType": "string", + "newType": "integer" + }, + { + "name": "schema_registry_enable_qualified_subjects", + "oldType": "string", + "newType": "boolean" + } + ], + "deprecatedProperties": [], + "removedDeprecatedProperties": [ + { + "name": "alter_topic_cfg_timeout_ms", + "type": "integer", + "description": "The duration, in milliseconds, that Redpanda waits for the replication of entries in the controller log when executing a request to alter topic configurations. This timeout ensures that configuration changes are replicated across the cluster before the alteration request is considered complete." + }, + { + "name": "cloud_storage_disable_metadata_consistency_checks", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "cloud_storage_reconciliation_ms", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "coproc_max_batch_size", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "coproc_max_inflight_bytes", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "coproc_max_ingest_bytes", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "coproc_offset_flush_interval_ms", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "create_topic_timeout_ms", + "type": "integer", + "description": "Timeout, in milliseconds, to wait for new topic creation." + }, + { + "name": "datalake_disk_space_monitor_interval", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "enable_admin_api", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "enable_coproc", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "find_coordinator_timeout_ms", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "full_raft_configuration_recovery_pattern", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "id_allocator_replication", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "kafka_admin_topic_api_rate", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "kafka_client_group_byte_rate_quota", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "kafka_client_group_fetch_byte_rate_quota", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "kafka_memory_batch_size_estimate_for_fetch", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "kafka_quota_balancer_min_shard_throughput_bps", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "kafka_quota_balancer_min_shard_throughput_ratio", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "kafka_quota_balancer_node_period", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "kafka_quota_balancer_window", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "kafka_throughput_throttling_v2", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "log_compaction_adjacent_merge_self_compaction_count", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "log_message_timestamp_alert_after_ms", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "log_message_timestamp_alert_before_ms", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "max_version", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "metadata_status_wait_timeout_ms", + "type": "integer", + "description": "Maximum time to wait in metadata request for cluster health to be refreshed." + }, + { + "name": "min_version", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "node_management_operation_timeout_ms", + "type": "integer", + "description": "Timeout for executing node management operations." + }, + { + "name": "raft_max_concurrent_append_requests_per_follower", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "raft_recovery_default_read_size", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "recovery_append_timeout_ms", + "type": "integer", + "description": "Timeout for append entry requests issued while updating a stale follower." + }, + { + "name": "rm_sync_timeout_ms", + "type": "integer", + "description": "Resource manager's synchronization timeout. Specifies the maximum time for this node to wait for the internal state machine to catch up with all events written by previous leaders before rejecting a request." + }, + { + "name": "rm_violation_recovery_policy", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "schema_registry_protobuf_renderer_v2", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "seed_server_meta_topic_partitions", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "seq_table_min_size", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "target_fetch_quota_byte_rate", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "target_quota_byte_rate", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "tm_sync_timeout_ms", + "type": "integer", + "description": "Transaction manager's synchronization timeout. Maximum time to wait for internal state machine to catch up before rejecting a request." + }, + { + "name": "tm_violation_recovery_policy", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "transaction_coordinator_replication", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "tx_registry_log_capacity", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "tx_registry_sync_timeout_ms", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "use_scheduling_groups", + "type": "deprecated_property", + "description": "No description" + }, + { + "name": "wait_for_leader_timeout_ms", + "type": "integer", + "description": "Timeout to wait for leadership in metadata cache." + } + ], + "removedProperties": [], + "emptyDescriptions": [ + { + "name": "redpanda.remote.allowgaps", + "type": "boolean" + }, + { + "name": "redpanda.virtual.cluster.id", + "type": "string" + } + ] + } +} \ No newline at end of file diff --git a/docs-data/redpanda-property-changes-v26.1.1-to-v26.1.1-rc5.json b/docs-data/redpanda-property-changes-v26.1.1-to-v26.1.1-rc5.json new file mode 100644 index 0000000000..0f0d4b108b --- /dev/null +++ b/docs-data/redpanda-property-changes-v26.1.1-to-v26.1.1-rc5.json @@ -0,0 +1,34 @@ +{ + "comparison": { + "oldVersion": "v26.1.1", + "newVersion": "v26.1.1-rc5", + "timestamp": "2026-03-31T14:37:03.774Z" + }, + "summary": { + "newProperties": 0, + "changedDefaults": 0, + "changedDescriptions": 0, + "changedTypes": 0, + "deprecatedProperties": 0, + "removedProperties": 0, + "emptyDescriptions": 2 + }, + "details": { + "newProperties": [], + "changedDefaults": [], + "changedDescriptions": [], + "changedTypes": [], + "deprecatedProperties": [], + "removedProperties": [], + "emptyDescriptions": [ + { + "name": "redpanda.remote.allowgaps", + "type": "boolean" + }, + { + "name": "redpanda.virtual.cluster.id", + "type": "string" + } + ] + } +} \ No newline at end of file diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc index 3c8abcbce6..f6cdd63007 100644 --- a/modules/ROOT/nav.adoc +++ b/modules/ROOT/nav.adoc @@ -36,7 +36,9 @@ ** xref:develop:kafka-clients.adoc[Kafka Compatibility] ** xref:develop:benchmark.adoc[] ** xref:develop:http-proxy.adoc[] -** xref:develop:config-topics.adoc[] +** xref:develop:manage-topics/index.adoc[] +*** xref:develop:manage-topics/config-topics.adoc[] +*** xref:develop:manage-topics/cloud-topics.adoc[] ** xref:console:ui/edit-topic-configuration.adoc[Edit Topic Configuration] ** xref:develop:produce-data/index.adoc[Produce Data] *** xref:develop:produce-data/configure-producers.adoc[] @@ -112,6 +114,7 @@ ***** xref:manage:kubernetes/storage/k-emptydir.adoc[emptyDir] **** xref:manage:kubernetes/storage/k-resize-persistentvolumes.adoc[Expand PersistentVolumes] **** xref:manage:kubernetes/storage/k-delete-persistentvolume.adoc[Delete PersistentVolumes] +*** xref:manage:kubernetes/k-cloud-topics.adoc[Cloud Topics] *** xref:manage:kubernetes/tiered-storage/index.adoc[Tiered Storage] **** xref:manage:kubernetes/tiered-storage/k-tiered-storage.adoc[Use Tiered Storage] **** xref:manage:kubernetes/tiered-storage/k-fast-commission-decommission.adoc[] @@ -134,6 +137,8 @@ ***** xref:manage:kubernetes/security/authentication/k-authentication.adoc[Enable Authentication] ***** xref:manage:kubernetes/security/authentication/k-user-controller.adoc[Manage Users and ACLs (Operator)] ***** xref:manage:kubernetes/security/authorization/k-role-controller.adoc[Manage Roles and ACLs (Operator)] +***** xref:manage:kubernetes/security/authorization/k-group-controller.adoc[Manage Groups and ACLs (Operator)] +***** xref:manage:kubernetes/security/authentication/k-schema-registry-acls.adoc[Manage Schema Registry ACLs (Operator)] **** xref:manage:kubernetes/security/k-audit-logging.adoc[Audit Logging] *** xref:manage:kubernetes/k-rack-awareness.adoc[Rack Awareness] *** xref:manage:kubernetes/k-remote-read-replicas.adoc[Remote Read Replicas] @@ -163,6 +168,7 @@ *** xref:manage:audit-logging.adoc[Audit Logging] **** xref:manage:audit-logging/audit-log-samples.adoc[Sample Audit Log Messages] *** xref:manage:cluster-maintenance/disk-utilization.adoc[] +*** xref:manage:cluster-maintenance/about-throughput-quotas.adoc[] *** xref:manage:cluster-maintenance/manage-throughput.adoc[Manage Throughput] *** xref:manage:cluster-maintenance/compaction-settings.adoc[Compaction Settings] *** xref:manage:cluster-maintenance/configure-client-connections.adoc[] @@ -172,6 +178,7 @@ *** xref:manage:security/authentication.adoc[Authentication] *** xref:manage:security/authorization/index.adoc[Authorization] **** xref:manage:security/authorization/rbac.adoc[Role-Based Access Control (RBAC)] +**** xref:manage:security/authorization/gbac.adoc[Group-Based Access Control (GBAC)] **** xref:manage:security/authorization/acl.adoc[Access Control Lists (ACLs)] *** xref:manage:security/fips-compliance.adoc[FIPS Compliance] *** xref:manage:security/encryption.adoc[] diff --git a/modules/console/pages/ui/edit-topic-configuration.adoc b/modules/console/pages/ui/edit-topic-configuration.adoc index 75dcd24042..ab9207a367 100644 --- a/modules/console/pages/ui/edit-topic-configuration.adoc +++ b/modules/console/pages/ui/edit-topic-configuration.adoc @@ -18,4 +18,4 @@ == Suggested reading - xref:reference:properties/topic-properties.adoc[] -- xref:develop:config-topics.adoc[] \ No newline at end of file +- xref:develop:manage-topics/config-topics.adoc[] \ No newline at end of file diff --git a/modules/deploy/pages/console/kubernetes/deploy.adoc b/modules/deploy/pages/console/kubernetes/deploy.adoc index c63e6337c5..83cce815bb 100644 --- a/modules/deploy/pages/console/kubernetes/deploy.adoc +++ b/modules/deploy/pages/console/kubernetes/deploy.adoc @@ -667,6 +667,8 @@ spec: == Monitoring +Configure metrics exposure and Prometheus scraping for Redpanda Console. + Enable monitoring for Redpanda Console: [,yaml] @@ -678,6 +680,88 @@ config: port: 9090 ---- +=== Prometheus ServiceMonitor + +If you use the https://github.com/prometheus-operator/prometheus-operator[Prometheus Operator^], deploy a `ServiceMonitor` resource alongside Redpanda Console. Prometheus then discovers and scrapes Console metrics from the `/admin/metrics` endpoint. + +[tabs] +====== +Operator:: ++ +-- + +To enable the ServiceMonitor in the Console custom resource, set `monitoring.enabled` to `true`: + +[,yaml] +---- +apiVersion: cluster.redpanda.com/v1alpha2 +kind: Console +metadata: + name: redpanda-console + namespace: redpanda +spec: + monitoring: + enabled: true <1> + scrapeInterval: "30s" <2> + labels: <3> + release: kube-prometheus-stack + cluster: + clusterRef: + name: redpanda +---- + +<1> Set to `true` to create a `ServiceMonitor` resource. Default: `false`. +<2> How often Prometheus scrapes the metrics endpoint. Default: `1m`. +<3> Additional labels to apply to the `ServiceMonitor`. Match your Prometheus Operator's `serviceMonitorSelector` by applying the same labels here. + +Apply the Console CR: + +[,bash] +---- +kubectl apply -f console.yaml --namespace redpanda +---- + +-- +Helm:: ++ +-- + +To enable the ServiceMonitor in the Console Helm chart, add the following to your `console-values.yaml`: + +[,yaml] +---- +monitoring: + enabled: true <1> + scrapeInterval: "30s" <2> + labels: {} <3> +---- + +<1> Set to `true` to create a `ServiceMonitor` resource. Default: `false`. +<2> How often Prometheus scrapes the metrics endpoint. Default: `1m`. +<3> Additional labels to apply to the `ServiceMonitor`. Match your Prometheus Operator's `serviceMonitorSelector` by applying the same labels here. For example: ++ +[,yaml] +---- +monitoring: + enabled: true + labels: + release: kube-prometheus-stack +---- + +If you deploy Redpanda Console as a subchart of the Redpanda Helm chart, configure monitoring under the `console` key. All `monitoring` options are available under this key. + +[,yaml] +---- +console: + monitoring: + enabled: true +---- + +-- +====== + +When the Console server is configured with TLS (`config.server.tls.enabled: true`), the ServiceMonitor uses HTTPS and configures CA validation for scraping. + == Troubleshooting * **Connection refused**: Verify Redpanda broker addresses and network policies diff --git a/modules/deploy/pages/redpanda/manual/production/dev-deployment.adoc b/modules/deploy/pages/redpanda/manual/production/dev-deployment.adoc index 36e5db81e1..44aaf8cd91 100644 --- a/modules/deploy/pages/redpanda/manual/production/dev-deployment.adoc +++ b/modules/deploy/pages/redpanda/manual/production/dev-deployment.adoc @@ -8,7 +8,7 @@ You can deploy Redpanda using well-known configuration properties optimized for [NOTE] ==== -* Development mode enables write caching by default. This is a relaxed mode of xref:develop:produce-data/configure-producers.adoc#acksall[`acks=all`] that acknowledges a message as soon as it is received and acknowledged on a majority of brokers, without waiting for it to fsync to disk. Write caching provides lower latency while still ensuring that a majority of brokers acknowledge the write. For more information, or to disable this, see xref:develop:config-topics.adoc#configure-write-caching[write caching]. +* Development mode enables write caching by default. This is a relaxed mode of xref:develop:produce-data/configure-producers.adoc#acksall[`acks=all`] that acknowledges a message as soon as it is received and acknowledged on a majority of brokers, without waiting for it to fsync to disk. Write caching provides lower latency while still ensuring that a majority of brokers acknowledge the write. For more information, or to disable this, see xref:develop:manage-topics/config-topics.adoc#configure-write-caching[write caching]. * Development mode also bypasses `fsync`, acknowledging messages before they're stored to disk. This reduces the durability of messages, could cause potential data loss, and could give unrealistic performance characteristics for a production environment. ==== diff --git a/modules/deploy/partials/high-availability.adoc b/modules/deploy/partials/high-availability.adoc index 5801bf4f76..3e0b843de6 100644 --- a/modules/deploy/partials/high-availability.adoc +++ b/modules/deploy/partials/high-availability.adoc @@ -111,7 +111,7 @@ endif::[] A multi-region deployment is similar to a multi-AZ deployment, in that it needs at least three regions to counter the loss of a single region. Note that this deployment strategy increases latency due to the physical distance between regions. In addition to higher produce and end-to-end latency and increased costs, multi-region deployments require careful tuning. Redpanda recommends that you work closely with Redpanda’s Customer Success team when implementing a multi-region deployment. Also consider the following strategies to mitigate these challenges: -* Configure xref:develop:produce-data/leader-pinning.adoc#configure-leader-pinning[leader pinning] to ensure that topic partition leaders are geographically closer to clients. This can help lower network costs and latency by routing produce requests to brokers located in specific AZs. +* Configure xref:develop:produce-data/leader-pinning.adoc[Leader Pinning] to ensure that topic partition leaders are geographically closer to clients. This can help lower network costs and latency by routing produce requests to brokers located in specific AZs. * If your produce latency exceeds your requirements, you can configure producers to have `acks=1` instead of `acks=all`. This reduces latency by only waiting for the leader to acknowledge, rather than waiting for all brokers to respond. However, using this configuration can decrease message durability. If the partition leader goes offline, you may lose any messages that are acknowledged but not yet replicated. === Multi-cluster deployment diff --git a/modules/develop/pages/kafka-clients.adoc b/modules/develop/pages/kafka-clients.adoc index 3f7815ded2..ba55a37e30 100644 --- a/modules/develop/pages/kafka-clients.adoc +++ b/modules/develop/pages/kafka-clients.adoc @@ -60,7 +60,7 @@ endif::[] * HTTP Proxy (pandaproxy): Unlike other REST proxy implementations in the Kafka ecosystem, Redpanda HTTP Proxy does not support topic and ACLs CRUD through the HTTP Proxy. HTTP Proxy is designed for clients producing and consuming data that do not perform administrative functions. ifdef::env-cloud[] + -* The `delete.retention.ms` topic configuration in Kafka is not supported. Tombstone markers are not removed for topics with a `compact` xref:get-started:config-topics.adoc#change-the-cleanup-policy[cleanup policy]. Redpanda only deletes tombstone markers when topics with a cleanup policy of `compact,delete` have reached their xref:get-started:create-topic.adoc[retention limits]. +* The `delete.retention.ms` topic configuration in Kafka is not supported for Tiered Storage topics. Cloud Topics and local storage topics support Tombstone marker deletion using `delete.retention.ms`, but in Tiered Storage topics, Tombstone markers are only removed in accordance with normal topic retention, and only if the cleanup policy is `delete` or `compact, delete`. endif::[] ifndef::env-cloud[] + diff --git a/modules/develop/pages/manage-topics/cloud-topics.adoc b/modules/develop/pages/manage-topics/cloud-topics.adoc new file mode 100644 index 0000000000..be21d252df --- /dev/null +++ b/modules/develop/pages/manage-topics/cloud-topics.adoc @@ -0,0 +1,86 @@ += Manage Cloud Topics +:description: Cloud Topics are "diskless" Redpanda topics that enable you to store data directly to object storage to trade off latency for lower costs. +:page-topic-type: how-to +:personas: streaming_developer, platform_admin +:learning-objective-1: Describe the latency and cost trade-offs of Cloud Topics compared to standard Redpanda topics +:learning-objective-2: Create a Cloud Topic using rpk after enabling Cloud Topics on your cluster +:learning-objective-3: Identify Cloud Topics limitations and configurations that reduce cross-AZ networking costs +// tag::single-source[] + +ifndef::env-cloud[] +[NOTE] +==== +include::shared:partial$enterprise-license.adoc[] +==== +endif::[] + +include::ROOT:develop:partial$cloud-topics-overview.adoc[tag=intro] + +include::ROOT:develop:partial$cloud-topics-overview.adoc[tag=latency-explanation] + +== Prerequisites + +ifdef::env-cloud[] +- xref:manage:rpk/rpk-install.adoc[Install rpk] v26.1 or later. +endif::[] +ifndef::env-cloud[] +- xref:get-started:rpk-install.adoc[] v26.1 or later. +- xref:manage:tiered-storage.adoc#set-up-tiered-storage[Enable cloud storage] on your Redpanda cluster. ++ +[NOTE] +==== +If you plan to use Cloud Topics for all new topics in a Redpanda cluster, be sure to set the following cluster-level property: +[,bash] +---- +default_redpanda_storage_mode=cloud +---- +This ensures that newly-created Redpanda topics are Cloud Topics by default. +For details, see xref:manage:tiered-storage.adoc#enable-tiered-storage-for-a-cluster[Enable Tiered Storage for a cluster]. +==== + +- xref:manage:tiered-storage.adoc#configure-object-storage[Configure object storage]. +- Ensure that you have an Enterprise license. ++ +To check your license status, run: ++ +[,bash] +---- +rpk cluster license info +---- + +endif::[] + +== Limitations + +include::ROOT:develop:partial$cloud-topics-limitations.adoc[tag=limitations] + +== Enable Cloud Topics + +To enable Cloud Topics for a cluster: + +[,bash] +---- +rpk cluster config set cloud_topics_enabled=true +---- + +NOTE: This configuration update requires a restart to take effect. + + +After enabling Cloud Topics, you can proceed to create new Cloud Topics: + +[,bash] +---- +rpk topic create -c redpanda.storage.mode=cloud +---- + +[,console] +---- +TOPIC STATUS +audit.analytics.may2025 OK +---- + +You can make a topic a Cloud Topic only at topic creation time. + +In addition to replication, cross-AZ ingress (producer) and egress (consumer) traffic can also contribute substantially to cloud networking costs. When running multi-AZ clusters in general, Redpanda strongly recommends using xref:develop:consume-data/follower-fetching.adoc[Follower Fetching], which allows consumers to avoid crossing network zones. When possible, you can use xref:develop:produce-data/leader-pinning.adoc[leader pinning], which positions a topic's partition leader close to the producers, providing a similar benefit for ingress traffic. These features can add additional savings to the replication cost savings of Cloud Topics. + +// end::single-source[] diff --git a/modules/develop/pages/manage-topics/config-topics.adoc b/modules/develop/pages/manage-topics/config-topics.adoc new file mode 100644 index 0000000000..09d5c0b191 --- /dev/null +++ b/modules/develop/pages/manage-topics/config-topics.adoc @@ -0,0 +1,368 @@ += Manage Topics +:page-categories: Clients, Development +:description: Learn how to create topics, update topic configurations, and delete topics or records. +tag::single-source[] + +include::develop:partial$topic-defaults.adoc[] + +== Create a topic + +Creating a topic can be as simple as specifying a name for your topic on the command line. For example, to create a topic named `xyz`, run: + +[,bash] +---- +rpk topic create xyz +---- + +ifndef::env-cloud[] +This command creates a topic named `xyz` with one partition and one replica, because these are the default values set in the cluster configuration file. Replicas are copies of partitions that are distributed across different brokers, so if one broker goes down, other brokers still have a copy of the data. + +endif::[] + +ifdef::env-cloud[] +This command creates a topic named `xyz` with one partition and three replicas, because these are the default values set in the cluster configuration file. Replicas are copies of partitions that are distributed across different brokers, so if one broker goes down, other brokers still have a copy of the data. + +Redpanda Cloud supports 40,000 topics per cluster. + +endif::[] + +=== Choose the number of partitions + +A partition acts as a log file where topic data is written. Dividing topics into partitions allows producers to write messages in parallel and consumers to read messages in parallel. The higher the number of partitions, the greater the throughput. + +TIP: As a general rule, select a number of partitions that corresponds to the maximum number of consumers in any consumer group that will consume the data. + +For example, suppose you plan to create a consumer group with 10 consumers. To create topic `xyz` with 10 partitions, run: + +[,bash] +---- +rpk topic create xyz -p 10 +---- + +ifndef::env-cloud[] +[[choose-the-replication-factor]] +=== Choose the replication factor + +The default replication factor in the cluster configuration is set to 1. By choosing a replication factor greater than 1, you ensure that each partition has a copy of its data on at least one other broker. One replica acts as the leader, and the other replicas are followers. + +To specify a replication factor of 3 for topic `xyz`, run: + +[,bash] +---- +rpk topic create xyz -r 3 +---- + +NOTE: The replication factor must be an odd number. Redpanda Data recommends a replication factor of 3 for most use cases. Administrators may set a minimum required replication factor for any new topic in the cluster through the cluster-level xref:reference:cluster-properties.adoc#minimum_topic_replications[`minimum_topic_replications`] property. + +TIP: If you enable xref:manage:tiered-storage.adoc[Tiered Storage] on a topic, you can then use xref:manage:topic-recovery.adoc[topic recovery] to restore data for a deleted topic. + +endif::[] + +ifndef::env-cloud[] +[[choose-a-storage-mode]] +=== Choose a storage mode + +Starting in Redpanda v26.1, you can set the `redpanda.storage.mode` topic property to control how a topic stores data: + +[cols="1,3"] +|=== +| Value | Behavior + +| `unset` (default) +| Legacy behavior. Tiered Storage is controlled by the `redpanda.remote.read` and `redpanda.remote.write` topic properties and their cluster-level defaults (`cloud_storage_enable_remote_read` and `cloud_storage_enable_remote_write`). + +| `local` +| Topic data is stored only on the broker's local disk. Object storage upload is disabled for the topic, regardless of `redpanda.remote.read` and `redpanda.remote.write` values. + +| `tiered` +| Data is stored on local disk and uploaded to object storage. Enables xref:manage:tiered-storage.adoc[Tiered Storage] for the topic regardless of `redpanda.remote.read` and `redpanda.remote.write` values. + +| `cloud` +| Data is stored durably in object storage using the glossterm:Cloud Topic[,Cloud Topics] architecture. Local storage is used only as a write buffer. See xref:develop:manage-topics/cloud-topics.adoc[]. +|=== + +To set the storage mode at topic creation time: + +[,bash] +---- +rpk topic create -c redpanda.storage.mode=tiered +---- + +When `redpanda.storage.mode` is set to `local`, `tiered`, or `cloud`, the `redpanda.remote.read` and `redpanda.remote.write` topic properties have no effect on the topic. + +To apply a default storage mode to all new topics in a cluster, set the `default_redpanda_storage_mode` cluster property: + +[,bash] +---- +rpk cluster config set default_redpanda_storage_mode=tiered +---- + +To set `local` as the default storage mode for all new topics in a cluster: + +[,bash] +---- +rpk cluster config set default_redpanda_storage_mode=local +---- +If `default_redpanda_storage_mode` is not configured (the default), new topics use `unset` mode and Tiered Storage behavior is inherited from the cluster-level `cloud_storage_enable_remote_write` and `cloud_storage_enable_remote_read` properties. + +endif::[] + +== Update topic configurations + +After you create a topic, you can update the topic property settings for all new data written to it. For example, you can add partitions or change the cleanup policy. + +=== Add partitions + +You can assign a certain number of partitions when you create a topic, and add partitions later. For example, suppose you add brokers to your cluster, and you want to take advantage of the additional processing power. To increase the number of partitions for existing topics, run: + +[,bash] +---- +rpk topic add-partitions [TOPICS...] --num [#] +---- + +Note that `--num <#>` is the number of partitions to _add_, not the total number of partitions. + +include::develop:partial$balance-existing-topic-redistribution.adoc[] + +ifndef::env-cloud[] +[[change-the-replication-factor]] +=== Change the replication factor + +Suppose you create a topic with the default replication factor of 1 (which is specified in the cluster properties configuration file). Now you want to change the replication factor to 3, so you can have two backups of topic data in case a broker goes down. To set the replication factor to 3, run: + +[,bash] +---- +rpk topic alter-config [TOPICS...] --set replication.factor=3 +---- + +NOTE: The replication factor can't exceed the number of Redpanda brokers. If you try to set a replication factor greater than the number of brokers, the request is rejected. + +endif::[] + +ifndef::env-cloud[] +[[change-the-storage-mode]] +=== Change the storage mode + +You can change a topic's `redpanda.storage.mode` after creation, with the following restrictions: + +[cols="1,1,1,3"] +|=== +| From | To | Permitted | Notes + +| `local` +| `tiered` +| Yes +| Enables Tiered Storage for the topic. Object storage must be configured. + +| `tiered` +| `local` +| With caution +| Disables object storage uploads. Redpanda strongly recommends against repeatedly toggling this setting, as it can result in data gaps in Tiered Storage. + +| Any +| `cloud` +| No +| Cloud Topics can only be set at topic creation time. + +| `cloud` +| Any +| No +| A Cloud Topic cannot be converted to a local or Tiered Storage topic. +|=== + +For example, to transition an existing local topic to Tiered Storage: + +[,bash] +---- +rpk topic alter-config --set redpanda.storage.mode=tiered +---- + +endif::[] + + +=== Change the cleanup policy + +The cleanup policy determines how to clean up the partition log files when they reach a certain size: + +* `delete` deletes data based on age or log size. Topics retain all records until then. +* `compact` compacts the data by only keeping the latest values for each KEY. +* `compact,delete` combines both methods. + +Unlike compacted topics, which keep only the most recent message for a given key, topics configured with a `delete` cleanup policy provide a running history of all changes for those topics. + +include::develop:partial$topic-properties-warning.adoc[] + +For example, to change a topic's policy to `compact`, run: + +[,bash] +---- +rpk topic alter-config [TOPICS…] —-set cleanup.policy=compact +---- + +ifndef::env-cloud[] +For details on compaction in Redpanda, see xref:manage:cluster-maintenance/compaction-settings.adoc[Compaction settings]. + +endif::[] + +[[configure-write-caching]] +=== Configure write caching + +Write caching is a relaxed mode of xref:develop:produce-data/configure-producers.adoc#acksall[`acks=all`] that provides better performance at the expense of durability. It acknowledges a message as soon as it is received and acknowledged on a majority of brokers, without waiting for it to be written to disk. This provides lower latency while still ensuring that a majority of brokers acknowledge the write. + +Write caching applies to user topics. It does not apply to transactions or consumer offsets: data written in the context of a transaction and consumer offset commits is always written to disk and fsynced before being acknowledged to the client. + +ifndef::env-cloud[] +NOTE: For clusters in xref:reference:rpk/rpk-redpanda/rpk-redpanda-mode.adoc#development-mode[development mode], write caching is enabled by default. For clusters in production mode, it is disabled by default. + +endif::[] + +Only enable write caching on workloads that can tolerate some data loss in the case of multiple, simultaneous broker failures. Leaving write caching disabled safeguards your data against complete data center or availability zone failures. + +ifndef::env-cloud[] + +==== Configure at cluster level + +To enable write caching by default in all user topics, set the cluster-level property xref:reference:cluster-properties.adoc#write_caching_default[`write_caching_default`]: + +`rpk cluster config set write_caching_default=true` + +With `write_caching_default` set to true at the cluster level, Redpanda fsyncs to disk according to xref:reference:cluster-properties.adoc#raft_replica_max_pending_flush_bytes[`raft_replica_max_pending_flush_bytes`] and xref:reference:cluster-properties.adoc#raft_replica_max_flush_delay_ms[`raft_replica_max_flush_delay_ms`], whichever is reached first. + +endif::[] + +==== Configure at topic level + +To override the cluster-level setting at the topic level, set the topic-level property `write.caching`: + +`rpk topic alter-config my_topic --set write.caching=true` + +With `write.caching` enabled at the topic level, Redpanda fsyncs to disk according to `flush.ms` and `flush.bytes`, whichever is reached first. + +=== Remove a configuration setting + +You can remove a configuration that overrides the default setting, and the setting will use the default value again. For example, suppose you altered the cleanup policy to use `compact` instead of the default, `delete`. Now you want to return the policy setting to the default. To remove the configuration setting `cleanup.policy=compact`, run `rpk topic alter-config` with the `--delete` flag: + +[,bash] +---- +rpk topic alter-config [TOPICS...] --delete cleanup.policy +---- + +== List topic configuration settings + +To display all the configuration settings for a topic, run: + +[,bash] +---- +rpk topic describe -c +---- + +The `-c` flag limits the command output to just the topic configurations. This command is useful for checking the default configuration settings before you make any changes and for verifying changes after you make them. + +The following command output displays after running `rpk topic describe test-topic`, where `test-topic` was created with default settings: + +ifndef::env-cloud[] +[,bash] +---- +rpk topic describe test_topic +SUMMARY +======= +NAME test_topic +PARTITIONS 1 +REPLICAS 1 + +CONFIGS +======= +KEY VALUE SOURCE +cleanup.policy delete DYNAMIC_TOPIC_CONFIG +compression.type producer DEFAULT_CONFIG +max.message.bytes 1048576 DEFAULT_CONFIG +message.timestamp.type CreateTime DEFAULT_CONFIG +redpanda.datapolicy function_name: script_name: DEFAULT_CONFIG +redpanda.remote.delete true DEFAULT_CONFIG +redpanda.remote.read false DEFAULT_CONFIG +redpanda.remote.write false DEFAULT_CONFIG +redpanda.storage.mode unset DEFAULT_CONFIG +retention.bytes -1 DEFAULT_CONFIG +retention.local.target.bytes -1 DEFAULT_CONFIG +retention.local.target.ms 86400000 DEFAULT_CONFIG +retention.ms 604800000 DEFAULT_CONFIG +segment.bytes 1073741824 DEFAULT_CONFIG +---- + +Suppose you add two partitions, and increase the number of replicas to 3. The new command output confirms the changes in the `SUMMARY` section: + +[.no-copy] +---- +SUMMARY +======= +NAME test_topic +PARTITIONS 3 +REPLICAS 3 +---- + +endif::[] + +ifdef::env-cloud[] +[,bash] +---- +rpk topic describe test_topic +SUMMARY +======= +NAME test_topic +PARTITIONS 1 +REPLICAS 3 + +CONFIGS +======= +KEY VALUE SOURCE +cleanup.policy delete DYNAMIC_TOPIC_CONFIG +compression.type producer DEFAULT_CONFIG +max.message.bytes 20971520 DEFAULT_CONFIG +message.timestamp.type CreateTime DEFAULT_CONFIG +redpanda.datapolicy function_name: script_name: DEFAULT_CONFIG +redpanda.remote.delete true DEFAULT_CONFIG +redpanda.remote.read false DEFAULT_CONFIG +redpanda.remote.write false DEFAULT_CONFIG +retention.bytes -1 DEFAULT_CONFIG +retention.local.target.bytes -1 DEFAULT_CONFIG +retention.local.target.ms 86400000 DEFAULT_CONFIG +retention.ms 604800000 DEFAULT_CONFIG +segment.bytes 1073741824 DEFAULT_CONFIG +---- + +endif::[] + +== Delete a topic + +To delete a topic, run: + +[,bash] +---- +rpk topic delete +---- + +When a topic is deleted, its underlying data is deleted, too. + +To delete multiple topics at a time, provide a space-separated list. For example, to delete two topics named `topic1` and `topic2`, run: + +[,bash] +---- +rpk topic delete topic1 topic2 +---- + +You can also use the `-r` flag to specify one or more regular expressions; then, any topic names that match the pattern you specify are deleted. For example, to delete topics with names that start with "`f`" and end with "`r`", run: + +[,bash] +---- +rpk topic delete -r '^f.*' '.*r$' +---- + +Note that the first regular expression must start with the `^` symbol, and the last expression must end with the `$` symbol. This requirement helps prevent accidental deletions. + +include::develop:partial$delete-topic-records.adoc[] + +== Next steps + +xref:develop:produce-data/configure-producers.adoc[] + +end::single-source[] diff --git a/modules/develop/pages/manage-topics/index.adoc b/modules/develop/pages/manage-topics/index.adoc new file mode 100644 index 0000000000..d1eb45c4f9 --- /dev/null +++ b/modules/develop/pages/manage-topics/index.adoc @@ -0,0 +1,4 @@ += Topics +:page-categories: Clients, Development +:description: Learn how to manage topics in Redpanda, including creation, configuration, and advanced features. +:page-layout: index diff --git a/modules/develop/pages/produce-data/configure-producers.adoc b/modules/develop/pages/produce-data/configure-producers.adoc index 18de11b961..77a9bc4a3a 100644 --- a/modules/develop/pages/produce-data/configure-producers.adoc +++ b/modules/develop/pages/produce-data/configure-producers.adoc @@ -74,7 +74,7 @@ Kafka, a message is considered acknowledged without the requirement that it has been fsynced. Messages that have not been fsynced to disk may be lost in the event of a broker crash. So when using `acks=all`, the Redpanda default configuration is more resilient than Kafka's. You can also consider -using xref:develop:config-topics.adoc#configure-write-caching[write caching], which is a relaxed mode of `acks=all` that acknowledges a message as soon as it is received and acknowledged on a majority of brokers, without waiting for it to fsync to disk. This provides lower latency while still ensuring that a majority of brokers acknowledge the write. +using xref:develop:manage-topics/config-topics.adoc#configure-write-caching[write caching], which is a relaxed mode of `acks=all` that acknowledges a message as soon as it is received and acknowledged on a majority of brokers, without waiting for it to fsync to disk. This provides lower latency while still ensuring that a majority of brokers acknowledge the write. endif::[] diff --git a/modules/develop/pages/produce-data/leader-pinning.adoc b/modules/develop/pages/produce-data/leader-pinning.adoc index 2d88adab35..74cb3a0da8 100644 --- a/modules/develop/pages/produce-data/leader-pinning.adoc +++ b/modules/develop/pages/produce-data/leader-pinning.adoc @@ -1,10 +1,22 @@ -= Leader Pinning -:description: Learn about leader pinning and how to configure a preferred partition leader location based on cloud availability zones or regions. += Configure Leader Pinning +:description: Learn about Leader Pinning and how to configure a preferred partition leader location based on cloud availability zones or regions. +:page-topic-type: how-to +:personas: streaming_developer, platform_admin // tag::single-source[] +:learning-objective-1: Configure preferred partition leader placement using rack labels +:learning-objective-2: Configure ordered rack preference for priority-based leader failover +:learning-objective-3: Identify conditions where Leader Pinning cannot place leaders in preferred racks -Produce requests that write data to Redpanda topics go through the topic partition leader, which syncs messages across its follower replicas. For a Redpanda cluster deployed across multiple availability zones (AZs), leader pinning ensures that a topic's partition leaders are geographically closer to clients, which helps decrease networking costs and guarantees lower latency. +Produce requests that write data to Redpanda topics are routed through the topic partition leader, which syncs messages across its follower replicas. For a Redpanda cluster deployed across multiple availability zones (AZs), Leader Pinning ensures that a topic's partition leaders are geographically closer to clients, which helps decrease networking costs and guarantees lower latency. + +If consumers are located in the same preferred region or AZ for Leader Pinning, and you have not set up xref:develop:consume-data/follower-fetching.adoc[follower fetching], Leader Pinning can also help reduce networking costs on consume requests. + +After reading this page, you will be able to: + +* [ ] {learning-objective-1} +* [ ] {learning-objective-2} +* [ ] {learning-objective-3} -If consumers are located in the same preferred region or AZ for leader pinning, and you have not set up xref:develop:consume-data/follower-fetching.adoc[follower fetching], leader pinning can also help reduce networking costs on consume requests. ifndef::env-cloud[] == Prerequisites @@ -14,64 +26,160 @@ ifndef::env-cloud[] include::shared:partial$enterprise-license.adoc[] ==== -Before you can enable leader pinning, you must xref:manage:rack-awareness.adoc#configure-rack-awareness[configure rack awareness] on the cluster. If the config_ref:enable_rack_awareness,true,properties/cluster-properties[] cluster configuration property is set to `false`, leader pinning is disabled across the cluster. +Before you can enable Leader Pinning, you must xref:manage:rack-awareness.adoc#configure-rack-awareness[configure rack awareness] on the cluster. If the config_ref:enable_rack_awareness,true,properties/cluster-properties[] cluster configuration property is set to `false`, Leader Pinning is disabled across the cluster. endif::[] ifndef::env-cloud[] -== Configure leader pinning - -You can use both a topic configuration property and a cluster configuration property to configure leader pinning. +== Set leader rack preferences -You can set the topic configuration property for individual topics only, or set the cluster-wide configuration property that will enable leader pinning by default for all topics. You can also use a combination in which a default setting applies across the cluster, and you toggle the setting on or off for specific topics. +You can configure Leader Pinning at the topic level, the cluster level, or both. Set the topic configuration property to configure individual topics, or set the cluster configuration property to apply a default for all topics. You can also combine both: apply a cluster-wide default, then override specific topics with the topic property. This configuration is based on the following scenario: you have Redpanda deployed in a multi-AZ or multi-region cluster, and you have configured each broker so that the config_ref:rack,true,properties/broker-properties[] configuration property contains racks corresponding to the AZs: -* Set the topic configuration property xref:reference:properties/topic-properties.adoc#redpandaleaderspreference[`redpanda.leaders.preference`]. The property accepts the following string values: +* Set the topic configuration property xref:reference:properties/topic-properties.adoc#redpanda-leaders-preference[`redpanda.leaders.preference`]. This property accepts the following string values: + -- -** `none`: Opt out the topic from leader pinning. +** `none`: Disable Leader Pinning for the topic. ** `racks:[,,...]`: Specify the preferred location (rack) of all topic partition leaders. The list can contain one or more racks, and you can list the racks in any order. Spaces in the list are ignored, for example: `racks:rack1,rack2` and `racks: rack1, rack2` are equivalent. You cannot specify empty racks, for example: `racks: rack1,,rack2`. If you specify multiple racks, Redpanda tries to distribute the partition leader locations equally across brokers in these racks. +** `ordered_racks:[,,...]`: Supported in Redpanda v26.1 or later. Specify the preferred racks in priority order. Redpanda places leaders in the first listed rack when available, failing over to each subsequent rack when higher-priority racks are unavailable. If all listed racks are unavailable, leaders fall back to any other available brokers. Brokers with no rack assignment are treated as lowest priority. + -To find the rack identifier, run `rpk cluster info`. +Use `ordered_racks` for multi-region deployments with a primary region for leaders and explicit failover to a disaster recovery site. + +The xref:reference:properties/topic-properties.adoc#redpanda-leaders-preference[`redpanda.leaders.preference`] property inherits the default value from the cluster property `default_leaders_preference`. + +To find the rack identifiers of all brokers, run: + +[,bash] +---- +rpk cluster info +---- + +.Expected output +[,bash,role="no-copy"] +---- +CLUSTER +======= +redpanda.be267958-279d-49cd-ae86-98fc7ed2de48 + +BROKERS +======= +ID HOST PORT RACK +0* 54.70.51.189 9092 us-west-2a +1 35.93.178.18 9092 us-west-2b +2 35.91.121.126 9092 us-west-2c +---- + +To set the topic property: + +[,bash] +---- +rpk topic alter-config --set redpanda.leaders.preference=ordered_racks:, +---- -- -+ -This property inherits the default value from the cluster property `default_leaders_preference`. -* Set the cluster configuration property config_ref:default_leaders_preference,true,properties/cluster-properties[], which specifies the default leader pinning configuration for all topics that don’t have `redpanda.leaders.preference` explicitly set. It accepts values in the same format as `redpanda.leaders.preference`. Default: `none` +* Set the cluster configuration property config_ref:default_leaders_preference,true,properties/cluster-properties[], which specifies the default Leader Pinning configuration for all topics that don’t have `redpanda.leaders.preference` explicitly set. It accepts values in the same format as `redpanda.leaders.preference`, where the default is `none`. + This property also affects internal topics, such as `__consumer_offsets` and transaction coordinators. All offset tracking and transaction coordination requests get placed within the preferred regions or AZs for all clients, so you see end-to-end latency and networking cost benefits. ++ +To set the cluster property: ++ +[,bash] +---- +rpk cluster config set default_leaders_preference ordered_racks:, +---- -If there is more than one broker in the preferred AZ (or AZs), leader pinning distributes partition leaders uniformly across brokers in the AZ. +If there is more than one broker in the preferred AZ (or AZs), Leader Pinning distributes partition leaders uniformly across brokers in the AZ. endif::[] ifdef::env-cloud[] -== Configure leader pinning +== Set leader rack preferences -Configure leader pinning if you have Redpanda deployed in a multi-AZ or multi-region cluster and your ingress is concentrated in a particular AZ or region. +Configure Leader Pinning if you have Redpanda deployed in a multi-AZ or multi-region cluster and your ingress is concentrated in a particular AZ or region. -Use the topic configuration property `redpanda.leaders.preference` to configure leader pinning for individual topics. The property accepts the following string values: +Use the topic configuration property `redpanda.leaders.preference` to configure Leader Pinning for individual topics. The property accepts the following string values: -** `none`: Opt out the topic from leader pinning. +** `none`: Disable Leader Pinning for the topic. ** `racks:[,,...]`: Specify the preferred location (rack) of all topic partition leaders. The list can contain one or more racks, and you can list the racks in any order. Spaces in the list are ignored, for example: `racks:rack1,rack2` and `racks: rack1, rack2` are equivalent. You cannot specify empty racks, for example: `racks: rack1,,rack2`. If you specify multiple racks, Redpanda tries to distribute the partition leader locations equally across brokers in these racks. -+ -To find the rack identifier, run `rpk cluster info`. +** `ordered_racks:[,,...]`: Supported in Redpanda v26.1 or later. Specify the preferred racks in priority order. Redpanda places leaders in the first listed rack when available, failing over to each subsequent rack when higher-priority racks are unavailable. If all listed racks are unavailable, leaders fall back to any other available brokers. Brokers with no rack assignment are treated as lowest priority. + +To find the rack identifiers of all brokers, run: + +[,bash] +---- +rpk cluster info +---- + +.Expected output +[,bash,role="no-copy"] +---- +CLUSTER +======= +redpanda.be267958-279d-49cd-ae86-98fc7ed2de48 + +BROKERS +======= +ID HOST PORT RACK +0* 54.70.51.189 9092 us-west-2a +1 35.93.178.18 9092 us-west-2b +2 35.91.121.126 9092 us-west-2c +---- + +To set the topic property: + +[,bash] +---- +rpk topic alter-config --set redpanda.leaders.preference=ordered_racks:, +---- + +If there is more than one broker in the preferred AZ (or AZs), Leader Pinning distributes partition leaders uniformly across brokers in the AZ. -If there is more than one broker in the preferred AZ (or AZs), leader pinning distributes partition leaders uniformly across brokers in the AZ. +endif::[] + +== Limitations + +Leader Pinning controls which replica is elected as leader, and does not move replicas to different brokers. If all of a topic's replicas are on brokers in non-preferred racks, no replica exists in the preferred racks to elect as leader, and Redpanda may elect a non-preferred leader indefinitely. + +For example, consider a cluster deployed across four racks (A, B, C, D) with Leader Pinning configured as `ordered_racks:A,B,C,D`. With a replication factor of 3, rack awareness can only place replicas in three of the four racks. If the highest-priority rack (A) does not receive a replica, no replica exists there to elect as leader, and Redpanda may elect a non-preferred leader indefinitely. + +ifndef::env-cloud[] +To prevent this scenario: + +* Enable config_ref:enable_rack_awareness,true,properties/cluster-properties[`enable_rack_awareness`] to distribute replicas across racks automatically. +* Ensure the topic's replication factor at least equals the total number of racks in the cluster, so every rack, including the highest-priority rack, receives a replica. endif::[] +ifdef::env-cloud[] +To prevent this scenario, ensure the topic's replication factor at least equals the total number of racks in the cluster, so every rack, including the highest-priority rack, receives a replica. + +endif::[] + +== Leader Pinning failover across availability zones + +If there are three AZs: A, B, and C, and A becomes unavailable, the failover behavior with `racks` is as follows: + +* The topic with `A` as the preferred leader AZ will have its partition leaders uniformly distributed across B and C. +* The topic with `A,B` as the preferred leader AZs will have its partition leaders in B. +* The topic with `B` as the preferred leader AZ will have its partition leaders in B as well. + +=== Failover with ordered rack preference + +With `ordered_racks`, the failover order follows the configured priority list. Leaders move to the next available rack in the list when higher-priority racks become unavailable. -== Leader pinning failover across availability zones +For a topic configured with `ordered_racks:A,B,C`: -If there are three AZs: A, B, and C, and A becomes unavailable, the failover behavior is as follows: +* The topic with `A` as the first-priority rack will have its partition leaders in A. +* If A becomes unavailable, leaders move to B. +* If A and B become unavailable, leaders move to C. +* If A, B, and C all become unavailable, leaders fall back to any available brokers. -* A topic with "A" as the preferred leader AZ will have its partition leaders uniformly distributed across B and C. -* A topic with "A,B" as the preferred leader AZs will have its partition leaders in B. -* A topic with “B” as the preferred leader AZ will have its partition leaders in B as well. +If a higher-priority rack recovers and the topic's replication factor ensures that rack receives a replica, Redpanda automatically moves leaders back to the highest available preferred rack. == Suggested reading +// TODO: Add link to Cloud Topics +// * For latency-tolerant, high-throughput workloads where cross-AZ networking charges are a major cost driver, also consider xref:develop:manage-topics/cloud-topics.adoc[Cloud Topics] * xref:develop:consume-data/follower-fetching.adoc[] // end::single-source[] \ No newline at end of file diff --git a/modules/develop/partials/cloud-topics-limitations.adoc b/modules/develop/partials/cloud-topics-limitations.adoc new file mode 100644 index 0000000000..6ce7a40902 --- /dev/null +++ b/modules/develop/partials/cloud-topics-limitations.adoc @@ -0,0 +1,8 @@ +// tag::limitations[] +- Shadow links do not currently support Cloud Topics. +- Once created, a Cloud Topic cannot be converted back to a standard Redpanda topic that uses local or Tiered Storage. Conversely, existing topics created as local or Tiered Storage topics cannot be converted to Cloud Topics. +// end::limitations[] + +// tag::latency-limitation[] +- Higher produce latency compared to standard topics (expect 1-2 seconds with public cloud stores). +// end::latency-limitation[] diff --git a/modules/develop/partials/cloud-topics-overview.adoc b/modules/develop/partials/cloud-topics-overview.adoc new file mode 100644 index 0000000000..2ec20c0c28 --- /dev/null +++ b/modules/develop/partials/cloud-topics-overview.adoc @@ -0,0 +1,18 @@ +// tag::intro[] +Starting in v26.1, Redpanda provides glossterm:Cloud Topic[,Cloud Topics] to support multi-modal streaming workloads in the most cost-effective way possible: as a per-topic configuration running mixed latency workloads. While standard Redpanda +ifdef::env-cloud[] +xref:get-started:config-topics.adoc[topics] +endif::[] +ifndef::env-cloud[] +xref:develop:manage-topics/config-topics.adoc[topics] +endif::[] +that use local storage or Tiered Storage are ideal for latency-sensitive workloads (for example, for audit logs or analytics), Cloud Topics are optimized for latency-tolerant, high-throughput workloads where cross-AZ networking charges are a major consideration that can become the dominant cost driver at high throughput. These workloads can include observability streams, offline analytics, AI/ML model training data feeds, or development environments that have flexible latency requirements. + +Instead of replicating every byte across expensive network links, Cloud Topics leverage durable, inexpensive cloud storage (S3, ADLS, GCS, MinIO) as the primary mechanism to both replicate data and serve it to consumers. This eliminates over 90% of the cost of replicating data over network links in multi-AZ clusters. +// end::intro[] + +// tag::latency-explanation[] +The end-to-end latency experienced when using Cloud Topics can range from 500 ms to as high as a few seconds with different object stores. Lower latencies may be achievable in certain environments, but Cloud Topics is optimized for throughput rather than low latency or tightly constrained tail latency. This latency profile is often acceptable for many streaming workloads, and can unlock new streaming use cases that previously were not cost effective. + +With Cloud Topics, data from the client is not acknowledged until it is uploaded to object storage. This maintains durability in the face of infrastructure failures, but results in an increase in both produce latency and end-to-end latency, driven by both batching of produced data and the inherent latency of the underlying object store. You should generally expect end-to-end latencies of 1-2 seconds with public cloud stores. +// end::latency-explanation[] diff --git a/modules/develop/partials/cloud-topics-use-cases.adoc b/modules/develop/partials/cloud-topics-use-cases.adoc new file mode 100644 index 0000000000..44800c4d61 --- /dev/null +++ b/modules/develop/partials/cloud-topics-use-cases.adoc @@ -0,0 +1,8 @@ +// tag::use-cases[] +Cloud Topics are best suited for latency-tolerant workloads, including: + +- Observability and logging streams +- Offline analytics pipelines +- AI/ML training data ingestion +- Development and staging environments with flexible latency requirements +// end::use-cases[] diff --git a/modules/get-started/pages/quick-start.adoc b/modules/get-started/pages/quick-start.adoc index 5b0bdb92ea..5173dcd955 100644 --- a/modules/get-started/pages/quick-start.adoc +++ b/modules/get-started/pages/quick-start.adoc @@ -445,7 +445,7 @@ docker exec -it redpanda-0 rpk topic create chat-room \ -X pass=secretpassword ``` + -<1> Set a replication factor of 3 to replicate the topic across all 3 brokers. This replication factor provides high availability and data durability. For more details, see xref:develop:config-topics.adoc#choose-the-replication-factor[Choose the replication factor]. +<1> Set a replication factor of 3 to replicate the topic across all 3 brokers. This replication factor provides high availability and data durability. For more details, see xref:develop:manage-topics/config-topics.adoc#choose-the-replication-factor[Choose the replication factor]. <2> Enable remote reads for this topic to read offloaded records from object storage. <3> Enable remote writes for this topic to offload older records to object storage. For more details, see xref:manage:tiered-storage.adoc[]. + diff --git a/modules/get-started/pages/release-notes/helm-charts.adoc b/modules/get-started/pages/release-notes/helm-charts.adoc index 9aa030bc00..dd7c48ee12 100644 --- a/modules/get-started/pages/release-notes/helm-charts.adoc +++ b/modules/get-started/pages/release-notes/helm-charts.adoc @@ -12,6 +12,18 @@ See also: * xref:upgrade:k-compatibility.adoc[] * xref:upgrade:k-rolling-upgrade.adoc[] -== Redpanda chart v25.3.x +== Console chart v26.1.x -link:https://github.com/redpanda-data/redpanda-operator/blob/release/v25.3.x/charts/redpanda/CHANGELOG.md[Changelog^]. +link:https://github.com/redpanda-data/redpanda-operator/blob/release/v26.1.x/charts/console/CHANGELOG.md[Changelog^]. + +=== Prometheus ServiceMonitor + +The Console Helm chart now supports deploying a Prometheus `ServiceMonitor` using `monitoring.enabled`, `monitoring.scrapeInterval`, and `monitoring.labels`. See xref:deploy:console/kubernetes/deploy.adoc#prometheus-servicemonitor[Prometheus ServiceMonitor]. + +== Redpanda chart v26.1.x + +link:https://github.com/redpanda-data/redpanda-operator/blob/release/v26.1.x/charts/redpanda/CHANGELOG.md[Changelog^]. + +=== Config-watcher sidecar resource configuration + +You can now configure explicit CPU and memory resource requests and limits for the config-watcher sidecar using `statefulset.sideCars.configWatcher.resources`. This is required in namespaces that enforce LimitRange or ResourceQuota policies. See xref:manage:kubernetes/k-manage-resources.adoc#config-watcher[Configure config-watcher sidecar resources]. \ No newline at end of file diff --git a/modules/get-started/pages/release-notes/operator.adoc b/modules/get-started/pages/release-notes/operator.adoc index 294cd53cfa..32e1d2687d 100644 --- a/modules/get-started/pages/release-notes/operator.adoc +++ b/modules/get-started/pages/release-notes/operator.adoc @@ -10,16 +10,28 @@ See also: * xref:upgrade:k-rolling-upgrade.adoc[] -== Redpanda Operator v25.3.x +== Redpanda Operator v26.1.x -link:https://github.com/redpanda-data/redpanda-operator/blob/release/v25.3.x/operator/CHANGELOG.md[Changelog^] +link:https://github.com/redpanda-data/redpanda-operator/blob/release/v26.1.x/operator/CHANGELOG.md[Changelog^] -=== ShadowLink resource for disaster recovery +=== Prometheus ServiceMonitor for Console -Redpanda Operator v25.3.x introduces the ShadowLink custom resource for managing shadow links in Kubernetes. The ShadowLink resource allows you to declaratively configure and manage disaster recovery replication between Redpanda clusters. +The Console custom resource supports a `monitoring` configuration that deploys a Prometheus `ServiceMonitor` to automatically discover and scrape Console metrics. See xref:deploy:console/kubernetes/deploy.adoc#prometheus-servicemonitor[Prometheus ServiceMonitor]. -* **Declarative configuration**: Define shadow links as Kubernetes resources with full lifecycle management. -* **Status monitoring**: View shadow link health and replication status directly from Kubernetes. -* **Integrated failover**: Delete the ShadowLink resource to fail over all topics. +=== Schema Registry ACLs -See xref:manage:kubernetes/shadowing/k-shadow-linking.adoc[Shadow Linking in Kubernetes] for setup and xref:manage:kubernetes/shadowing/k-monitor-shadowing.adoc[monitoring] documentation. \ No newline at end of file +The Redpanda custom resource supports configuring Schema Registry ACLs through the `schemaRegistry.authenticationMethod` field. This enables fine-grained access control for Schema Registry operations in clusters deployed with Kubernetes. + +When you enable Schema Registry authentication, you can control which users and service accounts can register, modify, and read schemas. See xref:manage:kubernetes/security/authentication/k-authentication.adoc[] for configuration details. + +=== Cloud Topics for Kubernetes + +The Redpanda custom resource supports Cloud Topics in Kubernetes deployments. You can configure topics to use cloud storage as the primary backing store by setting the appropriate storage mode properties. + +Cloud Topics in Kubernetes provide the same cost savings and architectural benefits as self-managed deployments. You configure them declaratively through the Redpanda custom resource. For setup instructions, see xref:develop:manage-topics/cloud-topics.adoc[]. + +=== Group-based access control (GBAC) + +The Redpanda Operator supports group-based access control (GBAC) for Kubernetes deployments with OIDC authentication. You can assign roles and ACLs to OIDC groups, and users automatically inherit permissions from their group memberships. + +GBAC simplifies permission management in Kubernetes environments by integrating with your identity provider's group structure. See xref:manage:security/authorization/gbac.adoc[] for configuration and usage details. \ No newline at end of file diff --git a/modules/get-started/pages/release-notes/redpanda.adoc b/modules/get-started/pages/release-notes/redpanda.adoc index 86dfaf23f8..f0cd9a2f78 100644 --- a/modules/get-started/pages/release-notes/redpanda.adoc +++ b/modules/get-started/pages/release-notes/redpanda.adoc @@ -7,173 +7,216 @@ This topic includes new content added in version {page-component-version}. For a * xref:redpanda-cloud:get-started:whats-new-cloud.adoc[] * xref:redpanda-cloud:get-started:cloud-overview.adoc#redpanda-cloud-vs-self-managed-feature-compatibility[Redpanda Cloud vs Self-Managed feature compatibility] -NOTE: Redpanda v25.3 introduces breaking schema changes for Iceberg topics. If you are using Iceberg topics and want to retain the data in the corresponding Iceberg tables, review xref:upgrade:iceberg-schema-changes-and-migration-guide.adoc[] before upgrading your cluster, and follow the required migration steps to avoid sending new records to a dead-letter queue table. -== Iceberg topics with GCP BigLake +== Cloud Topics -A new xref:manage:iceberg/iceberg-topics-gcp-biglake.adoc[REST catalog integration] with Google Cloud BigLake allows you to add Redpanda topics as Iceberg tables in your data lakehouse. +xref:develop:manage-topics/cloud-topics.adoc[Cloud Topics] are now available, making it possible to use durable cloud storage (S3, ADLS, GCS) as the primary backing store instead of local disk, eliminating over 90% of cross-AZ replication costs. This makes them ideal for latency-tolerant, high-throughput workloads such as observability streams, analytics pipelines, and AI/ML training data feeds, where cross-AZ networking charges are the dominant cost driver. -See xref:manage:iceberg/use-iceberg-catalogs.adoc[] for details on configuring Iceberg REST catalog integrations with Redpanda. +You can use Cloud Topics exclusively in Redpanda Streaming clusters, or in combination with traditional Tiered Storage and local storage topics on a shared cluster supporting low latency workloads. -== Shadowing +Cloud Topics require Tiered Storage and an Enterprise license. For setup instructions and limitations, see xref:develop:manage-topics/cloud-topics.adoc[]. -Redpanda v25.3 introduces xref:deploy:redpanda/manual/disaster-recovery/shadowing/index.adoc[], an enterprise-licensed disaster recovery solution that provides asynchronous, offset-preserving replication between distinct Redpanda clusters. Shadowing enables cross-region data protection by replicating topic data, configurations, consumer group offsets, ACLs, and Schema Registry data with byte-level fidelity. +== Group-based access control (GBAC) -The shadow cluster operates in read-only mode while continuously receiving updates from the source cluster. During a disaster, you can failover individual topics or an entire shadow link to make resources fully writable for production traffic. See xref:deploy:redpanda/manual/disaster-recovery/shadowing/failover-runbook.adoc[] for emergency procedures. +Redpanda {page-component-version} introduces xref:manage:security/authorization/gbac.adoc[group-based access control (GBAC)], which extends OIDC authentication to support group-based permissions. In addition to assigning roles or ACLs to individual users, you can assign them to OIDC groups. Users inherit permissions from all groups reported by their identity provider (IdP) in the OIDC token claims. -Shadowing includes comprehensive metrics for monitoring replication health. See xref:manage:disaster-recovery/shadowing/monitor.adoc[] and xref:reference:public-metrics-reference.adoc#shadow-link-metrics[Shadow Link metrics reference]. +GBAC supports two authorization patterns: -== Connected client monitoring +* Assign a group as a member of an RBAC role so that all users in the group inherit the role's ACLs. +* Create ACLs directly with a `Group:` principal. -You can view details about Kafka client connections using `rpk` or the Admin API ListKafkaConnections endpoint. This allows you to view detailed information about active client connections on a cluster, and identify and troubleshoot problematic clients. For more information, see the xref:manage:cluster-maintenance/manage-throughput.adoc#view-connected-client-details[connected client details] example in the Manage Throughput guide. +Group membership is managed entirely by your IdP. Redpanda reads group information from the OIDC token at authentication time and works across the Kafka API, Schema Registry, and HTTP Proxy. -== New Admin API style +== FIPS 140-3 validation and FIPS Docker image -Redpanda v25.3 introduces a new API style for the Admin API, powered by https://connectrpc.com/docs/introduction[ConnectRPC]. New Redpanda features and operations in v25.3 are available as ConnectRPC services, allowing you to use autogenerated Protobuf clients in addition to using HTTP clients such as `curl`. +Redpanda's cryptographic module has been upgraded from FIPS 140-2 to https://csrc.nist.gov/pubs/fips/140-3/final[FIPS 140-3^] validation. Additionally, Redpanda now provides a FIPS-specific Docker image (`docker.redpanda.com/redpandadata/redpanda:-fips`) for `amd64` and `arm64` architectures, with the required OpenSSL FIPS module pre-configured. -Use the new ConnectRPC endpoints with the following v25.3 features: +NOTE: If you are upgrading with FIPS mode enabled, ensure all SASL/SCRAM user passwords are at least 14 characters before upgrading. FIPS 140-3 enforces stricter HMAC key size requirements. -* Shadowing -* Connected client monitoring +See xref:manage:security/fips-compliance.adoc[] for configuration details. -Existing Admin API endpoints from versions earlier than 25.3 remain supported, and you can continue to use them as usual. See xref:manage:use-admin-api.adoc[Manage Redpanda with the Admin API] to learn more about Admin API, and the link:/api/doc/admin/v2/[Admin API reference] to view the new endpoints. +== Iceberg: Expanded JSON Schema support -== Schema Registry import mode +Redpanda now supports additional JSON Schema patterns when translating to Iceberg tables: -Redpanda Schema Registry now supports an import mode that allows you to import existing schemas and retain their current IDs and version numbers. Import mode is useful when migrating from another schema registry. +* `$ref` support: Internal references using `$ref` (for example, `"$ref": "#/definitions/myType"`) are resolved from schema resources declared in the same document. External references are not yet supported. +* Map type from `additionalProperties`: `additionalProperties` objects that contain subschemas now translate to Iceberg `map`. +* `oneOf` nullable pattern: The `oneOf` keyword is now supported for the standard nullable pattern if exactly one branch is `{"type":"null"}` and the other is a non-null schema. -Starting with this release, import mode must be used when importing schemas. Read-write mode no longer allows specifying a schema ID and version when registering a schema. -See xref:manage:schema-reg/schema-reg-api.adoc#set-schema-registry-mode[Use the Schema Registry API]. +See xref:manage:iceberg/specify-iceberg-schema.adoc#how-iceberg-modes-translate-to-table-format[Specify Iceberg Schema] for JSON types mapping and updated requirements. -== Security report +== Ordered rack preference for Leader Pinning -You can now generate a security report for your Redpanda cluster using the link:/api/doc/admin/operation/operation-get_security_report[`/v1/security/report`] Admin API endpoint. The report provides detailed information about TLS configuration, authentication methods, authorization status, and security alerts across all Redpanda interfaces (Kafka, RPC, Admin, Schema Registry, HTTP Proxy). +xref:develop:produce-data/leader-pinning.adoc[Leader Pinning] now supports the `ordered_racks` configuration value, which lets you specify preferred racks in priority order. Unlike `racks`, which distributes leaders uniformly across all listed racks, `ordered_racks` places leaders in the highest-priority available rack and fails over to subsequent racks only when higher-priority racks become unavailable. -== Topic identifiers +== User-based throughput quotas -Redpanda v25.3 implements topic identifiers using 16 byte UUIDs as proposed in https://cwiki.apache.org/confluence/display/KAFKA/KIP-516%3A+Topic+Identifiers[KIP-516^]. +Redpanda now supports throughput quotas based on authenticated user principals. Unlike client-based quotas (which rely on self-declared `client-id` values), user-based quotas enforce limits using verified identities from SASL, mTLS, or OIDC authentication. -== Shadowing metrics +You can set quotas for individual users, default users, or fine-grained user/client combinations. See xref:manage:cluster-maintenance/about-throughput-quotas.adoc[] for conceptual details, and xref:manage:cluster-maintenance/manage-throughput.adoc#set-user-based-quotas[Set user-based quotas] to get started. -Redpanda v25.3 introduces comprehensive xref:reference:public-metrics-reference.adoc#shadow-link-metrics[Shadowing metrics] for monitoring disaster recovery replication: +== Cross-region Remote Read Replicas -* xref:reference:public-metrics-reference.adoc#redpanda_shadow_link_client_errors[`redpanda_shadow_link_client_errors`] - Track Kafka client errors during shadow link operations -* xref:reference:public-metrics-reference.adoc#redpanda_shadow_link_shadow_lag[`redpanda_shadow_link_shadow_lag`] - Monitor replication lag between source and shadow partitions -* xref:reference:public-metrics-reference.adoc#redpanda_shadow_link_shadow_topic_state[`redpanda_shadow_link_shadow_topic_state`] - Track shadow topic state distribution across links -* xref:reference:public-metrics-reference.adoc#redpanda_shadow_link_total_bytes_fetched[`redpanda_shadow_link_total_bytes_fetched`] - Monitor data transfer volume from source cluster -* xref:reference:public-metrics-reference.adoc#redpanda_shadow_link_total_bytes_written[`redpanda_shadow_link_total_bytes_written`] - Track data written to shadow cluster -* xref:reference:public-metrics-reference.adoc#redpanda_shadow_link_total_records_fetched[`redpanda_shadow_link_total_records_fetched`] - Monitor total records fetched from source cluster -* xref:reference:public-metrics-reference.adoc#redpanda_shadow_link_total_records_written[`redpanda_shadow_link_total_records_written`] - Track total messages written to shadow cluster +Remote Read Replica topics on AWS can be deployed in a different region from the origin cluster's S3 bucket. This enables cross-region disaster recovery and data locality scenarios while maintaining the read-only replication model. -For monitoring guidance and alert recommendations, see xref:manage:disaster-recovery/shadowing/monitor.adoc[]. +To create cross-region Remote Read Replica topics, configure dynamic upstreams that point to the origin cluster's S3 bucket location. Redpanda manages the number of concurrent dynamic upstreams based on your `cloud_storage_url_style` setting (virtual_host or path style). -== New commands +See xref:manage:tiered-storage.adoc#remote-read-replicas[Remote Read Replicas] for setup instructions and configuration details. -Redpanda v25.3 introduces the following xref:reference:rpk/rpk-shadow/rpk-shadow.adoc[`rpk shadow`] commands for managing Redpanda shadow links: +== Automatic broker decommissioning -* xref:reference:rpk/rpk-shadow/rpk-shadow-config-generate.adoc[`rpk shadow config generate`] - Generate configuration files for shadow links -* xref:reference:rpk/rpk-shadow/rpk-shadow-create.adoc[`rpk shadow create`] - Create new shadow links -* xref:reference:rpk/rpk-shadow/rpk-shadow-update.adoc[`rpk shadow update`] - Update existing shadow link configurations -* xref:reference:rpk/rpk-shadow/rpk-shadow-list.adoc[`rpk shadow list`] - List all shadow links -* xref:reference:rpk/rpk-shadow/rpk-shadow-describe.adoc[`rpk shadow describe`] - View shadow link configuration details -* xref:reference:rpk/rpk-shadow/rpk-shadow-status.adoc[`rpk shadow status`] - Monitor shadow link replication status -* xref:reference:rpk/rpk-shadow/rpk-shadow-failover.adoc[`rpk shadow failover`] - Perform emergency failover operations -* xref:reference:rpk/rpk-shadow/rpk-shadow-delete.adoc[`rpk shadow delete`] - Delete shadow links +When continuous partition balancing is enabled, Redpanda can automatically decommission brokers that remain unavailable for a configured duration. The xref:reference:properties/cluster-properties.adoc#partition_autobalancing_node_autodecommission_timeout_sec[`partition_autobalancing_node_autodecommission_timeout_sec`] property triggers permanent broker removal, unlike xref:reference:properties/cluster-properties.adoc#partition_autobalancing_node_availability_timeout_sec[`partition_autobalancing_node_availability_timeout_sec`] which only moves partitions temporarily. -In addition, the following commands have been added: +Key characteristics: -* xref:reference:rpk/rpk-cluster/rpk-cluster-connections.adoc[`rpk cluster connections`] - Monitor cluster connections and client statistics. -* xref:reference:rpk/rpk-redpanda/rpk-redpanda-config-print.adoc[`rpk redpanda config print`] - Display node configuration. +* Disabled by default +* Requires `partition_autobalancing_mode` set to `continuous` +* Permanently removes the node from the cluster (the node cannot rejoin automatically) +* Processes one decommission at a time to maintain cluster stability +* Manual intervention required if decommission stalls -== New configuration properties - -Redpanda 25.3 introduces the following configuration properties: - -**Shadowing:** - -* xref:reference:properties/cluster-properties.adoc#enable_shadow_linking[`enable_shadow_linking`]: Enable shadow links (Enterprise license required) - -**Timestamp validation:** - -* xref:reference:properties/cluster-properties.adoc#log_message_timestamp_after_max_ms[`log_message_timestamp_after_max_ms`]: Maximum timestamp difference for future records -* xref:reference:properties/cluster-properties.adoc#log_message_timestamp_before_max_ms[`log_message_timestamp_before_max_ms`]: Maximum timestamp difference for past records -* xref:reference:properties/topic-properties.adoc#messagetimestampaftermaxms[`message.timestamp.after.max.ms`]: Topic-level timestamp validation (future) -* xref:reference:properties/topic-properties.adoc#messagetimestampbeforemaxms[`message.timestamp.before.max.ms`]: Topic-level timestamp validation (past) +See xref:manage:cluster-maintenance/continuous-data-balancing.adoc[] for configuration details. -**Audit logging:** +== New configuration properties -* xref:reference:properties/cluster-properties.adoc#audit_use_rpc[`audit_use_rpc`]: Use internal RPCs for audit logging +**Storage mode:** + +* xref:reference:properties/cluster-properties.adoc#default_redpanda_storage_mode[`default_redpanda_storage_mode`]: Set the default storage mode for new topics (`local`, `tiered`, `cloud`, or `unset`) +* xref:reference:properties/topic-properties.adoc#redpandastoragemode[`redpanda.storage.mode`]: Set the storage mode for an individual topic, superseding the legacy `redpanda.remote.read` and `redpanda.remote.write` properties + +**Cloud Topics:** + +NOTE: Cloud Topics requires an Enterprise license. For more information, contact https://redpanda.com/try-redpanda?section=enterprise[Redpanda sales^]. + +* xref:reference:properties/cluster-properties.adoc#cloud_topics_allow_materialization_failure[`cloud_topics_allow_materialization_failure`]: Enable recovery from missing L0 extent objects +* xref:reference:properties/cluster-properties.adoc#cloud_topics_compaction_interval_ms[`cloud_topics_compaction_interval_ms`]: Interval for background compaction +* xref:reference:properties/cluster-properties.adoc#cloud_topics_compaction_key_map_memory[`cloud_topics_compaction_key_map_memory`]: Maximum memory per shard for compaction key-offset maps +* xref:reference:properties/cluster-properties.adoc#cloud_topics_compaction_max_object_size[`cloud_topics_compaction_max_object_size`]: Maximum size for L1 objects produced by compaction +* xref:reference:properties/cluster-properties.adoc#cloud_topics_epoch_service_max_same_epoch_duration[`cloud_topics_epoch_service_max_same_epoch_duration`]: Maximum duration a node can use the same epoch +* xref:reference:properties/cluster-properties.adoc#cloud_topics_fetch_debounce_enabled[`cloud_topics_fetch_debounce_enabled`]: Enable fetch debouncing +* xref:reference:properties/cluster-properties.adoc#cloud_topics_gc_health_check_interval[`cloud_topics_gc_health_check_interval`]: L0 garbage collector health check interval +* xref:reference:properties/cluster-properties.adoc#cloud_topics_l1_indexing_interval[`cloud_topics_l1_indexing_interval`]: Byte interval for index entries in long-term storage objects +* xref:reference:properties/cluster-properties.adoc#cloud_topics_long_term_file_deletion_delay[`cloud_topics_long_term_file_deletion_delay`]: Delay before deleting stale long-term files +* xref:reference:properties/cluster-properties.adoc#cloud_topics_long_term_flush_interval[`cloud_topics_long_term_flush_interval`]: Interval for flushing long-term storage metadata to object storage +* xref:reference:properties/cluster-properties.adoc#cloud_topics_metastore_lsm_apply_timeout_ms[`cloud_topics_metastore_lsm_apply_timeout_ms`]: Timeout for applying replicated writes to LSM database +* xref:reference:properties/cluster-properties.adoc#cloud_topics_metastore_replication_timeout_ms[`cloud_topics_metastore_replication_timeout_ms`]: Timeout for L1 metastore Raft replication +* xref:reference:properties/cluster-properties.adoc#cloud_topics_num_metastore_partitions[`cloud_topics_num_metastore_partitions`]: Number of partitions for the metastore topic +* xref:reference:properties/cluster-properties.adoc#cloud_topics_parallel_fetch_enabled[`cloud_topics_parallel_fetch_enabled`]: Enable parallel fetching +* xref:reference:properties/cluster-properties.adoc#cloud_topics_preregistered_object_ttl[`cloud_topics_preregistered_object_ttl`]: Time-to-live for pre-registered L1 objects +* xref:reference:properties/cluster-properties.adoc#cloud_topics_produce_no_pid_concurrency[`cloud_topics_produce_no_pid_concurrency`]: Concurrent Raft requests for producers without a producer ID +* xref:reference:properties/cluster-properties.adoc#cloud_topics_produce_write_inflight_limit[`cloud_topics_produce_write_inflight_limit`]: Maximum in-flight write requests per shard +* xref:reference:properties/cluster-properties.adoc#cloud_topics_reconciliation_max_interval[`cloud_topics_reconciliation_max_interval`]: Maximum reconciliation interval for adaptive scheduling +* xref:reference:properties/cluster-properties.adoc#cloud_topics_reconciliation_max_object_size[`cloud_topics_reconciliation_max_object_size`]: Maximum size for L1 objects produced by the reconciler +* xref:reference:properties/cluster-properties.adoc#cloud_topics_reconciliation_min_interval[`cloud_topics_reconciliation_min_interval`]: Minimum reconciliation interval for adaptive scheduling +* xref:reference:properties/cluster-properties.adoc#cloud_topics_reconciliation_parallelism[`cloud_topics_reconciliation_parallelism`]: Maximum concurrent objects built by reconciliation per shard +* xref:reference:properties/cluster-properties.adoc#cloud_topics_reconciliation_slowdown_blend[`cloud_topics_reconciliation_slowdown_blend`]: Blend factor for slowing down reconciliation +* xref:reference:properties/cluster-properties.adoc#cloud_topics_reconciliation_speedup_blend[`cloud_topics_reconciliation_speedup_blend`]: Blend factor for speeding up reconciliation +* xref:reference:properties/cluster-properties.adoc#cloud_topics_reconciliation_target_fill_ratio[`cloud_topics_reconciliation_target_fill_ratio`]: Target fill ratio for L1 objects +* xref:reference:properties/cluster-properties.adoc#cloud_topics_upload_part_size[`cloud_topics_upload_part_size`]: Part size for multipart uploads +* xref:reference:properties/cluster-properties.adoc#cloud_topics_epoch_service_epoch_increment_interval[`cloud_topics_epoch_service_epoch_increment_interval`]: Interval for cluster epoch incrementation +* xref:reference:properties/cluster-properties.adoc#cloud_topics_epoch_service_local_epoch_cache_duration[`cloud_topics_epoch_service_local_epoch_cache_duration`]: Cache duration for local epoch data +* xref:reference:properties/cluster-properties.adoc#cloud_topics_long_term_garbage_collection_interval[`cloud_topics_long_term_garbage_collection_interval`]: Interval for long-term storage garbage collection +* xref:reference:properties/cluster-properties.adoc#cloud_topics_produce_batching_size_threshold[`cloud_topics_produce_batching_size_threshold`]: Object size threshold that triggers upload +* xref:reference:properties/cluster-properties.adoc#cloud_topics_produce_cardinality_threshold[`cloud_topics_produce_cardinality_threshold`]: Partition cardinality threshold that triggers upload +* xref:reference:properties/cluster-properties.adoc#cloud_topics_produce_upload_interval[`cloud_topics_produce_upload_interval`]: Time interval that triggers upload +* xref:reference:properties/cluster-properties.adoc#cloud_topics_reconciliation_interval[`cloud_topics_reconciliation_interval`]: Interval for moving data from short-term to long-term storage +* xref:reference:properties/cluster-properties.adoc#cloud_topics_short_term_gc_backoff_interval[`cloud_topics_short_term_gc_backoff_interval`]: Backoff interval for short-term storage garbage collection +* xref:reference:properties/cluster-properties.adoc#cloud_topics_short_term_gc_interval[`cloud_topics_short_term_gc_interval`]: Interval for short-term storage garbage collection +* xref:reference:properties/cluster-properties.adoc#cloud_topics_short_term_gc_minimum_object_age[`cloud_topics_short_term_gc_minimum_object_age`]: Minimum age for objects to be eligible for short-term garbage collection **Object storage:** -* xref:reference:properties/object-storage-properties.adoc#cloud_storage_client_lease_timeout_ms[`cloud_storage_client_lease_timeout_ms`]: Object storage connection timeout -* xref:reference:properties/object-storage-properties.adoc#cloud_storage_gc_max_segments_per_run[`cloud_storage_gc_max_segments_per_run`]: Limits segment deletion rate during xref:manage:tiered-storage.adoc#object-storage-housekeeping[object storage housekeeping] - -**Iceberg:** - -* xref:reference:properties/cluster-properties.adoc#iceberg_default_catalog_namespace[`iceberg_default_catalog_namespace`]: Default Iceberg catalog namespace for tables -* xref:reference:properties/cluster-properties.adoc#iceberg_dlq_table_suffix[`iceberg_dlq_table_suffix`]: Iceberg DLQ table name suffix -* xref:reference:properties/cluster-properties.adoc#iceberg_rest_catalog_gcp_user_project[`iceberg_rest_catalog_gcp_user_project`]: GCP project for Iceberg REST catalog billing -* xref:reference:properties/cluster-properties.adoc#iceberg_topic_name_dot_replacement[`iceberg_topic_name_dot_replacement`]: Dot replacement in Iceberg table names - -**TLS:** - -* xref:reference:properties/cluster-properties.adoc#tls_v1_2_cipher_suites[`tls_v1_2_cipher_suites`]: TLS 1.2 cipher suites for client connections -* xref:reference:properties/cluster-properties.adoc#tls_v1_3_cipher_suites[`tls_v1_3_cipher_suites`]: TLS 1.3 cipher suites for client connections +* xref:reference:properties/object-storage-properties.adoc#cloud_storage_gc_max_segments_per_run[`cloud_storage_gc_max_segments_per_run`]: Maximum number of log segments to delete from object storage during each xref:manage:tiered-storage.adoc#object-storage-housekeeping[housekeeping run] +* xref:reference:properties/object-storage-properties.adoc#cloud_storage_prefetch_segments_max[`cloud_storage_prefetch_segments_max`]: Maximum number of small segments to prefetch during sequential reads -**Tiered Storage:** +**Authentication:** -* xref:reference:properties/cluster-properties.adoc#cloud_topics_epoch_service_epoch_increment_interval[`cloud_topics_epoch_service_epoch_increment_interval`]: Cluster epoch increment interval -* xref:reference:properties/cluster-properties.adoc#cloud_topics_epoch_service_local_epoch_cache_duration[`cloud_topics_epoch_service_local_epoch_cache_duration`]: Local epoch cache duration -* xref:reference:properties/cluster-properties.adoc#cloud_topics_short_term_gc_backoff_interval[`cloud_topics_short_term_gc_backoff_interval`]: Short-term garbage collection backoff interval -* xref:reference:properties/cluster-properties.adoc#cloud_topics_short_term_gc_interval[`cloud_topics_short_term_gc_interval`]: Short-term garbage collection interval -* xref:reference:properties/cluster-properties.adoc#cloud_topics_short_term_gc_minimum_object_age[`cloud_topics_short_term_gc_minimum_object_age`]: Minimum object age for garbage collection +* xref:reference:properties/cluster-properties.adoc#nested_group_behavior[`nested_group_behavior`]: Control how Redpanda handles nested groups extracted from authentication tokens +* xref:reference:properties/cluster-properties.adoc#oidc_group_claim_path[`oidc_group_claim_path`]: JSON path to extract groups from the JWT payload +* xref:reference:properties/cluster-properties.adoc#schema_registry_enable_qualified_subjects[`schema_registry_enable_qualified_subjects`]: Enable parsing of qualified subject syntax in Schema Registry -**Other configuration:** +**Other:** -* xref:reference:properties/cluster-properties.adoc#controller_backend_reconciliation_concurrency[`controller_backend_reconciliation_concurrency`]: Maximum concurrent controller reconciliation operations -* xref:reference:properties/cluster-properties.adoc#fetch_max_read_concurrency[`fetch_max_read_concurrency`]: Maximum concurrent partition reads per fetch request -* xref:reference:properties/cluster-properties.adoc#kafka_max_message_size_upper_limit_bytes[`kafka_max_message_size_upper_limit_bytes`]: Maximum allowed `max.message.size` topic property value -* xref:reference:properties/cluster-properties.adoc#kafka_produce_batch_validation[`kafka_produce_batch_validation`]: Validation level for produced batches -* xref:reference:properties/cluster-properties.adoc#log_compaction_max_priority_wait_ms[`log_compaction_max_priority_wait_ms`]: Maximum time a priority partition can wait for compaction before preempting regular compaction -* xref:reference:properties/cluster-properties.adoc#log_compaction_tx_batch_removal_enabled[`log_compaction_tx_batch_removal_enabled`]: Enable transactional batch removal during compaction -* xref:reference:properties/cluster-properties.adoc#sasl_mechanisms_overrides[`sasl_mechanisms_overrides`]: SASL authentication mechanisms per listener +* xref:reference:properties/cluster-properties.adoc#delete_topic_enable[`delete_topic_enable`]: Enable or disable topic deletion via the Kafka DeleteTopics API +* xref:reference:properties/cluster-properties.adoc#internal_rpc_request_timeout_ms[`internal_rpc_request_timeout_ms`]: Default timeout for internal RPC requests between nodes +* xref:reference:properties/cluster-properties.adoc#log_compaction_max_priority_wait_ms[`log_compaction_max_priority_wait_ms`]: Maximum time a priority partition (such as `__consumer_offsets`) waits before preempting regular compaction +* xref:reference:properties/cluster-properties.adoc#partition_autobalancing_node_autodecommission_timeout_sec[`partition_autobalancing_node_autodecommission_timeout_sec`]: Duration a node must be unavailable before Redpanda automatically decommissions it === Changes to default values -The following configuration properties have new default values in v25.3: - -* xref:reference:properties/cluster-properties.adoc#core_balancing_continuous[`core_balancing_continuous`]: Changed from `false` to `true` (Enterprise license required). -* xref:reference:properties/cluster-properties.adoc#partition_autobalancing_mode[`partition_autobalancing_mode`]: Changed from `node_add` to `continuous` (Enterprise license required). -* xref:reference:properties/cluster-properties.adoc#iceberg_throttle_backlog_size_ratio[`iceberg_throttle_backlog_size_ratio`]: Changed from `0.3` to `null`. - -[[behavior-changes]] -=== Behavior changes - -The following topic properties now support enhanced tristate behavior: - -* xref:reference:properties/topic-properties.adoc#segment-ms[`segment.ms`] -* xref:reference:properties/topic-properties.adoc#retention-bytes[`retention.bytes`] -* xref:reference:properties/topic-properties.adoc#retention-ms[`retention.ms`] -* xref:reference:properties/topic-properties.adoc#retention-local-target-bytes[`retention.local.target.bytes`] -* xref:reference:properties/topic-properties.adoc#retention-local-target-ms[`retention.local.target.ms`] -* xref:reference:properties/topic-properties.adoc#initial-retention-local-target-bytes[`initial.retention.local.target.bytes`] -* xref:reference:properties/topic-properties.adoc#initial-retention-local-target-ms[`initial.retention.local.target.ms`] -* xref:reference:properties/topic-properties.adoc#delete-retention-ms[`delete.retention.ms`] -* xref:reference:properties/topic-properties.adoc#min-cleanable-dirty-ratio[`min.cleanable.dirty.ratio`] - -Previously, these properties treated zero and negative values the same way. Now they support three distinct states: positive values set specific limits, zero provides immediate eligibility for cleanup/compaction, and negative values disable the feature entirely. Review your topic configurations if you currently use zero values for these properties. - -=== Deprecations - -The following configuration properties have been deprecated in v25.3 and will be removed in a future release: - -* `kafka_memory_batch_size_estimate_for_fetch`: No replacement. Remove from configuration. -* `log_compaction_disable_tx_batch_removal`: Use xref:reference:properties/cluster-properties.adoc#log_compaction_tx_batch_removal_enabled[`log_compaction_tx_batch_removal_enabled`] instead. Note the inverted logic: the new property enables the behavior when set to `true`. -* `log_message_timestamp_alert_after_ms`: Use xref:reference:properties/cluster-properties.adoc#log_message_timestamp_after_max_ms[`log_message_timestamp_after_max_ms`] instead. -* `log_message_timestamp_alert_before_ms`: Use xref:reference:properties/cluster-properties.adoc#log_message_timestamp_before_max_ms[`log_message_timestamp_before_max_ms`] instead. -* `raft_recovery_default_read_size`: No replacement. Remove from configuration. - -== Deprecated features - -Redpanda has deprecated support for specific TLSv1.2 and TLSv1.3 cipher suites and now uses more secure defaults. See xref:upgrade:deprecated/index.adoc[Deprecated Features] for the complete list. +* xref:reference:properties/cluster-properties.adoc#log_compaction_tx_batch_removal_enabled[`log_compaction_tx_batch_removal_enabled`]: Changed from `false` to `true`. +* xref:reference:properties/cluster-properties.adoc#tls_v1_2_cipher_suites[`tls_v1_2_cipher_suites`]: Changed from OpenSSL cipher names to IANA cipher names. + +=== Removed properties + +The following deprecated configuration properties have been removed in v26.1.1. If you have any of these in your configuration files, update them according to the guidance below. + +**RPC timeout properties:** + +Replace with xref:reference:properties/cluster-properties.adoc#internal_rpc_request_timeout_ms[`internal_rpc_request_timeout_ms`]. + +* `alter_topic_cfg_timeout_ms` +* `create_topic_timeout_ms` +* `metadata_status_wait_timeout_ms` +* `node_management_operation_timeout_ms` +* `recovery_append_timeout_ms` +* `rm_sync_timeout_ms` +* `tm_sync_timeout_ms` +* `wait_for_leader_timeout_ms` + +**Client throughput quota properties:** + +Use xref:reference:rpk/rpk-cluster/rpk-cluster-quotas.adoc[`rpk cluster quotas`] to manage xref:manage:cluster-maintenance/manage-throughput.adoc#client-throughput-limits[client throughput limits]. + +* `kafka_admin_topic_api_rate` +* `kafka_client_group_byte_rate_quota` +* `kafka_client_group_fetch_byte_rate_quota` +* `target_fetch_quota_byte_rate` +* `target_quota_byte_rate` + +**Quota balancer properties:** + +Use xref:manage:cluster-maintenance/manage-throughput.adoc#broker-wide-throughput-limit-properties[broker-wide throughput limit properties]. + +* `kafka_quota_balancer_min_shard_throughput_bps` +* `kafka_quota_balancer_min_shard_throughput_ratio` +* `kafka_quota_balancer_node_period` +* `kafka_quota_balancer_window` +* `kafka_throughput_throttling_v2` + +**Timestamp alert properties:** + +* `log_message_timestamp_alert_after_ms`: Use xref:reference:properties/cluster-properties.adoc#log_message_timestamp_after_max_ms[`log_message_timestamp_after_max_ms`] +* `log_message_timestamp_alert_before_ms`: Use xref:reference:properties/cluster-properties.adoc#log_message_timestamp_before_max_ms[`log_message_timestamp_before_max_ms`] + +**Other removed properties:** + +No replacement needed. These properties were deprecated placeholders that have been silently ignored and will continue to be ignored even after removal. + +* `cloud_storage_disable_metadata_consistency_checks` +* `cloud_storage_reconciliation_ms` +* `coproc_max_batch_size` +* `coproc_max_inflight_bytes` +* `coproc_max_ingest_bytes` +* `coproc_offset_flush_interval_ms` +* `datalake_disk_space_monitor_interval` +* `enable_admin_api` +* `enable_coproc` +* `find_coordinator_timeout_ms` +* `full_raft_configuration_recovery_pattern` +* `id_allocator_replication` +* `kafka_memory_batch_size_estimate_for_fetch` +* `log_compaction_adjacent_merge_self_compaction_count` +* `max_version` +* `min_version` +* `raft_max_concurrent_append_requests_per_follower` +* `raft_recovery_default_read_size` +* `rm_violation_recovery_policy` +* `schema_registry_protobuf_renderer_v2` +* `seed_server_meta_topic_partitions` +* `seq_table_min_size` +* `tm_violation_recovery_policy` +* `transaction_coordinator_replication` +* `tx_registry_log_capacity` +* `tx_registry_sync_timeout_ms` +* `use_scheduling_groups` diff --git a/modules/manage/examples/kubernetes/group-crds.feature b/modules/manage/examples/kubernetes/group-crds.feature new file mode 100644 index 0000000000..0ff6536558 --- /dev/null +++ b/modules/manage/examples/kubernetes/group-crds.feature @@ -0,0 +1,40 @@ +@cluster:sasl @variant:vectorized +Feature: Group CRDs + Background: Cluster available + Given cluster "sasl" is available + + @skip:gke @skip:aks @skip:eks + Scenario: Manage group ACLs + When I apply Kubernetes manifest: + """ +# tag::manage-group-acls[] + # In this example manifest, ACLs are created for an OIDC group called "engineering" + # in a cluster called "sasl". The group is granted read access to topics matching "team-" + # and read access to Schema Registry subjects matching "team-". + --- + apiVersion: cluster.redpanda.com/v1alpha2 + kind: Group + metadata: + name: engineering + spec: + cluster: + clusterRef: + name: sasl + authorization: + acls: + - type: allow + resource: + type: topic + name: team- + patternType: prefixed + operations: [Read, Describe] + - type: allow + resource: + type: subject + name: team- + patternType: prefixed + operations: [Read, Describe] +# end::manage-group-acls[] + """ + And group "engineering" is successfully synced + Then group "engineering" should have 2 ACLs for topic pattern "team-" in cluster "sasl" diff --git a/modules/manage/examples/kubernetes/role-crds.feature b/modules/manage/examples/kubernetes/role-crds.feature index 0f9dfc9ccf..2707bf84ff 100644 --- a/modules/manage/examples/kubernetes/role-crds.feature +++ b/modules/manage/examples/kubernetes/role-crds.feature @@ -43,8 +43,6 @@ Feature: Role CRDs And I apply Kubernetes manifest: """ # tag::manage-roles-with-authorization[] - # In this example manifest, a role called "read-only-role" is created in a cluster called "sasl". - # The role includes authorization rules that allow reading from topics with names starting with "public-". --- apiVersion: cluster.redpanda.com/v1alpha2 kind: RedpandaRole @@ -64,6 +62,12 @@ Feature: Role CRDs name: public- patternType: prefixed operations: [Read, Describe] + - type: allow + resource: + type: subject + name: public- + patternType: prefixed + operations: [Read, Describe] # end::manage-roles-with-authorization[] """ And role "read-only-role" is successfully synced diff --git a/modules/manage/examples/kubernetes/user-crds.feature b/modules/manage/examples/kubernetes/user-crds.feature index ada3478265..b49ae0db83 100644 --- a/modules/manage/examples/kubernetes/user-crds.feature +++ b/modules/manage/examples/kubernetes/user-crds.feature @@ -59,9 +59,6 @@ Feature: User CRDs When I apply Kubernetes manifest: """ # tag::manage-authz-only-manifest[] - # In this example manifest, an ACL called "travis" is created in a cluster called "sasl". - # The ACL give an existing user called "travis" permissions to read from all topics whose names start with some-topic. - # This example assumes that you already have a user called "travis" in your cluster. --- apiVersion: cluster.redpanda.com/v1alpha2 kind: User @@ -86,8 +83,121 @@ Feature: User CRDs name: some-topic patternType: prefixed operations: [Read] + - type: allow + resource: + type: subject + name: some-topic + patternType: prefixed + operations: [Read] # end::manage-authz-only-manifest[] """ And user "travis" is successfully synced And I delete the CRD user "travis" Then "travis" should be able to authenticate to the "sasl" cluster with password "password" and mechanism "SCRAM-SHA-256" + + @skip:gke @skip:aks @skip:eks + Scenario: Grant a user read access to a subject + Given there is no user "consumer-app" in cluster "sasl" + When I apply Kubernetes manifest: + """ +# tag::grant-user-read-access[] + --- + apiVersion: cluster.redpanda.com/v1alpha2 + kind: User + metadata: + name: consumer-app + spec: + cluster: + clusterRef: + name: redpanda + authorization: + acls: + - type: allow + resource: + type: topic + name: orders + patternType: literal + operations: [Read] + - type: allow + resource: + type: subject + name: orders-value + patternType: literal + operations: [Read] +# end::grant-user-read-access[] + """ + And user "consumer-app" is successfully synced + And I delete the CRD user "consumer-app" + + @skip:gke @skip:aks @skip:eks + Scenario: Grant a producer write access using prefix patterns + Given there is no user "producer-app" in cluster "sasl" + When I apply Kubernetes manifest: + """ +# tag::grant-producer-write-access[] + --- + apiVersion: cluster.redpanda.com/v1alpha2 + kind: User + metadata: + name: producer-app + spec: + cluster: + clusterRef: + name: redpanda + authentication: + type: scram-sha-512 + password: + valueFrom: + secretKeyRef: + name: producer-app-secret + key: password + authorization: + acls: + - type: allow + resource: + type: topic + name: events- + patternType: prefixed + operations: [Write, Describe] + - type: allow + resource: + type: subject + name: events- + patternType: prefixed + operations: [Write, Describe] +# end::grant-producer-write-access[] + """ + And user "producer-app" is successfully synced + And I delete the CRD user "producer-app" + + @skip:gke @skip:aks @skip:eks + Scenario: Grant global Schema Registry access + Given there is no user "schema-admin" in cluster "sasl" + When I apply Kubernetes manifest: + """ +# tag::grant-global-sr-access[] + --- + apiVersion: cluster.redpanda.com/v1alpha2 + kind: User + metadata: + name: schema-admin + spec: + cluster: + clusterRef: + name: redpanda + authorization: + acls: + - type: allow + resource: + type: registry + operations: [Read, Write, Delete, Describe, DescribeConfigs, AlterConfigs] + - type: allow + resource: + type: subject + name: "" + patternType: prefixed + operations: [Read, Write, Delete, Describe, DescribeConfigs, AlterConfigs] +# end::grant-global-sr-access[] + """ + And user "schema-admin" is successfully synced + And I delete the CRD user "schema-admin" diff --git a/modules/manage/pages/audit-logging/audit-log-samples.adoc b/modules/manage/pages/audit-logging/audit-log-samples.adoc index 7ba297e6b8..6681c21047 100644 --- a/modules/manage/pages/audit-logging/audit-log-samples.adoc +++ b/modules/manage/pages/audit-logging/audit-log-samples.adoc @@ -3,6 +3,9 @@ :page-categories: Management, Security // tag::single-source[] +ifdef::env-cloud[:gbac-doc: security:authorization/gbac.adoc] +ifndef::env-cloud[:gbac-doc: manage:security/authorization/gbac.adoc] + ifndef::env-cloud[] [NOTE] ==== @@ -80,6 +83,59 @@ This scenario shows the message resulting from an admin using rpk with successfu ---- ==== +.Authentication successful (OIDC with group claims) +[%collapsible] +==== +This scenario shows a successful OIDC authentication event that includes the user's IdP group memberships in the `user.groups` field. Group memberships are extracted from the OIDC token and included in all authentication events for OIDC users. +[,json] +---- +{ + "category_uid": 3, + "class_uid": 3002, + "metadata": { + "product": { + "name": "Redpanda", + "uid": "0", + "vendor_name": "Redpanda Data, Inc.", + "version": "v26.1.1" + }, + "version": "1.0.0" + }, + "severity_id": 1, + "time": 1700533469078, + "type_uid": 300201, + "activity_id": 1, + "auth_protocol": "SASL-OAUTHBEARER", + "auth_protocol_id": 99, + "dst_endpoint": { + "ip": "127.0.0.1", + "port": 9092, + "svc_name": "kafka rpc protocol" + }, + "is_cleartext": false, + "is_mfa": false, + "service": { + "name": "kafka rpc protocol" + }, + "src_endpoint": { + "ip": "10.0.1.50", + "name": "kafka-client", + "port": 48210 + }, + "status_id": 1, + // IdP group memberships extracted from the OIDC token + "user": { + "name": "alice@example.com", + "type_id": 1, + "groups": [ + {"type": "idp_group", "name": "engineering"}, + {"type": "idp_group", "name": "analytics"} + ] + } +} +---- +==== + .Authentication failed [%collapsible] ==== @@ -237,6 +293,93 @@ This example illustrates an ACL update that also requires a superuser authentica ---- ==== +.Authorization matched on a group ACL +[%collapsible] +==== +This example shows an API Activity (6003) where the authorization decision matched an ALLOW ACL on a `Group:` principal. The `actor.user.groups` field includes the matched group with type `idp_group`, and the `authorization_metadata` shows the group ACL that granted access. See xref:{gbac-doc}[Group-Based Access Control]. + +[,json] +---- +{ + "category_uid": 6, + "class_uid": 6003, + "metadata": { + "product": { + "name": "Redpanda", + "uid": "0", + "vendor_name": "Redpanda Data, Inc.", + "version": "v26.1.0" + }, + "version": "1.0.0" + }, + "severity_id": 1, + "time": 1774544504327, + "type_uid": 600303, + "activity_id": 3, + "actor": { + "authorizations": [ + { + "decision": "authorized", + "policy": { + "desc": "acl: {principal type {group} name {/sales} host {{any_host}} op all perm allow}, resource: type {topic} name {sales-topic} pattern {literal}", + "name": "aclAuthorization" + } + } + ], + // The matched group appears in the user's groups field + "user": { + "name": "alice", + "type_id": 1, + "groups": [ + { + "type": "idp_group", + "name": "/sales" + } + ] + } + }, + "api": { + "operation": "produce", + "service": { + "name": "kafka rpc protocol" + } + }, + "dst_endpoint": { + "ip": "127.0.1.1", + "port": 9092, + "svc_name": "kafka rpc protocol" + }, + "resources": [ + { + "name": "sales-topic", + "type": "topic" + } + ], + "src_endpoint": { + "ip": "127.0.0.1", + "name": "rdkafka", + "port": 42728 + }, + "status_id": 1, + "unmapped": { + "authorization_metadata": { + "acl_authorization": { + "host": "{{any_host}}", + "op": "all", + "permission_type": "allow", + "principal": "type {group} name {/sales}" + }, + "resource": { + "name": "sales-topic", + "pattern": "literal", + "type": "topic" + } + } + } +} +---- +==== + .Metadata request (with counts) [%collapsible] ==== diff --git a/modules/manage/pages/cluster-maintenance/about-throughput-quotas.adoc b/modules/manage/pages/cluster-maintenance/about-throughput-quotas.adoc new file mode 100644 index 0000000000..6626ae99e8 --- /dev/null +++ b/modules/manage/pages/cluster-maintenance/about-throughput-quotas.adoc @@ -0,0 +1,282 @@ += About Client Throughput Quotas +:description: Understand how Redpanda's user-based and client ID-based throughput quotas work, including entity hierarchy, precedence rules, and quota tracking behavior. +:page-topic-type: concepts +:page-aliases: +:personas: platform_admin, developer +:learning-objective-1: Describe the difference between user-based and client ID-based quotas +:learning-objective-2: Determine which quota type to use for your use case +:learning-objective-3: Explain quota precedence rules and how Redpanda tracks quota usage + +// tag::single-source[] +ifdef::env-cloud[] +:authentication-doc: security:cloud-authentication.adoc +endif::[] +ifndef::env-cloud[] +:authentication-doc: manage:security/authentication.adoc +endif::[] + +Redpanda uses throughput quotas to limit the rate of produce and consume requests from clients. Understanding how quotas work helps you prevent individual clients from disproportionately consuming resources and causing performance degradation for other clients (also known as the "noisy-neighbor" problem), and ensure fair resource sharing across users and applications. + +After reading this page, you will be able to: + +* [ ] {learning-objective-1} +* [ ] {learning-objective-2} +* [ ] {learning-objective-3} + +To configure and manage throughput quotas, see xref:manage:cluster-maintenance/manage-throughput.adoc[]. + +== Throughput control overview + +Redpanda provides two ways to control throughput: + +* Broker-wide limits: Configured using cluster properties. For details, see xref:manage:cluster-maintenance/manage-throughput.adoc#broker-wide-throughput-limits[Broker-wide throughput limits]. +* Client throughput quotas: Configured using the Kafka API. Client quotas enable per-user and per-client rate limiting with fine-grained control through entity hierarchy and precedence rules. This page focuses on client quotas. + +== Supported quota types + +Redpanda supports three Kafka API-based quota types: + +|=== +| Quota type | Description + +| `producer_byte_rate` +| Limit throughput of produce requests (bytes per second) + +| `consumer_byte_rate` +| Limit throughput of fetch requests (bytes per second) + +| `controller_mutation_rate` +| Limit rate of topic mutation requests (partitions created or deleted per second) +|=== + +All quota types can be applied to groups of client connections based on user principals, client IDs, or combinations of both. + +== Quota entities + +Redpanda uses two pieces of identifying information from each client connection to determine which quota applies: + +* Client ID: An ID that clients self-declare. Quotas can target an exact client ID (`client-id`) or a prefix (`client-id-prefix`). Multiple client connections that share a client ID or ID prefix are grouped into a single quota entity. +* User glossterm:principal[]: An authenticated identity verified through SASL, mTLS, or OIDC. Connections that share the same user are considered one entity. + +You can configure quotas that target either entity type, or combine both for fine-grained control. + +=== Client ID-based quotas + +Client ID-based quotas apply to clients identified by their `client-id` field, which is set by the client application. The client ID is typically a configurable property when you create a client with Kafka libraries. When using client ID-based quotas, multiple clients using the same client ID share the same quota tracking. + +Client ID-based quotas rely on clients honestly reporting their identity and correctly setting the `client-id` property. This makes client ID-based quotas unsuitable for guaranteeing isolation between tenants. + +Use client ID-based quotas when: + +* Authentication is not enabled. +* Grouping by application or service name is sufficient. +* You operate a single-tenant environment where all clients are trusted. +* You need simple rate limiting without user-level isolation. + +=== User-based quotas + +IMPORTANT: User-based quotas require xref:manage:security/authentication.adoc[authentication] to be enabled on your cluster. + +User-based quotas apply to authenticated user principals. Each user has a separate quota, providing a way to limit the impact of individual users on the cluster. + +User-based quotas rely on Redpanda's authentication system to verify user identity. The user principal is extracted from SASL credentials, mTLS certificates, or OIDC tokens and cannot be forged by clients. + +Use user-based quotas when: + +* You operate a multi-tenant environment, such as SaaS platforms or enterprises with departments. +* You require isolation between users or tenants, to avoid noisy-neighbor issues. +* You need per-user billing or metering. + +=== Combined user and client quotas + +You can combine user and client identities for fine-grained control over specific (user, client) combinations. + +Use combined quotas when: + +* You need fine-grained control, for example: user `alice` using a specific application. +* Different rate limits apply to different apps used by the same user. For example, `alice`+'s+ `payment-processor` gets 10 MB/s, but `alice`+'s+ `analytics-consumer` gets 50 MB/s. See <> for examples. + +== Quota precedence and tracking + +When a request arrives, Redpanda resolves which quota to apply by matching the request's authenticated user principal and client ID against configured quotas. Redpanda applies the most specific match, using the precedence order in the following table (highest priority first). + +The precedence level that matches also determines how quota usage is tracked. Redpanda tracks quota usage using a tracker key that determines which connections share the same quota bucket. How connections are grouped into buckets depends on the type of entity the quota targets. + +To get independent quota tracking per user and client ID combination, configure quotas that include both dimensions, such as `/config/users//clients/` or `/config/users//clients/`. + +.Quota precedence, tracking, and isolation by configuration level +[cols="1,2,3,2,3", options="header"] +|=== +| Level | Match type | Config path | Tracker key | Isolation behavior + +| 1 +| Exact user + exact client +| `/config/users//clients/` +| `(user, client-id)` +| Each unique (user, client-id) pair tracked independently + +| 2 +| Exact user + client prefix +| `/config/users//client-id-prefix/` +| `(user, client-id-prefix)` +| Clients matching the prefix share tracking within that user + +| 3 +| Exact user + default client +| `/config/users//clients/` +| `(user, client-id)` +| Each unique (user, client-id) pair tracked independently + +| 4 +| Exact user only +| `/config/users/` +| `user` +| All clients for that user share a single tracking bucket + +| 5 +| Default user + exact client +| `/config/users//clients/` +| `(user, client-id)` +| Each unique (user, client-id) pair tracked independently + +| 6 +| Default user + client prefix +| `/config/users//client-id-prefix/` +| `(user, client-id-prefix)` +| Clients matching the prefix share tracking within each user + +| 7 +| Default user + default client +| `/config/users//clients/` +| `(user, client-id)` +| Each unique (user, client-id) pair tracked independently + +| 8 +| Default user only +| `/config/users/` +| `user` +| All clients for each user share a single tracking bucket (per user) + +| 9 +| Exact client only +| `/config/clients/` +| `client-id` +| All users with that client ID share a single tracking bucket + +| 10 +| Client prefix only +| `/config/client-id-prefix/` +| `client-id-prefix` +| All clients matching the prefix share a single bucket across all users + +| 11 +| Default client only +| `/config/clients/` +| `client-id` +| Each unique client ID tracked independently + +| 12 +| No quota configured +| N/A +| N/A +| No tracking / unlimited throughput +|=== + +IMPORTANT: The `` entity matches any user or client that doesn't have a more specific quota configured. This is different from an empty/unauthenticated user (`user=""`), or undeclared client ID (`client-id=""`), which are treated as specific entities. + +=== Unauthenticated connections + +Unauthenticated connections have an empty user principal (`user=""`) and are not treated as `user=`. + +Unauthenticated connections: + +* Fall back to client-only quotas. +* Have unlimited throughput only if no client-only quota matches. + +=== Example: Precedence resolution + +Given these configured quotas: + +[,bash] +---- +rpk cluster quotas alter --add consumer_byte_rate=5000000 --name user=alice --name client-id=app-1 +rpk cluster quotas alter --add consumer_byte_rate=10000000 --name user=alice +rpk cluster quotas alter --add consumer_byte_rate=20000000 --name client-id=app-1 +---- + +|=== +| User + Client ID | Precedence match + +| `user=alice`, `client-id=app-1` +| Level 1: Exact user + exact client + +| `user=alice`, `client-id=app-2` +| Level 4: Exact user only + +| `user=bob`, `client-id=app-1` +| Level 9: Exact client only + +| `user=bob`, `client-id=app-2` +| Level 12: No quota configured +|=== + +When no quota matches (level 12), the connection is not throttled. + +=== Example: User-only quota + +If you configure a 10 MB/s produce quota for user `alice`: + +[,bash] +---- +rpk cluster quotas alter --add producer_byte_rate=10000000 --name user=alice +---- + +Then `alice` connecting with client ID `app-1` and `alice` connecting with client ID `app-2` share the same 10 MB/s produce limit. + +To give each of `alice`+'s+ clients an independent 10 MB/s limit, configure: + +[,bash] +---- +rpk cluster quotas alter --add producer_byte_rate=10000000 --name user=alice --default client-id +---- + +=== Example: User default quota + +If you configure a default 10 MB/s produce quota for all users: + +[,bash] +---- +rpk cluster quotas alter --add producer_byte_rate=10000000 --default user +---- + +This quota applies to all users who don't have a more specific quota configured. Each user is tracked independently: `alice` gets her own 10 MB/s bucket, `bob` gets his own 10 MB/s bucket, and so on. + +Within each user, all client ID values share that user's bucket. `alice` connecting with client ID `app-1` and `alice` connecting with client ID `app-2` share the same 10 MB/s produce limit, while `bob`+'s+ connections have a separate 10 MB/s limit. + +[[throttling-enforcement]] +== Throughput throttling enforcement + +NOTE: As of v24.2, Redpanda enforces all throughput limits per broker, including client throughput. + +Redpanda enforces throughput limits by applying backpressure to clients. When a connection exceeds its throughput limit, Redpanda throttles the connection to bring the rate back within the allowed level: + +. Redpanda adds a `throttle_time_ms` field to responses, indicating how long the client should wait. +. If the client doesn't honor the throttle time, Redpanda inserts delays on the connection's next read operation. + +ifndef::env-cloud[] +The throttling delay may not exceed the limit set by the `max_kafka_throttle_delay_ms` tunable property. +endif::[] + +ifdef::env-cloud[] +In Redpanda Cloud, the throttling delay is set to 30 seconds. +endif::[] + +== Default behavior + +Quotas are opt-in restrictions and not enforced by default. When no quotas are configured, clients have unlimited throughput. + +== Next steps + +* xref:manage:cluster-maintenance/manage-throughput.adoc[Configure throughput quotas] +* xref:{authentication-doc}[Enable authentication for user-based quotas] +// end::single-source[] diff --git a/modules/manage/pages/cluster-maintenance/compaction-settings.adoc b/modules/manage/pages/cluster-maintenance/compaction-settings.adoc index a4c55a2f79..e0d528b982 100644 --- a/modules/manage/pages/cluster-maintenance/compaction-settings.adoc +++ b/modules/manage/pages/cluster-maintenance/compaction-settings.adoc @@ -12,7 +12,7 @@ image::shared:compaction-example.png[Example of topic compaction] This diagram illustrates a compacted topic. Imagine a remote sensor network that uses image recognition to track appearances of red pandas in a geographic area. The sensor network employs special devices that send records to a topic when they detect one. You might enable compaction to reduce topic storage while still maintaining a record in the topic of the last time each device saw a red panda, perhaps to see if they stop frequenting a given area. The left side of the diagram shows all records sent across the topic. The right side illustrates the results of compaction; older records for certain keys are deleted from the log. -NOTE: If your application requires consuming every record for a given key, consider using the `delete` xref:develop:config-topics#change-the-cleanup-policy.adoc[cleanup policy] instead. +NOTE: If your application requires consuming every record for a given key, consider using the `delete` xref:develop:manage-topics/config-topics.adoc#change-the-cleanup-policy[cleanup policy] instead. IMPORTANT: When using xref:manage:tiered-storage.adoc[Tiered Storage], compaction functions at the local storage level. As long as a segment remains in local storage, its records are eligible for compaction. Once a segment is uploaded to object storage and removed from local storage it is not retrieved for further compaction operations. A key may therefore appear in multiple segments between Tiered Storage and local storage. diff --git a/modules/manage/pages/cluster-maintenance/continuous-data-balancing.adoc b/modules/manage/pages/cluster-maintenance/continuous-data-balancing.adoc index 5813068b43..56fdf9182b 100644 --- a/modules/manage/pages/cluster-maintenance/continuous-data-balancing.adoc +++ b/modules/manage/pages/cluster-maintenance/continuous-data-balancing.adoc @@ -2,19 +2,30 @@ :description: Continuous Data Balancing simplifies operations with self-healing clusters that dynamically balance partitions. :page-aliases: cluster-administration:continuous-data-balancing.adoc :page-categories: Management +:page-topic-type: how-to +:personas: infrastructure_operator +:learning-objective-1: Enable Continuous Data Balancing on a Redpanda cluster +:learning-objective-2: Check data balancing status using rpk +:learning-objective-3: Cancel partition balancing moves for a specific node [NOTE] ==== include::shared:partial$enterprise-license.adoc[] ==== -Continuous Data Balancing continuously monitors your node and rack availability and disk usage. This enables self-healing clusters that dynamically balance partitions, ensuring smooth operations and optimal cluster performance. +Continuous Data Balancing continuously monitors your node and rack availability and disk usage, dynamically balancing partitions to maintain smooth operations and optimal cluster performance. -It also maintains the configured replication level, even after infrastructure failure. Node availability has the highest priority in data balancing. After a rack (with all nodes belonging to it) becomes unavailable, Redpanda moves partition replicas to the remaining nodes. This violates the rack awareness constraint. But after this rack (or a new one) becomes available, Redpanda repairs the rack awareness constraint by moving excess replicas from racks that have more than one replica to the newly-available rack. +Continuous Data Balancing also maintains the configured replication level, even after infrastructure failure. Node availability has the highest priority in data balancing. After a rack (with all nodes belonging to it) becomes unavailable, Redpanda moves partition replicas to the remaining nodes. This violates the rack awareness constraint. After the rack (or a replacement rack) becomes available, Redpanda repairs the constraint by moving excess replicas from racks that have more than one replica to the newly-available rack. + +After reading this page, you will be able to: + +* [ ] {learning-objective-1} +* [ ] {learning-objective-2} +* [ ] {learning-objective-3} == Set Continuous Data Balancing properties -To enable Continuous Data Balancing, set the `partition_autobalancing_mode` property to `continuous`. You can then customize properties for monitoring your node availability and disk usage. +To enable Continuous Data Balancing, set the `partition_autobalancing_mode` property to `continuous`. Customize the following properties to monitor node availability and disk usage. |=== | Property | Description @@ -22,29 +33,42 @@ To enable Continuous Data Balancing, set the `partition_autobalancing_mode` prop | `partition_autobalancing_node_availability_timeout_sec` | When a node is unreachable for the specified amount of time, Redpanda acts as if the node had been decommissioned: rebalancing begins, re-creating all of its replicas on other nodes in the cluster. + + -*Note:* The node remains part of the cluster, and it can rejoin when it comes back online. A node that was actually decommissioned is removed from the cluster. + +The node remains part of the cluster and can rejoin when it comes back online. A node that was actually decommissioned is removed from the cluster. + + Default is 900 seconds (15 minutes). +[[partition_autobalancing_node_autodecommission_timeout_sec]] +| `partition_autobalancing_node_autodecommission_timeout_sec` +| When a node is unavailable for this timeout duration, Redpanda automatically and permanently decommissions the node. This property only applies when `partition_autobalancing_mode` is set to `continuous`. Unlike `partition_autobalancing_node_availability_timeout_sec`, which moves partitions while keeping the node in the cluster, this property removes the node from the cluster entirely. A decommissioned node cannot rejoin the cluster. + + + +Only one node is decommissioned at a time. If a decommission is already in progress, automatic decommission does not trigger until it completes. If the decommission stalls (for example, because the node holds the only replica of a partition), manual intervention is required. See xref:manage:cluster-maintenance/nodewise-partition-recovery.adoc[]. + + + +By default, this property is null and automatic decommission is disabled. + | `partition_autobalancing_max_disk_usage_percent` | When a node fills up to this disk usage percentage, Redpanda starts moving replicas off the node to other nodes with disk utilization below the percentage. + + Default is 80%. |=== -For information about other modes with `partition_autobalancing_mode`, see xref:./cluster-balancing.adoc[Cluster Balancing]. +For the other `partition_autobalancing_mode` options, see xref:manage:cluster-maintenance/cluster-balancing.adoc[Cluster balancing]. + +== Use data balancing commands -== Use Data Balancing commands +Use the following `rpk` commands to monitor and control data balancing. === Check data balancing status To see the status, run: -`rpk cluster partitions balancer-status` +[,bash] +---- +rpk cluster partitions balancer-status +---- This shows the time since the last data balancing, the number of replica movements in progress, the nodes that are unavailable, and the nodes that are over the disk space threshold (default = 80%). -It also returns a data balancing status: `off`, `ready`, `starting`, `in-progress`, or `stalled`. If the command reports a `stalled` status, check the following: +It also returns a data balancing status: `off`, `ready`, `starting`, `in-progress`, or `stalled`. If the command reports a `stalled` status, verify: * Are there enough healthy nodes? For example, in a three node cluster, no movements are possible for partitions with three replicas. * Does the cluster have sufficient space? Partitions are not moved if all nodes in the cluster are utilizing more than their disk space threshold. @@ -55,10 +79,16 @@ It also returns a data balancing status: `off`, `ready`, `starting`, `in-progres To cancel the current partition balancing moves, run: -`rpk cluster partitions movement-cancel` +[,bash] +---- +rpk cluster partitions movement-cancel +---- -To cancel the partition moves in a specific node, add `--node`. For example: +To cancel partition moves on a specific node, use the `--node` flag. For example: -`rpk cluster partitions movement-cancel --node 1` +[,bash] +---- +rpk cluster partitions movement-cancel --node 1 +---- -NOTE: If continuous balancing hasn't been turned off, and if the system is still unbalanced, then it schedules another partition balancing. To stop all balancing, first set `partition_autobalancing_mode` to `off`. Then cancel current data balancing moves. +NOTE: If continuous balancing is still enabled and the cluster remains unbalanced, Redpanda schedules another partition balancing round. To stop all balancing, first set `partition_autobalancing_mode` to `off`, then cancel the current data balancing moves. diff --git a/modules/manage/pages/cluster-maintenance/decommission-brokers.adoc b/modules/manage/pages/cluster-maintenance/decommission-brokers.adoc index b3a155a2a6..e3c1d7927f 100644 --- a/modules/manage/pages/cluster-maintenance/decommission-brokers.adoc +++ b/modules/manage/pages/cluster-maintenance/decommission-brokers.adoc @@ -10,6 +10,16 @@ When you decommission a broker, its partition replicas are reallocated across th CAUTION: When a broker is decommissioned, it cannot rejoin the cluster. If a broker with the same ID tries to rejoin the cluster, it is rejected. +== Decommissioning methods + +There are two ways to decommission brokers in Redpanda: + +* Manual decommissioning (described in this guide): Use `rpk` commands to explicitly decommission a broker when you need full control over the timing and selection of brokers to remove. + +* Automatic decommissioning: When xref:manage:cluster-maintenance/continuous-data-balancing.adoc[Continuous Data Balancing] is enabled, you can configure the xref:manage:cluster-maintenance/continuous-data-balancing.adoc#partition_autobalancing_node_autodecommission_timeout_sec[partition_autobalancing_node_autodecommission_timeout_sec] property to automatically decommission brokers that remain unavailable for a specified duration. + +Both methods permanently remove the broker from the cluster. Decommissioned brokers cannot rejoin. + == What happens when a broker is decommissioned? When a broker is decommissioned, the controller leader creates a reallocation plan for all partition replicas that are allocated to that broker. By default, this reallocation is done in batches of 50 to avoid overwhelming the remaining brokers with Raft recovery. See xref:reference:tunable-properties.adoc#partition_autobalancing_concurrent_moves[`partition_autobalancing_concurrent_moves`]. diff --git a/modules/manage/pages/cluster-maintenance/disk-utilization.adoc b/modules/manage/pages/cluster-maintenance/disk-utilization.adoc index 8d66849891..b1484a49a7 100644 --- a/modules/manage/pages/cluster-maintenance/disk-utilization.adoc +++ b/modules/manage/pages/cluster-maintenance/disk-utilization.adoc @@ -85,7 +85,7 @@ Redpanda runs a log cleanup process in the background to apply these policy sett See also: * xref:manage:tiered-storage.adoc#manage-local-capacity-for-tiered-storage-topics[Manage local capacity for Tiered Storage topics] -* xref:develop:config-topics.adoc#delete-records-from-a-topic[Delete records from a topic] +* xref:develop:manage-topics/config-topics.adoc#delete-records-from-a-topic[Delete records from a topic] [[set-time-based-retention]] === Set time-based retention diff --git a/modules/manage/pages/cluster-maintenance/manage-throughput.adoc b/modules/manage/pages/cluster-maintenance/manage-throughput.adoc index dab8e18374..ce1e6da001 100644 --- a/modules/manage/pages/cluster-maintenance/manage-throughput.adoc +++ b/modules/manage/pages/cluster-maintenance/manage-throughput.adoc @@ -1,32 +1,47 @@ = Manage Throughput -:description: Learn how to manage the throughput of Kafka traffic. +:description: Configure broker-wide and client-specific throughput quotas to prevent resource exhaustion and noisy-neighbor issues. :page-categories: Management, Networking +:page-topic-type: how-to +:personas: platform_admin, developer +:learning-objective-1: Set user-based throughput quotas +:learning-objective-2: Set client ID-based quotas +:learning-objective-3: Monitor quota usage and throttling behavior // tag::single-source[] ifdef::env-cloud[] :monitor-doc: manage:monitor-cloud.adoc#throughput :connected-clients-api-doc-ref: link:/api/doc/cloud-dataplane/operation/operation-monitoringservice_listkafkaconnections +:authentication-doc: security:cloud-authentication.adoc endif::[] ifndef::env-cloud[] :monitor-doc: manage:monitoring.adoc#throughput :connected-clients-api-doc-ref: link:/api/doc/admin/v2/operation/operation-redpanda-core-admin-v2-clusterservice-listkafkaconnections +:authentication-doc: manage:security/authentication.adoc endif::[] -Redpanda supports throughput throttling on both ingress and egress independently, and allows configuration at the broker and client levels. This helps prevent clients from causing unbounded network and disk usage on brokers. You can configure limits at two levels: +Redpanda throttles throughput on ingress and egress independently, and you can configure limits at the broker and client levels. This prevents clients from causing unbounded network and disk usage on brokers. -* *Broker limits*: These apply to all clients connected to the broker and restrict total traffic on the broker. See <>. +You can configure limits at two levels: + +* Broker limits: These apply to all clients connected to the broker and restrict total traffic on the broker. See <>. ifndef::env-cloud[] -* *Client limits*: These apply to a set of clients defined by their `client_id` and help prevent a set of clients from starving other clients using the same broker. You can manage client quotas with xref:reference:rpk/rpk-cluster/rpk-cluster-quotas.adoc[`rpk cluster quotas`], with {ui}, or with the Kafka API. When no quotas apply, the client has unlimited throughput. +* Client limits: These apply to authenticated users or clients defined by their client ID. You can manage client quotas with xref:reference:rpk/rpk-cluster/rpk-cluster-quotas.adoc[`rpk cluster quotas`], with {ui}, or with the Kafka API. When no quotas apply, the client has unlimited throughput. endif::[] ifdef::env-cloud[] -* *Client limits*: These apply to a set of clients defined by their `client_id` and help prevent a set of clients from starving other clients using the same broker. You can manage client quotas with xref:reference:rpk/rpk-cluster/rpk-cluster-quotas.adoc[`rpk cluster quotas`], with the {ui} UI, with the link:https://docs.redpanda.com/api/doc/cloud-dataplane/operation/operation-quotaservice_listquotas[Redpanda Cloud Data Plane API], or with the Kafka API. When no quotas apply, the client has unlimited throughput. +* Client limits: These apply to authenticated users or clients defined by their client ID. You can manage client quotas with xref:reference:rpk/rpk-cluster/rpk-cluster-quotas.adoc[`rpk cluster quotas`], with the {ui} UI, with the link:https://docs.redpanda.com/api/doc/cloud-dataplane/operation/operation-quotaservice_listquotas[Redpanda Cloud Data Plane API], or with the Kafka API. When no quotas apply, the client has unlimited throughput. NOTE: Throughput throttling is supported for BYOC and Dedicated clusters only. endif::[] +After reading this page, you will be able to: + +* [ ] {learning-objective-1} +* [ ] {learning-objective-2} +* [ ] {learning-objective-3} + == View connected client details -You may find it helpful to check the xref:{monitor-doc}[current produce and consume throughput] of a client before you configure throughput quotas. +Before configuring throughput quotas, check the xref:{monitor-doc}[current produce and consume throughput] of a client. ifndef::env-cloud[] Use the xref:reference:rpk/rpk-cluster/rpk-cluster-connections-list.adoc[`rpk cluster connections list`] command or the {connected-clients-api-doc-ref}[ListKafkaConnections] Admin API endpoint to view detailed information about active Kafka client connections. @@ -258,6 +273,8 @@ UID STATE USER CLIENT-ID b41584f3-2662-4185-a4b8-0d8510f5c780 OPEN UNAUTHENTICATED perf-producer-client 127.0.0.1:55002 0 0 8s 7.743592270s 0B 0B 1 b20601a3-624c-4a8c-ab88-717643f01d56 OPEN UNAUTHENTICATED perf-producer-client 127.0.0.1:55012 0 0 9s 0s 78.9MB 0B 292 ---- + +The `USER` field in the connection list shows the authenticated principal. Unauthenticated connections show `UNAUTHENTICATED`, which corresponds to an empty user principal (`user=""`) in quota configurations, not `user=`. -- ifndef::env-cloud[] @@ -417,24 +434,19 @@ curl \ } ---- ==== +The user principal field in the connection list shows the authenticated principal. Unauthenticated connections show `AUTHENTICATION_STATE_UNAUTHENTICATED`, which corresponds to an empty user principal (`user=""`) in quota configurations, not `user=`. -- endif::[] ====== +To view connections for a specific authenticated user: -== Throughput throttling enforcement - -NOTE: As of v24.2, Redpanda enforces all throughput limits per broker, including client throughput. - -Throughput limits are enforced by applying backpressure to clients. When a connection is in breach of the throughput limit, the throttler advises the client about the delay (throttle time) that would bring the rate back to the allowed level. Redpanda starts by adding a `throttle_time_ms` field to responses. If that isn't honored, delays are inserted on the connection's next read operation. +[,bash] +---- +rpk cluster connections list --user alice +---- -ifdef::env-cloud[] -In Redpanda Cloud, the throttling delay is set to 30 seconds. -endif::[] - -ifndef::env-cloud[] -The throttling delay may not exceed the limit set by xref:reference:tunable-properties.adoc#max_kafka_throttle_delay_ms[`max_kafka_throttle_delay_ms`]. -endif::[] +This shows all connections from user `alice`, useful for monitoring clients that are subject to user-based quotas. == Broker-wide throughput limits @@ -469,47 +481,96 @@ The properties for broker-wide throughput quota balancing are configured at the ==== By default, both `kafka_throughput_limit_node_in_bps` and `kafka_throughput_limit_node_out_bps` are disabled, and no throughput limits are applied. You must manually set them to enable throughput throttling. ==== + +To set broker-wide throughput limits, use xref:reference:rpk/rpk-cluster/rpk-cluster-config-set.adoc[`rpk cluster config set`] to configure the cluster properties: + +[,bash] +---- +# Set ingress limit to 100 MB/s per broker +rpk cluster config set kafka_throughput_limit_node_in_bps 100000000 + +# Set egress limit to 200 MB/s per broker +rpk cluster config set kafka_throughput_limit_node_out_bps 200000000 +---- endif::[] == Client throughput limits -Redpanda provides configurable throughput quotas that apply to an individual client or a group of clients. You can apply a quota for an individual client based on an exact match with its `client_id`, or a group of clients based on IDs that start with a given prefix. +Redpanda provides configurable throughput quotas for individual clients or authenticated users. Quotas are managed through the Kafka-compatible AlterClientQuotas and DescribeClientQuotas APIs, accessible with `rpk`, Redpanda Console, or Kafka client libraries. -As of v24.2, client throughput quotas are compatible with the https://cwiki.apache.org/confluence/display/KAFKA/KIP-546%3A+Add+Client+Quota+APIs+to+the+Admin+Client[AlterClientQuotas and DescribeClientQuotas^] Kafka APIs, and are separate from quotas configured through cluster configuration in earlier Redpanda versions. The client throughput quotas no longer apply on a per-shard basis, and now limit the rates across a Redpanda broker's node. The quotas are neither shared nor balanced between brokers. +Redpanda supports two types of client throughput quotas: -Redpanda supports the following Kafka API-based quota types on clients: +* Client ID-based quotas: Limit throughput based on the self-declared `client-id` field. +* User-based quotas: Limit throughput based on authenticated user glossterm:principal[]. Requires xref:{authentication-doc}[authentication]. -|=== -| Quota type | Description +You can also combine both types for fine-grained control (for example, limiting a specific user when using a specific client application). -| `producer_byte_rate` -| Limit throughput of produce requests +For conceptual information about quota types, entity hierarchy, precedence rules, and how Redpanda tracks and enforces quotas through throttling, see xref:manage:cluster-maintenance/about-throughput-quotas.adoc[]. -| `consumer_byte_rate` -| Limit throughput of fetch requests +=== Set user-based quotas -| `controller_mutation_rate` -| Limit rate of topic mutation requests, including create, add, and delete partition, in number of partitions per second +IMPORTANT: User-based quotas require authentication to be enabled. To set up authentication, see xref:{authentication-doc}[]. -|=== +==== Quota for a specific user + +To limit throughput for a specific authenticated user across all clients: + +[,bash] +---- +rpk cluster quotas alter --add producer_byte_rate=2000000 --name user=alice +---- -You can also apply a default quota for all other client requests that don't have a specific quota based on an exact match or `client_id` prefix. +This limits user `alice` to 2 MB/s for produce requests regardless of the client ID used. -It is possible to create conflicting quotas if you configure the same quotas through both the Kafka API and a cluster configuration. Redpanda resolves these conflicts by following an order of preference in finding a matching quota for a request: +To view quotas for a user: -. Quota configured through the Kafka API for an exact match on `client_id` -. Quota configured through the Kafka API for a prefix match on `client_id` -ifndef::env-cloud[] -. Quota configured through cluster configuration properties (`kafka_client_group_byte_rate_quota`, `kafka_client_group_fetch_byte_rate_quota`-deprecated in v24.2) for a prefix match on `client_id` -endif::[] -. Default quota configured through the Kafka API on `client_id` -ifndef::env-cloud[] -. Default quota configured through cluster configuration properties (`target_quota_byte_rate`, `target_fetch_quota_byte_rate`, `kafka_admin_topic_api_rate`-deprecated in v24.2) on `client_id` +[,bash] +---- +rpk cluster quotas describe --name user=alice +---- -Redpanda recommends <> over from cluster configuration-managed quotas to Kafka-compatible quotas. You can re-create the configuration-based quotas with `rpk`, and then remove the cluster configurations. -endif::[] +Expected output: + +[,bash,role=no-copy] +---- +user=alice + producer_byte_rate=2000000 +---- + +==== Default quota for all users + +To set a fallback quota for any user without a more specific quota: + +[,bash] +---- +rpk cluster quotas alter --add consumer_byte_rate=5000000 --default user +---- + +This applies a 5 MB/s fetch quota to all authenticated users who don't have a more specific quota configured. + +=== Remove a user quota + +To remove a quota for a specific user: + +[,bash] +---- +rpk cluster quotas alter --delete consumer_byte_rate --name user=alice +---- + +To remove all quotas for a user: + +[,bash] +---- +rpk cluster quotas delete --name user=alice +---- + +=== Set client ID-based quotas -=== Individual client throughput limit +Client ID-based quotas apply to all users using a specific client ID. These quotas do not require authentication. Because the client ID is self-declared, client ID-based quotas are not suitable for guaranteeing isolation between tenants. + +For multi-tenant environments, Redpanda recommends user-based quotas for per-tenant isolation. + +==== Individual client ID throughput limit ifdef::env-cloud[] NOTE: The following sections show how to manage throughput with `rpk`. You can also manage throughput with the link:https://docs.redpanda.com/api/doc/cloud-dataplane/operation/operation-quotaservice_listquotas[Redpanda Cloud Data Plane API]. @@ -531,7 +592,7 @@ client-id=consumer-1 ---- -To set a throughput quota for a single client, use the xref:reference:rpk/rpk-cluster/rpk-cluster-quotas-alter.adoc[`rpk cluster quotas alter`] command. +To set a throughput quota for a single client, use the xref:reference:rpk/rpk-cluster/rpk-cluster-quotas-alter.adoc[`rpk cluster quotas alter`] command. [,bash] ---- @@ -544,7 +605,7 @@ ENTITY STATUS client-id=consumer-1 OK ---- -=== Group of clients throughput limit +==== Group of clients throughput limit Alternatively, you can view or configure throughput quotas for a group of clients based on a match on client ID prefix. The following example sets the `consumer_byte_rate` quota to client IDs prefixed with `consumer-`: @@ -553,12 +614,11 @@ Alternatively, you can view or configure throughput quotas for a group of client rpk cluster quotas alter --add consumer_byte_rate=200000 --name client-id-prefix=consumer- ---- -NOTE: A client group specified with `client-id-prefix` is not the equivalent of a Kafka consumer group. It is used only to match requests based on the `client_id` prefix. The `client_id` field is typically a configurable property when you create a client with Kafka libraries. - +NOTE: A `client-id-prefix` quota group is not related to Kafka consumer groups. The client ID is an application-defined identifier sent with every request. Client libraries typically default to their own name (such as `kgo`, `rdkafka`, `sarama`, or `perf-producer-client`), but applications can set it using the https://kafka.apache.org/documentation/#consumerconfigs_client.id[`client.id`^] configuration property. This makes prefix-based quotas useful for grouping related applications (for example, `inventory-service-` to match `inventory-service-1`, `inventory-service-2`, etc.). -=== Default client throughput limit +==== Default client throughput limit -You can apply default throughput limits to clients. Redpanda applies the default limits if no quotas are configured for a specific `client_id` or prefix. +You can apply default throughput limits to clients. Redpanda applies the default limits if no quotas are configured for a specific client ID or prefix. To specify a produce quota of 1 GB/s through the Kafka API (applies across all produce requests to a single broker), run: @@ -567,94 +627,70 @@ To specify a produce quota of 1 GB/s through the Kafka API (applies across all p rpk cluster quotas alter --default client-id --add producer_byte_rate=1000000000 ---- -=== Bulk manage client throughput limits +=== Set combined user and client quotas -To more easily manage multiple quotas, you can use the `cluster quotas describe` and xref:reference:rpk/rpk-cluster/rpk-cluster-quotas-import.adoc[`cluster quotas import`] commands to do a bulk export and update. +You can set quotas for specific (user, client ID) combinations for fine-grained control. -For example, to export all client quotas in JSON format: +==== User with specific client + +To limit a specific user when using a specific client: [,bash] ---- -rpk cluster quotas describe --format json +rpk cluster quotas alter --add consumer_byte_rate=1000000 --name user=alice --name client-id=consumer-1 ---- -`rpk cluster quotas import` accepts the output string from `rpk cluster quotas describe --format `: +User `alice` using `client-id=consumer-1` is limited to a 1 MB/s fetch rate. The same user with a different client ID would use a different quota (or fall back to less specific matches). + +To view combined quotas: [,bash] ---- -rpk cluster quotas import --from '{"quotas":[{"entity":[{"name":"foo","type":"client-id"}],"values":[{"key":"consumer_byte_rate","values":"12123123"}]},{"entity":[{"name":"foo-","type":"client-id-prefix"}],"values":[{"key":"producer_byte_rate","values":"12123123"},{"key":"consumer_byte_rate","values":"4444444"}]}]}' +rpk cluster quotas describe --name user=alice --name client-id=consumer-1 ---- -You can also save the JSON or YAML output to a file and pass the file path in the `--from` flag. +==== User with client prefix -[[migrate]] -=== Migrate cluster configuration quotas to Kafka API-based quotas +To set a shared quota for a user across multiple clients matching a prefix: -. Use xref:reference:rpk/rpk-cluster/rpk-cluster-config-get.adoc[`rpk cluster config get`] to view current client quotas managed with cluster configuration. The following example shows how to retrieve the `kafka_client_group_byte_rate_quota` for two groups of producers: -+ [,bash] ---- -rpk cluster config get kafka_client_group_byte_rate_quota - ----- -+ -[,bash,role=no-copy] ----- -"kafka_client_group_byte_rate_quota": [ - { - "group_name": "group_1", - "clients_prefix": "producer_group_alone_producer", - "quota": 10240 - }, - { "group_name": "group_2", - "clients_prefix": "producer_group_multiple", - "quota": 20480 - } -] +rpk cluster quotas alter --add producer_byte_rate=3000000 --name user=bob --name client-id-prefix=app- ---- -ifndef::env-cloud[] -. Each client quota cluster property (xref:upgrade:deprecated/index.adoc[deprecated in v24.2]) corresponds to a quota type in Kafka. Check the corresponding `rpk` arguments to use when setting the new quota values: -+ -|=== -| Cluster configuration property | `rpk cluster quotas` arguments -| `target_quota_byte_rate` -| `--default client-id --add producer_byte_rate=` +All clients used by user `bob` with a client ID starting with `app-` share a combined 3 MB/s produce quota. -| `target_fetch_quota_byte_rate` -| `--default client-id --add consumer_byte_rate=` +==== Default user with specific client -| `kafka_admin_topic_api_rate` -| `--default client-id --add controller_mutation_rate=` +To set a quota for a specific client across all users: -| `kafka_client_group_byte_rate_quota` -| `--name client-id-prefix= --add producer_byte_rate=` +[,bash] +---- +rpk cluster quotas alter --add producer_byte_rate=500000 --default user --name client-id=payment-processor +---- -| `kafka_client_group_fetch_byte_rate_quota` -| `--name client-id-prefix= --add consumer_byte_rate=` +Any user using `client-id=payment-processor` is limited to a 500 KB/s produce rate, unless they have a more specific quota configured. -|=== -+ -The client throughput quotas set through the Kafka API apply per broker, so you must convert the cluster configuration values that were applied on a per-shard (logical CPU core) basis. For example, if you set `target_fetch_quota_byte_rate` to 100 MBps/shard, and you run Redpanda on 16-core brokers, you can set the new consumer_byte_rate quota to 100 * 16 = 1600 MBps. -endif::[] +=== Bulk manage client throughput limits + +To more easily manage multiple quotas, you can use the `cluster quotas describe` and xref:reference:rpk/rpk-cluster/rpk-cluster-quotas-import.adoc[`cluster quotas import`] commands to do a bulk export and update. + +For example, to export all client quotas in JSON format: -. Use `rpk cluster quotas alter` to set the corresponding client throughput quotas based on the Kafka API: -+ [,bash] ---- -rpk cluster quotas alter --name client-id-prefix=producer_group_alone_producer --add producer_byte_rate= -rpk cluster quotas alter --name client-id-prefix=producer_group_multiple --add producer_byte_rate= +rpk cluster quotas describe --format json ---- -+ -Replace the placeholder values with the new quota values, accounting for the conversion to per-broker limits. For example, 10240 * broker core count = new quota. -. Use xref:reference:rpk/rpk-cluster/rpk-cluster-config-set.adoc[`rpk cluster config set`] to remove the configuration-based quotas: -+ +`rpk cluster quotas import` accepts the output string from `rpk cluster quotas describe --format `: + [,bash] ---- -rpk cluster config set kafka_client_group_byte_rate_quota= +rpk cluster quotas import --from '{"quotas":[{"entity":[{"name":"analytics-consumer","type":"client-id"}],"values":[{"key":"consumer_byte_rate","values":"10000000"}]},{"entity":[{"name":"analytics-","type":"client-id-prefix"}],"values":[{"key":"producer_byte_rate","values":"10000000"},{"key":"consumer_byte_rate","values":"5000000"}]}]}' ---- +You can also save the JSON or YAML output to a file and pass the file path in the `--from` flag. + === View throughput limits in {ui} You can also use {ui} to view enforced limits. In the side menu, go to **Quotas**. @@ -674,6 +710,11 @@ ifndef::env-cloud[] ** `/metrics` - xref:reference:internal-metrics-reference.adoc#vectorized_kafka_quotas_client_quota_throttle_time[`vectorized_kafka_quotas_client_quota_throttle_time`] endif::[] +To identify which clients are actively connected and generating traffic, see <>. + +Quota metrics use the `redpanda_quota_rule` label to identify which quota was applied to a request. The label distinguishes between different entity types (user, client, or combinations). See the label values in xref:reference:public-metrics-reference.adoc#redpanda_kafka_quotas_client_quota_throughput[`redpanda_kafka_quotas_client_quota_throughput`]. + +ifndef::env-cloud[] The `kafka_quotas` logger provides details at the trace level on client quota throttling: [,bash] @@ -684,9 +725,12 @@ TRACE 2024-06-14 15:37:44,835 [shard 2:main] kafka_quotas - quota_manager.cc:36 TRACE 2024-06-14 15:37:59,195 [shard 2:main] kafka_quotas - quota_manager.cc:361 - request: ctx:{quota_type: produce_quota, client_id: {rpk}}, key:k_client_id{rpk}, value:{limit: {1111}, rule: kafka_client_default}, bytes: 1316, delay:184518451ns, capped_delay:184518451ns TRACE 2024-06-14 15:37:59,195 [shard 2:main] kafka_quotas - connection_context.cc:605 - [127.0.0.1:58636] throttle request:{snc:0, client:184}, enforce:{snc:-14359, client:-14359}, key:0, request_size:1316 ---- +endif::[] == See also -- xref:manage:cluster-maintenance/configure-client-connections.adoc[Configure Client Connections] +- xref:manage:cluster-maintenance/about-throughput-quotas.adoc[] +- xref:manage:cluster-maintenance/configure-client-connections.adoc[] +- xref:{authentication-doc}[] // end::single-source[] diff --git a/modules/manage/pages/cluster-maintenance/topic-property-configuration.adoc b/modules/manage/pages/cluster-maintenance/topic-property-configuration.adoc index 8a182e138a..7f1d6410f1 100644 --- a/modules/manage/pages/cluster-maintenance/topic-property-configuration.adoc +++ b/modules/manage/pages/cluster-maintenance/topic-property-configuration.adoc @@ -220,6 +220,6 @@ For complete details about all available topic properties, see xref:reference:pr * xref:reference:properties/topic-properties.adoc[Topic Configuration Properties] - Complete reference of all available topic properties * xref:manage:cluster-maintenance/cluster-property-configuration.adoc[Configure Cluster Properties] - Configure cluster-wide defaults -* xref:develop:config-topics.adoc[Manage Topics] - Create and manage topics +* xref:develop:manage-topics/config-topics.adoc[Manage Topics] - Create and manage topics * xref:manage:kubernetes/k-manage-topics.adoc[Manage Topics in Kubernetes] - Topic management in Kubernetes deployments * xref:console:ui/edit-topic-configuration.adoc[Edit Topic Configuration in Redpanda Console] - Graphical topic configuration \ No newline at end of file diff --git a/modules/manage/pages/iceberg/about-iceberg-topics.adoc b/modules/manage/pages/iceberg/about-iceberg-topics.adoc index cc18710e04..4f74450f63 100644 --- a/modules/manage/pages/iceberg/about-iceberg-topics.adoc +++ b/modules/manage/pages/iceberg/about-iceberg-topics.adoc @@ -77,7 +77,7 @@ ifdef::env-cloud[] To create an Iceberg table for a Redpanda topic, you must set the cluster configuration property config_ref:iceberg_enabled,true,properties/cluster-properties[`iceberg_enabled`] to `true`, and also configure the topic property `redpanda.iceberg.mode`. You can choose to provide a schema if you need the Iceberg table to be structured with defined columns. endif::[] -. Set the `iceberg_enabled` configuration option on your cluster to `true`. +. Set the `iceberg_enabled` configuration option on your cluster to `true`. ifdef::env-cloud[] + [tabs] @@ -88,7 +88,7 @@ rpk:: [,bash] ---- rpk cloud login -rpk profile create --from-cloud +rpk profile create --from-cloud rpk cluster config set iceberg_enabled true ---- -- @@ -122,9 +122,13 @@ The link:/api/doc/cloud-controlplane/operation/operation-clusterservice_updatecl endif::[] ifndef::env-cloud[] + +When multiple clusters write to the same catalog, each cluster must use a distinct namespace to avoid table name collisions. This is especially critical for REST catalog providers that offer a single global catalog per account (such as AWS Glue), where there is no other isolation mechanism. By default, Redpanda creates Iceberg tables in a namespace called `redpanda`. To use a unique namespace for your cluster's REST catalog integration, set config_ref:iceberg_default_catalog_namespace,true,properties/cluster-properties[`iceberg_default_catalog_namespace`] at the same time. This property cannot be changed after you enable Iceberg topics on the cluster. ++ [,bash] ---- -rpk cluster config set iceberg_enabled true +rpk cluster config set iceberg_enabled true +# Optional: set a custom namespace (default is "redpanda") +# rpk cluster config set iceberg_default_catalog_namespace '[""]' ---- + [,bash,role=no-copy] diff --git a/modules/manage/pages/iceberg/iceberg-topics-aws-glue.adoc b/modules/manage/pages/iceberg/iceberg-topics-aws-glue.adoc index f25ab71625..a530eeec06 100644 --- a/modules/manage/pages/iceberg/iceberg-topics-aws-glue.adoc +++ b/modules/manage/pages/iceberg/iceberg-topics-aws-glue.adoc @@ -130,11 +130,15 @@ To configure your Redpanda cluster to enable Iceberg on a topic and integrate wi . Edit your cluster configuration to set the `iceberg_enabled` property to `true`, and set the catalog integration properties listed in the example below. ifndef::env-cloud[] + +By default, Redpanda creates Iceberg tables in a namespace called `redpanda`. Because AWS Glue provides a single catalog per account, each Redpanda cluster that writes to the same Glue catalog must use a distinct namespace to avoid table name collisions. To set a unique namespace, set config_ref:iceberg_default_catalog_namespace,true,properties/cluster-properties[`iceberg_default_catalog_namespace`] at the same time. This property cannot be changed after Iceberg is enabled. ++ Run `rpk cluster config edit` to update these properties: + [,bash] ---- iceberg_enabled: true +# Set a custom namespace instead of the default "redpanda" +iceberg_default_catalog_namespace: [""] # Glue requires Redpanda Iceberg tables to be manually deleted iceberg_delete: false iceberg_catalog_type: rest diff --git a/modules/manage/pages/iceberg/iceberg-topics-databricks-unity.adoc b/modules/manage/pages/iceberg/iceberg-topics-databricks-unity.adoc index 72a3318b4c..3fcef32425 100644 --- a/modules/manage/pages/iceberg/iceberg-topics-databricks-unity.adoc +++ b/modules/manage/pages/iceberg/iceberg-topics-databricks-unity.adoc @@ -191,8 +191,13 @@ echo "hello world\nfoo bar\nbaz qux" | rpk topic produce --format=' You should see the topic as a table with data in Unity Catalog. The data may take some time to become visible, depending on your config_ref:iceberg_target_lag_ms,true,properties/cluster-properties[`iceberg_target_lag_ms`] setting. +ifndef::env-cloud[] +. In Catalog Explorer, open your catalog. You should see a `redpanda` schema (or the namespace you configured with config_ref:iceberg_default_catalog_namespace,true,properties/cluster-properties[`iceberg_default_catalog_namespace`]), in addition to `default` and `information_schema`. +endif::[] +ifdef::env-cloud[] . In Catalog Explorer, open your catalog. You should see a `redpanda` schema, in addition to `default` and `information_schema`. -. The `redpanda` schema and the table residing within this schema are automatically added for you. The table name is the same as the topic name. +endif::[] +. The schema and the table residing within it are automatically added for you. The table name is the same as the topic name. == Query Iceberg table using Databricks SQL diff --git a/modules/manage/pages/iceberg/query-iceberg-topics.adoc b/modules/manage/pages/iceberg/query-iceberg-topics.adoc index 413cace9f6..939674dff5 100644 --- a/modules/manage/pages/iceberg/query-iceberg-topics.adoc +++ b/modules/manage/pages/iceberg/query-iceberg-topics.adoc @@ -97,6 +97,10 @@ endif::[] {"user_id": 2324, "event_type": "BUTTON_CLICK", "ts": "2024-11-25T20:23:59.380Z"} ---- +ifndef::env-cloud[] +NOTE: The query examples on this page use `redpanda` as the Iceberg namespace, which is the default. If you configured a different namespace using config_ref:iceberg_default_catalog_namespace,true,properties/cluster-properties[`iceberg_default_catalog_namespace`], replace `redpanda` with your configured namespace. +endif::[] + === Topic with schema (`value_schema_id_prefix` mode) NOTE: The steps in this section also apply to the `value_schema_latest` mode, except the produce step. The `value_schema_latest` mode is not compatible with the Schema Registry wire format. The xref:reference:rpk/rpk-topic/rpk-topic-produce[`rpk topic produce`] command embeds the wire format header, so you must use your own producer code with `value_schema_latest`. diff --git a/modules/manage/pages/iceberg/redpanda-topics-iceberg-snowflake-catalog.adoc b/modules/manage/pages/iceberg/redpanda-topics-iceberg-snowflake-catalog.adoc index bc53d3755f..7488014154 100644 --- a/modules/manage/pages/iceberg/redpanda-topics-iceberg-snowflake-catalog.adoc +++ b/modules/manage/pages/iceberg/redpanda-topics-iceberg-snowflake-catalog.adoc @@ -169,7 +169,12 @@ echo "hello world\nfoo bar\nbaz qux" | rpk topic produce --format=' You should see the topic as a table in Open Catalog. . In Open Catalog, select *Catalogs*, then open your catalog. -. Under your catalog, you should see the `redpanda` namespace, and a table with the name of your topic. The `redpanda` namespace and the table are automatically added for you. +ifndef::env-cloud[] +. Under your catalog, you should see the `redpanda` namespace (or the namespace you configured with config_ref:iceberg_default_catalog_namespace,true,properties/cluster-properties[`iceberg_default_catalog_namespace`]), and a table with the name of your topic. The namespace and the table are automatically added for you. +endif::[] +ifdef::env-cloud[] +. Under your catalog, you should see the `redpanda` namespace and a table with the name of your topic. The namespace and the table are automatically added for you. +endif::[] == Query Iceberg table in Snowflake diff --git a/modules/manage/pages/iceberg/specify-iceberg-schema.adoc b/modules/manage/pages/iceberg/specify-iceberg-schema.adoc index b98c0a1793..d8ccd38c37 100644 --- a/modules/manage/pages/iceberg/specify-iceberg-schema.adoc +++ b/modules/manage/pages/iceberg/specify-iceberg-schema.adoc @@ -275,7 +275,7 @@ Requirements: - Only JSON Schema Draft-07 is currently supported. - You must declare the JSON Schema dialect using the `$schema` keyword, for example `"$schema": "http://json-schema.org/draft-07/schema#"`. -- You must use a JSON Schema that constrains JSON documents to a strict type in order for Redpanda to translate to Iceberg; that is, each subschema must use the `type` keyword. +- You must use a JSON Schema that constrains JSON documents to a strict type so Redpanda can translate to Iceberg. In most cases this means each subschema uses the `type` keyword, but a subschema can also use `$ref` if the referenced schema resolves to a strict type. .Valid JSON Schema example [,json] @@ -310,7 +310,7 @@ Requirements: | null | -| The `null` type is not supported except when it is paired with another type to indicate nullability. +| The `null` type is only supported as a nullability marker, either in a `type` array (for example, `["string", "null"]`) or in an exclusive `oneOf` nullable pattern. | number | double @@ -325,9 +325,11 @@ Requirements: | The `format` keyword can be used for custom Iceberg types. See <> for details. | object -| struct -| The `properties` keyword must be used to define `struct` fields and constrain their types. The `additionalProperties` keyword is accepted only when it is set to `false`. - +| struct or map +a| * Use `properties` to define `struct` fields and constrain their types. `additionalProperties: false` is supported for closed objects. +* If `additionalProperties` contains a schema, it translates to an Iceberg `map`. +* You cannot combine `properties` and `additionalProperties` in an object if `additionalProperties` is set to a schema. + |=== [[format-translation]] @@ -341,13 +343,20 @@ Requirements: |=== +The following keywords have specific behavior: + +* The `$ref` keyword is supported for internal references resolved from schema resources declared in the same document (using `$id`), including relative and absolute URI forms. References to external resources and references to unknown keywords are not supported. A root-level `$ref` schema is not supported. +* The `oneOf` keyword is supported only for the nullable serializer pattern where exactly one branch is `{"type":"null"}` and the other branch is a non-null schema (`T|null`). +* In Iceberg output, Redpanda writes all fields as nullable regardless of serializer nullability annotations. + The following are not supported for JSON Schema: -* Relative and absolute (including external) references using `$ref` and `$dynamicRef` keywords +* The `$dynamicRef` keyword * The `default` keyword -* Conditional typing (`if`, `then`, `else`, `dependent` keywords) -* Boolean JSON Schema combinations (`allOf`, `anyOf`, `oneOf` keywords) -* Dynamic object members (`patternProperties` and `additionalProperties` (except when it is set to `false`) keywords) +* Conditional typing (`if`, `then`, `else`, `dependencies` keywords) +* Boolean JSON Schema combinations (`allOf`, `anyOf`, and non-nullable `oneOf` patterns) +* Dynamic object members with the `patternProperties` keyword +* The `additionalProperties` keyword when set to `true` -- ====== diff --git a/modules/manage/pages/iceberg/use-iceberg-catalogs.adoc b/modules/manage/pages/iceberg/use-iceberg-catalogs.adoc index b5f615ae13..4aa3f32a27 100644 --- a/modules/manage/pages/iceberg/use-iceberg-catalogs.adoc +++ b/modules/manage/pages/iceberg/use-iceberg-catalogs.adoc @@ -90,6 +90,14 @@ To connect to a REST catalog, set the following cluster configuration properties NOTE: You must set `iceberg_rest_catalog_endpoint` at the same time that you set `iceberg_catalog_type` to `rest`. +ifndef::env-cloud[] +==== Configure table namespace + +Check if your REST catalog provider has specific requirements or recommendations for namespaces. For example, AWS Glue offers only a single global catalog per account, and each cluster that writes to the same Glue catalog must use a distinct namespace to avoid table name collisions. + +By default, Redpanda creates Iceberg tables in a namespace called `redpanda`. To use a unique namespace, configure the config_ref:iceberg_default_catalog_namespace,true,properties/cluster-properties[`iceberg_default_catalog_namespace`] cluster property. You must set this property before enabling the Iceberg integration or at the same time. After you have enabled Iceberg, do not change this property value. +endif::[] + ==== Configure authentication To authenticate with the REST catalog, set the following cluster properties: @@ -272,7 +280,10 @@ The Spark engine can use the REST catalog to automatically discover the topic's SELECT * FROM streaming.redpanda.; ---- -The Iceberg table name is the name of your Redpanda topic. Redpanda puts the Iceberg table into a namespace called `redpanda`, creating the namespace if necessary. +The Iceberg table name is the name of your Redpanda topic. +ifndef::env-cloud[] +If you configured a different namespace using config_ref:iceberg_default_catalog_namespace,true,properties/cluster-properties[`iceberg_default_catalog_namespace`], replace `redpanda` with your configured namespace. +endif::[] TIP: You may need to explicitly create a table for the Iceberg data in your query engine. For an example, see xref:manage:iceberg/redpanda-topics-iceberg-snowflake-catalog.adoc[]. diff --git a/modules/manage/pages/kubernetes/k-cloud-topics.adoc b/modules/manage/pages/kubernetes/k-cloud-topics.adoc new file mode 100644 index 0000000000..b21ab21425 --- /dev/null +++ b/modules/manage/pages/kubernetes/k-cloud-topics.adoc @@ -0,0 +1,487 @@ += Cloud Topics on Kubernetes +:description: Configure Cloud Topics on Kubernetes to optimize for latency-tolerant, high-throughput workloads using object storage as the primary data tier. +:page-topic-type: how-to +:personas: platform_admin, streaming_developer +:learning-objective-1: Configure object storage for Cloud Topics on Kubernetes +:learning-objective-2: Enable and create Cloud Topics using the Redpanda Operator or Helm +:learning-objective-3: Verify topic storage mode configuration +:env-kubernetes: true + +include::develop:partial$cloud-topics-overview.adoc[tag=intro] + +After reading this page, you will be able to: + +* [ ] {learning-objective-1} +* [ ] {learning-objective-2} +* [ ] {learning-objective-3} + +== How Cloud Topics work + +With standard Redpanda topics, data is replicated across brokers using Raft consensus and stored locally on each replica. Cloud Topics change this model: data is acknowledged only after it is uploaded to object storage, making object storage the source of truth for both replication and consumption. + +include::develop:partial$cloud-topics-overview.adoc[tag=latency-explanation] + +=== Storage modes + +Redpanda supports multiple storage modes that you can set at the cluster or topic level using the `redpanda.storage.mode` property: + +include::manage:partial$storage-modes-table.adoc[tag=storage-modes] + +=== Ideal use cases + +include::develop:partial$cloud-topics-use-cases.adoc[tag=use-cases] + +=== Limitations + +include::develop:partial$cloud-topics-limitations.adoc[tag=limitations] +include::develop:partial$cloud-topics-limitations.adoc[tag=latency-limitation] + +== Prerequisites + +You must have the following: + +- **kubectl**: Ensure you have the https://kubernetes.io/docs/tasks/tools/#kubectl[`kubectl`^] command-line tool installed and configured to communicate with your cluster. +- **Redpanda**: A xref:deploy:deployment-option/self-hosted/kubernetes/kubernetes-deploy.adoc[Redpanda Operator and a Redpanda resource deployed] in your Kubernetes cluster running Redpanda v26.1 or later. +- **Object storage**: A configured object storage backend (Amazon S3, Google Cloud Storage, Azure Blob Storage, or an S3-compatible store such as MinIO). +- **Enterprise license**: A valid Redpanda Enterprise license applied to the cluster. ++ +[NOTE] +==== +include::shared:partial$enterprise-license.adoc[] +==== ++ +To check if you already have a license key applied to your cluster: + +[,bash] +---- +rpk cluster license info +---- + +== Configure object storage + +Cloud Topics use the same object storage configuration as Tiered Storage. If you have already configured object storage for Tiered Storage, you can skip this step and proceed to <>. + +For detailed instructions including IAM role configuration, access key management, and encryption options, see xref:manage:kubernetes/tiered-storage/k-tiered-storage.adoc#configure-object-storage[Configure object storage] in the Tiered Storage documentation. + +The following examples show the minimum required configuration for each cloud provider. + +=== Amazon S3 + +[tabs] +====== +Helm + Operator:: ++ +-- +.`redpanda-cluster.yaml` +[,yaml] +---- +apiVersion: cluster.redpanda.com/v1alpha2 +kind: Redpanda +metadata: + name: redpanda +spec: + clusterSpec: + storage: + tiered: + credentialsSecretRef: + accessKey: + name: storage-secrets + key: access-key + secretKey: + name: storage-secrets + key: secret-key + config: + cloud_storage_enabled: true + cloud_storage_credentials_source: config_file # <1> + cloud_storage_region: + cloud_storage_bucket: +---- +<1> Use `aws_instance_metadata` instead if you are using an IAM role attached to your nodes. +-- +Helm:: ++ +-- +.`cloud-storage.yaml` +[,yaml] +---- +storage: + tiered: + credentialsSecretRef: + accessKey: + name: storage-secrets + key: access-key + secretKey: + name: storage-secrets + key: secret-key + config: + cloud_storage_enabled: true + cloud_storage_credentials_source: config_file # <1> + cloud_storage_region: + cloud_storage_bucket: +---- +<1> Use `aws_instance_metadata` instead if you are using an IAM role attached to your nodes. +-- +====== + +For details on IAM roles, access keys, and AWS KMS encryption, see xref:manage:kubernetes/tiered-storage/k-tiered-storage.adoc#amazon-s3[Amazon S3] in the Tiered Storage documentation. + +=== Google Cloud Storage (GCS) + +[tabs] +====== +Helm + Operator:: ++ +-- +.`redpanda-cluster.yaml` +[,yaml] +---- +apiVersion: cluster.redpanda.com/v1alpha2 +kind: Redpanda +metadata: + name: redpanda +spec: + clusterSpec: + storage: + tiered: + credentialsSecretRef: + accessKey: + name: storage-secrets + key: access-key + secretKey: + name: storage-secrets + key: secret-key + config: + cloud_storage_enabled: true + cloud_storage_credentials_source: config_file # <1> + cloud_storage_api_endpoint: storage.googleapis.com + cloud_storage_region: + cloud_storage_bucket: +---- +<1> Use `gcp_instance_metadata` instead if you are using a GCP service account attached to your nodes. +-- +Helm:: ++ +-- +.`cloud-storage.yaml` +[,yaml] +---- +storage: + tiered: + credentialsSecretRef: + accessKey: + name: storage-secrets + key: access-key + secretKey: + name: storage-secrets + key: secret-key + config: + cloud_storage_enabled: true + cloud_storage_credentials_source: config_file # <1> + cloud_storage_api_endpoint: storage.googleapis.com + cloud_storage_region: + cloud_storage_bucket: +---- +<1> Use `gcp_instance_metadata` instead if you are using a GCP service account attached to your nodes. +-- +====== + +For details on IAM roles, HMAC access keys, and customer-managed encryption keys, see xref:manage:kubernetes/tiered-storage/k-tiered-storage.adoc#google-cloud-storage[Google Cloud Storage] in the Tiered Storage documentation. + +=== Azure Blob Storage + +[tabs] +====== +Helm + Operator:: ++ +-- +.`redpanda-cluster.yaml` +[,yaml] +---- +apiVersion: cluster.redpanda.com/v1alpha2 +kind: Redpanda +metadata: + name: redpanda +spec: + clusterSpec: + storage: + tiered: + credentialsSecretRef: + accessKey: + name: storage-secrets + key: shared-key + config: + cloud_storage_enabled: true + cloud_storage_credentials_source: config_file # <1> + cloud_storage_azure_storage_account: + cloud_storage_azure_container: +---- +<1> Use `azure_aks_oidc_federation` instead if you are using an Azure managed identity. When using managed identities, omit `credentialsSecretRef` and configure workload identity annotations on the service account. +-- +Helm:: ++ +-- +.`cloud-storage.yaml` +[,yaml] +---- +storage: + tiered: + credentialsSecretRef: + accessKey: + name: storage-secrets + key: shared-key + config: + cloud_storage_enabled: true + cloud_storage_credentials_source: config_file # <1> + cloud_storage_azure_storage_account: + cloud_storage_azure_container: +---- +<1> Use `azure_aks_oidc_federation` instead if you are using an Azure managed identity. When using managed identities, omit `credentialsSecretRef` and configure workload identity annotations on the service account. +-- +====== + +For details on managed identities and account access keys, see xref:manage:kubernetes/tiered-storage/k-tiered-storage.adoc#microsoft-absadls[Microsoft ABS/ADLS] in the Tiered Storage documentation. + +== Enable Cloud Topics + +To enable Cloud Topics, set the `cloud_topics_enabled` cluster property to `true` and set the default storage mode for all new topics to `cloud`. + +[tabs] +====== +Helm + Operator:: ++ +-- + +. Add the following to your Redpanda custom resource to enable Cloud Topics and set the default storage mode: ++ +.`redpanda-cluster.yaml` +[,yaml] +---- +apiVersion: cluster.redpanda.com/v1alpha2 +kind: Redpanda +metadata: + name: redpanda +spec: + clusterSpec: + storage: + tiered: + mountType: none + credentialsSecretRef: + accessKey: + name: cloud-storage-creds + key: access-key + secretKey: + name: cloud-storage-creds + key: secret-key + config: + cloud_storage_enabled: true + cloud_storage_region: + cloud_storage_bucket: + config: + cluster: + cloud_topics_enabled: true + default_redpanda_storage_mode: cloud # <1> +---- +<1> Optional. Set to `cloud` to make all new topics Cloud Topics by default. Omit this to create Cloud Topics individually. + +. Apply the configuration: ++ +[,bash] +---- +kubectl apply -f redpanda-cluster.yaml -n +---- + +. The Redpanda Operator reconciles the configuration. Wait for the cluster to be ready: ++ +[,bash] +---- +kubectl rollout status statefulset redpanda -n --watch +---- + +. Verify that Cloud Topics are enabled: ++ +[,bash] +---- +kubectl exec -n redpanda-0 -c redpanda -- \ + rpk cluster config get cloud_topics_enabled +---- ++ +Expected output: `true` + +-- +Helm:: ++ +-- + +. Add the following to your Helm values file: ++ +.`cloud-topics-values.yaml` +[,yaml] +---- +storage: + tiered: + mountType: none + credentialsSecretRef: + accessKey: + name: cloud-storage-creds + key: access-key + secretKey: + name: cloud-storage-creds + key: secret-key + config: + cloud_storage_enabled: true + cloud_storage_region: + cloud_storage_bucket: +config: + cluster: + cloud_topics_enabled: true + default_redpanda_storage_mode: cloud # <1> +---- +<1> Optional. Set to `cloud` to make all new topics Cloud Topics by default. Omit this to create Cloud Topics individually. + +. Deploy or upgrade the Helm chart: ++ +[,bash] +---- +helm upgrade --install redpanda redpanda/redpanda \ + -n --create-namespace \ + -f cloud-topics-values.yaml +---- + +. Wait for the cluster to be ready: ++ +[,bash] +---- +kubectl rollout status statefulset redpanda -n --watch +---- + +. Verify that Cloud Topics are enabled: ++ +[,bash] +---- +kubectl exec -n redpanda-0 -c redpanda -- \ + rpk cluster config get cloud_topics_enabled +---- ++ +Expected output: `true` +-- +====== + +== Create a Cloud Topic + +You can create Cloud Topics either by setting the cluster default storage mode to `cloud`, or by configuring individual topics. + +=== Create Cloud Topics by default + +If you set `default_redpanda_storage_mode` to `cloud` in the cluster configuration, all new topics inherit the `cloud` storage mode automatically. + +Verify the cluster default: + +[,bash] +---- +kubectl exec -n redpanda-0 -c redpanda -- \ + rpk cluster config get default_redpanda_storage_mode +---- + +Expected output: `cloud` + +Any new topic created without an explicit storage mode inherits this default: + +.`cloud-topic.yaml` +[,yaml] +---- +apiVersion: cluster.redpanda.com/v1alpha2 +kind: Topic +metadata: + name: my-cloud-topic + namespace: +spec: + partitions: 3 + replicationFactor: 3 + cluster: + clusterRef: + name: redpanda +---- + +[,bash] +---- +kubectl apply -f cloud-topic.yaml +---- + +=== Create individual Cloud Topics + +If the cluster default storage mode is not set to `cloud`, you can create individual Cloud Topics by setting the `redpanda.storage.mode` property on the topic. + +.`cloud-topic.yaml` +[,yaml] +---- +apiVersion: cluster.redpanda.com/v1alpha2 +kind: Topic +metadata: + name: my-cloud-topic + namespace: +spec: + partitions: 3 + replicationFactor: 3 + additionalConfig: + redpanda.storage.mode: "cloud" + cluster: + clusterRef: + name: redpanda +---- + +[,bash] +---- +kubectl apply -f cloud-topic.yaml +---- + +=== Override the cluster default + +Topic-level storage mode settings override the cluster default. For example, if the cluster default is `cloud`, you can create a topic that uses Tiered Storage instead: + +.`tiered-topic.yaml` +[,yaml] +---- +apiVersion: cluster.redpanda.com/v1alpha2 +kind: Topic +metadata: + name: my-tiered-topic + namespace: +spec: + partitions: 3 + replicationFactor: 3 + additionalConfig: + redpanda.storage.mode: "tiered" + cluster: + clusterRef: + name: redpanda +---- + +== Verify topic storage mode + +To verify the storage mode of a topic, inspect its configuration through the Topic resource status: + +[,bash] +---- +kubectl get topic -n -o jsonpath=\ + '{range .status.topicConfiguration[*]}{.name}={.value} ({.source}){"\n"}{end}' \ + | grep storage.mode +---- + +Expected output for a Cloud Topic: + +---- +storage.mode=cloud (DEFAULT_CONFIG) +---- + +Or, if explicitly set on the topic: + +---- +storage.mode=cloud (DYNAMIC_TOPIC_CONFIG) +---- + +The `source` field indicates whether the value was inherited from the cluster default (`DEFAULT_CONFIG`) or explicitly set on the topic (`DYNAMIC_TOPIC_CONFIG`). + +== Suggested reading + +- xref:manage:kubernetes/tiered-storage/k-tiered-storage.adoc[Use Tiered Storage on Kubernetes] +- xref:manage:kubernetes/k-manage-topics.adoc[Manage Topics with the Redpanda Operator] diff --git a/modules/manage/pages/kubernetes/k-decommission-brokers.adoc b/modules/manage/pages/kubernetes/k-decommission-brokers.adoc index bf71604385..fb0410ab7e 100644 --- a/modules/manage/pages/kubernetes/k-decommission-brokers.adoc +++ b/modules/manage/pages/kubernetes/k-decommission-brokers.adoc @@ -15,6 +15,16 @@ You may want to decommission a broker in the following situations: NOTE: When a broker is decommissioned, it cannot rejoin the cluster. If a broker with the same ID tries to rejoin the cluster, it is rejected. +== Decommissioning methods + +There are two ways to decommission brokers in Redpanda: + +* Manual decommissioning (described in this guide): Use `rpk` commands or Kubernetes automation to explicitly decommission a broker when you need full control over the timing and selection of brokers to remove. + +* Automatic decommissioning: When xref:manage:cluster-maintenance/continuous-data-balancing.adoc[Continuous Data Balancing] is enabled, you can configure the xref:manage:cluster-maintenance/continuous-data-balancing.adoc#partition_autobalancing_node_autodecommission_timeout_sec[partition_autobalancing_node_autodecommission_timeout_sec] property to automatically decommission brokers that remain unavailable for a specified duration. + +Both methods permanently remove the broker from the cluster. Decommissioned brokers cannot rejoin. + == Prerequisites You must have the following: diff --git a/modules/manage/pages/kubernetes/k-schema-controller.adoc b/modules/manage/pages/kubernetes/k-schema-controller.adoc index e2e5772aba..4062cd33dd 100644 --- a/modules/manage/pages/kubernetes/k-schema-controller.adoc +++ b/modules/manage/pages/kubernetes/k-schema-controller.adoc @@ -278,4 +278,6 @@ internal-rpk registry subject list For more details on using schemas in Redpanda, see: * xref:manage:schema-reg/index.adoc[] +* xref:manage:kubernetes/security/authentication/k-schema-registry-acls.adoc[Manage Schema Registry ACLs (Operator)] +* xref:manage:schema-reg/schema-reg-authorization.adoc[] diff --git a/modules/manage/pages/kubernetes/security/authentication/k-schema-registry-acls.adoc b/modules/manage/pages/kubernetes/security/authentication/k-schema-registry-acls.adoc new file mode 100644 index 0000000000..89d23a8b85 --- /dev/null +++ b/modules/manage/pages/kubernetes/security/authentication/k-schema-registry-acls.adoc @@ -0,0 +1,133 @@ += Manage Schema Registry ACLs with the Redpanda Operator +:description: Manage Schema Registry ACLs declaratively in Kubernetes using User, RedpandaRole, and Group custom resources with the Redpanda Operator. +:page-categories: Management, Security +:page-topic-type: how-to +:personas: platform_operator +:env-kubernetes: true + +With the Redpanda Operator, you can declaratively manage Schema Registry ACLs alongside standard Kafka ACLs using the existing xref:reference:k-crd.adoc#k8s-api-github-com-redpanda-data-redpanda-operator-operator-api-redpanda-v1alpha2-user[User], xref:reference:k-crd.adoc#k8s-api-github-com-redpanda-data-redpanda-operator-operator-api-redpanda-v1alpha2-role[RedpandaRole], and Group custom resources. This allows you to control which users and roles perform specific operations within Schema Registry. + +For Schema Registry Authorization concepts and the available operations, see xref:manage:schema-reg/schema-reg-authorization.adoc[]. + +== Prerequisites + +You must have the following: + +* *kubectl*: The https://kubernetes.io/docs/tasks/tools/#kubectl[kubectl^] command-line tool, installed and configured to communicate with your cluster. +* *Redpanda Operator v25.3 or later*: See xref:deploy:deployment-option/self-hosted/kubernetes/k-production-deployment.adoc[]. +* *Redpanda cluster with SASL enabled*: See xref:manage:kubernetes/security/authentication/k-authentication.adoc#enable[Enable SASL authentication]. +* *Schema Registry Authorization enabled*: See xref:manage:schema-reg/schema-reg-authorization.adoc#enable-schema-registry-authorization[Enable Schema Registry Authorization]. + +== Schema Registry ACL resource types + +The Redpanda Operator supports two Schema Registry ACL resource types in addition to the standard Kafka ACL resource types (`topic`, `group`, `cluster`, `transactionalId`): + +* `subject`: Controls ACL access for specific Schema Registry subjects. Specify the subject name in `resource.name`. Supports both `literal` and `prefixed` pattern types. +* `registry`: Controls access to global Schema Registry operations. The `registry` resource type does not require a `name` because it applies to all global registry operations. + +For a full list of supported operations by resource type, see xref:manage:schema-reg/schema-reg-authorization.adoc#supported-operations[Supported operations]. + +== Define Schema Registry ACLs in a User resource + +The xref:manage:kubernetes/security/authentication/k-user-controller.adoc[User resource] supports Schema Registry ACLs alongside standard Kafka ACLs. + +.`user-with-sr-acls.yaml` +[,yaml,indent=0] +---- +include::manage:example$kubernetes/user-crds.feature[tags=manage-authz-only-manifest,indent=0] +---- + +In this example, the User resource creates ACLs for an existing user called `travis` in the cluster called `sasl`. The first ACL rule grants read access to all topics whose names start with `some-topic` using a `prefixed` pattern type. The second ACL rule grants read access to Schema Registry subjects matching the same prefix. + +When both Kafka and Schema Registry ACLs are defined in the same User resource, the operator syncs them independently. Kafka ACLs are applied through the Kafka API and Schema Registry ACLs are applied through the Schema Registry API. + +== Define Schema Registry ACLs in a RedpandaRole resource + +The xref:manage:kubernetes/security/authorization/k-role-controller.adoc[RedpandaRole resource] groups Schema Registry ACLs into reusable permission sets for multiple users. + +.`role-with-sr-acls.yaml` +[,yaml,indent=0] +---- +include::manage:example$kubernetes/role-crds.feature[tags=manage-roles-with-authorization,indent=0] +---- + +In this example, a RedpandaRole called `read-only-role` is created in the cluster called `sasl`. The user `charlie` is assigned as a principal. The authorization rules grant `Read` and `Describe` access to all topics with names starting with `public-` using a `prefixed` pattern type, and the same `Read` and `Describe` access to Schema Registry subjects matching the same prefix. + +== Define Schema Registry ACLs in a Group resource + +The Group resource supports Schema Registry ACLs for OIDC groups. + +.`group-with-sr-acls.yaml` +[,yaml,indent=0] +---- +include::manage:example$kubernetes/group-crds.feature[tags=manage-group-acls,indent=0] +---- + +In this example, ACLs are created for an OIDC group called `engineering` in the cluster called `sasl`. The authorization rules grant `Read` and `Describe` access to all topics with names starting with `team-` using a `prefixed` pattern type, and the same `Read` and `Describe` access to Schema Registry subjects matching the same prefix. + +== Common use cases + +The following examples show common patterns for configuring Schema Registry ACLs using the User resource. + +=== Grant a user read access to a subject + +This example gives a consumer application read access to the `orders` topic and its associated Schema Registry subject `orders-value`. Both ACLs use a `literal` pattern type to match exact resource names. + +.`consumer-app.yaml` +[,yaml,indent=0] +---- +include::manage:example$kubernetes/user-crds.feature[tags=grant-user-read-access,indent=0] +---- + +=== Grant a producer write access using prefix patterns + +This example creates a user called `producer-app` with both authentication credentials and authorization rules. The ACLs grant `Write` and `Describe` access to all topics and Schema Registry subjects whose names start with `events-` using a `prefixed` pattern type. This allows the producer to register new schema versions for any subject matching the prefix. + +.`producer-app.yaml` +[,yaml,indent=0] +---- +include::manage:example$kubernetes/user-crds.feature[tags=grant-producer-write-access,indent=0] +---- + +=== Grant global Schema Registry access + +This example gives a schema administrator full access to all Schema Registry operations. The first ACL rule uses the `registry` resource type, which applies to global operations such as getting or setting the global compatibility level. The `registry` resource type does not require a `name` field. The second ACL rule uses a `subject` resource type with an empty name and `prefixed` pattern type to match all subjects. + +.`schema-admin.yaml` +[,yaml,indent=0] +---- +include::manage:example$kubernetes/user-crds.feature[tags=grant-global-sr-access,indent=0] +---- + +== Partial sync behavior + +When a resource includes both Kafka and Schema Registry ACLs, the operator syncs them independently. If the Kafka ACLs sync successfully but the Schema Registry ACLs fail (for example, if Schema Registry Authorization is not enabled), the resource enters a `PartiallySynced` state. Check the resource status conditions for details: + +[,bash] +---- +kubectl get user -o jsonpath='{.status.conditions}' --namespace +---- + +== Deploy and verify + +To deploy a resource with Schema Registry ACLs, apply the manifest to the same namespace as your Redpanda cluster: + +[,bash] +---- +kubectl apply -f .yaml --namespace +---- + +After deploying, verify that the Redpanda Operator reconciled the resource: + +[,bash] +---- +kubectl logs -l app.kubernetes.io/name=operator -c manager --namespace +---- + +== Next steps + +* xref:manage:schema-reg/schema-reg-authorization.adoc[] +* xref:manage:kubernetes/security/authentication/k-user-controller.adoc[] +* xref:manage:kubernetes/security/authorization/k-role-controller.adoc[] +* xref:reference:k-crd.adoc#k8s-api-github-com-redpanda-data-redpanda-operator-operator-api-redpanda-v1alpha2-aclresourcespec[ACLResourceSpec] +* xref:manage:security/authorization/acl.adoc[] diff --git a/modules/manage/pages/kubernetes/security/authorization/k-group-controller.adoc b/modules/manage/pages/kubernetes/security/authorization/k-group-controller.adoc new file mode 100644 index 0000000000..1c21d6cbcd --- /dev/null +++ b/modules/manage/pages/kubernetes/security/authorization/k-group-controller.adoc @@ -0,0 +1,248 @@ += Manage Groups and ACLs with the Redpanda Operator +:description: Use the Group resource to declaratively manage ACLs for OIDC groups as part of a Redpanda deployment. Each Group resource maps to an external OIDC group identity and manages the Kafka ACLs for that group principal. +:page-categories: Management, Security +:env-kubernetes: true +:page-topic-type: how-to +:learning-objective-1: Create Group resources to manage ACLs for OIDC groups +:learning-objective-2: Configure authorization rules for group principals +:learning-objective-3: Deploy and verify Group resources in Kubernetes +:personas: platform_admin, security_admin + +ifndef::env-cloud[] +[NOTE] +==== +include::shared:partial$enterprise-license.adoc[] +==== +endif::[] + +With the Redpanda Operator, you can declaratively manage Kafka ACLs for OIDC groups using xref:reference:k-crd.adoc#k8s-api-github-com-redpanda-data-redpanda-operator-operator-api-redpanda-v1alpha2-group[Group custom resources] in Kubernetes. Each Group resource maps to an external OIDC group identity and manages the Kafka ACLs for that group's principal. The group controller (a component of the Redpanda Operator) keeps ACLs in sync with the Group resource. + +NOTE: Groups are external identities sourced from your OIDC identity provider (IdP) via JWT token claims. Unlike xref:manage:kubernetes/security/authentication/k-user-controller.adoc[User resources], Group resources do not create entities in Redpanda -- they only manage ACLs for the `Group:` principal. + +After reading this page, you will be able to: + +* [ ] {learning-objective-1} +* [ ] {learning-objective-2} +* [ ] {learning-objective-3} + +== What are groups and why use them? + +Groups let you assign Kafka ACL permissions to OIDC identities without creating individual Redpanda users. When a user authenticates through your OIDC provider, the JWT token includes group claims. Redpanda maps those group claims to ACL principals of the form `Group:`, so any user in that OIDC group inherits the group's permissions. + +This is useful when: + +* *You use an external identity provider*: Your organization manages identities in an IdP such as Okta, Azure AD, or Keycloak, and you want Redpanda permissions to follow those group memberships. +* *You want centralized access management*: Instead of creating individual Redpanda users and assigning permissions one by one, you grant permissions to groups. When someone joins or leaves a group in your IdP, their Redpanda access updates automatically. +* *You want to combine with roles*: Groups can be assigned as principals in xref:manage:kubernetes/security/authorization/k-role-controller.adoc[RedpandaRole resources], allowing OIDC groups to inherit role-based permissions. + +== Prerequisites + +You must have the following: + +* *Kubectl*: Ensure you have the https://kubernetes.io/docs/tasks/tools/#kubectl[kubectl^] command-line tool installed and configured to communicate with your cluster. +* *Redpanda Operator*: Ensure you have at least version 25.3 of the xref:deploy:kubernetes/k-production-deployment.adoc[Redpanda Operator]. +* *Redpanda cluster with OIDC enabled*: Ensure you have a Redpanda cluster deployed with xref:manage:security/authentication/oidc/index.adoc[OIDC authentication] configured. Group-based access control (GBAC) is an enterprise feature that requires OIDC. +* *Redpanda v26.1+*: The cluster must be running Redpanda v26.1 or later, which supports the v2 Security API required for group principals. + +== Create a Group resource + +You can use the Group resource to: + +- <> +- <> + +Each Group resource manages the ACLs for a single group principal (`Group:`). Only one Group resource is allowed per group name in the Redpanda cluster. + +[[with-acls]] +=== Create ACLs for an OIDC group + +- *Use case*: You want to grant specific Kafka permissions to all users who belong to an OIDC group. This is the most common use case, where your IdP manages group membership and Redpanda enforces the permissions. +- *What happens when deleted*: All ACLs managed by this Group resource are removed. Users in the OIDC group lose the permissions granted by this resource. + +This example shows how to create ACLs for an OIDC group called `engineering`, granting read access to topics and Schema Registry subjects matching `team-`. + +.`engineering-group.yaml` +[,yaml,indent=0] +---- +include::manage:example$kubernetes/group-crds.feature[tags=manage-group-acls,indent=0] +---- + +[[without-acls]] +=== Register a group without ACLs + +- *Use case*: You want to ensure no stale ACLs exist for a group, or you want to create the Group resource first and add ACLs later. +- *What happens when deleted*: No ACLs are affected because none were defined. + +When you create a Group resource without an `authorization` section, the operator ensures that no ACLs exist for the group principal. If any pre-existing ACLs were previously set for this group, they are removed. + +.`empty-group.yaml` +[,yaml,indent=0] +---- +apiVersion: cluster.redpanda.com/v1alpha2 +kind: Group +metadata: + name: empty-group +spec: + cluster: + clusterRef: + name: redpanda +---- + +== Configuration + +The following sections provide guidance on configuring Group resources and defining ACLs for OIDC groups. + +=== Choose a group name + +The `metadata.name` field in the Group resource must match the group name in your OIDC provider's JWT token claims. Keep in mind the following: + +- *Match your IdP*: The name must exactly match the group claim value from your OIDC provider. For example, if your IdP sends `engineering` as a group claim, the Group resource must be named `engineering`. +- *Unique*: Each group name must be unique within the Redpanda cluster. +- *Stable*: Avoid changing group names, as this involves deleting and recreating the Group resource. The cluster source is immutable. + +[,yaml] +---- +metadata: + name: engineering +---- + +=== Reference the Redpanda cluster + +The `spec.cluster` field specifies which Redpanda cluster the ACLs should be applied to. This field is immutable after creation. + +[,yaml] +---- +spec: + cluster: + clusterRef: + name: redpanda +---- + +=== Define authorization rules + +The `spec.authorization` field allows you to manage ACLs for the group. ACLs define the permissions that all members of the OIDC group have over specific resources in Redpanda, such as topics, consumer groups, and Schema Registry subjects. + +[,yaml] +---- +spec: + authorization: + acls: + - type: allow + resource: + type: topic + name: team- + patternType: prefixed + operations: [Read, Describe] + - type: allow + resource: + type: subject + name: team- + patternType: prefixed + operations: [Read, Describe] +---- + +- `type`: Defines whether the ACL allows or denies access. Acceptable values: `allow`, `deny`. +- `resource.type`: Specifies the resource type. Acceptable values: `topic`, `group`, `cluster`, `transactionalId`, `subject`. +- `patternType`: Specifies how the resource name is interpreted. Acceptable values: `literal`, `prefixed`. Default: `literal`. ++ +TIP: Using `literal` names for resources ensures that only the exact resources you intend are accessible. Use `prefixed` patterns cautiously to avoid accidental permission grants. +- `operations`: Lists the allowed operations, such as `Read`, `Write`, `Create`, `Describe`, and `Delete`. + +When the `authorization` field is omitted or contains an empty `acls` list, the operator removes any existing ACLs for the group principal. + +For more details about ACLs, including supported operations and resources in Redpanda, see xref:manage:security/authorization/acl.adoc[]. + +== Deploy a Group resource + +To deploy a Group resource, apply the manifest to the same namespace as your Redpanda cluster: + +[,bash] +---- +kubectl apply -f .yaml --namespace +---- + +- Replace `` with the filename of your manifest. +- Replace `` with the namespace in which you deployed Redpanda. + +== Verify a group + +After deploying a Group resource, verify that the Redpanda Operator reconciled it: + +[,bash] +---- +kubectl get groups --namespace +---- + +The `SYNCED` column should show `True` when the group's ACLs have been successfully applied. + +You can also check the operator logs for details: + +[,bash] +---- +kubectl logs -l app.kubernetes.io/name=operator -c manager --namespace +---- + +== Update a group + +To update a group's ACLs, edit the Group resource configuration and apply the changes. The operator will reconcile the ACLs to match the updated specification, adding new ACLs and removing any that are no longer defined. + +[,bash] +---- +kubectl apply -f .yaml --namespace +---- + +For example, to expand permissions for a group from read-only to read-write: + +[,yaml] +---- +spec: + authorization: + acls: + - type: allow + resource: + type: topic + name: platform- + patternType: prefixed + operations: [Read, Write, Describe] +---- + +== Delete a group + +To delete a group, delete the Group resource: + +[,bash] +---- +kubectl delete -f .yaml --namespace +---- + +When a Group resource is deleted, all ACLs managed by that resource are removed from the Redpanda cluster. The group itself continues to exist in your OIDC provider -- only the Redpanda ACLs are affected. + +== Best practices + +When working with Group resources, consider the following best practices: + +=== Group design + +- *Align with your IdP*: Structure your Group resources to mirror the group hierarchy in your OIDC provider. This makes it easier to reason about who has access to what. +- *Principle of least privilege*: Grant only the minimum permissions necessary for the group's function. Start with read-only access and expand as needed. +- *Naming conventions*: Use the same group names as your IdP to avoid confusion. Document which IdP groups map to which Redpanda permissions. + +=== Permission management + +- *Prefer group-based over user-based ACLs*: When your organization uses OIDC, manage permissions through groups rather than individual users for easier administration. +- *Use specific resource patterns*: Prefer `literal` patterns over `prefixed` patterns unless you specifically need pattern matching. +- *Combine with roles*: For complex permission models, use Group resources for direct ACLs and xref:manage:kubernetes/security/authorization/k-role-controller.adoc[RedpandaRole resources] to assign groups to roles. +- *Regular reviews*: Periodically review group permissions to ensure they remain appropriate and necessary. + +=== Integration with other resources + +- *Groups and roles*: OIDC groups can be assigned as principals in RedpandaRole resources using the `Group:` format. This allows groups to inherit role-level permissions in addition to any direct ACLs. +- *Groups and users*: Group ACLs and user ACLs are independent. A user who belongs to an OIDC group receives both the group's permissions and any individual ACLs defined through a User resource. +- *Avoid conflicts*: Be careful not to create conflicting ACLs (allow vs. deny) between Group, Role, and User resources for the same principals and resources. + +== Suggested reading + +* xref:reference:k-crd.adoc#k8s-api-github-com-redpanda-data-redpanda-operator-operator-api-redpanda-v1alpha2-group[Group resource] +* xref:manage:kubernetes/security/authentication/k-user-controller.adoc[Manage Users and ACLs] +* xref:manage:kubernetes/security/authorization/k-role-controller.adoc[Manage Roles and ACLs] +* xref:manage:security/authorization/acl.adoc[Access Control Lists (ACLs)] diff --git a/modules/manage/pages/security/authorization/acl.adoc b/modules/manage/pages/security/authorization/acl.adoc index e371c402a3..35fec7a6f3 100644 --- a/modules/manage/pages/security/authorization/acl.adoc +++ b/modules/manage/pages/security/authorization/acl.adoc @@ -62,8 +62,8 @@ Understanding these terms helps you configure least-privilege access. | Term | Definition | Example | Principal -| The entity (user or role) requesting access -| `User:analytics-user`, `RedpandaRole:data-engineers` +| The entity (user, role, or group) requesting access +| `User:analytics-user`, `RedpandaRole:data-engineers`, `Group:engineering` | Resource | The Redpanda component being accessed (cluster, topic, consumer group, transactional ID, Schema Registry glossterm:subject[], and Schema Registry operation) @@ -91,7 +91,13 @@ ACL commands work on a multiplicative basis. If you specify two principals and t [[principals]] === Principals -All ACLs require a principal. A principal is composed of two parts: the type, and the name. Redpanda supports the types "User" and "RedpandaRole". When you create user "bar", Redpanda expects you to add ACLs for "User:bar". +All ACLs require a principal. A principal is composed of two parts: the type, and the name. Redpanda supports the types "User", "RedpandaRole", and "Group". When you create user "bar", Redpanda expects you to add ACLs for "User:bar". To grant permissions to an OIDC group, use the `Group:` prefix (for example, `Group:engineering`). +ifndef::env-cloud[] +See xref:manage:security/authorization/gbac.adoc[]. +endif::[] +ifdef::env-cloud[] +See xref:security:authorization/gbac.adoc[]. +endif::[] The `--allow-principal` and `--deny-principal` flags add this prefix for you, if necessary. diff --git a/modules/manage/pages/security/authorization/gbac.adoc b/modules/manage/pages/security/authorization/gbac.adoc new file mode 100644 index 0000000000..8bc8f876b8 --- /dev/null +++ b/modules/manage/pages/security/authorization/gbac.adoc @@ -0,0 +1,17 @@ += Configure Group-Based Access Control +:description: Manage Redpanda permissions at scale using your identity provider's groups. Define access once per group and let your IdP control membership, with no per-user configuration in Redpanda. +:page-topic-type: how-to +:page-categories: Management, Security +:personas: security_engineer, platform_engineer +:learning-objective-1: Configure the cluster properties that enable GBAC +:learning-objective-2: Assign an OIDC group to an RBAC role +:learning-objective-3: Create a group-based ACL using the Group: principal prefix + +ifndef::env-cloud[] +[NOTE] +==== +include::shared:partial$enterprise-license.adoc[] +==== +endif::[] + +include::manage:partial$gbac-dp.adoc[] diff --git a/modules/manage/pages/security/authorization/index.adoc b/modules/manage/pages/security/authorization/index.adoc index 16ff7a1f26..d426846189 100644 --- a/modules/manage/pages/security/authorization/index.adoc +++ b/modules/manage/pages/security/authorization/index.adoc @@ -1,8 +1,8 @@ = Configure Authorization -:description: Redpanda provides two mechanisms for controlling user permissions. +:description: Redpanda provides mechanisms for controlling user permissions, including ACLs, role-based access control, and group-based access control. :page-aliases: security:authorization/index.adoc, manage:security/authorization.adoc :page-categories: Management, Security :page-layout: index -Authorization works in tandem with xref:security/authentication.adoc[authentication]. Authentication grants permission to interact with Redpanda resources while authorization controls what a principal is permitted to do once authenticated. +Authorization works in tandem with xref:security/authentication.adoc[authentication]. Authentication verifies who a principal is. Authorization controls what that principal can do once authenticated. diff --git a/modules/manage/pages/security/fips-compliance.adoc b/modules/manage/pages/security/fips-compliance.adoc index 208af643d6..3e2d268c6e 100644 --- a/modules/manage/pages/security/fips-compliance.adoc +++ b/modules/manage/pages/security/fips-compliance.adoc @@ -1,8 +1,19 @@ -= Configure Redpanda for FIPS -:description: Configure Redpanda to operate in FIPS compliance mode. += Configure Redpanda for FIPS +:description: Configure Redpanda to operate in FIPS-compliant mode. :page-aliases: security:fips-compliance.adoc +:page-topic-type: how-to +:personas: platform_operator +:learning-objective-1: Configure a Redpanda broker to run in FIPS-compliant mode +:learning-objective-2: Set the required OpenSSL properties for FIPS mode +:learning-objective-3: Deploy Redpanda in FIPS-compliant mode using Docker -Redpanda provides FIPS-compliant cipher enforcement for brokers using OpenSSL 3.0.9, which is https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/4282[validated^] for https://csrc.nist.gov/pubs/fips/140-2/upd2/final[140-2^] and is undergoing validation by NIST for https://csrc.nist.gov/pubs/fips/140-3/final[140-3^]. Both Redpanda and `rpk` leverage validated OpenSSL libraries for all security-related cryptography operations. +Redpanda provides Federal Information Processing Standards (FIPS)-compliant cipher enforcement for brokers using a https://csrc.nist.gov/pubs/fips/140-3/final[FIPS 140-3^]-validated OpenSSL cryptographic module. Redpanda and `rpk` both use the OpenSSL library for security-related cryptographic operations. + +After reading this page, you will be able to: + +* [ ] {learning-objective-1} +* [ ] {learning-objective-2} +* [ ] {learning-objective-3} [NOTE] ==== @@ -18,19 +29,27 @@ rpk cluster license info == Prerequisites -Before configuring brokers to run in FIPS compliance mode (FIPS mode), check to make sure the `redpanda-rpk-fips` and `redpanda-fips` packages are xref:deploy:deployment-option/self-hosted/manual/production/production-deployment.adoc#install-redpanda-for-fips-compliance[installed]. These packages are required by both the `redpanda` and `redpanda-tuner` install packages. +Before configuring brokers to run in FIPS mode on Linux, install the `redpanda-rpk-fips` and `redpanda-fips` xref:deploy:redpanda/manual/production/production-deployment.adoc#install-redpanda-for-fips-compliance[packages]. + +For Docker deployments, use the FIPS-specific image instead: `docker.redpanda.com/redpandadata/redpanda:-fips`. + +[WARNING] +==== +Before upgrading to Redpanda 26.1 with FIPS mode enabled, change any SASL/SCRAM user passwords shorter than 14 characters to at least 14 characters. FIPS 140-3 enforces stricter HMAC key size requirements than FIPS 140-2. Because Redpanda stores passwords in encrypted form, it cannot check the length of existing passwords. Clients with passwords shorter than 14 characters will fail to authenticate after the upgrade. +==== == Limitations -- Redpanda is not fully FIPS-compliant when used with the Redpanda Helm chart and Operator in a Kubernetes deployment. +- Redpanda FIPS mode requires a FIPS-enabled host when deployed with the Redpanda Helm chart or Operator. - Redpanda Console is not FIPS-compliant. -- PKCS#12 keys for xref:manage:security/encryption.adoc[TLS encryption] are not supported when FIPS mode is enabled in Redpanda. The PKCS12KDF algorithm used in PKCS#12 is not FIPS-compliant. To use Redpanda in FIPS mode with TLS enabled, configure your certificates and keys in PEM format instead. +- Redpanda does not support PKCS#12 keys for xref:manage:security/encryption.adoc[TLS encryption] when FIPS mode is enabled. The PKCS12KDF algorithm used in PKCS#12 is not FIPS-compliant. To use Redpanda in FIPS mode with TLS enabled, configure your certificates and keys in PEM format instead. +- When FIPS mode is `enabled` or `permissive`, SASL/SCRAM passwords must be at least 14 characters. == Configure FIPS mode When you configure a broker to run in FIPS mode: -* FIPS compliance is enforced _immediately_ upon the startup of Redpanda. +* Redpanda enforces FIPS compliance _immediately_ on startup. * Redpanda and its dependencies only use FIPS-validated cryptographic modules for all cryptographic algorithms used in a security context. @@ -42,43 +61,100 @@ Redpanda logs an error and exits immediately if: * It cannot detect a FIPS-validated library. -To place a broker in FIPS compliance mode, enable xref:reference:properties/broker-properties.adoc#fips_mode[`fips_mode`] in the Redpanda broker configuration file (typically located in `/etc/redpanda/redpanda.yaml`). All fields are within the `redpanda` object: +To place a broker in FIPS-compliant mode, enable xref:reference:properties/broker-properties.adoc#fips_mode[`fips_mode`] in the Redpanda broker configuration file (typically located in `/etc/redpanda/redpanda.yaml`). All fields are within the `redpanda` object: -```yaml +[,yaml] +---- redpanda: # .... fips_mode: enabled -``` +---- Available `fips_mode` values are: -* `disabled`: Redpanda is not running in FIPS compliance mode. +* `disabled`: Redpanda is not running in FIPS-compliant mode. * `enabled`: When Redpanda starts up, it looks for a value of `1` in the file `/proc/sys/crypto/fips_enabled`. If the file doesn't exist or doesn't contain `1`, Redpanda logs an error and exits immediately. -* `permissive`: This setting is a safety value option only, and _should not be used in a production environment_. If specified, Redpanda logs a WARNING, but continues operations even if the underlying operating system is not configured for FIPS. If set, your Redpanda instance is _not_ running in FIPS compliance mode. +* `permissive`: This setting is a safety value option only. Do not use it in a production environment. If specified, Redpanda logs a WARNING, but continues operations even if the underlying operating system is not configured for FIPS. If set, your Redpanda instance is _not_ running in FIPS-compliant mode. + +You must also configure OpenSSL properties for FIPS mode. === FIPS OpenSSL configuration -You must specify the following SSL configurations for brokers you want to run in FIPS compliance mode: +You must specify the following SSL configurations for brokers you want to run in FIPS-compliant mode: -* xref:reference:properties/broker-properties.adoc#openssl_config_file[`openssl_config_file`]: Specifies the path to the OpenSSL configuration file that was created as part of the `redpanda-fips` package installation. This file is used when OpenSSL is initialized to find the `fipsmodule.cnf` file that was created by the `openssl fipsinstall` command. Typically, this value should be `/opt/redpanda/openssl/openssl.cnf`. +* xref:reference:properties/broker-properties.adoc#openssl_config_file[`openssl_config_file`]: Specifies the path to the OpenSSL configuration file created during `redpanda-fips` package installation. OpenSSL uses this file during initialization to find the `fipsmodule.cnf` file that `openssl fipsinstall` creates. Typically, this value is `/opt/redpanda/openssl/openssl.cnf`. -* xref:reference:properties/broker-properties.adoc#openssl_module_directory[`openssl_module_directory`]: Specifies the path to the directory that contains the `fips.so` cryptographic provider. Typically, this value should be: `/opt/redpanda/lib/ossl-modules/`. +* xref:reference:properties/broker-properties.adoc#openssl_module_directory[`openssl_module_directory`]: Specifies the path to the directory that contains the `fips.so` cryptographic provider. Typically, this value is: `/opt/redpanda/lib/ossl-modules/`. + -The following configuration starts Redpanda in FIPS mode: +The following configuration starts Redpanda in FIPS mode: + -```yaml +[,yaml] +---- redpanda: # .... fips_mode: enabled openssl_config_file: /opt/redpanda/openssl/openssl.cnf openssl_module_directory: /opt/redpanda/lib/ossl-modules/ -``` - -== Suggested reading - -* xref:deploy:deployment-option/self-hosted/manual/production/production-deployment.adoc#install-redpanda-for-fips-compliance[Install Redpanda for FIPS Compliance] -* https://github.com/openssl/openssl/blob/openssl-3.0.9/README-FIPS.md[OpenSSL FIPS Readme^] -* https://www.openssl.org/docs/man3.0/man7/fips_module.html[OpenSSL fips_module^] -* https://csrc.nist.gov/CSRC/media/projects/cryptographic-module-validation-program/documents/security-policies/140sp4282.pdf[OpenSSL FIPS Security Policy^] +---- + +== Configure FIPS mode with Docker + +The Redpanda FIPS Docker image (`docker.redpanda.com/redpandadata/redpanda:-fips`) is available for `amd64` and `arm64` architectures. The image includes the required OpenSSL files, pre-configured. + +Pass the FIPS broker configuration to the container the same way as any other Redpanda Docker deployment: either by mounting a configuration file or by passing settings as flags. + +[tabs] +====== +Mount a configuration file:: ++ +-- +. Create a `redpanda.yaml` with the required FIPS settings: ++ +[,yaml] +---- +redpanda: + fips_mode: enabled + openssl_config_file: /opt/redpanda/openssl/openssl.cnf + openssl_module_directory: /opt/redpanda/lib/ossl-modules/ +---- + +. Mount the file when starting the container: ++ +[,bash] +---- +docker run -d \ + --name=redpanda \ + -p 9092:9092 \ + -p 9644:9644 \ + -v /path/to/redpanda.yaml:/etc/redpanda/redpanda.yaml \ + docker.redpanda.com/redpandadata/redpanda:-fips \ + redpanda start --overprovisioned --smp 1 +---- +-- +Pass settings as flags:: ++ +-- +Pass the FIPS settings directly to `redpanda start`: + +[,bash] +---- +docker run -d \ + --name=redpanda \ + -p 9092:9092 \ + -p 9644:9644 \ + docker.redpanda.com/redpandadata/redpanda:-fips \ + redpanda start --overprovisioned --smp 1 \ + --set redpanda.fips_mode=enabled \ + --set redpanda.openssl_config_file=/opt/redpanda/openssl/openssl.cnf \ + --set redpanda.openssl_module_directory=/opt/redpanda/lib/ossl-modules/ +---- +-- +====== + +== Next steps + +* xref:deploy:redpanda/manual/production/production-deployment.adoc#install-redpanda-for-fips-compliance[Install Redpanda for FIPS Compliance] +// TODO: Confirm OpenSSL version and FIPS 140-3 certificate number with engineering (ENG-307). Update links below accordingly. +* https://github.com/openssl/openssl/blob/master/README-FIPS.md[OpenSSL FIPS Readme^] diff --git a/modules/manage/partials/audit-logging.adoc b/modules/manage/partials/audit-logging.adoc index ad1fe36a02..52020f974a 100644 --- a/modules/manage/partials/audit-logging.adoc +++ b/modules/manage/partials/audit-logging.adoc @@ -159,10 +159,11 @@ NOTE: The Included column captures whether the event itself is included (for exa |=== |Data Logging Level |Audit Event |Included? |Details -.11+|System Level +.12+|System Level |Date and time stamp for each entry |Yes |`time` field on each event |Successful and failed access attempts |Yes |The `status_id` field shows success/failure for all access attempts for which auditing is enabled |User ID |Yes |`user.name` +|User group memberships |Yes |`user.groups` field with type `idp_group`. Included in authentication events for OIDC users and in authorization events when a group ACL matches. See xref:manage:security/authorization/gbac.adoc[]. |User connect and disconnect time |No |Connect and disconnect time may be inferred from the presence or absence of activity. |Password change |Yes |For SCRAM users managed through Redpanda core, the Admin API call associated with the password change is logged. Note that this does not cover users synced from external IdPs, such as through OIDC. |Changes of security settings |Yes |For example, ACL creation is logged (kafka `create_acls`), and cluster configuration changes are logged (Admin API events) @@ -433,6 +434,6 @@ xref:manage:audit-logging/audit-log-samples.adoc[See samples of audit log messag include::shared:partial$suggested-reading.adoc[] - xref:reference:topic-properties.adoc[] -- xref:develop:config-topics.adoc[] +- xref:develop:manage-topics/config-topics.adoc[] endif::[] diff --git a/modules/manage/partials/authentication.adoc b/modules/manage/partials/authentication.adoc index ef2ae49d0e..aac5fdbe74 100644 --- a/modules/manage/partials/authentication.adoc +++ b/modules/manage/partials/authentication.adoc @@ -867,6 +867,8 @@ but can instead rely on the trusted authentication capabilities of established I Redpanda's implementation of OIDC provides SASL/OAUTHBEARER support for the Kafka API, and supports standard OIDC authentication across all other HTTP APIs, including Schema Registry, HTTP Proxy, and the Admin API. +TIP: With OIDC enabled, you can also use xref:manage:security/authorization/gbac.adoc[group-based access control (GBAC)] to assign Redpanda permissions to OIDC groups instead of individual users. To use GBAC, configure your IdP to include group claims in the access token (for example, a `groups` claim). See your IdP's documentation for how to add group claims to tokens. + include::manage:partial$security/oidc/limitations.adoc[leveloffset=+3] ===== OIDC credentials flow and access token validation diff --git a/modules/manage/partials/gbac-assign-group-role.adoc b/modules/manage/partials/gbac-assign-group-role.adoc new file mode 100644 index 0000000000..fb94b541be --- /dev/null +++ b/modules/manage/partials/gbac-assign-group-role.adoc @@ -0,0 +1,23 @@ +To assign a group to a role in {ui}: + +. From *Security* on the left navigation menu, select the *Roles* tab. + +. Select the role you want to assign the group to. + +. Click *Edit*. + +. In the *Principals* section, enter the group name using the `Group:` format. For example, `Group:engineering`. + +. Click *Update*. + +To remove a group from a role: + +. From *Security* on the left navigation menu, select the *Roles* tab. + +. Select the role that has the group assignment you want to remove. + +. Click *Edit*. + +. In the *Principals* section, remove the `Group:` entry. + +. Click *Update*. diff --git a/modules/manage/partials/gbac-create-group-acl.adoc b/modules/manage/partials/gbac-create-group-acl.adoc new file mode 100644 index 0000000000..6d80c65ad2 --- /dev/null +++ b/modules/manage/partials/gbac-create-group-acl.adoc @@ -0,0 +1,13 @@ +In {ui}, group-based ACLs are managed through roles. To create an ACL for an OIDC group: + +. From *Security* on the left navigation menu, select the *Roles* tab. + +. Click *Create role* to open the role creation form, or select an existing role and click *Edit*. + +. In the *Principals* field, enter the group principal using the `Group:` format. For example, `Group:engineering`. + +. Define the permissions (ACLs) you want to grant to users in the group. You can configure ACLs for clusters, topics, consumer groups, transactional IDs, Schema Registry subjects, and Schema Registry operations. + +. Click *Create* (or *Update* if editing an existing role). + +NOTE: {ui} assigns ACLs through roles. To grant permissions to a group, create a role for that group, add the group as a principal, and define the ACLs on the role. To create ACLs with a `Group:` principal directly (without creating a role), use `rpk`. diff --git a/modules/manage/partials/gbac-dp.adoc b/modules/manage/partials/gbac-dp.adoc new file mode 100644 index 0000000000..cfa7d0875d --- /dev/null +++ b/modules/manage/partials/gbac-dp.adoc @@ -0,0 +1,554 @@ +// tag::single-source[] +ifdef::env-cloud[:oidc-doc: security:cloud-authentication.adoc#single-sign-on] +ifndef::env-cloud[:oidc-doc: manage:security/authentication.adoc#oidc] +ifdef::env-cloud[:acl-doc: security:authorization/acl.adoc] +ifndef::env-cloud[:acl-doc: manage:security/authorization/acl.adoc] +ifdef::env-cloud[:rbac-doc: security:authorization/rbac/index.adoc] +ifndef::env-cloud[:rbac-doc: manage:security/authorization/rbac.adoc] + +Group-based access control (GBAC) lets you manage Redpanda permissions at scale using the groups that already exist in your identity provider (IdP). Instead of creating and maintaining per-user permissions in Redpanda, you define access once for a group and your IdP controls who belongs to it. When users join or leave a team, their Redpanda access updates automatically at next login with no changes needed in Redpanda. + +GBAC extends xref:{oidc-doc}[OIDC authentication] and supports two ways to grant permissions to groups: create xref:{acl-doc}[ACLs] with `Group:` principals, or assign groups as members of xref:{rbac-doc}[RBAC] roles. Both approaches can be used independently or together. + +After reading this page, you will be able to: + +* [ ] {learning-objective-1} +* [ ] {learning-objective-2} +* [ ] {learning-objective-3} + +== Prerequisites + +To use GBAC, you need: + +ifndef::env-cloud[] +* An xref:get-started:licensing/overview.adoc[Enterprise Edition] license applied to your cluster. +* Superuser access to configure cluster properties and manage ACLs. +endif::[] +* xref:{oidc-doc}[OIDC authentication] configured and enabled on your cluster. +* Your IdP configured to include group claims in the OIDC access token (for example, a `groups` claim). + +== How GBAC works + +When a user authenticates with OIDC, Redpanda reads a configurable claim from the JWT access token (for example, `$.groups`) and extracts the list of groups the user belongs to. Redpanda then matches those group names against `Group:` principals in its ACLs and role assignments. + +Group membership is managed entirely by your IdP. Redpanda never stores or manages group membership directly. It reads group information from the OIDC token at authentication time. Changes you make in the IdP (adding or removing group memberships) take effect at the user's next authentication, when a new token is issued. + +GBAC works across the following Redpanda APIs: + +* Kafka API +* Schema Registry +* HTTP Proxy + +=== Authorization patterns + +GBAC supports two usage patterns: + +* Group as an ACL principal: Create an ACL with a `Group:` principal. Users in that group receive that permission directly. +* Group assigned to a role: Assign a group as a member of a role-based access control (RBAC) role. All users in the group inherit the role's ACLs. + +Both patterns can be used together. When a user belongs to multiple groups, they inherit the combined permissions of all groups. + +Redpanda evaluates all authorization sources (user ACLs, role ACLs, group ACLs, and group-to-role ACLs) in a single unified flow. Deny rules are checked first across all sources. If any source produces a deny, Redpanda rejects the request regardless of allows from other sources. If no deny is found, Redpanda checks for an allow across all sources. If no allow is found, Redpanda denies the request by default. + +.Authorization evaluation flow +[mermaid,width=100%] +.... +flowchart LR + A[Request] --> B{"Check all sources\nfor deny"} + + B -- "Deny found" --> DENY["❌ Deny"] + B -- "No deny found" --> C{"Check all sources\nfor allow"} + + C -- "Allow found" --> ALLOW["✅ Allow"] + C -- "No allow found" --> DEFAULT["❌ Default deny"] + + style DENY fill:#f44,color:#fff + style ALLOW fill:#4a4,color:#fff + style DEFAULT fill:#f44,color:#fff + + subgraph sources [" "] + direction LR + S1["User ACLs"] + S2["Role ACLs\n(RBAC)"] + S3["Group ACLs"] + S4["Group→Role\nACLs"] + end +.... + +== Supported identity providers + +GBAC works with any OIDC-compliant identity provider. These providers are commonly used with Redpanda: + +* https://auth0.com/docs/secure/tokens/json-web-tokens/create-custom-claims[Auth0^]: Configure group claims in Auth0 Actions or Rules. +* https://developer.okta.com/docs/concepts/universal-directory/[Okta^]: Assign groups to applications and include them in token claims. +* https://learn.microsoft.com/en-us/entra/identity/hybrid/connect/how-to-connect-fed-group-claims[Microsoft Entra ID (Azure AD)^]: Configure group claims in the application manifest. + +For IdP-specific configuration steps, see your provider's documentation. + +== Limitations + +* Azure AD group limit: Users with more than 200 group memberships in Azure AD receive a URL reference in their token instead of a list of group names. Redpanda does not follow that URL and cannot resolve groups in this case. ++ +Mitigation: Filter token claims to include only the groups relevant to Redpanda. + +* Nested groups: Redpanda does not recursively resolve nested group hierarchies. If group A contains group B, only the direct memberships reported in the token are used. Use xref:reference:properties/cluster-properties.adoc#nested_group_behavior[`nested_group_behavior: suffix`] to extract the last path segment from hierarchical group names when needed. + +* No wildcard ACLs for groups: ACL matching for `Group:` principals uses literal string comparison only. Wildcard patterns are not supported. + +ifdef::env-cloud[] +== Register groups in Redpanda Cloud + +To assign an IdP group to a role or ACL, you must first register the group: + +[tabs] +==== +Cloud UI:: ++ +-- +. In the left navigation menu, select *Organization IAM*, then select the *Groups* tab. +. Click *Create group*. +. Enter a *Name* that matches the group in your IdP exactly (for example, `engineering`). +. Optionally, enter a *Description*, and configure a *Role binding* to assign the group to a role with a specific scope and resource. +. Click *Create*. +-- + +Control Plane API:: ++ +-- +Make a link:/api/doc/cloud-controlplane/operation/operation-groupservice_creategroup[`POST /v1/groups`] request to the xref:redpanda-cloud:manage:api/cloud-byoc-controlplane-api.adoc[Control Plane API]: + +[,bash] +---- +curl -X POST 'https://api.redpanda.com/v1/groups' \ + -H 'Content-Type: application/json' \ + -H 'Authorization: Bearer ' \ + -d '{ + "group": { + "name": "", + "description": "" + } + }' +---- + +Replace `` with the name that matches the group in your IdP (for example, `engineering`). The name must match exactly for GBAC to map the group correctly. +-- +==== +endif::[] + +== Create group-based ACLs + +You can grant permissions directly to a group by creating an xref:{acl-doc}[ACL] with a `Group:` principal. This works the same as creating an ACL for a user, but uses the `Group:` prefix instead of `User:`. + +[tabs] +==== +rpk:: ++ +-- +To grant cluster-level access to the `engineering` group: + +[,bash] +---- +rpk security acl create --allow-principal Group:engineering --operation describe --cluster +---- + +To grant topic-level access: + +[,bash] +---- +rpk security acl create \ + --allow-principal Group:engineering \ + --operation read,describe \ + --topic 'analytics-' \ + --resource-pattern-type prefixed +---- + +-- +{ui}:: ++ +-- +include::manage:partial$gbac-create-group-acl.adoc[] +-- +ifdef::env-cloud[] +Data Plane API:: ++ +-- +Make a link:/api/doc/cloud-dataplane/operation/operation-aclservice_createacl[`POST /v1/acls`] request with a `Group:` principal. For example, to grant the `engineering` group read access to a topic: + +[,bash] +---- +curl -X POST "${DATAPLANE_API_URL}/v1/acls" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "resource_type": "RESOURCE_TYPE_TOPIC", + "resource_name": "analytics-events", + "resource_pattern_type": "RESOURCE_PATTERN_TYPE_LITERAL", + "principal": "Group:engineering", + "host": "*", + "operation": "OPERATION_READ", + "permission_type": "PERMISSION_TYPE_ALLOW" + }' +---- +-- +endif::[] +==== + +== Assign groups to roles + +To manage permissions at scale, assign a group to an xref:{rbac-doc}[RBAC] role. All users in the group inherit the role's ACLs automatically. + +[tabs] +==== +rpk:: ++ +-- +To assign a group to a role: + +[,bash] +---- +rpk security role assign --principal Group: +---- + +For example, to assign the `engineering` group to the `DataEngineers` role: + +[,bash] +---- +rpk security role assign DataEngineers --principal Group:engineering +---- + +To remove a group from a role: + +[,bash] +---- +rpk security role unassign --principal Group: +---- + +For example: + +[,bash] +---- +rpk security role unassign DataEngineers --principal Group:engineering +---- +-- +{ui}:: ++ +-- +include::manage:partial$gbac-assign-group-role.adoc[] +-- +ifdef::env-cloud[] +Data Plane API:: ++ +-- +First, retrieve your cluster's Data Plane API URL: + +[,bash] +---- +export DATAPLANE_API_URL=$(curl -s https://api.redpanda.com/v1/clusters/ \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer " | jq -r .cluster.dataplane_api) +---- + +Make a link:/api/doc/cloud-dataplane/operation/operation-securityservice_updaterolemembership[`PUT /v1/roles/\{role_name}`] request to assign a group to a role: + +[,bash] +---- +curl -X PUT "${DATAPLANE_API_URL}/v1/roles/DataEngineers" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "add": [{"principal": "Group:engineering"}] + }' +---- + +To remove a group from a role, use the `remove` field: + +[,bash] +---- +curl -X PUT "${DATAPLANE_API_URL}/v1/roles/DataEngineers" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "remove": [{"principal": "Group:engineering"}] + }' +---- +-- +endif::[] +ifndef::env-cloud[] +Admin API:: ++ +-- +Use the Admin API link:/api/doc/admin/v2/group/endpoint-securityservice[`SecurityService`] operations to manage group-to-role assignments. Send all requests as `POST` with a JSON body. For guidance on using the Admin API (ConnectRPC), see xref:manage:use-admin-api.adoc[]. + +To assign a group to a role, make a link:/api/doc/admin/v2/operation/operation-redpanda-core-admin-v2-securityservice-addrolemembers[`POST AddRoleMembers`] request: + +[,bash] +---- +curl -u : \ + --request POST 'http://localhost:9644/redpanda.core.admin.v2.SecurityService/AddRoleMembers' \ + --header 'Content-Type: application/json' \ + --data '{ + "roleName": "DataEngineers", + "members": [{"group": {"name": "engineering"}}] + }' +---- + +To remove a group from a role, make a link:/api/doc/admin/v2/operation/operation-redpanda-core-admin-v2-securityservice-removerolemembers[`POST RemoveRoleMembers`] request: + +[,bash] +---- +curl -u : \ + --request POST 'http://localhost:9644/redpanda.core.admin.v2.SecurityService/RemoveRoleMembers' \ + --header 'Content-Type: application/json' \ + --data '{ + "roleName": "DataEngineers", + "members": [{"group": {"name": "engineering"}}] + }' +---- +-- +endif::[] +==== + +== View groups and roles + +Use the following commands to inspect group assignments and role memberships. + +=== List groups assigned to a role + +[tabs] +==== +rpk:: ++ +-- +To see which groups are assigned to a role, use `--print-members`. Groups are listed alongside other principals such as `User:` and appear as `Group:` entries: + +[,bash] +---- +rpk security role describe --print-members +---- + +For example: + +[,bash] +---- +rpk security role describe DataEngineers --print-members +---- + +To list all roles assigned to a specific group: + +[,bash] +---- +rpk security role list --principal Group: +---- + +For example: + +[,bash] +---- +rpk security role list --principal Group:engineering +---- +-- +{ui}:: ++ +-- +To view groups assigned to a role in {ui}: + +. From *Security* on the left navigation menu, select the *Roles* tab. + +. Select the role you want to inspect. + +. The role details page lists all principals, including any `Group:` entries. +-- +ifdef::env-cloud[] +Data Plane API:: ++ +-- +To list all members of a role (including groups), make a link:/api/doc/cloud-dataplane/operation/operation-securityservice_listrolemembers[`GET /v1/roles/\{role_name}/members`] request: + +[,bash] +---- +curl -X GET "${DATAPLANE_API_URL}/v1/roles/DataEngineers/members" \ + -H "Authorization: Bearer " +---- + +The response includes a `members` array. Group members appear with the `Group:` prefix in the `principal` field. + +To list all roles assigned to a specific group, make a link:/api/doc/cloud-dataplane/operation/operation-securityservice_listroles[`GET /v1/roles`] request with a principal filter: + +[,bash] +---- +curl -X GET "${DATAPLANE_API_URL}/v1/roles?filter.principal=Group:engineering" \ + -H "Authorization: Bearer " +---- +-- +endif::[] +ifndef::env-cloud[] +Admin API:: ++ +-- +These operations use the link:/api/doc/admin/v2/group/endpoint-securityservice[Admin API v2] `SecurityService`. Send all requests as `POST` with a JSON body. + +To retrieve a role's details including all members (users and groups), make a link:/api/doc/admin/v2/operation/operation-redpanda-core-admin-v2-securityservice-getrole[`POST GetRole`] request: + +[,bash] +---- +curl -u : \ + --request POST 'http://localhost:9644/redpanda.core.admin.v2.SecurityService/GetRole' \ + --header 'Content-Type: application/json' \ + --data '{"name": "DataEngineers"}' +---- + +The response includes a `members` array with both `user` and `group` entries. + +To list all roles, make a link:/api/doc/admin/v2/operation/operation-redpanda-core-admin-v2-securityservice-listroles[`POST ListRoles`] request: + +[,bash] +---- +curl -u : \ + --request POST 'http://localhost:9644/redpanda.core.admin.v2.SecurityService/ListRoles' \ + --header 'Content-Type: application/json' \ + --data '{}' +---- + +To verify how Redpanda resolves groups from an OIDC token, make a link:/api/doc/admin/v2/operation/operation-redpanda-core-admin-v2-securityservice-resolveoidcidentity[`POST ResolveOidcIdentity`] request. Pass the token in the `Authorization` header. The response includes the resolved `principal`, token expiry, and a `groups` field listing all groups extracted from the token: + +[,bash] +---- +curl \ + --request POST 'http://localhost:9644/redpanda.core.admin.v2.SecurityService/ResolveOidcIdentity' \ + --header 'Content-Type: application/json' \ + --header 'Authorization: Bearer ' \ + --data '{}' +---- +-- +endif::[] +==== + +== Customize token claim extraction + +Different identity providers store group information in different locations within the JWT token. + +ifdef::env-cloud[] +In Redpanda Cloud, group claim extraction is configured through your SSO connection settings. + +. In the Cloud UI, navigate to *Organization IAM > Single sign-on*, then select your IdP connection. +. Ensure the mapping mode is set to *use_map*. +. Configure *Attributes (JSON)* to map attribute names to claim paths, including `federated_groups` for group claims. ++ +A claim path is a https://goessner.net/articles/JsonPath/[JSON path^] expression that tells Redpanda where to find group information in the OIDC token. The appropriate claim path for each attribute may vary per IdP. ++ +For example, Okta exposes group claims in `${context.userinfo.groups}`. In this case, you must also include `groups` in *Userinfo scope*. +endif::[] + +ifndef::env-cloud[] +Two cluster properties control how Redpanda extracts group names: + +* xref:reference:properties/cluster-properties.adoc#oidc_group_claim_path[`oidc_group_claim_path`]: A https://goessner.net/articles/JsonPath/[JSON path^] expression that tells Redpanda where to find group information in the OIDC token. For example, Auth0 and Okta typically use a top-level `groups` claim (`$.groups`), while Keycloak nests roles inside `realm_access` (`$.realm_access.roles`). Default: `$.groups`. +* xref:reference:properties/cluster-properties.adoc#nested_group_behavior[`nested_group_behavior`]: Controls how Redpanda handles group names that use path-style notation (for example, `/departments/eng/platform`). Set to `none` to use the full path as-is, or `suffix` to extract only the last segment. Default: `none`. ++ +NOTE: When `nested_group_behavior` is set to `suffix`, groups that share a leaf name (for example, `/departments/eng/groupA` and `/departments/sales/groupA`) both resolve to `Group:groupA`. ACLs or role assignments for that principal apply to members of both groups. Design your group naming conventions to avoid unintended collisions. + +To update these properties, use xref:manage:cluster-maintenance/cluster-property-configuration.adoc[any configuration method] (`rpk cluster config set`, the Admin API, or Redpanda Console). Changes take effect immediately without a restart. +endif::[] + +=== Token structure examples + +The following examples show how Redpanda extracts group principals from different token formats. + +==== Flat group values (default) + +With `oidc_group_claim_path: "$.groups"`, Redpanda extracts principals `Group:engineering` and `Group:analytics` from the token. + +[,json] +---- +{"groups": ["engineering", "analytics"]} +---- + +==== Nested claim + +With `oidc_group_claim_path: "$.realm_access.roles"`, Redpanda extracts principals `Group:eng` and `Group:fin` from the token. + +[,json] +---- +{"realm_access": {"roles": ["eng", "fin"]}} +---- + +==== Path-style group names with no suffix extraction (default) + +With `nested_group_behavior: "none"` (the default), Redpanda maps the full path to principals `Group:/departments/eng/platform` and `Group:/departments/eng/infra`. + +[,json] +---- +{"groups": ["/departments/eng/platform", "/departments/eng/infra"]} +---- + +// Not supported in Cloud +ifndef::env-cloud[] +==== Path-style group names with suffix extraction + +When xref:reference:properties/cluster-properties.adoc#nested_group_behavior[`nested_group_behavior`] is set to `suffix`, Redpanda maps the last path segment to principals `Group:platform` and `Group:infra`. + +[,json] +---- +{"groups": ["/departments/eng/platform", "/departments/eng/infra"]} +---- +endif::[] + +==== CSV-formatted group claim + +Some identity providers return group claims as a single comma-separated string instead of an array. + +[,json] +---- +{"groups": "engineering,analytics,finance"} +---- + +Redpanda automatically splits comma-separated values and extracts principals `Group:engineering`, `Group:analytics`, and `Group:finance`. + +ifndef::env-cloud[] +== Troubleshoot GBAC + +If group-based permissions are not working as expected: + +* Decode the JWT access token from your IdP and verify that the expected group claims are present: ++ +[,bash] +---- +echo $ACCESS_TOKEN | cut -d. -f2 | base64 -d 2>/dev/null | jq . +---- ++ +Look for the claim that matches your xref:reference:properties/cluster-properties.adoc#oidc_group_claim_path[`oidc_group_claim_path`] configuration (default: `$.groups`). +* Use the `ResolveOidcIdentity` Admin API endpoint to verify which groups Redpanda extracts from a token. See <>. +* Verify that your cluster configuration matches the token structure: ++ +[,bash] +---- +rpk cluster config get oidc_group_claim_path +rpk cluster config get nested_group_behavior +---- +* Temporarily enable debug logging for the `security` logger to see all claims in the validated JWT: ++ +[,bash] +---- +rpk redpanda admin config log-level set security --level debug +---- ++ +This helps diagnose incorrect claim paths, missing groups, or token content issues. The debug level reverts automatically after the expiry period (default: 300 seconds). +endif::[] + +== Audit logging + +When xref:manage:audit-logging.adoc[audit logging] is enabled, Redpanda includes group information in the following event types: + +ifndef::env-cloud[] +* Authentication events: Events across Kafka API, HTTP Proxy, Schema Registry, and Admin API include the user's IdP group memberships in the `user.groups` field with type `idp_group`. +endif::[] +ifdef::env-cloud[] +* Authentication events: Events across Kafka API, HTTP Proxy, and Schema Registry include the user's IdP group memberships in the `user.groups` field with type `idp_group`. +endif::[] +* Authorization events: When an authorization decision matches a group ACL, the matched group appears in the `actor.user.groups` field with type `idp_group`. + +== Next steps + +* xref:manage:audit-logging.adoc[Set up audit logging] to monitor group-based access events. + +// end::single-source[] diff --git a/modules/manage/partials/remote-read-replicas.adoc b/modules/manage/partials/remote-read-replicas.adoc index ab0ee72961..027a59235b 100644 --- a/modules/manage/partials/remote-read-replicas.adoc +++ b/modules/manage/partials/remote-read-replicas.adoc @@ -36,9 +36,9 @@ You need the following: * An origin cluster with xref:{tiered-storage-link}#set-up-tiered-storage[Tiered Storage] set up. Multi-region buckets or containers are not supported. * A topic on the origin cluster, which you can use as a Remote Read Replica topic on the remote cluster. * A separate remote cluster. -** AWS: The remote cluster must be in the same region as the origin cluster's storage bucket/container. +** AWS: The remote cluster can be in the same or a different region as the origin cluster's S3 bucket. For cross-region Remote Read Replica topics, see <>. ** GCP: The remote cluster can be in the same or a different region as the bucket/container. -** Azure: Remote read replicas are not supported. +** Azure: Remote read replicas are not supported. include::shared:partial$enterprise-license.adoc[] @@ -56,7 +56,7 @@ You must configure access to the same object storage as the origin cluster. ifndef::env-kubernetes[] To set up a Remote Read Replica topic on a separate remote cluster: -. Create a remote cluster for the Remote Read Replica topic. For AWS, the remote cluster must be in the same region as the origin cluster's storage bucket/container. For GCP, the remote cluster can be in the same or a different region as the bucket/container. +. Create a remote cluster for the Remote Read Replica topic. For GCP, the remote cluster can be in the same or a different region as the bucket/container. For AWS, the remote cluster can be in the same or a different region, but cross-region Remote Read Replica topics require additional configuration. See <>. . Run `rpk cluster config edit`, and then specify properties specific to your object storage provider (your cluster will require a restart after any changes to these properties): + @@ -437,6 +437,50 @@ rpk topic create -c redpanda.remote.readreplica= * Do not use `redpanda.remote.read` or `redpanda.remote.write` with `redpanda.remote.readreplica`. Redpanda ignores the values for remote read and remote write properties on read replica topics. ==== +[[create-cross-region-rrr-topic]] +=== Create a cross-region Remote Read Replica topic on AWS + +Use this configuration only when the remote cluster is in a *different AWS region* than the origin cluster's S3 bucket. For same-region AWS or GCP deployments, use the standard <>. + +==== Prerequisites + +You must explicitly set the xref:reference:properties/object-storage-properties.adoc#cloud_storage_url_style[`cloud_storage_url_style`] cluster property to `virtual_host` or `path` on the remote cluster. The default value does not support cross-region Remote Read Replicas. + +==== Create the topic + +To create a cross-region Remote Read Replica topic, append `region` and `endpoint` query-string parameters to the bucket name. + +In the following example, replace the placeholders: + +- ``: The name of the topic in the cluster hosting the Remote Read Replica. +- ``: The S3 bucket configured on the origin cluster (`cloud_storage_bucket`). +- ``: The AWS region of the origin cluster's S3 bucket (not the remote cluster's region). + +[,bash] +---- +rpk topic create \ + -c redpanda.remote.readreplica=?region=&endpoint=s3..amazonaws.com +---- + +For example, if the origin cluster stores data in a bucket called `my-bucket` in `us-east-1`: + +[,bash] +---- +rpk topic create my-topic \ + -c redpanda.remote.readreplica=my-bucket?region=us-east-1&endpoint=s3.us-east-1.amazonaws.com +---- + +NOTE: The `endpoint` value must not include the bucket name. When using `virtual_host` URL style, Redpanda automatically prepends the bucket name to the endpoint. When using `path` URL style, Redpanda appends the bucket name as a path segment. + +==== Limits + +Each unique combination of region and endpoint creates a separate object storage target on the remote cluster. A cluster supports a maximum of 10 targets. + +How targets are counted depends on `cloud_storage_url_style`: + +- `virtual_host`: Each unique combination of bucket, region, and endpoint counts as one target. You can create up to 10 distinct cross-region Remote Read Replica topics for each cluster. +- `path`: Each unique combination of region and endpoint counts as one target (the bucket name is not part of the key). You can create cross-region Remote Read Replica topics for multiple buckets using the same region/endpoint combination, with a maximum of 10 distinct region/endpoint combinations for each cluster. + == Reduce lag in data availability :config-ref: cloud_storage_segment_max_upload_interval_sec diff --git a/modules/manage/partials/storage-modes-table.adoc b/modules/manage/partials/storage-modes-table.adoc new file mode 100644 index 0000000000..37decd1b1e --- /dev/null +++ b/modules/manage/partials/storage-modes-table.adoc @@ -0,0 +1,18 @@ +// tag::storage-modes[] +[cols="1,3"] +|=== +| Mode | Behavior + +| `unset` +| Follows legacy behavior. Topics inherit cluster-level remote read/write settings. + +| `local` +| Data is stored only on local disk. No remote storage is used. + +| `tiered` +| Data is written locally and offloaded to object storage asynchronously using Tiered Storage. + +| `cloud` +| Data is managed primarily in object storage. Local storage acts as a cache. +|=== +// end::storage-modes[] diff --git a/modules/manage/partials/tiered-storage.adoc b/modules/manage/partials/tiered-storage.adoc index d2434598bc..a3f8fc6dc8 100644 --- a/modules/manage/partials/tiered-storage.adoc +++ b/modules/manage/partials/tiered-storage.adoc @@ -869,32 +869,50 @@ To enable Tiered Storage for a cluster (in addition to setting `cloud_storage_en * config_ref:cloud_storage_enable_remote_write,true,properties/object-storage-properties[] * config_ref:cloud_storage_enable_remote_read,true,properties/object-storage-properties[] -When you enable Tiered Storage for a cluster, you enable it for all existing topics in the cluster. When cluster-level properties are changed, the changes apply only to new topics, not existing topics. You must restart your cluster after enabling Tiered Storage. +When you enable Tiered Storage for a cluster, you enable it for all existing topics in the cluster. You must restart your cluster after enabling Tiered Storage. -NOTE: The `cloud_storage_enable_remote_write` and `cloud_storage_enable_remote_read` cluster-level properties are essentially creation-time defaults for the `redpanda.remote.write` and `redpanda.remote.read` topic-level properties. +NOTE: `cloud_storage_enable_remote_write` and `cloud_storage_enable_remote_read` act as creation-time defaults only for topics with `redpanda.storage.mode=unset`. They have no effect on topics where `redpanda.storage.mode` is explicitly set to `local`, `tiered`, or `cloud`. To apply a default storage mode to all new topics, use the config_ref:default_redpanda_storage_mode,true,properties/cluster-properties[] cluster property instead. ==== Enable Tiered Storage for specific topics -To enable Tiered Storage for a new or existing topic (in addition to setting `cloud_storage_enabled` to `true`), set the following topic-level properties to `true`: +Starting in Redpanda v26.1, the recommended way to enable Tiered Storage for a topic is to set the `redpanda.storage.mode` topic property to `tiered`: -* `redpanda.remote.write` -* `redpanda.remote.read` +[,bash] +---- +rpk topic create -c redpanda.storage.mode=tiered +---- -For example, to create a new topic with Tiered Storage: +To enable Tiered Storage on an existing topic that was created in `local` mode: + +[,bash] +---- +rpk topic alter-config --set redpanda.storage.mode=tiered +---- + +When `redpanda.storage.mode=tiered` is set, Tiered Storage is fully enabled for the topic. The `redpanda.remote.read` and `redpanda.remote.write` topic properties have no effect on a topic's storage when `redpanda.storage.mode` is set to any value other than `unset`. + +==== Configure Tiered Storage using legacy topic properties + +For topics with `redpanda.storage.mode=unset` (the default when `default_redpanda_storage_mode` is not configured), Tiered Storage is controlled by the `redpanda.remote.read` and `redpanda.remote.write` topic properties: + +* `redpanda.remote.write`: Uploads data from Redpanda to object storage. +* `redpanda.remote.read`: Fetches data from object storage to Redpanda. + +For example, to create a topic using the legacy properties when the storage mode is `unset`: [,bash] ---- rpk topic create -c redpanda.remote.read=true -c redpanda.remote.write=true ---- -To enable Tiered Storage on an existing topic, run: +To enable Tiered Storage on an existing topic where the storage mode is `unset`: [,bash] ---- rpk topic alter-config --set redpanda.remote.read=true --set redpanda.remote.write=true ---- -Topic-level properties override cluster-level properties. For example, for new topics, if `cloud_storage_enable_remote_write` is set to `true`, you can set `redpanda.remote.write` to `false` to turn it off for a particular topic. +For newly-created unset topics, the cluster-level `cloud_storage_enable_remote_write` and `cloud_storage_enable_remote_read` properties dictate the topic-level properties `redpanda.remote.write` and `redpanda.remote.read` at topic creation time, respectively. Altering the cluster properties has no effect on existing topics, only newly-created ones. To alter the permissions for existing topics, you can set these topic properties directly. For example, `redpanda.remote.write=false` to disable uploads for a specific topic. Tiered Storage topic-level properties: @@ -902,23 +920,25 @@ Tiered Storage topic-level properties: | Property | Description | `redpanda.remote.write` -| Uploads data from Redpanda to object storage. Overrides the cluster-level `cloud_storage_enable_remote_write` configuration for the topic. +| Uploads data from Redpanda to object storage. For topics with `redpanda.storage.mode=unset`, overrides the cluster-level `cloud_storage_enable_remote_write` setting. Has no effect on topics with `redpanda.storage.mode` set to `local`, `tiered`, or `cloud`. | `redpanda.remote.read` -| Fetches data from object storage to Redpanda. Overrides the cluster-level `cloud_storage_enable_remote_read` configuration for the topic. +| Fetches data from object storage to Redpanda. For topics with `redpanda.storage.mode=unset`, overrides the cluster-level `cloud_storage_enable_remote_read` setting. Has no effect on topics with `redpanda.storage.mode` set to `local`, `tiered`, or `cloud`. | `redpanda.remote.recovery` | Recovers or reproduces a topic from object storage. Use this property during topic creation. It does not apply to existing topics. | `redpanda.remote.delete` -| When set to `true`, deleting a topic also deletes its objects in object storage. Both `redpanda.remote.write` and `redpanda.remote.read` must be enabled, and the topic must not be a Remote Read Replica topic. +| When set to `true`, deleting a topic also deletes its objects in object storage. Applies to both Tiered Storage topics and Cloud Topics. + +For Tiered Storage topics, the topic must not be a Remote Read Replica topic. When set to `false`, deleting a topic does not delete its objects in object storage. Default is `true` for new topics. |=== -The following tables list outcomes for combinations of cluster-level and topic-level configurations: +The following tables show outcomes for combinations of cluster-level and topic-level configurations for topics with `redpanda.storage.mode=unset` at topic creation time: |=== | Cluster-level configuration with `cloud_storage_enable_remote_write` | Topic-level configuration with `redpanda.remote.write` | Outcome: whether remote write is enabled or disabled on the topic diff --git a/modules/reference/attachments/redpanda-properties-v25.3.11.json b/modules/reference/attachments/redpanda-properties-v26.1.1-rc5.json similarity index 96% rename from modules/reference/attachments/redpanda-properties-v25.3.11.json rename to modules/reference/attachments/redpanda-properties-v26.1.1-rc5.json index 7210a97fc6..f72db52b73 100644 --- a/modules/reference/attachments/redpanda-properties-v25.3.11.json +++ b/modules/reference/attachments/redpanda-properties-v26.1.1-rc5.json @@ -36,8 +36,23 @@ "config::leaders_preference": { "defined_in": "config/leaders_preference.h", "properties": { + "none_str": { + "type": "string" + }, + "ordered_racks_prefix": { + "type": "string" + }, + "ordered_racks_str": { + "type": "string" + }, "racks": { "type": "array" + }, + "racks_prefix": { + "type": "string" + }, + "racks_str": { + "type": "string" } }, "type": "object" @@ -221,6 +236,18 @@ }, "type": "enum" }, + "model::leader_balancer_mode": { + "defined_in": "model/metadata.h", + "enum": [ + "calibrated", + "random" + ], + "enum_string_mappings": { + "calibrated": "calibrated", + "random": "random" + }, + "type": "enum" + }, "model::node_id": { "alias_for": "named_type", "defined_in": "model/fundamental.h", @@ -256,6 +283,22 @@ }, "type": "enum" }, + "model::redpanda_storage_mode": { + "defined_in": "model/metadata.h", + "enum": [ + "local", + "tiered", + "cloud", + "unset" + ], + "enum_string_mappings": { + "cloud": "cloud", + "local": "local", + "tiered": "tiered", + "unset": "unset" + }, + "type": "enum" + }, "model::timestamp_type": { "defined_in": "model/timestamp.h", "enum": [ @@ -525,27 +568,6 @@ "type": "integer", "visibility": "tunable" }, - "alter_topic_cfg_timeout_ms": { - "c_type": "std::chrono::milliseconds", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "default": 5000, - "default_human_readable": "5 seconds", - "defined_in": "src/v/config/configuration.cc", - "description": "The duration, in milliseconds, that Redpanda waits for the replication of entries in the controller log when executing a request to alter topic configurations. This timeout ensures that configuration changes are replicated across the cluster before the alteration request is considered complete.", - "is_deprecated": false, - "is_enterprise": false, - "maximum": 17592186044415, - "minimum": -17592186044416, - "name": "alter_topic_cfg_timeout_ms", - "needs_restart": false, - "nullable": false, - "type": "integer", - "visibility": "tunable" - }, "api_doc_dir": { "c_type": "ss::sstring", "category": "pandaproxy", @@ -1766,22 +1788,6 @@ "type": "boolean", "visibility": "tunable" }, - "cloud_storage_disable_metadata_consistency_checks": { - "c_type": "deprecated_property", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, - "is_enterprise": false, - "name": "cloud_storage_disable_metadata_consistency_checks", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" - }, "cloud_storage_disable_read_replica_loop_for_tests": { "c_type": "bool", "cloud_byoc_only": false, @@ -2702,42 +2708,47 @@ "type": "integer", "visibility": "tunable" }, - "cloud_storage_readreplica_manifest_sync_timeout_ms": { - "c_type": "std::chrono::milliseconds", + "cloud_storage_prefetch_segments_max": { + "c_type": "uint16_t", "cloud_byoc_only": false, "cloud_editable": false, "cloud_readonly": false, "cloud_supported": false, "config_scope": "cluster", - "default": 30000, - "default_human_readable": "30 seconds", + "default": 3, "defined_in": "src/v/config/configuration.cc", - "description": "Timeout to check if new data is available for partitions in object storage for read replicas.", + "description": "Maximum number of small segments (size <= chunk size) to prefetch ahead during sequential reads. Set to 0 to disable cross-segment prefetch.", "is_deprecated": false, "is_enterprise": false, - "maximum": 17592186044415, - "minimum": -17592186044416, - "name": "cloud_storage_readreplica_manifest_sync_timeout_ms", + "maximum": 65535, + "minimum": 0, + "name": "cloud_storage_prefetch_segments_max", "needs_restart": false, "nullable": false, "type": "integer", + "version": "v26.1.1", "visibility": "tunable" }, - "cloud_storage_reconciliation_ms": { - "c_type": "deprecated_property", + "cloud_storage_readreplica_manifest_sync_timeout_ms": { + "c_type": "std::chrono::milliseconds", "cloud_byoc_only": false, "cloud_editable": false, "cloud_readonly": false, "cloud_supported": false, "config_scope": "cluster", + "default": 30000, + "default_human_readable": "30 seconds", "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, + "description": "Timeout to check if new data is available for partitions in object storage for read replicas.", + "is_deprecated": false, "is_enterprise": false, - "name": "cloud_storage_reconciliation_ms", - "needs_restart": true, + "maximum": 17592186044415, + "minimum": -17592186044416, + "name": "cloud_storage_readreplica_manifest_sync_timeout_ms", + "needs_restart": false, "nullable": false, - "type": "deprecated_property" + "type": "integer", + "visibility": "tunable" }, "cloud_storage_recovery_temporary_retention_bytes_default": { "c_type": "size_t", @@ -3216,6 +3227,126 @@ "type": "string", "visibility": "user" }, + "cloud_topics_allow_materialization_failure": { + "c_type": "bool", + "cloud_byoc_only": false, + "cloud_editable": false, + "cloud_readonly": false, + "cloud_supported": false, + "config_scope": "cluster", + "default": false, + "defined_in": "src/v/config/configuration.cc", + "description": "When enabled, the reconciler tolerates missing L0 extent objects (404 errors) during materialization. Failed extents are skipped, producing L1 state with empty offset ranges where deleted data was. Use this to recover partitions after accidental deletion of live extent objects.", + "is_deprecated": false, + "is_enterprise": false, + "name": "cloud_topics_allow_materialization_failure", + "needs_restart": false, + "nullable": false, + "type": "boolean", + "version": "v26.1.1", + "visibility": "tunable" + }, + "cloud_topics_compaction_interval_ms": { + "c_type": "std::chrono::milliseconds", + "cloud_byoc_only": false, + "cloud_editable": false, + "cloud_readonly": false, + "cloud_supported": false, + "config_scope": "cluster", + "default": 30000, + "default_human_readable": "30 seconds", + "defined_in": "src/v/config/configuration.cc", + "description": "How often to trigger background compaction for cloud topics.", + "is_deprecated": false, + "is_enterprise": false, + "maximum": 17592186044415, + "minimum": -17592186044416, + "name": "cloud_topics_compaction_interval_ms", + "needs_restart": false, + "nullable": false, + "type": "integer", + "version": "v26.1.1", + "visibility": "tunable" + }, + "cloud_topics_compaction_key_map_memory": { + "c_type": "uint64_t", + "cloud_byoc_only": false, + "cloud_editable": false, + "cloud_readonly": false, + "cloud_supported": false, + "config_scope": "cluster", + "default": 134217728, + "defined_in": "src/v/config/configuration.cc", + "description": "Maximum number of bytes that may be used on each shard by cloud topics compaction key-offset maps.", + "example": "`134217728`", + "is_deprecated": false, + "is_enterprise": false, + "maximum": 18446744073709551615, + "minimum": 0, + "name": "cloud_topics_compaction_key_map_memory", + "needs_restart": true, + "nullable": false, + "type": "integer", + "version": "v26.1.1", + "visibility": "tunable" + }, + "cloud_topics_compaction_max_object_size": { + "c_type": "size_t", + "cloud_byoc_only": false, + "cloud_editable": false, + "cloud_readonly": false, + "cloud_supported": false, + "config_scope": "cluster", + "default": 134217728, + "defined_in": "src/v/config/configuration.cc", + "description": "Maximum size in bytes for L1 objects produced by cloud topics compaction.", + "is_deprecated": false, + "is_enterprise": false, + "name": "cloud_topics_compaction_max_object_size", + "needs_restart": false, + "nullable": false, + "type": "integer", + "version": "v26.1.1", + "visibility": "tunable" + }, + "cloud_topics_disable_level_zero_gc_for_tests": { + "c_type": "bool", + "cloud_byoc_only": false, + "cloud_editable": false, + "cloud_readonly": false, + "cloud_supported": false, + "config_scope": "cluster", + "default": false, + "defined_in": "src/v/config/configuration.cc", + "description": "Disables the level-zero garbage collector in cloud topics. This property exists to simplify testing and shouldn't be set in production.", + "is_deprecated": false, + "is_enterprise": false, + "name": "cloud_topics_disable_level_zero_gc_for_tests", + "needs_restart": true, + "nullable": false, + "type": "boolean", + "version": "v26.1.1", + "visibility": "tunable" + }, + "cloud_topics_disable_metastore_flush_loop_for_tests": { + "c_type": "bool", + "cloud_byoc_only": false, + "cloud_editable": false, + "cloud_readonly": false, + "cloud_supported": false, + "config_scope": "cluster", + "default": false, + "defined_in": "src/v/config/configuration.cc", + "description": "Disables the metastore flush loop in cloud topics. The property exists to simplify testing of read replicas and shouldn't be set in production.", + "is_deprecated": false, + "is_enterprise": false, + "name": "cloud_topics_disable_metastore_flush_loop_for_tests", + "needs_restart": true, + "nullable": false, + "type": "boolean", + "version": "v26.1.1", + "visibility": "tunable" + }, "cloud_topics_disable_reconciliation_loop": { "c_type": "bool", "cloud_byoc_only": false, @@ -3226,7 +3357,6 @@ "default": false, "defined_in": "src/v/config/configuration.cc", "description": "Disables the cloud topics reconciliation loop. Disabling the loop can negatively impact performance and stability of the cluster.", - "exclude_from_docs": true, "is_deprecated": false, "is_enterprise": false, "name": "cloud_topics_disable_reconciliation_loop", @@ -3244,7 +3374,7 @@ "config_scope": "cluster", "default": false, "defined_in": "src/v/config/configuration.cc", - "description": "Enable cloud topics.", + "description": "Enable Cloud Topics for the cluster. Cloud Topics are optimized for high-throughput, cost-sensitive workloads that can tolerate higher latencies compared to standard Kafka topics.", "enterprise_constructor": "restricted_only", "enterprise_restricted_value": [ "true" @@ -3252,12 +3382,14 @@ "enterprise_value": [ "true" ], - "exclude_from_docs": true, "is_deprecated": false, "is_enterprise": true, "name": "cloud_topics_enabled", - "needs_restart": false, + "needs_restart": true, "nullable": false, + "related_topics": [ + "xref:develop:manage-topics/cloud-topics.adoc[Cloud Topics]" + ], "type": "boolean", "visibility": "user" }, @@ -3305,152 +3437,550 @@ "version": "v25.3.3", "visibility": "tunable" }, - "cloud_topics_long_term_garbage_collection_interval": { + "cloud_topics_epoch_service_max_same_epoch_duration": { "c_type": "std::chrono::milliseconds", "cloud_byoc_only": false, "cloud_editable": false, "cloud_readonly": false, "cloud_supported": false, "config_scope": "cluster", - "default": 300000, - "default_human_readable": "5 minutes", + "default": 86400000, + "default_human_readable": "1 day", "defined_in": "src/v/config/configuration.cc", - "description": "Time interval after which data is garbage collected from long term storage.", - "exclude_from_docs": true, + "description": "The duration of time that a node can use the exact same epoch.", "is_deprecated": false, "is_enterprise": false, "maximum": 17592186044415, "minimum": -17592186044416, - "name": "cloud_topics_long_term_garbage_collection_interval", + "name": "cloud_topics_epoch_service_max_same_epoch_duration", "needs_restart": false, "nullable": false, "type": "integer", + "version": "v26.1.1", "visibility": "tunable" }, - "cloud_topics_produce_batching_size_threshold": { - "c_type": "size_t", + "cloud_topics_fetch_debounce_enabled": { + "c_type": "bool", "cloud_byoc_only": false, "cloud_editable": false, "cloud_readonly": false, "cloud_supported": false, "config_scope": "cluster", - "default": 4194304, + "default": true, "defined_in": "src/v/config/configuration.cc", - "description": "The size limit for the object size in cloud topics. When the amount of data on a shard reaches this limit, an upload is triggered.", - "exclude_from_docs": true, + "description": "Enables fetch debouncing in cloud topics. This mechanism guarantees that the broker fetches every object only once improving the performance and lowering the cost.", "is_deprecated": false, "is_enterprise": false, - "name": "cloud_topics_produce_batching_size_threshold", - "needs_restart": false, + "name": "cloud_topics_fetch_debounce_enabled", + "needs_restart": true, "nullable": false, - "type": "integer", + "type": "boolean", + "version": "v26.1.1", "visibility": "user" }, - "cloud_topics_produce_cardinality_threshold": { - "c_type": "size_t", + "cloud_topics_gc_health_check_interval": { + "c_type": "std::chrono::milliseconds", "cloud_byoc_only": false, "cloud_editable": false, "cloud_readonly": false, "cloud_supported": false, "config_scope": "cluster", - "default": 1000, + "default": 10000, + "default_human_readable": "10 seconds", "defined_in": "src/v/config/configuration.cc", - "description": "Threshold for the object cardinality in cloud topics. When the number of partitions in waiting for the upload reach this limit, an upload is triggered.", - "exclude_from_docs": true, + "description": "The interval at which the L0 garbage collector checks cluster health. GC will not proceed while the cluster is unhealthy.", "is_deprecated": false, "is_enterprise": false, - "name": "cloud_topics_produce_cardinality_threshold", + "maximum": 17592186044415, + "minimum": -17592186044416, + "name": "cloud_topics_gc_health_check_interval", "needs_restart": false, "nullable": false, "type": "integer", - "visibility": "user" + "version": "v26.1.1", + "visibility": "tunable" }, - "cloud_topics_produce_upload_interval": { - "c_type": "std::chrono::milliseconds", + "cloud_topics_l1_indexing_interval": { + "c_type": "size_t", "cloud_byoc_only": false, "cloud_editable": false, "cloud_readonly": false, "cloud_supported": false, "config_scope": "cluster", - "default": 250, - "default_human_readable": "250 milliseconds", + "default": 4194304, "defined_in": "src/v/config/configuration.cc", - "description": "Time interval after which the upload is triggered.", - "exclude_from_docs": true, + "description": "The byte interval at which index entries are created within long term storage objects for cloud topics. Index entries are stored in the object metadata and enable efficient seeking by offset or timestamp within a partition. Lower values produce more index entries (better seek granularity) at the cost of a larger footer.", "is_deprecated": false, "is_enterprise": false, - "maximum": 17592186044415, - "minimum": -17592186044416, - "name": "cloud_topics_produce_upload_interval", + "name": "cloud_topics_l1_indexing_interval", "needs_restart": false, "nullable": false, "type": "integer", - "visibility": "user" + "version": "v26.1.1", + "visibility": "tunable" }, - "cloud_topics_reconciliation_interval": { + "cloud_topics_long_term_file_deletion_delay": { "c_type": "std::chrono::milliseconds", "cloud_byoc_only": false, "cloud_editable": false, "cloud_readonly": false, "cloud_supported": false, "config_scope": "cluster", - "default": 10000, - "default_human_readable": "10 seconds", + "default": 3600000, + "default_human_readable": "1 hour", "defined_in": "src/v/config/configuration.cc", - "description": "Time interval after which data is moved from short term storage to long term storage.", - "exclude_from_docs": true, + "description": "Delay before deleting stale long term files, allowing concurrent readers (e.g. read replica topics) to finish reading them before removal.", "is_deprecated": false, "is_enterprise": false, "maximum": 17592186044415, "minimum": -17592186044416, - "name": "cloud_topics_reconciliation_interval", + "name": "cloud_topics_long_term_file_deletion_delay", "needs_restart": false, "nullable": false, "type": "integer", + "version": "v26.1.1", "visibility": "tunable" }, - "cloud_topics_short_term_gc_backoff_interval": { + "cloud_topics_long_term_flush_interval": { "c_type": "std::chrono::milliseconds", "cloud_byoc_only": false, "cloud_editable": false, "cloud_readonly": false, "cloud_supported": false, "config_scope": "cluster", - "default": 60000, - "default_human_readable": "1 minute", + "default": 600000, + "default_human_readable": "10 minutes", "defined_in": "src/v/config/configuration.cc", - "description": "The interval, in milliseconds, between invocations of the L0 garbage collection work loop when no progress is being made or errors are occurring.\n\nL0 (level-zero) objects are short-term data objects in Tiered Storage that are periodically garbage collected. When GC encounters errors or cannot make progress (for example, if there are no objects eligible for deletion), this backoff interval prevents excessive retries.\n\nIncrease this value to reduce system load when GC cannot make progress. Decrease it if you need faster retry attempts after transient errors.", + "description": "Time interval at which long term storage metadata is flushed to object storage.", "is_deprecated": false, "is_enterprise": false, "maximum": 17592186044415, "minimum": -17592186044416, - "name": "cloud_topics_short_term_gc_backoff_interval", + "name": "cloud_topics_long_term_flush_interval", "needs_restart": false, "nullable": false, "type": "integer", - "version": "v25.3.3", + "version": "v26.1.1", "visibility": "tunable" }, - "cloud_topics_short_term_gc_interval": { + "cloud_topics_long_term_garbage_collection_interval": { "c_type": "std::chrono::milliseconds", "cloud_byoc_only": false, "cloud_editable": false, "cloud_readonly": false, "cloud_supported": false, "config_scope": "cluster", - "default": 10000, - "default_human_readable": "10 seconds", + "default": 300000, + "default_human_readable": "5 minutes", "defined_in": "src/v/config/configuration.cc", - "description": "The interval, in milliseconds, between invocations of the L0 (level-zero) garbage collection work loop when progress is being made.\n\nL0 objects are short-term data objects in Tiered Storage associated with global epochs. This property controls how frequently GC runs when it successfully deletes objects. Lower values increase GC frequency, which can help maintain lower object counts but may increase S3 API usage.\n\nDecrease this value if L0 object counts are growing too quickly and you need more aggressive garbage collection. Increase it to reduce S3 API costs in clusters with lower ingestion rates.", + "description": "Time interval after which data is garbage collected from long term storage.", "is_deprecated": false, "is_enterprise": false, "maximum": 17592186044415, "minimum": -17592186044416, - "name": "cloud_topics_short_term_gc_interval", + "name": "cloud_topics_long_term_garbage_collection_interval", "needs_restart": false, "nullable": false, "type": "integer", - "version": "v25.3.3", + "visibility": "tunable" + }, + "cloud_topics_metastore_lsm_apply_timeout_ms": { + "c_type": "std::chrono::milliseconds", + "cloud_byoc_only": false, + "cloud_editable": false, + "cloud_readonly": false, + "cloud_supported": false, + "config_scope": "cluster", + "default": 300000, + "default_human_readable": "5 minutes", + "defined_in": "src/v/config/configuration.cc", + "description": "Timeout for applying a replicated write batch to the local LSM database. This may take longer than usual when L0 compaction is behind and writes are being throttled.", + "is_deprecated": false, + "is_enterprise": false, + "maximum": 17592186044415, + "minimum": -17592186044416, + "name": "cloud_topics_metastore_lsm_apply_timeout_ms", + "needs_restart": false, + "nullable": false, + "type": "integer", + "version": "v26.1.1", + "visibility": "tunable" + }, + "cloud_topics_metastore_replication_timeout_ms": { + "c_type": "std::chrono::milliseconds", + "cloud_byoc_only": false, + "cloud_editable": false, + "cloud_readonly": false, + "cloud_supported": false, + "config_scope": "cluster", + "default": 30000, + "default_human_readable": "30 seconds", + "defined_in": "src/v/config/configuration.cc", + "description": "Timeout for L1 metastore Raft replication and waiting for the STM to apply the replicated write batch.", + "is_deprecated": false, + "is_enterprise": false, + "maximum": 17592186044415, + "minimum": -17592186044416, + "name": "cloud_topics_metastore_replication_timeout_ms", + "needs_restart": false, + "nullable": false, + "type": "integer", + "version": "v26.1.1", + "visibility": "tunable" + }, + "cloud_topics_num_metastore_partitions": { + "c_type": "int32_t", + "cloud_byoc_only": false, + "cloud_editable": false, + "cloud_readonly": false, + "cloud_supported": false, + "config_scope": "cluster", + "default": 3, + "defined_in": "src/v/config/configuration.cc", + "description": "Number of partitions for the cloud topics metastore topic, used to spread metastore load across the cluster. Higher values allow more parallel metadata operations but reduce the amount of work each partition can batch together. Only takes effect when the metastore topic is first created.", + "is_deprecated": false, + "is_enterprise": false, + "maximum": 2147483647, + "minimum": -2147483648, + "name": "cloud_topics_num_metastore_partitions", + "needs_restart": true, + "nullable": false, + "type": "integer", + "version": "v26.1.1", + "visibility": "tunable" + }, + "cloud_topics_parallel_fetch_enabled": { + "c_type": "bool", + "cloud_byoc_only": false, + "cloud_editable": false, + "cloud_readonly": false, + "cloud_supported": false, + "config_scope": "cluster", + "default": true, + "defined_in": "src/v/config/configuration.cc", + "description": "Enable parallel fetching in cloud topics. This mechanism improves the throughput by allowing the broker to download data needed by the fetch request using multiple shards.", + "is_deprecated": false, + "is_enterprise": false, + "name": "cloud_topics_parallel_fetch_enabled", + "needs_restart": true, + "nullable": false, + "type": "boolean", + "version": "v26.1.1", + "visibility": "user" + }, + "cloud_topics_preregistered_object_ttl": { + "c_type": "std::chrono::milliseconds", + "cloud_byoc_only": false, + "cloud_editable": false, + "cloud_readonly": false, + "cloud_supported": false, + "config_scope": "cluster", + "default": 3600000, + "default_human_readable": "1 hour", + "defined_in": "src/v/config/configuration.cc", + "description": "Time-to-live for pre-registered L1 objects before they are expired.", + "is_deprecated": false, + "is_enterprise": false, + "maximum": 17592186044415, + "minimum": -17592186044416, + "name": "cloud_topics_preregistered_object_ttl", + "needs_restart": false, + "nullable": false, + "type": "integer", + "version": "v26.1.1", + "visibility": "tunable" + }, + "cloud_topics_produce_batching_size_threshold": { + "c_type": "size_t", + "cloud_byoc_only": false, + "cloud_editable": false, + "cloud_readonly": false, + "cloud_supported": false, + "config_scope": "cluster", + "default": 4194304, + "defined_in": "src/v/config/configuration.cc", + "description": "The size limit for the object size in cloud topics. When the amount of data on a shard reaches this limit, an upload is triggered.", + "is_deprecated": false, + "is_enterprise": false, + "name": "cloud_topics_produce_batching_size_threshold", + "needs_restart": false, + "nullable": false, + "type": "integer", + "visibility": "user" + }, + "cloud_topics_produce_cardinality_threshold": { + "c_type": "size_t", + "cloud_byoc_only": false, + "cloud_editable": false, + "cloud_readonly": false, + "cloud_supported": false, + "config_scope": "cluster", + "default": 1000, + "defined_in": "src/v/config/configuration.cc", + "description": "Threshold for the object cardinality in cloud topics. When the number of partitions in waiting for the upload reach this limit, an upload is triggered.", + "is_deprecated": false, + "is_enterprise": false, + "name": "cloud_topics_produce_cardinality_threshold", + "needs_restart": false, + "nullable": false, + "type": "integer", + "visibility": "user" + }, + "cloud_topics_produce_no_pid_concurrency": { + "c_type": "size_t", + "cloud_byoc_only": false, + "cloud_editable": false, + "cloud_readonly": false, + "cloud_supported": false, + "config_scope": "cluster", + "default": 32, + "defined_in": "src/v/config/configuration.cc", + "description": "Maximum number of concurrent raft replication requests for producers without a producer ID (idempotency disabled). Limits how many no-PID writes can proceed past the producer queue into raft simultaneously.", + "is_deprecated": false, + "is_enterprise": false, + "name": "cloud_topics_produce_no_pid_concurrency", + "needs_restart": true, + "nullable": false, + "type": "integer", + "version": "v26.1.1", + "visibility": "tunable" + }, + "cloud_topics_produce_upload_interval": { + "c_type": "std::chrono::milliseconds", + "cloud_byoc_only": false, + "cloud_editable": false, + "cloud_readonly": false, + "cloud_supported": false, + "config_scope": "cluster", + "default": 250, + "default_human_readable": "250 milliseconds", + "defined_in": "src/v/config/configuration.cc", + "description": "Time interval after which the upload is triggered.", + "is_deprecated": false, + "is_enterprise": false, + "maximum": 17592186044415, + "minimum": -17592186044416, + "name": "cloud_topics_produce_upload_interval", + "needs_restart": false, + "nullable": false, + "type": "integer", + "visibility": "user" + }, + "cloud_topics_produce_write_inflight_limit": { + "c_type": "size_t", + "cloud_byoc_only": false, + "cloud_editable": false, + "cloud_readonly": false, + "cloud_supported": false, + "config_scope": "cluster", + "default": 1024, + "defined_in": "src/v/config/configuration.cc", + "description": "Maximum number of in-flight write requests per shard in the cloud topics write pipeline. Requests that exceed this limit are queued until a slot becomes available.", + "is_deprecated": false, + "is_enterprise": false, + "name": "cloud_topics_produce_write_inflight_limit", + "needs_restart": true, + "nullable": false, + "type": "integer", + "version": "v26.1.1", + "visibility": "tunable" + }, + "cloud_topics_reconciliation_interval": { + "cloud_byoc_only": false, + "cloud_editable": false, + "cloud_readonly": false, + "cloud_supported": false, + "config_scope": "cluster", + "default": null, + "defined_in": "override", + "description": "Configuration property: cloud_topics_reconciliation_interval", + "is_deprecated": false, + "is_topic_property": false, + "name": "cloud_topics_reconciliation_interval", + "type": "string", + "visibility": "user" + }, + "cloud_topics_reconciliation_max_interval": { + "c_type": "std::chrono::milliseconds", + "cloud_byoc_only": false, + "cloud_editable": false, + "cloud_readonly": false, + "cloud_supported": false, + "config_scope": "cluster", + "default": 10000, + "default_human_readable": "10 seconds", + "defined_in": "src/v/config/configuration.cc", + "description": "Maximum reconciliation interval for adaptive scheduling.", + "is_deprecated": false, + "is_enterprise": false, + "maximum": 17592186044415, + "minimum": -17592186044416, + "name": "cloud_topics_reconciliation_max_interval", + "needs_restart": false, + "nullable": false, + "type": "integer", + "version": "v26.1.1", + "visibility": "tunable" + }, + "cloud_topics_reconciliation_max_object_size": { + "c_type": "size_t", + "cloud_byoc_only": false, + "cloud_editable": false, + "cloud_readonly": false, + "cloud_supported": false, + "config_scope": "cluster", + "default": 83886080, + "defined_in": "src/v/config/configuration.cc", + "description": "Maximum size in bytes for L1 objects produced by the reconciler. With the default target fill ratio of 0.8, this gives an effective target object size of 64 MiB.", + "is_deprecated": false, + "is_enterprise": false, + "name": "cloud_topics_reconciliation_max_object_size", + "needs_restart": false, + "nullable": false, + "type": "integer", + "version": "v26.1.1", + "visibility": "tunable" + }, + "cloud_topics_reconciliation_min_interval": { + "c_type": "std::chrono::milliseconds", + "cloud_byoc_only": false, + "cloud_editable": false, + "cloud_readonly": false, + "cloud_supported": false, + "config_scope": "cluster", + "default": 250, + "default_human_readable": "250 milliseconds", + "defined_in": "src/v/config/configuration.cc", + "description": "Minimum reconciliation interval for adaptive scheduling. The reconciler will not run more frequently than this.", + "is_deprecated": false, + "is_enterprise": false, + "maximum": 17592186044415, + "minimum": -17592186044416, + "name": "cloud_topics_reconciliation_min_interval", + "needs_restart": false, + "nullable": false, + "type": "integer", + "version": "v26.1.1", + "visibility": "tunable" + }, + "cloud_topics_reconciliation_parallelism": { + "c_type": "size_t", + "cloud_byoc_only": false, + "cloud_editable": false, + "cloud_readonly": false, + "cloud_supported": false, + "config_scope": "cluster", + "default": 8, + "defined_in": "src/v/config/configuration.cc", + "description": "Maximum number, per shard, of concurrent objects built by reconciliation", + "is_deprecated": false, + "is_enterprise": false, + "name": "cloud_topics_reconciliation_parallelism", + "needs_restart": true, + "nullable": false, + "type": "integer", + "version": "v26.1.1", + "visibility": "tunable" + }, + "cloud_topics_reconciliation_slowdown_blend": { + "c_type": "double", + "cloud_byoc_only": false, + "cloud_editable": false, + "cloud_readonly": false, + "cloud_supported": false, + "config_scope": "cluster", + "default": 0.4, + "defined_in": "src/v/config/configuration.cc", + "description": "Blend factor for slowing down reconciliation (0.0 to 1.0). Higher values mean reconciliation lowers its frequency faster when trying to find a frequency that produces well-sized objects. Generally this should be lower than the speedup blend, because reconciliation has less opportunities to adapt its frequency when it runs less frequently.", + "is_deprecated": false, + "is_enterprise": false, + "name": "cloud_topics_reconciliation_slowdown_blend", + "needs_restart": false, + "nullable": false, + "type": "number", + "version": "v26.1.1", + "visibility": "tunable" + }, + "cloud_topics_reconciliation_speedup_blend": { + "c_type": "double", + "cloud_byoc_only": false, + "cloud_editable": false, + "cloud_readonly": false, + "cloud_supported": false, + "config_scope": "cluster", + "default": 0.9, + "defined_in": "src/v/config/configuration.cc", + "description": "Blend factor for speeding up reconciliation (0.0 to 1.0). Higher values mean reconciliation increases its frequency faster when trying to find a frequency that produces well-sized objects.", + "is_deprecated": false, + "is_enterprise": false, + "name": "cloud_topics_reconciliation_speedup_blend", + "needs_restart": false, + "nullable": false, + "type": "number", + "version": "v26.1.1", + "visibility": "tunable" + }, + "cloud_topics_reconciliation_target_fill_ratio": { + "c_type": "double", + "cloud_byoc_only": false, + "cloud_editable": false, + "cloud_readonly": false, + "cloud_supported": false, + "config_scope": "cluster", + "default": 0.8, + "defined_in": "src/v/config/configuration.cc", + "description": "Target fill ratio for L1 objects. The reconciler adapts its interval to produce objects at approximately this fill level (0.0 to 1.0).", + "is_deprecated": false, + "is_enterprise": false, + "name": "cloud_topics_reconciliation_target_fill_ratio", + "needs_restart": false, + "nullable": false, + "type": "number", + "version": "v26.1.1", + "visibility": "tunable" + }, + "cloud_topics_short_term_gc_backoff_interval": { + "c_type": "std::chrono::milliseconds", + "cloud_byoc_only": false, + "cloud_editable": false, + "cloud_readonly": false, + "cloud_supported": false, + "config_scope": "cluster", + "default": 60000, + "default_human_readable": "1 minute", + "defined_in": "src/v/config/configuration.cc", + "description": "The interval, in milliseconds, between invocations of the L0 garbage collection work loop when no progress is being made or errors are occurring.\n\nL0 (level-zero) objects are short-term data objects in Tiered Storage that are periodically garbage collected. When GC encounters errors or cannot make progress (for example, if there are no objects eligible for deletion), this backoff interval prevents excessive retries.\n\nIncrease this value to reduce system load when GC cannot make progress. Decrease it if you need faster retry attempts after transient errors.", + "is_deprecated": false, + "is_enterprise": false, + "maximum": 17592186044415, + "minimum": -17592186044416, + "name": "cloud_topics_short_term_gc_backoff_interval", + "needs_restart": false, + "nullable": false, + "type": "integer", + "version": "v25.3.3", + "visibility": "tunable" + }, + "cloud_topics_short_term_gc_interval": { + "c_type": "std::chrono::milliseconds", + "cloud_byoc_only": false, + "cloud_editable": false, + "cloud_readonly": false, + "cloud_supported": false, + "config_scope": "cluster", + "default": 10000, + "default_human_readable": "10 seconds", + "defined_in": "src/v/config/configuration.cc", + "description": "The interval, in milliseconds, between invocations of the L0 (level-zero) garbage collection work loop when progress is being made.\n\nL0 objects are short-term data objects in Tiered Storage associated with global epochs. This property controls how frequently GC runs when it successfully deletes objects. Lower values increase GC frequency, which can help maintain lower object counts but may increase S3 API usage.\n\nDecrease this value if L0 object counts are growing too quickly and you need more aggressive garbage collection. Increase it to reduce S3 API costs in clusters with lower ingestion rates.", + "is_deprecated": false, + "is_enterprise": false, + "maximum": 17592186044415, + "minimum": -17592186044416, + "name": "cloud_topics_short_term_gc_interval", + "needs_restart": false, + "nullable": false, + "type": "integer", + "version": "v25.3.3", "visibility": "tunable" }, "cloud_topics_short_term_gc_minimum_object_age": { @@ -3475,6 +4005,25 @@ "version": "v25.3.3", "visibility": "tunable" }, + "cloud_topics_upload_part_size": { + "c_type": "size_t", + "cloud_byoc_only": false, + "cloud_editable": false, + "cloud_readonly": false, + "cloud_supported": false, + "config_scope": "cluster", + "default": 16777216, + "defined_in": "src/v/config/configuration.cc", + "description": "The part size in bytes used for multipart uploads. The minimum of 5 MiB is the smallest non-terminal part size allowed by cloud object storage providers.", + "is_deprecated": false, + "is_enterprise": false, + "name": "cloud_topics_upload_part_size", + "needs_restart": true, + "nullable": false, + "type": "integer", + "version": "v26.1.1", + "visibility": "tunable" + }, "cluster_id": { "c_type": "ss::sstring", "cloud_byoc_only": false, @@ -4233,70 +4782,6 @@ "type": "integer", "visibility": "tunable" }, - "coproc_max_batch_size": { - "c_type": "deprecated_property", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, - "is_enterprise": false, - "name": "coproc_max_batch_size", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" - }, - "coproc_max_inflight_bytes": { - "c_type": "deprecated_property", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, - "is_enterprise": false, - "name": "coproc_max_inflight_bytes", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" - }, - "coproc_max_ingest_bytes": { - "c_type": "deprecated_property", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, - "is_enterprise": false, - "name": "coproc_max_ingest_bytes", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" - }, - "coproc_offset_flush_interval_ms": { - "c_type": "deprecated_property", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, - "is_enterprise": false, - "name": "coproc_offset_flush_interval_ms", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" - }, "coproc_supervisor_server": { "c_type": "deprecated_property", "cloud_byoc_only": false, @@ -4311,7 +4796,7 @@ "name": "coproc_supervisor_server", "needs_restart": true, "nullable": false, - "type": "deprecated_property" + "type": "object" }, "core_balancing_continuous": { "c_type": "bool", @@ -4463,27 +4948,6 @@ "version": "v24.3.4", "visibility": "user" }, - "create_topic_timeout_ms": { - "c_type": "std::chrono::milliseconds", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "default": 2000, - "default_human_readable": "2 seconds", - "defined_in": "src/v/config/configuration.cc", - "description": "Timeout, in milliseconds, to wait for new topic creation.", - "is_deprecated": false, - "is_enterprise": false, - "maximum": 17592186044415, - "minimum": -17592186044416, - "name": "create_topic_timeout_ms", - "needs_restart": false, - "nullable": false, - "type": "integer", - "visibility": "tunable" - }, "dashboard_dir": { "c_type": "deprecated_property", "cloud_byoc_only": false, @@ -4498,7 +4962,7 @@ "name": "dashboard_dir", "needs_restart": true, "nullable": false, - "type": "deprecated_property" + "type": "object" }, "data_directory": { "c_type": "data_directory_path", @@ -4773,22 +5237,6 @@ "type": "boolean", "visibility": "user" }, - "datalake_disk_space_monitor_interval": { - "c_type": "deprecated_property", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, - "is_enterprise": false, - "name": "datalake_disk_space_monitor_interval", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" - }, "datalake_disk_usage_overage_coeff": { "c_type": "double", "cloud_byoc_only": false, @@ -5039,6 +5487,36 @@ "type": "integer", "visibility": "tunable" }, + "default_redpanda_storage_mode": { + "c_type": "model::redpanda_storage_mode", + "cloud_byoc_only": false, + "cloud_editable": true, + "cloud_readonly": false, + "cloud_supported": true, + "config_scope": "cluster", + "default": "unset", + "defined_in": "src/v/config/configuration.cc", + "description": "Set the default storage mode for new topics. This value applies to any topic created without an explicit <> setting (that is, when the topic's `redpanda.storage.mode` is `unset`).\n\nAccepted values:\n\n* `unset`: Defer to the legacy <> and <> topic properties for Tiered Storage configuration.\n* `local`: Store data only on local disks, with no object storage involvement.\n* `tiered`: Store data on local disks and replicate it to object storage using xref:manage:tiered-storage.adoc[Tiered Storage]. Equivalent to setting `redpanda.remote.read` and `redpanda.remote.write` to `true`.\n* `cloud`: Store data primarily in object storage using xref:manage:tiered-storage.adoc#cloud-topics[cloud topics].", + "enum": [ + "local", + "tiered", + "cloud", + "unset" + ], + "example": "`tiered`", + "is_deprecated": false, + "is_enterprise": false, + "name": "default_redpanda_storage_mode", + "needs_restart": false, + "nullable": false, + "related_topics": [ + "xref:manage:tiered-storage.adoc[Tiered Storage]", + "xref:develop:manage-topics/cloud-topics.adoc[Manage Cloud Topics]" + ], + "type": "string", + "version": "v26.1.1", + "visibility": "user" + }, "default_topic_partitions": { "c_type": "int32_t", "cloud_byoc_only": false, @@ -5125,6 +5603,32 @@ ], "type": "integer" }, + "delete_topic_enable": { + "c_type": "bool", + "cloud_byoc_only": false, + "cloud_editable": false, + "cloud_readonly": false, + "cloud_supported": false, + "config_scope": "cluster", + "default": true, + "defined_in": "src/v/config/configuration.cc", + "description": "Enable or disable topic deletion via the Kafka DeleteTopics API. When set to false, all topic deletion requests are rejected with error code 73 (TOPIC_DELETION_DISABLED). This is a cluster-wide safety setting that cannot be overridden by superusers. Topics in kafka_nodelete_topics are always protected regardless of this setting.", + "enterprise_constructor": "restricted_only", + "enterprise_restricted_value": [ + "false" + ], + "enterprise_value": [ + "false" + ], + "is_deprecated": false, + "is_enterprise": true, + "name": "delete_topic_enable", + "needs_restart": false, + "nullable": false, + "type": "boolean", + "version": "v26.1.1", + "visibility": "user" + }, "developer_mode": { "c_type": "bool", "category": "redpanda", @@ -5289,22 +5793,6 @@ "type": "boolean", "visibility": "user" }, - "enable_admin_api": { - "c_type": "deprecated_property", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, - "is_enterprise": false, - "name": "enable_admin_api", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" - }, "enable_auto_rebalance_on_node_add": { "c_type": "bool", "cloud_byoc_only": false, @@ -5337,7 +5825,7 @@ "name": "enable_central_config", "needs_restart": true, "nullable": false, - "type": "deprecated_property" + "type": "object" }, "enable_cluster_metadata_upload_loop": { "c_type": "bool", @@ -5411,22 +5899,6 @@ "type": "boolean", "visibility": "user" }, - "enable_coproc": { - "c_type": "deprecated_property", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, - "is_enterprise": false, - "name": "enable_coproc", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" - }, "enable_developmental_unrecoverable_data_corrupting_features": { "cloud_byoc_only": false, "cloud_editable": false, @@ -5602,7 +6074,7 @@ "config_scope": "cluster", "default": "none", "defined_in": "src/v/config/configuration.cc", - "description": "Controls whether Redpanda validates schema IDs in records and which topic properties are enforced.\n\nValues:\n\n* `none`: Schema validation is disabled (no schema ID checks are done). Associated topic properties cannot be modified.\n* `redpanda`: Schema validation is enabled. Only Redpanda topic properties are accepted.\n* `compat`: Schema validation is enabled. Both Redpanda and compatible topic properties are accepted.\n\nNOTE: Enabling this property will trigger decompression of message batches for topics on which validation is configured, which may lead to a modest increase in CPU load. Redpanda recommends monitoring CPU utilization after topics are configured.", + "description": "Controls whether Redpanda validates schema IDs in records and which topic properties are enforced.\n\nValues:\n\n* `none`: Schema validation is disabled (no schema ID checks are done). Associated topic properties cannot be modified.\n* `redpanda`: Schema validation is enabled. Only Redpanda topic properties are accepted.\n* `compat`: Schema validation is enabled. Both Redpanda and compatible topic properties are accepted.", "enterprise_constructor": "restricted_only", "enterprise_restricted_value": [ "compat", @@ -5900,22 +6372,6 @@ "type": "integer", "visibility": "tunable" }, - "find_coordinator_timeout_ms": { - "c_type": "deprecated_property", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, - "is_enterprise": false, - "name": "find_coordinator_timeout_ms", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" - }, "fips_mode": { "c_type": "fips_mode_flag", "category": "redpanda", @@ -5991,22 +6447,6 @@ ], "type": "integer" }, - "full_raft_configuration_recovery_pattern": { - "c_type": "deprecated_property", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, - "is_enterprise": false, - "name": "full_raft_configuration_recovery_pattern", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" - }, "group_initial_rebalance_delay": { "c_type": "std::chrono::milliseconds", "cloud_byoc_only": false, @@ -6352,7 +6792,7 @@ "redpanda" ], "defined_in": "src/v/config/configuration.cc", - "description": "The default Iceberg catalog namespace where Redpanda creates tables. Supports nested namespaces as an array of strings.\n\nIMPORTANT: This value must be configured before enabling Iceberg and must not be changed afterward. Changing it will cause Redpanda to lose track of existing tables.", + "description": "The default namespace (database name) for Iceberg tables. All tables created by Redpanda will be placed in this namespace within the Iceberg catalog. Supports nested namespaces as an array of strings.\n\nIMPORTANT: This value must be configured before enabling Iceberg and must not be changed afterward. Changing it will cause Redpanda to lose track of existing tables.", "is_deprecated": false, "is_enterprise": false, "items": { @@ -7076,22 +7516,6 @@ "type": "integer", "visibility": "tunable" }, - "id_allocator_replication": { - "c_type": "deprecated_property", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, - "is_enterprise": false, - "name": "id_allocator_replication", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" - }, "initial.retention.local.target.bytes": { "acceptable_values": "bytes (integer)", "alternate_cluster_property": "", @@ -7188,6 +7612,28 @@ "type": "integer", "visibility": "user" }, + "internal_rpc_request_timeout_ms": { + "c_type": "std::chrono::milliseconds", + "cloud_byoc_only": false, + "cloud_editable": false, + "cloud_readonly": false, + "cloud_supported": false, + "config_scope": "cluster", + "default": 10000, + "default_human_readable": "10 seconds", + "defined_in": "src/v/config/configuration.cc", + "description": "Default timeout for RPC requests between Redpanda nodes.", + "is_deprecated": false, + "is_enterprise": false, + "maximum": 17592186044415, + "minimum": -17592186044416, + "name": "internal_rpc_request_timeout_ms", + "needs_restart": false, + "nullable": false, + "type": "integer", + "version": "v26.1.1", + "visibility": "tunable" + }, "internal_topic_replication_factor": { "c_type": "int", "cloud_byoc_only": false, @@ -7229,22 +7675,6 @@ "type": "integer", "visibility": "tunable" }, - "kafka_admin_topic_api_rate": { - "c_type": "deprecated_property", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, - "is_enterprise": false, - "name": "kafka_admin_topic_api_rate", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" - }, "kafka_api": { "c_type": "config::broker_authn_endpoint", "category": "redpanda", @@ -7320,38 +7750,6 @@ "type": "integer", "visibility": "tunable" }, - "kafka_client_group_byte_rate_quota": { - "c_type": "deprecated_property", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, - "is_enterprise": false, - "name": "kafka_client_group_byte_rate_quota", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" - }, - "kafka_client_group_fetch_byte_rate_quota": { - "c_type": "deprecated_property", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, - "is_enterprise": false, - "name": "kafka_client_group_fetch_byte_rate_quota", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" - }, "kafka_connection_rate_limit": { "c_type": "int64_t", "cloud_byoc_only": false, @@ -7537,7 +7935,7 @@ "default": 5000, "default_human_readable": "5 seconds", "defined_in": "src/v/config/configuration.cc", - "description": "Target duration for a single fetch request. The broker tries to complete each fetch within this duration, even if fewer bytes are available than requested.", + "description": "Broker-side target for the duration of a single fetch request. The broker will try to complete fetches within the specified duration, even if it means returning less bytes in the fetch than are available.", "is_deprecated": false, "is_enterprise": false, "maximum": 17592186044415, @@ -7611,22 +8009,6 @@ "type": "integer", "visibility": "tunable" }, - "kafka_memory_batch_size_estimate_for_fetch": { - "c_type": "deprecated_property", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, - "is_enterprise": false, - "name": "kafka_memory_batch_size_estimate_for_fetch", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" - }, "kafka_memory_share_for_fetch": { "c_type": "double", "cloud_byoc_only": false, @@ -7886,111 +8268,47 @@ "name": "kafka_qdc_min_depth", "needs_restart": true, "nullable": false, - "type": "integer", - "visibility": "tunable" - }, - "kafka_qdc_window_count": { - "c_type": "size_t", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "default": 12, - "defined_in": "src/v/config/configuration.cc", - "description": "Number of windows used in Kafka queue depth control latency tracking.", - "is_deprecated": false, - "is_enterprise": false, - "name": "kafka_qdc_window_count", - "needs_restart": true, - "nullable": false, - "type": "integer", - "visibility": "tunable" - }, - "kafka_qdc_window_size_ms": { - "c_type": "std::chrono::milliseconds", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "default": 1500, - "default_human_readable": "1500 milliseconds", - "defined_in": "src/v/config/configuration.cc", - "description": "Window size for Kafka queue depth control latency tracking.", - "is_deprecated": false, - "is_enterprise": false, - "maximum": 17592186044415, - "minimum": -17592186044416, - "name": "kafka_qdc_window_size_ms", - "needs_restart": true, - "nullable": false, - "type": "integer", - "visibility": "tunable" - }, - "kafka_quota_balancer_min_shard_throughput_bps": { - "c_type": "deprecated_property", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, - "is_enterprise": false, - "name": "kafka_quota_balancer_min_shard_throughput_bps", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" - }, - "kafka_quota_balancer_min_shard_throughput_ratio": { - "c_type": "deprecated_property", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, - "is_enterprise": false, - "name": "kafka_quota_balancer_min_shard_throughput_ratio", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" + "type": "integer", + "visibility": "tunable" }, - "kafka_quota_balancer_node_period": { - "c_type": "deprecated_property", + "kafka_qdc_window_count": { + "c_type": "size_t", "cloud_byoc_only": false, "cloud_editable": false, "cloud_readonly": false, "cloud_supported": false, "config_scope": "cluster", + "default": 12, "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, + "description": "Number of windows used in Kafka queue depth control latency tracking.", + "is_deprecated": false, "is_enterprise": false, - "name": "kafka_quota_balancer_node_period", + "name": "kafka_qdc_window_count", "needs_restart": true, "nullable": false, - "type": "deprecated_property" + "type": "integer", + "visibility": "tunable" }, - "kafka_quota_balancer_window": { - "c_type": "deprecated_property", + "kafka_qdc_window_size_ms": { + "c_type": "std::chrono::milliseconds", "cloud_byoc_only": false, "cloud_editable": false, "cloud_readonly": false, "cloud_supported": false, "config_scope": "cluster", + "default": 1500, + "default_human_readable": "1500 milliseconds", "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, + "description": "Window size for Kafka queue depth control latency tracking.", + "is_deprecated": false, "is_enterprise": false, - "name": "kafka_quota_balancer_window", + "maximum": 17592186044415, + "minimum": -17592186044416, + "name": "kafka_qdc_window_size_ms", "needs_restart": true, "nullable": false, - "type": "deprecated_property" + "type": "integer", + "visibility": "tunable" }, "kafka_request_max_bytes": { "c_type": "uint32_t", @@ -8292,22 +8610,6 @@ "type": "integer", "visibility": "tunable" }, - "kafka_throughput_throttling_v2": { - "c_type": "deprecated_property", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, - "is_enterprise": false, - "name": "kafka_throughput_throttling_v2", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" - }, "kafka_topics_max": { "c_type": "uint32_t", "cloud_byoc_only": false, @@ -8388,20 +8690,27 @@ "visibility": "tunable" }, "leader_balancer_mode": { - "c_type": "deprecated_property", + "c_type": "model::leader_balancer_mode", "cloud_byoc_only": false, "cloud_editable": false, "cloud_readonly": false, "cloud_supported": false, "config_scope": "cluster", + "default": "calibrated", "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, + "description": "Mode of the leader balancer optimization strategy. `calibrated` uses a heuristic that balances leaders based on replica counts per shard. `random` randomly moves leaders to reduce load on heavily-loaded shards. Legacy values `greedy_balanced_shards` and `random_hill_climbing` are treated as `calibrated`.", + "enum": [ + "calibrated", + "random" + ], + "example": "`model::leader_balancer_mode_to_string( model::leader_balancer_mode::calibrated)`", + "is_deprecated": false, "is_enterprise": false, "name": "leader_balancer_mode", - "needs_restart": true, + "needs_restart": false, "nullable": false, - "type": "deprecated_property" + "type": "string", + "visibility": "user" }, "leader_balancer_mute_timeout": { "c_type": "std::chrono::milliseconds", @@ -8547,37 +8856,20 @@ "type": "string", "visibility": "user" }, - "log_compaction_adjacent_merge_self_compaction_count": { - "c_type": "deprecated_property", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, - "is_enterprise": false, - "name": "log_compaction_adjacent_merge_self_compaction_count", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" - }, "log_compaction_disable_tx_batch_removal": { - "c_type": "deprecated_property", "cloud_byoc_only": false, "cloud_editable": false, "cloud_readonly": false, "cloud_supported": false, "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", + "default": null, + "defined_in": "override", "description": "Prevents log compaction from removing transaction metadata. Only set this to `true` if you experience stability issues related to transaction cleanup during compaction.", - "is_deprecated": true, - "is_enterprise": false, + "is_deprecated": false, + "is_topic_property": false, "name": "log_compaction_disable_tx_batch_removal", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" + "type": "string", + "visibility": "user" }, "log_compaction_interval_ms": { "c_type": "std::chrono::milliseconds", @@ -8687,7 +8979,7 @@ "cloud_readonly": false, "cloud_supported": false, "config_scope": "cluster", - "default": false, + "default": true, "defined_in": "src/v/config/configuration.cc", "description": "Enables removal of transactional control batches during compaction. These batches are removed according to a topic's configured delete.retention.ms, and only if the topic's cleanup.policy allows compaction.", "is_deprecated": false, @@ -8789,38 +9081,6 @@ "type": "integer", "visibility": "user" }, - "log_message_timestamp_alert_after_ms": { - "c_type": "deprecated_property", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, - "is_enterprise": false, - "name": "log_message_timestamp_alert_after_ms", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" - }, - "log_message_timestamp_alert_before_ms": { - "c_type": "deprecated_property", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, - "is_enterprise": false, - "name": "log_message_timestamp_alert_before_ms", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" - }, "log_message_timestamp_before_max_ms": { "c_type": "std::chrono::milliseconds", "cloud_byoc_only": false, @@ -9265,22 +9525,6 @@ "type": "integer", "visibility": "tunable" }, - "max_version": { - "c_type": "deprecated_property", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, - "is_enterprise": false, - "name": "max_version", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" - }, "members_backend_retry_ms": { "c_type": "std::chrono::milliseconds", "cloud_byoc_only": false, @@ -9482,27 +9726,6 @@ "type": "integer", "visibility": "tunable" }, - "metadata_status_wait_timeout_ms": { - "c_type": "std::chrono::milliseconds", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "default": 2000, - "default_human_readable": "2 seconds", - "defined_in": "src/v/config/configuration.cc", - "description": "Maximum time to wait in metadata request for cluster health to be refreshed.", - "is_deprecated": false, - "is_enterprise": false, - "maximum": 17592186044415, - "minimum": -17592186044416, - "name": "metadata_status_wait_timeout_ms", - "needs_restart": true, - "nullable": false, - "type": "integer", - "visibility": "tunable" - }, "metrics_reporter_report_interval": { "c_type": "std::chrono::milliseconds", "cloud_byoc_only": false, @@ -9659,22 +9882,6 @@ "type": "integer", "visibility": "user" }, - "min_version": { - "c_type": "deprecated_property", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, - "is_enterprise": false, - "name": "min_version", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" - }, "minimum_topic_replication": { "c_type": "int16_t", "cloud_byoc_only": false, @@ -9713,6 +9920,25 @@ "nullable": false, "type": "boolean" }, + "nested_group_behavior": { + "c_type": "security::oidc::nested_group_behavior", + "cloud_byoc_only": false, + "cloud_editable": false, + "cloud_readonly": false, + "cloud_supported": false, + "config_scope": "cluster", + "default": "none", + "defined_in": "src/v/config/configuration.cc", + "description": "Behavior for handling nested groups when extracting groups from authentication tokens. Two options are available - none and suffix. With none, the group is left alone (e.g. '/group/child/grandchild'). Suffix will extract the final component from the nested group (e.g. '/group' -> 'group' and '/group/child/grandchild' -> 'grandchild').", + "is_deprecated": false, + "is_enterprise": false, + "name": "nested_group_behavior", + "needs_restart": false, + "nullable": false, + "type": "string", + "version": "v26.1.1", + "visibility": "user" + }, "node_id": { "c_type": "model::node_id", "category": "redpanda", @@ -9775,27 +10001,6 @@ "type": "integer", "visibility": "tunable" }, - "node_management_operation_timeout_ms": { - "c_type": "std::chrono::milliseconds", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "default": 5000, - "default_human_readable": "5 seconds", - "defined_in": "src/v/config/configuration.cc", - "description": "Timeout for executing node management operations.", - "is_deprecated": false, - "is_enterprise": false, - "maximum": 17592186044415, - "minimum": -17592186044416, - "name": "node_management_operation_timeout_ms", - "needs_restart": true, - "nullable": false, - "type": "integer", - "visibility": "tunable" - }, "node_status_interval": { "c_type": "std::chrono::milliseconds", "cloud_byoc_only": false, @@ -9876,6 +10081,25 @@ "type": "string", "visibility": "user" }, + "oidc_group_claim_path": { + "c_type": "ss::sstring", + "cloud_byoc_only": false, + "cloud_editable": false, + "cloud_readonly": false, + "cloud_supported": false, + "config_scope": "cluster", + "default": "$.groups", + "defined_in": "src/v/config/configuration.cc", + "description": "JSON path to extract groups from the JWT payload.", + "is_deprecated": false, + "is_enterprise": false, + "name": "oidc_group_claim_path", + "needs_restart": false, + "nullable": false, + "type": "string", + "version": "v26.1.1", + "visibility": "user" + }, "oidc_keys_refresh_interval": { "c_type": "std::chrono::seconds", "cloud_byoc_only": false, @@ -10138,6 +10362,27 @@ "type": "integer", "visibility": "deprecated" }, + "partition_autobalancing_node_autodecommission_timeout_sec": { + "c_type": "std::chrono::seconds", + "cloud_byoc_only": false, + "cloud_editable": false, + "cloud_readonly": false, + "cloud_supported": false, + "config_scope": "cluster", + "default": null, + "defined_in": "src/v/config/configuration.cc", + "description": "When a node is unavailable for at least this timeout duration, it triggers Redpanda to decommission the node. This property applies only when `partition_autobalancing_mode` is set to `continuous`.", + "is_deprecated": false, + "is_enterprise": false, + "maximum": 17179869183, + "minimum": -17179869184, + "name": "partition_autobalancing_node_autodecommission_timeout_sec", + "needs_restart": false, + "nullable": true, + "type": "integer", + "version": "v26.1.1", + "visibility": "user" + }, "partition_autobalancing_node_availability_timeout_sec": { "c_type": "std::chrono::seconds", "cloud_byoc_only": false, @@ -10148,7 +10393,7 @@ "default": 900, "default_human_readable": "15 minutes", "defined_in": "src/v/config/configuration.cc", - "description": "When a node is unavailable for at least this timeout duration, it triggers Redpanda to move partitions off of the node. This property applies only when `partition_autobalancing_mode` is set to `continuous`. ", + "description": "When a node is unavailable for at least this timeout duration, it triggers Redpanda to move partitions off of the node. This property applies only when `partition_autobalancing_mode` is set to `continuous`.", "is_deprecated": false, "is_enterprise": false, "maximum": 17179869183, @@ -10646,22 +10891,6 @@ "type": "integer", "visibility": "tunable" }, - "raft_max_concurrent_append_requests_per_follower": { - "c_type": "deprecated_property", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, - "is_enterprise": false, - "name": "raft_max_concurrent_append_requests_per_follower", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" - }, "raft_max_inflight_follower_append_entries_requests_per_shard": { "c_type": "size_t", "cloud_byoc_only": false, @@ -10717,22 +10946,6 @@ "type": "integer", "visibility": "tunable" }, - "raft_recovery_default_read_size": { - "c_type": "deprecated_property", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, - "is_enterprise": false, - "name": "raft_recovery_default_read_size", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" - }, "raft_recovery_throttle_disable_dynamic_mode": { "c_type": "bool", "cloud_byoc_only": false, @@ -10973,54 +11186,33 @@ "cloud_readonly": false, "cloud_supported": false, "config_scope": "cluster", - "default": 131072, - "defined_in": "src/v/config/configuration.cc", - "description": "Minimum batch cache reclaim size.", - "is_deprecated": false, - "is_enterprise": false, - "name": "reclaim_min_size", - "needs_restart": true, - "nullable": false, - "type": "integer", - "visibility": "tunable" - }, - "reclaim_stable_window": { - "c_type": "std::chrono::milliseconds", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "default": 10000, - "default_human_readable": "10 seconds", + "default": 131072, "defined_in": "src/v/config/configuration.cc", - "description": "If the duration since the last time memory was reclaimed is longer than the amount of time specified in this property, the memory usage of the batch cache is considered stable, so only the minimum size (<>) is set to be reclaimed.", + "description": "Minimum batch cache reclaim size.", "is_deprecated": false, "is_enterprise": false, - "maximum": 17592186044415, - "minimum": -17592186044416, - "name": "reclaim_stable_window", + "name": "reclaim_min_size", "needs_restart": true, "nullable": false, "type": "integer", "visibility": "tunable" }, - "recovery_append_timeout_ms": { + "reclaim_stable_window": { "c_type": "std::chrono::milliseconds", "cloud_byoc_only": false, "cloud_editable": false, "cloud_readonly": false, "cloud_supported": false, "config_scope": "cluster", - "default": 5000, - "default_human_readable": "5 seconds", + "default": 10000, + "default_human_readable": "10 seconds", "defined_in": "src/v/config/configuration.cc", - "description": "Timeout for append entry requests issued while updating a stale follower.", + "description": "If the duration since the last time memory was reclaimed is longer than the amount of time specified in this property, the memory usage of the batch cache is considered stable, so only the minimum size (<>) is set to be reclaimed.", "is_deprecated": false, "is_enterprise": false, "maximum": 17592186044415, "minimum": -17592186044416, - "name": "recovery_append_timeout_ms", + "name": "reclaim_stable_window", "needs_restart": true, "nullable": false, "type": "integer", @@ -11049,27 +11241,24 @@ "visibility": "user" }, "redpanda.cloud_topic.enabled": { - "acceptable_values": "", - "alternate_cluster_property": "", - "alternate_cluster_property_doc_file": "", "category": "tiered-storage", "cloud_byoc_only": false, "cloud_editable": false, "cloud_readonly": false, "cloud_supported": false, - "cluster_property_doc_file": "", "config_scope": "topic", - "corresponding_cluster_property": null, "default": null, - "defined_in": "src/v/kafka/protocol/topic_properties.h", - "description": "", - "exclude_from_docs": true, + "defined_in": "override", + "description": "Enable Cloud Topic storage mode for this topic. When enabled, topic data is stored primarily in object storage with local storage used only as a write buffer.\n\nTIP: To configure storage modes with more flexibility, use <> which supports `local`, `tiered`, `cloud`, and `unset` modes.", "is_deprecated": false, - "is_enterprise": false, "is_topic_property": true, "name": "redpanda.cloud_topic.enabled", - "needs_restart": false, - "type": "string" + "related_topics": [ + "xref:develop:manage-topics/cloud-topics.adoc[Cloud Topics]", + "xref:manage:tiered-storage.adoc[Tiered Storage]" + ], + "type": "string", + "visibility": "user" }, "redpanda.iceberg.delete": { "acceptable_values": "", @@ -11417,6 +11606,39 @@ ], "type": "boolean" }, + "redpanda.storage.mode": { + "acceptable_values": "", + "alternate_cluster_property": "", + "alternate_cluster_property_doc_file": "", + "category": "tiered-storage", + "cloud_byoc_only": false, + "cloud_editable": false, + "cloud_readonly": false, + "cloud_supported": false, + "cluster_property_doc_file": "cluster-properties.adoc", + "config_scope": "topic", + "corresponding_cluster_property": "default_redpanda_storage_mode", + "default": "unset", + "defined_in": "src/v/kafka/protocol/topic_properties.h", + "description": "The storage mode for a topic. Determines how topic data is stored and whether it is eligible for upload to object storage.\n\nAccepted values:\n\n* `local`: Topic data is stored only on the broker's local disk. Object storage upload is disabled for the topic, regardless of cluster-level Tiered Storage settings.\n* `tiered`: Topic data is stored on local disk and also uploaded to object storage. Enables xref:manage:tiered-storage.adoc[Tiered Storage] for the topic.\n* `cloud`: Topic data is stored in object storage using the glossterm:Cloud Topic[,Cloud Topics] architecture. Local storage is used only as a write buffer.\n* `unset`: Specifies that the topic's storage mode is unset, regardless of the cluster default. The topic may still have Tiered Storage enabled through the legacy properties `redpanda.remote.read` and `redpanda.remote.write`.\n\nThis property overrides the cluster-wide config_ref:default_redpanda_storage_mode,true,properties/cluster-properties[] setting for individual topics.", + "enum": [ + "local", + "tiered", + "cloud", + "unset" + ], + "is_deprecated": false, + "is_enterprise": false, + "is_topic_property": true, + "name": "redpanda.storage.mode", + "needs_restart": false, + "related_topics": [ + "xref:manage:tiered-storage.adoc[Tiered Storage]", + "xref:develop:manage-topics/cloud-topics.adoc[Manage Cloud Topics]" + ], + "type": "string", + "version": "v26.1.1" + }, "redpanda.value.schema.id.validation": { "acceptable_values": "", "alternate_cluster_property": "", @@ -11553,8 +11775,8 @@ "xref:reference:rpk/rpk-topic/rpk-topic-describe.adoc[`rpk topic describe`]", "xref:reference:rpk/rpk-topic/rpk-topic-alter-config.adoc[`rpk topic alter-config`]", "xref:reference:properties/cluster-properties.adoc#default_topic_replication[`default_topic_replication`]", - "xref:develop:config-topics.adoc#choose-the-replication-factor[Choose the replication factor]", - "xref:develop:config-topics.adoc#change-the-replication-factor[Change the replication factor]", + "xref:develop:manage-topics/config-topics.adoc#choose-the-replication-factor[Choose the replication factor]", + "xref:develop:manage-topics/config-topics.adoc#change-the-replication-factor[Change the replication factor]", "xref:reference:properties/cluster-properties.adoc#default_topic_replication[default_topic_replication]" ], "type": "integer" @@ -11904,43 +12126,6 @@ "type": "string", "visibility": "user" }, - "rm_sync_timeout_ms": { - "c_type": "std::chrono::milliseconds", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "default": 10000, - "default_human_readable": "10 seconds", - "defined_in": "src/v/config/configuration.cc", - "description": "Resource manager's synchronization timeout. Specifies the maximum time for this node to wait for the internal state machine to catch up with all events written by previous leaders before rejecting a request.", - "is_deprecated": false, - "is_enterprise": false, - "maximum": 17592186044415, - "minimum": -17592186044416, - "name": "rm_sync_timeout_ms", - "needs_restart": false, - "nullable": false, - "type": "integer", - "visibility": "user" - }, - "rm_violation_recovery_policy": { - "c_type": "deprecated_property", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, - "is_enterprise": false, - "name": "rm_violation_recovery_policy", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" - }, "rpc_client_connections_per_peer": { "c_type": "int", "cloud_byoc_only": false, @@ -12466,23 +12651,21 @@ "type": "array" }, "schema_registry_avro_use_named_references": { - "c_type": "bool", + "c_type": "deprecated_property", "cloud_byoc_only": false, "cloud_editable": false, "cloud_readonly": false, "cloud_supported": false, "config_scope": "cluster", - "default": false, "defined_in": "src/v/config/configuration.cc", - "description": "When enabled, Avro schemas with external references are compiled using named reference resolution instead of schema concatenation. This fixes issues with compatibility checks and schema validation for schemas with reference dependencies. This config will be deprecated and always enabled in v26.1.1.", - "is_deprecated": false, + "description": null, + "is_deprecated": true, "is_enterprise": false, "name": "schema_registry_avro_use_named_references", - "needs_restart": false, + "needs_restart": true, "nullable": false, - "type": "boolean", - "version": "v25.3.10", - "visibility": "tunable" + "type": "object", + "version": "v25.3.10" }, "schema_registry_enable_authorization": { "c_type": "bool", @@ -12510,21 +12693,24 @@ "type": "boolean", "visibility": "user" }, - "schema_registry_protobuf_renderer_v2": { - "c_type": "deprecated_property", + "schema_registry_enable_qualified_subjects": { + "c_type": "bool", "cloud_byoc_only": false, "cloud_editable": false, "cloud_readonly": false, "cloud_supported": false, "config_scope": "cluster", + "default": false, "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, + "description": "Enable parsing of qualified subject syntax (:.context:subject). When false, subjects are treated literally, as subjects in the default context. When true, qualified syntax is parsed to extract context and subject.", + "is_deprecated": false, "is_enterprise": false, - "name": "schema_registry_protobuf_renderer_v2", + "name": "schema_registry_enable_qualified_subjects", "needs_restart": true, "nullable": false, - "type": "deprecated_property" + "type": "boolean", + "version": "v26.1.1", + "visibility": "tunable" }, "schema_registry_replication_factor": { "c_type": "int16_t", @@ -12589,22 +12775,6 @@ "nullable": false, "type": "string" }, - "seed_server_meta_topic_partitions": { - "c_type": "deprecated_property", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, - "is_enterprise": false, - "name": "seed_server_meta_topic_partitions", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" - }, "seed_servers": { "c_type": "seed_server", "category": "redpanda", @@ -12724,22 +12894,6 @@ "type": "integer", "visibility": "tunable" }, - "seq_table_min_size": { - "c_type": "deprecated_property", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, - "is_enterprise": false, - "name": "seq_table_min_size", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" - }, "space_management_enable": { "c_type": "bool", "cloud_byoc_only": false, @@ -13157,38 +13311,6 @@ "type": "array", "visibility": "user" }, - "target_fetch_quota_byte_rate": { - "c_type": "deprecated_property", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, - "is_enterprise": false, - "name": "target_fetch_quota_byte_rate", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" - }, - "target_quota_byte_rate": { - "c_type": "deprecated_property", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, - "is_enterprise": false, - "name": "target_quota_byte_rate", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" - }, "tls_certificate_name_format": { "c_type": "tls_name_format", "cloud_byoc_only": false, @@ -13260,7 +13382,7 @@ "cloud_readonly": false, "cloud_supported": false, "config_scope": "cluster", - "default": "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:AES256-GCM-SHA384:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:AES128-SHA:AES128-CCM:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:AES256-SHA:AES256-CCM", + "default": "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256:TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256:TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384:TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384:TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256:TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256", "defined_in": "src/v/config/configuration.cc", "description": "The encryption algorithms available for TLS 1.2 client connections, specified as a colon-separated list in OpenSSL format. Use this to support older clients that require specific encryption methods.", "is_deprecated": false, @@ -13289,43 +13411,6 @@ "type": "string", "visibility": "user" }, - "tm_sync_timeout_ms": { - "c_type": "std::chrono::milliseconds", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "default": 10000, - "default_human_readable": "10 seconds", - "defined_in": "src/v/config/configuration.cc", - "description": "Transaction manager's synchronization timeout. Maximum time to wait for internal state machine to catch up before rejecting a request.", - "is_deprecated": false, - "is_enterprise": false, - "maximum": 17592186044415, - "minimum": -17592186044416, - "name": "tm_sync_timeout_ms", - "needs_restart": true, - "nullable": false, - "type": "integer", - "visibility": "user" - }, - "tm_violation_recovery_policy": { - "c_type": "deprecated_property", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, - "is_enterprise": false, - "name": "tm_violation_recovery_policy", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" - }, "tombstone_retention_ms": { "c_type": "std::chrono::milliseconds", "cloud_byoc_only": false, @@ -13550,22 +13635,6 @@ "type": "integer", "visibility": "tunable" }, - "transaction_coordinator_replication": { - "c_type": "deprecated_property", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, - "is_enterprise": false, - "name": "transaction_coordinator_replication", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" - }, "transaction_max_timeout_ms": { "c_type": "std::chrono::milliseconds", "cloud_byoc_only": false, @@ -13633,38 +13702,6 @@ "type": "integer", "visibility": "deprecated" }, - "tx_registry_log_capacity": { - "c_type": "deprecated_property", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, - "is_enterprise": false, - "name": "tx_registry_log_capacity", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" - }, - "tx_registry_sync_timeout_ms": { - "c_type": "deprecated_property", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, - "is_enterprise": false, - "name": "tx_registry_sync_timeout_ms", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" - }, "tx_timeout_delay_ms": { "c_type": "std::chrono::milliseconds", "cloud_byoc_only": false, @@ -13838,22 +13875,6 @@ "type": "boolean", "visibility": "tunable" }, - "use_scheduling_groups": { - "c_type": "deprecated_property", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "defined_in": "src/v/config/configuration.cc", - "description": null, - "is_deprecated": true, - "is_enterprise": false, - "name": "use_scheduling_groups", - "needs_restart": true, - "nullable": false, - "type": "deprecated_property" - }, "verbose_logging_timeout_sec_max": { "c_type": "std::chrono::seconds", "category": "redpanda", @@ -13899,27 +13920,6 @@ "type": "integer", "visibility": "tunable" }, - "wait_for_leader_timeout_ms": { - "c_type": "std::chrono::milliseconds", - "cloud_byoc_only": false, - "cloud_editable": false, - "cloud_readonly": false, - "cloud_supported": false, - "config_scope": "cluster", - "default": 5000, - "default_human_readable": "5 seconds", - "defined_in": "src/v/config/configuration.cc", - "description": "Timeout to wait for leadership in metadata cache.", - "is_deprecated": false, - "is_enterprise": false, - "maximum": 17592186044415, - "minimum": -17592186044416, - "name": "wait_for_leader_timeout_ms", - "needs_restart": false, - "nullable": false, - "type": "integer", - "visibility": "tunable" - }, "write.caching": { "acceptable_values": "", "alternate_cluster_property": "", diff --git a/modules/reference/pages/properties/cluster-properties.adoc b/modules/reference/pages/properties/cluster-properties.adoc index f3460c1adc..053173d11d 100644 --- a/modules/reference/pages/properties/cluster-properties.adoc +++ b/modules/reference/pages/properties/cluster-properties.adoc @@ -10,4 +10,4 @@ NOTE: Some cluster properties require that you restart the cluster for any updat == Cluster configuration -include::reference:partial$properties/cluster-properties.adoc[tags=!deprecated;!exclude-from-docs] \ No newline at end of file +include::reference:partial$properties/cluster-properties.adoc[tags=!deprecated;!exclude-from-docs] diff --git a/modules/reference/pages/properties/topic-properties.adoc b/modules/reference/pages/properties/topic-properties.adoc index ce669ba8e9..e99bff7283 100644 --- a/modules/reference/pages/properties/topic-properties.adoc +++ b/modules/reference/pages/properties/topic-properties.adoc @@ -33,6 +33,112 @@ These properties control disk flushing, replication, and how topics interact wit include::reference:partial$properties/topic-properties.adoc[tags=category-performance-cluster;!deprecated;!exclude-from-docs] +The maximum bytes not fsynced per partition. If this configured threshold is reached, the log is automatically fsynced, even though it wasn't explicitly requested. + +*Type:* integer + +*Unit:* bytes + +*Accepted values:* [`1`, `9223372036854775807`] + +*Default:* `262144` + +*Related cluster property:* xref:./cluster-properties.adoc#flush_bytes[`flush_bytes`] + +**Related topics**: + +- xref:develop:produce-data/configure-producers.adoc[] + +--- +[[flushms]] +=== flush.ms + +The maximum delay (in ms) between two subsequent fsyncs. After this delay, the log is automatically fsynced. + +*Type:* integer + +*Unit:* milliseconds + +*Accepted values:* [`1`, `9223372036854775`] + +*Default:* `100` + +*Related cluster property:* xref:./cluster-properties.adoc#flush_ms[`flush_ms`] + +**Related topics**: + +- xref:develop:produce-data/configure-producers.adoc[] + +--- +[[redpandaleaderspreference]] +=== redpanda.leaders.preference + +The preferred location (rack) for partition leaders of a topic. + +This property inherits the value from the config_ref:default_leaders_preference,true,properties/cluster-properties[] cluster configuration property. You may override the cluster-wide setting by specifying the value for individual topics. + +If the cluster configuration property config_ref:enable_rack_awareness,true,properties/cluster-properties[] is set to `false`, Leader Pinning is disabled across the cluster. + +*Type:* string + +*Default:* `none` + +**Values**: + +- `none`: Opt out the topic from Leader Pinning. +- `racks:[,,...]`: Specify the preferred location (rack) of all topic partition leaders. The list can contain one or more rack IDs. If you specify multiple IDs, Redpanda tries to distribute the partition leader locations equally across brokers in these racks. + +**Related topics**: + +- xref:develop:produce-data/leader-pinning.adoc[Leader pinning] + +--- +[[replicationfactor]] +=== replication.factor + +The number of replicas of a topic to save in different nodes (brokers) of a cluster. + +If `replication.factor` is set to a positive value, it overrides the cluster property xref:./cluster-properties.adoc#default_topic_replication[default_topic_replication] for the topic. + +NOTE: Although `replication.factor` isn't returned or displayed by xref:reference:rpk/rpk-topic/rpk-topic-describe.adoc[`rpk topic describe`] as a valid Kafka property, you can set it using xref:reference:rpk/rpk-topic/rpk-topic-alter-config.adoc[`rpk topic alter-config`]. When the `replication.factor` of a topic is altered, it isn't simply a property value that's updated, but rather the actual replica sets of topic partitions that are changed. + +*Type:* integer + +*Accepted values:* [`1`, `512`] + +*Default:* null + +*Related cluster property:* xref:./cluster-properties.adoc#default_topic_replication[`default_topic_replication`] + +**Related topics**: + +- xref:develop:manage-topics/config-topics.adoc#choose-the-replication-factor[Choose the replication factor] +- xref:develop:manage-topics/config-topics.adoc#change-the-replication-factor[Change the replication factor] + +--- +[[writecaching]] +=== write.caching + +The write caching mode to apply to a topic. + +When `write.caching` is set, it overrides the cluster property xref:cluster-properties.adoc#write_caching_default[`write_caching_default`]. Write caching acknowledges a message as soon as it is received and acknowledged on a majority of brokers, without waiting for it to be written to disk. With `acks=all`, this provides lower latency while still ensuring that a majority of brokers acknowledge the write. Fsyncs follow <> and <>, whichever is reached first. + +*Type:* boolean + +*Default:* `false` + +**Values**: + +- `true` - Enables write caching for a topic, according to <> and <>. +- `false` - Disables write caching for a topic, according to <> and <>. + +*Related cluster property:* xref:./cluster-properties.adoc#write_caching_default[`write_caching_default`] + +**Related topics**: + +- xref:develop:manage-topics/config-topics.adoc#configure-write-caching[Write caching] + +--- == Tiered Storage properties Configure properties to manage topics for xref:manage:tiered-storage.adoc[Tiered Storage]. diff --git a/modules/reference/pages/rpk/rpk-redpanda/rpk-redpanda-mode.adoc b/modules/reference/pages/rpk/rpk-redpanda/rpk-redpanda-mode.adoc index 9cabe78320..efd6b2a553 100644 --- a/modules/reference/pages/rpk/rpk-redpanda/rpk-redpanda-mode.adoc +++ b/modules/reference/pages/rpk/rpk-redpanda/rpk-redpanda-mode.adoc @@ -10,7 +10,7 @@ Development mode (`development` or `dev`) includes the following development-onl * Sets `developer_mode` to `true`. This starts Redpanda with dev-mode only settings, including: ** No minimal memory limits are enforced. ** No core assignment rules for Redpanda nodes are enforced. - ** Enables write caching, which is a relaxed mode of `acks=all` that acknowledges a message as soon as it is received and acknowledged on a majority of brokers, without waiting for it to fsync to disk. This provides lower latency while still ensuring that a majority of brokers acknowledge the write. For more information, or to disable this, see xref:develop:config-topics.adoc#configure-write-caching[write caching]. + ** Enables write caching, which is a relaxed mode of `acks=all` that acknowledges a message as soon as it is received and acknowledged on a majority of brokers, without waiting for it to fsync to disk. This provides lower latency while still ensuring that a majority of brokers acknowledge the write. For more information, or to disable this, see xref:develop:manage-topics/config-topics.adoc#configure-write-caching[write caching]. ** Bypasses `fsync` (from https://docs.seastar.io/master/structseastar_1_1reactor%5F%5Foptions.html#ad66cb23f59ed5dfa8be8189313988692[Seastar option `unsafe_bypass_fsync`^]), which results in unrealistically fast clusters and may result in data loss. * Sets `overprovisioned` to `true`. Redpanda expects a dev system to be an overprovisioned environment. Based on a https://docs.seastar.io/master/structseastar_1_1reactor%5F%5Foptions.html#a0caf6c2ad579b8c22e1352d796ec3c1d[Seastar option^], setting `overprovisioned` disables thread affinity, zeros idle polling time, and disables busy-poll for disk I/O. * Sets all autotuner xref:./rpk-redpanda-tune-list.adoc#tuners[tuners] to `false`. The tuners are intended to run only for production mode. diff --git a/modules/reference/pages/rpk/rpk-security/rpk-security-role-assign.adoc b/modules/reference/pages/rpk/rpk-security/rpk-security-role-assign.adoc index 5670fc9dd7..41aaf549c2 100644 --- a/modules/reference/pages/rpk/rpk-security/rpk-security-role-assign.adoc +++ b/modules/reference/pages/rpk/rpk-security/rpk-security-role-assign.adoc @@ -7,18 +7,30 @@ The `--principal` flag accepts principals with the format `:> setting (that is, when the topic's `redpanda.storage.mode` is `unset`). + +Accepted values: + +* `unset`: Defer to the legacy <> and <> topic properties for Tiered Storage configuration. +* `local`: Store data only on local disks, with no object storage involvement. +ifndef::env-cloud[] +* `tiered`: Store data on local disks and replicate it to object storage using xref:manage:tiered-storage.adoc[Tiered Storage]. Equivalent to setting `redpanda.remote.read` and `redpanda.remote.write` to `true`. +* `cloud`: Store data primarily in object storage using xref:manage:tiered-storage.adoc#cloud-topics[cloud topics]. +endif::[] +ifdef::env-cloud[] +* `tiered`: Store data on local disks and replicate it to object storage using Tiered Storage. Equivalent to setting `redpanda.remote.read` and `redpanda.remote.write` to `true`. +* `cloud`: Store data primarily in object storage using Cloud Topics. +endif::[] + +[cols="1s,2a"] +|=== +| Property | Value + +| Type +| `string` (enum) + +| Accepted values +| +ifndef::env-cloud[] +`local`, `tiered`, `cloud`, `unset` +endif::[] +ifdef::env-cloud[] +`local`, `tiered`, `cloud`, `unset` +endif::[] + + +| Default +| +ifdef::env-cloud[] +Available in the Redpanda Cloud Console (editable) +endif::[] +ifndef::env-cloud[] +`unset` +endif::[] + +| Nullable +| No + +| Requires restart +| No + +ifndef::env-cloud[] +| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] +| Yes +endif::[] + +ifndef::env-cloud[] +| Visibility +| User +endif::[] + +| Example +| +`tiered` + +| Related topics +| +* xref:manage:tiered-storage.adoc[Tiered Storage] + +* xref:develop:manage-topics/cloud-topics.adoc[Manage Cloud Topics] + +|=== + +// end::redpanda-cloud[] + === default_topic_partitions Default number of partitions per topic. @@ -4106,6 +5136,61 @@ endif::[] |=== +=== delete_topic_enable + +ifndef::env-cloud[] +*Introduced in v26.1.1* +endif::[] + +Enable or disable topic deletion via the Kafka DeleteTopics API. When set to false, all topic deletion requests are rejected with error code 73 (TOPIC_DELETION_DISABLED). This is a cluster-wide safety setting that cannot be overridden by superusers. Topics in kafka_nodelete_topics are always protected regardless of this setting. + +ifndef::env-cloud[] +.Enterprise license required +[NOTE] +==== +The following values require an Enterprise license: `false` + +For license details, see xref:get-started:licensing/index.adoc[Redpanda Licensing]. +==== +endif::[] + +[cols="1s,2a"] +|=== +| Property | Value + +| Type +| `boolean` + + + +| Default +| +ifdef::env-cloud[] +Available in the Redpanda Cloud Console +endif::[] +ifndef::env-cloud[] +`true` +endif::[] + +| Nullable +| No + +| Requires restart +| No + +ifndef::env-cloud[] +| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] +| Yes +endif::[] + +ifndef::env-cloud[] +| Visibility +| User +endif::[] + +|=== + + === disable_batch_cache Disable batch cache in log manager. @@ -4354,45 +5439,6 @@ endif::[] |=== -// tag::deprecated[] -=== enable_admin_api - -No description available. - - -[cols="1s,2a"] -|=== -| Property | Value - -| Type -| `deprecated_property` - - - -| Default -| -ifdef::env-cloud[] -Available in the Redpanda Cloud Console -endif::[] -ifndef::env-cloud[] -`null` -endif::[] - -| Nullable -| No - -| Requires restart -| Yes - -ifndef::env-cloud[] -| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] -| Yes -endif::[] - -|=== - -// end::deprecated[] - // tag::deprecated[] === enable_auto_rebalance_on_node_add @@ -4527,79 +5573,36 @@ endif::[] * xref:reference:public-metrics-reference.adoc#redpanda_kafka_consumer_group_topics[`redpanda_kafka_consumer_group_topics`] -* xref:reference:public-metrics-reference.adoc#redpanda_kafka_consumer_group_committed_offset[`redpanda_kafka_consumer_group_committed_offset`] - -* xref:reference:public-metrics-reference.adoc#redpanda_kafka_consumer_group_lag_max[`redpanda_kafka_consumer_group_lag_max`] - -* xref:reference:public-metrics-reference.adoc#redpanda_kafka_consumer_group_lag_sum[`redpanda_kafka_consumer_group_lag_sum`] - -* xref:reference:properties/cluster-properties.adoc#consumer_group_lag_collection_interval_sec[`consumer_group_lag_collection_interval_sec`] - -ifndef::env-cloud[] -* xref:manage:monitoring.adoc#consumers[Monitor consumer group lag] -endif::[] - -ifdef::env-cloud[] -* xref:manage:monitor-cloud.adoc#consumers[Monitor consumer group lag] -endif::[] - -|=== - -// end::redpanda-cloud[] - -=== enable_controller_log_rate_limiting - -Limits the write rate for the controller log. - -[cols="1s,2a"] -|=== -| Property | Value - -| Type -| `boolean` - - - -| Default -| -ifdef::env-cloud[] -Available in the Redpanda Cloud Console -endif::[] -ifndef::env-cloud[] -`false` -endif::[] - -| Nullable -| No +* xref:reference:public-metrics-reference.adoc#redpanda_kafka_consumer_group_committed_offset[`redpanda_kafka_consumer_group_committed_offset`] -| Requires restart -| No +* xref:reference:public-metrics-reference.adoc#redpanda_kafka_consumer_group_lag_max[`redpanda_kafka_consumer_group_lag_max`] + +* xref:reference:public-metrics-reference.adoc#redpanda_kafka_consumer_group_lag_sum[`redpanda_kafka_consumer_group_lag_sum`] + +* xref:reference:properties/cluster-properties.adoc#consumer_group_lag_collection_interval_sec[`consumer_group_lag_collection_interval_sec`] ifndef::env-cloud[] -| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] -| Yes +* xref:manage:monitoring.adoc#consumers[Monitor consumer group lag] endif::[] -ifndef::env-cloud[] -| Visibility -| User +ifdef::env-cloud[] +* xref:manage:monitor-cloud.adoc#consumers[Monitor consumer group lag] endif::[] |=== +// end::redpanda-cloud[] -// tag::deprecated[] -=== enable_coproc - -No description available. +=== enable_controller_log_rate_limiting +Limits the write rate for the controller log. [cols="1s,2a"] |=== | Property | Value | Type -| `deprecated_property` +| `boolean` @@ -4609,23 +5612,27 @@ ifdef::env-cloud[] Available in the Redpanda Cloud Console endif::[] ifndef::env-cloud[] -`null` +`false` endif::[] | Nullable | No | Requires restart -| Yes +| No ifndef::env-cloud[] | Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] | Yes endif::[] +ifndef::env-cloud[] +| Visibility +| User +endif::[] + |=== -// end::deprecated[] // tag::exclude-from-docs[] === enable_developmental_unrecoverable_data_corrupting_features @@ -5022,8 +6029,6 @@ Values: * `redpanda`: Schema validation is enabled. Only Redpanda topic properties are accepted. * `compat`: Schema validation is enabled. Both Redpanda and compatible topic properties are accepted. -NOTE: Enabling this property will trigger decompression of message batches for topics on which validation is configured, which may lead to a modest increase in CPU load. Redpanda recommends monitoring CPU utilization after topics are configured. - ifndef::env-cloud[] .Enterprise license required [NOTE] @@ -5706,87 +6711,6 @@ endif::[] |=== -// tag::deprecated[] -=== find_coordinator_timeout_ms - -No description available. - - -[cols="1s,2a"] -|=== -| Property | Value - -| Type -| `deprecated_property` - - - -| Default -| -ifdef::env-cloud[] -Available in the Redpanda Cloud Console -endif::[] -ifndef::env-cloud[] -`null` -endif::[] - -| Nullable -| No - -| Unit -| Milliseconds - -| Requires restart -| Yes - -ifndef::env-cloud[] -| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] -| Yes -endif::[] - -|=== - -// end::deprecated[] - -// tag::deprecated[] -=== full_raft_configuration_recovery_pattern - -No description available. - - -[cols="1s,2a"] -|=== -| Property | Value - -| Type -| `deprecated_property` - - - -| Default -| -ifdef::env-cloud[] -Available in the Redpanda Cloud Console -endif::[] -ifndef::env-cloud[] -`null` -endif::[] - -| Nullable -| No - -| Requires restart -| Yes - -ifndef::env-cloud[] -| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] -| Yes -endif::[] - -|=== - -// end::deprecated[] - === group_initial_rebalance_delay Delay added to the rebalance phase to wait for new members. @@ -6524,7 +7448,7 @@ ifndef::env-cloud[] *Introduced in v25.3.5* endif::[] -The default Iceberg catalog namespace where Redpanda creates tables. Supports nested namespaces as an array of strings. +The default namespace (database name) for Iceberg tables. All tables created by Redpanda will be placed in this namespace within the Iceberg catalog. Supports nested namespaces as an array of strings. IMPORTANT: This value must be configured before enabling Iceberg and must not be changed afterward. Changing it will cause Redpanda to lose track of existing tables. @@ -8240,45 +9164,6 @@ endif::[] |=== -// tag::deprecated[] -=== id_allocator_replication - -No description available. - - -[cols="1s,2a"] -|=== -| Property | Value - -| Type -| `deprecated_property` - - - -| Default -| -ifdef::env-cloud[] -Available in the Redpanda Cloud Console -endif::[] -ifndef::env-cloud[] -`null` -endif::[] - -| Nullable -| No - -| Requires restart -| Yes - -ifndef::env-cloud[] -| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] -| Yes -endif::[] - -|=== - -// end::deprecated[] - === initial_retention_local_target_bytes_default Initial local retention size target for partitions of topics with xref:manage:tiered-storage.adoc[Tiered Storage] enabled. If no initial local target retention is configured, then all locally-retained data will be delivered to learner when joining the partition replica set. @@ -8370,55 +9255,13 @@ endif::[] |=== -=== internal_topic_replication_factor - -Target replication factor for internal topics. - -*Unit*: number of replicas per topic. - -[cols="1s,2a"] -|=== -| Property | Value - -| Type -| `integer` - - - -| Range -| [`-2147483648`, `2147483647`] - -| Default -| -ifdef::env-cloud[] -Available in the Redpanda Cloud Console -endif::[] -ifndef::env-cloud[] -`3` -endif::[] - -| Nullable -| No - -| Requires restart -| Yes - -ifndef::env-cloud[] -| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] -| Yes -endif::[] +=== internal_rpc_request_timeout_ms ifndef::env-cloud[] -| Visibility -| User +*Introduced in v26.1.1* endif::[] -|=== - - -=== join_retry_timeout_ms - -Time between cluster join retries in milliseconds. +Default timeout for RPC requests between Redpanda nodes. [cols="1s,2a"] |=== @@ -8438,7 +9281,7 @@ ifdef::env-cloud[] Available in the Redpanda Cloud Console endif::[] ifndef::env-cloud[] -`5000` (5 seconds) +`10000` (10 seconds) endif::[] | Nullable @@ -8448,7 +9291,7 @@ endif::[] | Milliseconds | Requires restart -| Yes +| No ifndef::env-cloud[] | Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] @@ -8463,48 +9306,11 @@ endif::[] |=== -// tag::deprecated[] -=== kafka_admin_topic_api_rate - -No description available. - - -[cols="1s,2a"] -|=== -| Property | Value - -| Type -| `deprecated_property` - - - -| Default -| -ifdef::env-cloud[] -Available in the Redpanda Cloud Console -endif::[] -ifndef::env-cloud[] -`null` -endif::[] - -| Nullable -| No - -| Requires restart -| Yes - -ifndef::env-cloud[] -| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] -| Yes -endif::[] - -|=== - -// end::deprecated[] +=== internal_topic_replication_factor -=== kafka_batch_max_bytes +Target replication factor for internal topics. -The default maximum batch size for topics if the topic property xref:reference:properties/topic-properties.adoc[`message.max.bytes`] is not set. If the batch is compressed, the limit applies to the compressed batch size. +*Unit*: number of replicas per topic. [cols="1s,2a"] |=== @@ -8515,8 +9321,8 @@ The default maximum batch size for topics if the topic property xref:reference:p -| Maximum -| `4294967295` +| Range +| [`-2147483648`, `2147483647`] | Default | @@ -8524,17 +9330,14 @@ ifdef::env-cloud[] Available in the Redpanda Cloud Console endif::[] ifndef::env-cloud[] -`1048576` +`3` endif::[] | Nullable | No -| Unit -| Bytes - | Requires restart -| No +| Yes ifndef::env-cloud[] | Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] @@ -8543,42 +9346,43 @@ endif::[] ifndef::env-cloud[] | Visibility -| Tunable +| User endif::[] -| Related topics -|xref:reference:properties/topic-properties.adoc[`message.max.bytes`] - |=== -// tag::deprecated[] -=== kafka_client_group_byte_rate_quota - -No description available. +=== join_retry_timeout_ms +Time between cluster join retries in milliseconds. [cols="1s,2a"] |=== | Property | Value | Type -| `deprecated_property` +| `integer` +| Range +| [`-17592186044416`, `17592186044415`] + | Default | ifdef::env-cloud[] Available in the Redpanda Cloud Console endif::[] ifndef::env-cloud[] -`null` +`5000` (5 seconds) endif::[] | Nullable | No +| Unit +| Milliseconds + | Requires restart | Yes @@ -8587,48 +9391,63 @@ ifndef::env-cloud[] | Yes endif::[] -|=== +ifndef::env-cloud[] +| Visibility +| Tunable +endif::[] -// end::deprecated[] +|=== -// tag::deprecated[] -=== kafka_client_group_fetch_byte_rate_quota -No description available. +=== kafka_batch_max_bytes +The default maximum batch size for topics if the topic property xref:reference:properties/topic-properties.adoc[`message.max.bytes`] is not set. If the batch is compressed, the limit applies to the compressed batch size. [cols="1s,2a"] |=== | Property | Value | Type -| `deprecated_property` +| `integer` +| Maximum +| `4294967295` + | Default | ifdef::env-cloud[] Available in the Redpanda Cloud Console endif::[] ifndef::env-cloud[] -`null` +`1048576` endif::[] | Nullable | No +| Unit +| Bytes + | Requires restart -| Yes +| No ifndef::env-cloud[] | Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] | Yes endif::[] +ifndef::env-cloud[] +| Visibility +| Tunable +endif::[] + +| Related topics +|xref:reference:properties/topic-properties.adoc[`message.max.bytes`] + |=== -// end::deprecated[] === kafka_connection_rate_limit @@ -8984,7 +9803,7 @@ ifndef::env-cloud[] *Introduced in v25.3.7* endif::[] -Target duration for a single fetch request. The broker tries to complete each fetch within this duration, even if fewer bytes are available than requested. +Broker-side target for the duration of a single fetch request. The broker will try to complete fetches within the specified duration, even if it means returning less bytes in the fetch than are available. [cols="1s,2a"] |=== @@ -9167,45 +9986,6 @@ endif::[] |=== -// tag::deprecated[] -=== kafka_memory_batch_size_estimate_for_fetch - -No description available. - - -[cols="1s,2a"] -|=== -| Property | Value - -| Type -| `deprecated_property` - - - -| Default -| -ifdef::env-cloud[] -Available in the Redpanda Cloud Console -endif::[] -ifndef::env-cloud[] -`null` -endif::[] - -| Nullable -| No - -| Requires restart -| Yes - -ifndef::env-cloud[] -| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] -| Yes -endif::[] - -|=== - -// end::deprecated[] - === kafka_memory_share_for_fetch The share of Kafka subsystem memory that can be used for fetch read buffers, as a fraction of the Kafka subsystem memory amount. @@ -9699,189 +10479,14 @@ ifdef::env-cloud[] Available in the Redpanda Cloud Console endif::[] ifndef::env-cloud[] -`80` (80 milliseconds) -endif::[] - -| Nullable -| No - -| Unit -| Milliseconds - -| Requires restart -| Yes - -ifndef::env-cloud[] -| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] -| Yes -endif::[] - -ifndef::env-cloud[] -| Visibility -| User -endif::[] - -|=== - - -=== kafka_qdc_min_depth - -Minimum queue depth used in Kafka queue depth control. - -[cols="1s,2a"] -|=== -| Property | Value - -| Type -| `integer` - - - -| Default -| -ifdef::env-cloud[] -Available in the Redpanda Cloud Console -endif::[] -ifndef::env-cloud[] -`1` -endif::[] - -| Nullable -| No - -| Requires restart -| Yes - -ifndef::env-cloud[] -| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] -| Yes -endif::[] - -ifndef::env-cloud[] -| Visibility -| Tunable -endif::[] - -|=== - - -=== kafka_qdc_window_count - -Number of windows used in Kafka queue depth control latency tracking. - -[cols="1s,2a"] -|=== -| Property | Value - -| Type -| `integer` - - - -| Default -| -ifdef::env-cloud[] -Available in the Redpanda Cloud Console -endif::[] -ifndef::env-cloud[] -`12` -endif::[] - -| Nullable -| No - -| Requires restart -| Yes - -ifndef::env-cloud[] -| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] -| Yes -endif::[] - -ifndef::env-cloud[] -| Visibility -| Tunable -endif::[] - -|=== - - -=== kafka_qdc_window_size_ms - -Window size for Kafka queue depth control latency tracking. - -[cols="1s,2a"] -|=== -| Property | Value - -| Type -| `integer` - - - -| Range -| [`-17592186044416`, `17592186044415`] - -| Default -| -ifdef::env-cloud[] -Available in the Redpanda Cloud Console -endif::[] -ifndef::env-cloud[] -`1500` (1500 milliseconds) -endif::[] - -| Nullable -| No - -| Unit -| Milliseconds - -| Requires restart -| Yes - -ifndef::env-cloud[] -| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] -| Yes -endif::[] - -ifndef::env-cloud[] -| Visibility -| Tunable -endif::[] - -|=== - - -// tag::deprecated[] -=== kafka_quota_balancer_min_shard_throughput_bps - -No description available. - - -[cols="1s,2a"] -|=== -| Property | Value - -| Type -| `deprecated_property` - - - -| Default -| -ifdef::env-cloud[] -Available in the Redpanda Cloud Console -endif::[] -ifndef::env-cloud[] -`null` +`80` (80 milliseconds) endif::[] | Nullable | No | Unit -| Bytes per second +| Milliseconds | Requires restart | Yes @@ -9891,22 +10496,24 @@ ifndef::env-cloud[] | Yes endif::[] -|=== +ifndef::env-cloud[] +| Visibility +| User +endif::[] -// end::deprecated[] +|=== -// tag::deprecated[] -=== kafka_quota_balancer_min_shard_throughput_ratio -No description available. +=== kafka_qdc_min_depth +Minimum queue depth used in Kafka queue depth control. [cols="1s,2a"] |=== | Property | Value | Type -| `deprecated_property` +| `integer` @@ -9916,7 +10523,7 @@ ifdef::env-cloud[] Available in the Redpanda Cloud Console endif::[] ifndef::env-cloud[] -`null` +`1` endif::[] | Nullable @@ -9930,22 +10537,24 @@ ifndef::env-cloud[] | Yes endif::[] -|=== +ifndef::env-cloud[] +| Visibility +| Tunable +endif::[] -// end::deprecated[] +|=== -// tag::deprecated[] -=== kafka_quota_balancer_node_period -No description available. +=== kafka_qdc_window_count +Number of windows used in Kafka queue depth control latency tracking. [cols="1s,2a"] |=== | Property | Value | Type -| `deprecated_property` +| `integer` @@ -9955,7 +10564,7 @@ ifdef::env-cloud[] Available in the Redpanda Cloud Console endif::[] ifndef::env-cloud[] -`null` +`12` endif::[] | Nullable @@ -9969,24 +10578,29 @@ ifndef::env-cloud[] | Yes endif::[] -|=== +ifndef::env-cloud[] +| Visibility +| Tunable +endif::[] -// end::deprecated[] +|=== -// tag::deprecated[] -=== kafka_quota_balancer_window -No description available. +=== kafka_qdc_window_size_ms +Window size for Kafka queue depth control latency tracking. [cols="1s,2a"] |=== | Property | Value | Type -| `deprecated_property` +| `integer` + +| Range +| [`-17592186044416`, `17592186044415`] | Default | @@ -9994,12 +10608,15 @@ ifdef::env-cloud[] Available in the Redpanda Cloud Console endif::[] ifndef::env-cloud[] -`null` +`1500` (1500 milliseconds) endif::[] | Nullable | No +| Unit +| Milliseconds + | Requires restart | Yes @@ -10008,9 +10625,13 @@ ifndef::env-cloud[] | Yes endif::[] +ifndef::env-cloud[] +| Visibility +| Tunable +endif::[] + |=== -// end::deprecated[] === kafka_request_max_bytes @@ -10687,45 +11308,6 @@ endif::[] |=== -// tag::deprecated[] -=== kafka_throughput_throttling_v2 - -No description available. - - -[cols="1s,2a"] -|=== -| Property | Value - -| Type -| `deprecated_property` - - - -| Default -| -ifdef::env-cloud[] -Available in the Redpanda Cloud Console -endif::[] -ifndef::env-cloud[] -`null` -endif::[] - -| Nullable -| No - -| Requires restart -| Yes - -ifndef::env-cloud[] -| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] -| Yes -endif::[] - -|=== - -// end::deprecated[] - === kafka_topics_max Maximum number of Kafka user topics that can be created. If `null`, then no limit is enforced. @@ -10901,19 +11483,25 @@ endif::[] |=== -// tag::deprecated[] === leader_balancer_mode -No description available. - +Mode of the leader balancer optimization strategy. `calibrated` uses a heuristic that balances leaders based on replica counts per shard. `random` randomly moves leaders to reduce load on heavily-loaded shards. Legacy values `greedy_balanced_shards` and `random_hill_climbing` are treated as `calibrated`. [cols="1s,2a"] |=== | Property | Value | Type -| `deprecated_property` +| `string` (enum) +| Accepted values +| +ifndef::env-cloud[] +`calibrated`, `random` +endif::[] +ifdef::env-cloud[] +`calibrated`, `random` +endif::[] | Default @@ -10922,23 +11510,31 @@ ifdef::env-cloud[] Available in the Redpanda Cloud Console endif::[] ifndef::env-cloud[] -`null` +`calibrated` endif::[] | Nullable | No | Requires restart -| Yes +| No ifndef::env-cloud[] | Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] | Yes endif::[] +ifndef::env-cloud[] +| Visibility +| User +endif::[] + +| Example +| +`model::leader_balancer_mode_to_string( model::leader_balancer_mode::calibrated)` + |=== -// end::deprecated[] === leader_balancer_mute_timeout @@ -11258,18 +11854,16 @@ endif::[] |=== -// tag::deprecated[] -=== log_compaction_adjacent_merge_self_compaction_count - -No description available. +=== log_compaction_disable_tx_batch_removal +Prevents log compaction from removing transaction metadata. Only set this to `true` if you experience stability issues related to transaction cleanup during compaction. [cols="1s,2a"] |=== | Property | Value | Type -| `deprecated_property` +| `string` @@ -11285,55 +11879,18 @@ endif::[] | Nullable | No -| Requires restart -| Yes - ifndef::env-cloud[] | Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] | Yes endif::[] -|=== - -// end::deprecated[] - -// tag::deprecated[] -=== log_compaction_disable_tx_batch_removal - -Prevents log compaction from removing transaction metadata. Only set this to `true` if you experience stability issues related to transaction cleanup during compaction. - -[cols="1s,2a"] -|=== -| Property | Value - -| Type -| `deprecated_property` - - - -| Default -| -ifdef::env-cloud[] -Available in the Redpanda Cloud Console -endif::[] -ifndef::env-cloud[] -`null` -endif::[] - -| Nullable -| No - -| Requires restart -| Yes - ifndef::env-cloud[] -| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] -| Yes +| Visibility +| User endif::[] |=== -// end::deprecated[] === log_compaction_interval_ms @@ -11581,7 +12138,7 @@ ifdef::env-cloud[] Available in the Redpanda Cloud Console endif::[] ifndef::env-cloud[] -`false` +`true` endif::[] | Nullable @@ -11737,128 +12294,37 @@ ifndef::env-cloud[] | Yes endif::[] -ifndef::env-cloud[] -| Visibility -| Tunable -endif::[] - -|=== - - -=== log_message_timestamp_after_max_ms - -The maximum allowed time difference when a record's timestamp is in the future compared to the broker's clock. For topics using <> timestamps, Redpanda rejects records with timestamps that are too far in the future. This property has no effect on topics using <> timestamps. The topic property xref:./topic-properties.adoc#messagetimestampaftermaxms[`message.timestamp.after.max.ms`] overrides this cluster-level setting. - -[cols="1s,2a"] -|=== -| Property | Value - -| Type -| `integer` - - - -| Range -| [`-17592186044416`, `17592186044415`] - -| Default -| -ifdef::env-cloud[] -Available in the Redpanda Cloud Console -endif::[] -ifndef::env-cloud[] -`3600000` (1 hour) -endif::[] - -| Nullable -| No - -| Unit -| Milliseconds - -| Requires restart -| No - -ifndef::env-cloud[] -| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] -| Yes -endif::[] - -ifndef::env-cloud[] -| Visibility -| User -endif::[] - -| Related topics -|xref:reference:properties/topic-properties.adoc#messagetimestampaftermaxms[`message.timestamp.after.max.ms`] - -|=== - - -// tag::deprecated[] -=== log_message_timestamp_alert_after_ms - -No description available. - - -[cols="1s,2a"] -|=== -| Property | Value - -| Type -| `deprecated_property` - - - -| Default -| -ifdef::env-cloud[] -Available in the Redpanda Cloud Console -endif::[] -ifndef::env-cloud[] -`null` -endif::[] - -| Nullable -| No - -| Unit -| Milliseconds - -| Requires restart -| Yes - -ifndef::env-cloud[] -| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] -| Yes +ifndef::env-cloud[] +| Visibility +| Tunable endif::[] |=== -// end::deprecated[] - -// tag::deprecated[] -=== log_message_timestamp_alert_before_ms -No description available. +=== log_message_timestamp_after_max_ms +The maximum allowed time difference when a record's timestamp is in the future compared to the broker's clock. For topics using <> timestamps, Redpanda rejects records with timestamps that are too far in the future. This property has no effect on topics using <> timestamps. The topic property xref:./topic-properties.adoc#messagetimestampaftermaxms[`message.timestamp.after.max.ms`] overrides this cluster-level setting. [cols="1s,2a"] |=== | Property | Value | Type -| `deprecated_property` +| `integer` +| Range +| [`-17592186044416`, `17592186044415`] + | Default | ifdef::env-cloud[] Available in the Redpanda Cloud Console endif::[] ifndef::env-cloud[] -`null` +`3600000` (1 hour) endif::[] | Nullable @@ -11868,16 +12334,23 @@ endif::[] | Milliseconds | Requires restart -| Yes +| No ifndef::env-cloud[] | Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] | Yes endif::[] +ifndef::env-cloud[] +| Visibility +| User +endif::[] + +| Related topics +|xref:reference:properties/topic-properties.adoc#messagetimestampaftermaxms[`message.timestamp.after.max.ms`] + |=== -// end::deprecated[] === log_message_timestamp_before_max_ms @@ -12762,45 +13235,6 @@ endif::[] |=== -// tag::deprecated[] -=== max_version - -No description available. - - -[cols="1s,2a"] -|=== -| Property | Value - -| Type -| `deprecated_property` - - - -| Default -| -ifdef::env-cloud[] -Available in the Redpanda Cloud Console -endif::[] -ifndef::env-cloud[] -`null` -endif::[] - -| Nullable -| No - -| Requires restart -| Yes - -ifndef::env-cloud[] -| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] -| Yes -endif::[] - -|=== - -// end::deprecated[] - === members_backend_retry_ms Time between members backend reconciliation loop retries. @@ -13031,53 +13465,6 @@ endif::[] |=== -=== metadata_status_wait_timeout_ms - -Maximum time to wait in metadata request for cluster health to be refreshed. - -[cols="1s,2a"] -|=== -| Property | Value - -| Type -| `integer` - - - -| Range -| [`-17592186044416`, `17592186044415`] - -| Default -| -ifdef::env-cloud[] -Available in the Redpanda Cloud Console -endif::[] -ifndef::env-cloud[] -`2000` (2 seconds) -endif::[] - -| Nullable -| No - -| Unit -| Milliseconds - -| Requires restart -| Yes - -ifndef::env-cloud[] -| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] -| Yes -endif::[] - -ifndef::env-cloud[] -| Visibility -| Tunable -endif::[] - -|=== - - === metrics_reporter_report_interval Cluster metrics reporter report interval. @@ -13302,20 +13689,21 @@ endif::[] |=== -// tag::deprecated[] -=== min_version - -No description available. +=== minimum_topic_replication +Minimum allowable replication factor for topics in this cluster. The set value must be positive, odd, and equal to or less than the number of available brokers. Changing this parameter only restricts newly-created topics. Redpanda returns an `INVALID_REPLICATION_FACTOR` error on any attempt to create a topic with a replication factor less than this property. If you change the `minimum_topic_replications` setting, the replication factor of existing topics remains unchanged. However, Redpanda will log a warning on start-up with a list of any topics that have fewer replicas than this minimum. For example, you might see a message such as `Topic X has a replication factor less than specified minimum: 1 < 3`. [cols="1s,2a"] |=== | Property | Value | Type -| `deprecated_property` +| `integer` + +| Range +| [`-32768`, `32767`] | Default | @@ -13323,39 +13711,44 @@ ifdef::env-cloud[] Available in the Redpanda Cloud Console endif::[] ifndef::env-cloud[] -`null` +`1` endif::[] | Nullable | No | Requires restart -| Yes +| No ifndef::env-cloud[] | Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] | Yes endif::[] +ifndef::env-cloud[] +| Visibility +| User +endif::[] + |=== -// end::deprecated[] -=== minimum_topic_replication +=== nested_group_behavior -Minimum allowable replication factor for topics in this cluster. The set value must be positive, odd, and equal to or less than the number of available brokers. Changing this parameter only restricts newly-created topics. Redpanda returns an `INVALID_REPLICATION_FACTOR` error on any attempt to create a topic with a replication factor less than this property. If you change the `minimum_topic_replications` setting, the replication factor of existing topics remains unchanged. However, Redpanda will log a warning on start-up with a list of any topics that have fewer replicas than this minimum. For example, you might see a message such as `Topic X has a replication factor less than specified minimum: 1 < 3`. +ifndef::env-cloud[] +*Introduced in v26.1.1* +endif::[] + +Behavior for handling nested groups when extracting groups from authentication tokens. Two options are available - none and suffix. With none, the group is left alone (e.g. '/group/child/grandchild'). Suffix will extract the final component from the nested group (e.g. '/group' -> 'group' and '/group/child/grandchild' -> 'grandchild'). [cols="1s,2a"] |=== | Property | Value | Type -| `integer` - +| `string` -| Range -| [`-32768`, `32767`] | Default | @@ -13363,7 +13756,7 @@ ifdef::env-cloud[] Available in the Redpanda Cloud Console endif::[] ifndef::env-cloud[] -`1` +`none` endif::[] | Nullable @@ -13429,9 +13822,9 @@ endif::[] |=== -=== node_management_operation_timeout_ms +=== node_status_interval -Timeout for executing node management operations. +Time interval between two node status messages. Node status messages establish liveness status outside of the Raft protocol. [cols="1s,2a"] |=== @@ -13451,17 +13844,14 @@ ifdef::env-cloud[] Available in the Redpanda Cloud Console endif::[] ifndef::env-cloud[] -`5000` (5 seconds) +`100` (100 milliseconds) endif::[] | Nullable | No -| Unit -| Milliseconds - | Requires restart -| Yes +| No ifndef::env-cloud[] | Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] @@ -13476,9 +13866,9 @@ endif::[] |=== -=== node_status_interval +=== node_status_reconnect_max_backoff_ms -Time interval between two node status messages. Node status messages establish liveness status outside of the Raft protocol. +Maximum backoff (in milliseconds) to reconnect to an unresponsive peer during node status liveness checks. [cols="1s,2a"] |=== @@ -13498,12 +13888,15 @@ ifdef::env-cloud[] Available in the Redpanda Cloud Console endif::[] ifndef::env-cloud[] -`100` (100 milliseconds) +`15000` (15 seconds) endif::[] | Nullable | No +| Unit +| Milliseconds + | Requires restart | No @@ -13514,15 +13907,15 @@ endif::[] ifndef::env-cloud[] | Visibility -| Tunable +| User endif::[] |=== -=== node_status_reconnect_max_backoff_ms +=== oidc_clock_skew_tolerance -Maximum backoff (in milliseconds) to reconnect to an unresponsive peer during node status liveness checks. +The amount of time (in seconds) to allow for when validating the expiry claim in the token. [cols="1s,2a"] |=== @@ -13534,7 +13927,7 @@ Maximum backoff (in milliseconds) to reconnect to an unresponsive peer during no | Range -| [`-17592186044416`, `17592186044415`] +| [`-17179869184`, `17179869183`] | Default | @@ -13542,15 +13935,12 @@ ifdef::env-cloud[] Available in the Redpanda Cloud Console endif::[] ifndef::env-cloud[] -`15000` (15 seconds) +`null` endif::[] | Nullable | No -| Unit -| Milliseconds - | Requires restart | No @@ -13567,29 +13957,27 @@ endif::[] |=== -=== oidc_clock_skew_tolerance +// tag::redpanda-cloud[] +=== oidc_discovery_url -The amount of time (in seconds) to allow for when validating the expiry claim in the token. +The URL pointing to the well-known discovery endpoint for the OIDC provider. [cols="1s,2a"] |=== | Property | Value | Type -| `integer` - +| `string` -| Range -| [`-17179869184`, `17179869183`] | Default | ifdef::env-cloud[] -Available in the Redpanda Cloud Console +Available in the Redpanda Cloud Console (read-only) endif::[] ifndef::env-cloud[] -`null` +`https://auth.prd.cloud.redpanda.com/.well-known/openid-configuration` endif::[] | Nullable @@ -13610,11 +13998,15 @@ endif::[] |=== +// end::redpanda-cloud[] + +=== oidc_group_claim_path -// tag::redpanda-cloud[] -=== oidc_discovery_url +ifndef::env-cloud[] +*Introduced in v26.1.1* +endif::[] -The URL pointing to the well-known discovery endpoint for the OIDC provider. +JSON path to extract groups from the JWT payload. [cols="1s,2a"] |=== @@ -13628,10 +14020,10 @@ The URL pointing to the well-known discovery endpoint for the OIDC provider. | Default | ifdef::env-cloud[] -Available in the Redpanda Cloud Console (read-only) +Available in the Redpanda Cloud Console endif::[] ifndef::env-cloud[] -`https://auth.prd.cloud.redpanda.com/.well-known/openid-configuration` +`$.groups` endif::[] | Nullable @@ -13652,7 +14044,6 @@ endif::[] |=== -// end::redpanda-cloud[] === oidc_keys_refresh_interval @@ -13958,14 +14349,68 @@ endif::[] | Type | `string` (enum) -| Accepted values -| -ifndef::env-cloud[] -`off`, `node_add`, `continuous` (Enterprise) -endif::[] -ifdef::env-cloud[] -`off`, `node_add`, `continuous` -endif::[] +| Accepted values +| +ifndef::env-cloud[] +`off`, `node_add`, `continuous` (Enterprise) +endif::[] +ifdef::env-cloud[] +`off`, `node_add`, `continuous` +endif::[] + + +| Default +| +ifdef::env-cloud[] +Available in the Redpanda Cloud Console +endif::[] +ifndef::env-cloud[] +`continuous` +endif::[] + +| Nullable +| No + +| Requires restart +| No + +ifndef::env-cloud[] +| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] +| Yes +endif::[] + +ifndef::env-cloud[] +| Visibility +| User +endif::[] + +| Example +| +`node_add` + +| Related topics +| +* xref:manage:cluster-maintenance/cluster-balancing.adoc[partition balancing] + +* xref:manage:cluster-maintenance/continuous-data-balancing.adoc[Configure Continuous Data Balancing] + +* xref:get-started:licensing/index.adoc[enterprise license] + +|=== + + +// tag::deprecated[] +=== partition_autobalancing_movement_batch_size_bytes + +Total size of partitions that autobalancer is going to move in one batch (deprecated, use partition_autobalancing_concurrent_moves to limit the autobalancer concurrency) + +[cols="1s,2a"] +|=== +| Property | Value + +| Type +| `integer` + | Default @@ -13974,12 +14419,15 @@ ifdef::env-cloud[] Available in the Redpanda Cloud Console endif::[] ifndef::env-cloud[] -`continuous` +`5368709120` endif::[] | Nullable | No +| Unit +| Bytes + | Requires restart | No @@ -13990,28 +14438,20 @@ endif::[] ifndef::env-cloud[] | Visibility -| User +| Deprecated endif::[] -| Example -| -`node_add` - -| Related topics -| -* xref:manage:cluster-maintenance/cluster-balancing.adoc[partition balancing] - -* xref:manage:cluster-maintenance/continuous-data-balancing.adoc[Configure Continuous Data Balancing] - -* xref:get-started:licensing/index.adoc[enterprise license] - |=== +// end::deprecated[] -// tag::deprecated[] -=== partition_autobalancing_movement_batch_size_bytes +=== partition_autobalancing_node_autodecommission_timeout_sec -Total size of partitions that autobalancer is going to move in one batch (deprecated, use partition_autobalancing_concurrent_moves to limit the autobalancer concurrency) +ifndef::env-cloud[] +*Introduced in v26.1.1* +endif::[] + +When a node is unavailable for at least this timeout duration, it triggers Redpanda to decommission the node. This property applies only when `partition_autobalancing_mode` is set to `continuous`. [cols="1s,2a"] |=== @@ -14022,20 +14462,23 @@ Total size of partitions that autobalancer is going to move in one batch (deprec +| Range +| [`-17179869184`, `17179869183`] + | Default | ifdef::env-cloud[] Available in the Redpanda Cloud Console endif::[] ifndef::env-cloud[] -`5368709120` +`null` endif::[] | Nullable -| No +| Yes | Unit -| Bytes +| Seconds | Requires restart | No @@ -14047,16 +14490,15 @@ endif::[] ifndef::env-cloud[] | Visibility -| Deprecated +| User endif::[] |=== -// end::deprecated[] === partition_autobalancing_node_availability_timeout_sec -When a node is unavailable for at least this timeout duration, it triggers Redpanda to move partitions off of the node. This property applies only when `partition_autobalancing_mode` is set to `continuous`. +When a node is unavailable for at least this timeout duration, it triggers Redpanda to move partitions off of the node. This property applies only when `partition_autobalancing_mode` is set to `continuous`. [cols="1s,2a"] |=== @@ -14812,45 +15254,6 @@ endif::[] |=== -// tag::deprecated[] -=== raft_max_concurrent_append_requests_per_follower - -No description available. - - -[cols="1s,2a"] -|=== -| Property | Value - -| Type -| `deprecated_property` - - - -| Default -| -ifdef::env-cloud[] -Available in the Redpanda Cloud Console -endif::[] -ifndef::env-cloud[] -`null` -endif::[] - -| Nullable -| No - -| Requires restart -| Yes - -ifndef::env-cloud[] -| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] -| Yes -endif::[] - -|=== - -// end::deprecated[] - === raft_max_inflight_follower_append_entries_requests_per_shard The maximum number of append entry requests that may be sent from Raft groups on a Seastar shard to the current node, and are awaiting a reply. This property replaces `raft_max_concurrent_append_requests_per_follower`. @@ -14978,45 +15381,6 @@ endif::[] |=== -// tag::deprecated[] -=== raft_recovery_default_read_size - -No description available. - - -[cols="1s,2a"] -|=== -| Property | Value - -| Type -| `deprecated_property` - - - -| Default -| -ifdef::env-cloud[] -Available in the Redpanda Cloud Console -endif::[] -ifndef::env-cloud[] -`null` -endif::[] - -| Nullable -| No - -| Requires restart -| Yes - -ifndef::env-cloud[] -| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] -| Yes -endif::[] - -|=== - -// end::deprecated[] - === raft_recovery_throttle_disable_dynamic_mode include::reference:partial$internal-use-property.adoc[] @@ -15631,53 +15995,6 @@ endif::[] |=== -=== recovery_append_timeout_ms - -Timeout for append entry requests issued while updating a stale follower. - -[cols="1s,2a"] -|=== -| Property | Value - -| Type -| `integer` - - - -| Range -| [`-17592186044416`, `17592186044415`] - -| Default -| -ifdef::env-cloud[] -Available in the Redpanda Cloud Console -endif::[] -ifndef::env-cloud[] -`5000` (5 seconds) -endif::[] - -| Nullable -| No - -| Unit -| Milliseconds - -| Requires restart -| Yes - -ifndef::env-cloud[] -| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] -| Yes -endif::[] - -ifndef::env-cloud[] -| Visibility -| Tunable -endif::[] - -|=== - - === release_cache_on_segment_roll Flag for specifying whether or not to release cache when a full segment is rolled. @@ -16192,92 +16509,6 @@ endif::[] |=== -=== rm_sync_timeout_ms - -Resource manager's synchronization timeout. Specifies the maximum time for this node to wait for the internal state machine to catch up with all events written by previous leaders before rejecting a request. - -[cols="1s,2a"] -|=== -| Property | Value - -| Type -| `integer` - - - -| Range -| [`-17592186044416`, `17592186044415`] - -| Default -| -ifdef::env-cloud[] -Available in the Redpanda Cloud Console -endif::[] -ifndef::env-cloud[] -`10000` (10 seconds) -endif::[] - -| Nullable -| No - -| Unit -| Milliseconds - -| Requires restart -| No - -ifndef::env-cloud[] -| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] -| Yes -endif::[] - -ifndef::env-cloud[] -| Visibility -| User -endif::[] - -|=== - - -// tag::deprecated[] -=== rm_violation_recovery_policy - -No description available. - - -[cols="1s,2a"] -|=== -| Property | Value - -| Type -| `deprecated_property` - - - -| Default -| -ifdef::env-cloud[] -Available in the Redpanda Cloud Console -endif::[] -ifndef::env-cloud[] -`null` -endif::[] - -| Nullable -| No - -| Requires restart -| Yes - -ifndef::env-cloud[] -| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] -| Yes -endif::[] - -|=== - -// end::deprecated[] - === rpc_client_connections_per_peer The maximum number of connections a broker will open to each of its peers. @@ -17114,20 +17345,22 @@ endif::[] |=== +// tag::deprecated[] === schema_registry_avro_use_named_references ifndef::env-cloud[] *Introduced in v25.3.10* endif::[] -When enabled, Avro schemas with external references are compiled using named reference resolution instead of schema concatenation. This fixes issues with compatibility checks and schema validation for schemas with reference dependencies. This config will be deprecated and always enabled in v26.1.1. +No description available. + [cols="1s,2a"] |=== | Property | Value | Type -| `boolean` +| `object` @@ -17137,27 +17370,23 @@ ifdef::env-cloud[] Available in the Redpanda Cloud Console endif::[] ifndef::env-cloud[] -`false` +`null` endif::[] | Nullable | No | Requires restart -| No +| Yes ifndef::env-cloud[] | Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] | Yes endif::[] -ifndef::env-cloud[] -| Visibility -| Tunable -endif::[] - |=== +// end::deprecated[] // tag::redpanda-cloud[] === schema_registry_enable_authorization @@ -17213,18 +17442,20 @@ endif::[] // end::redpanda-cloud[] -// tag::deprecated[] -=== schema_registry_protobuf_renderer_v2 +=== schema_registry_enable_qualified_subjects -No description available. +ifndef::env-cloud[] +*Introduced in v26.1.1* +endif::[] +Enable parsing of qualified subject syntax (:.context:subject). When false, subjects are treated literally, as subjects in the default context. When true, qualified syntax is parsed to extract context and subject. [cols="1s,2a"] |=== | Property | Value | Type -| `deprecated_property` +| `boolean` @@ -17234,7 +17465,7 @@ ifdef::env-cloud[] Available in the Redpanda Cloud Console endif::[] ifndef::env-cloud[] -`null` +`false` endif::[] | Nullable @@ -17248,51 +17479,13 @@ ifndef::env-cloud[] | Yes endif::[] -|=== - -// end::deprecated[] - -// tag::deprecated[] -=== seed_server_meta_topic_partitions - -No description available. - - -[cols="1s,2a"] -|=== -| Property | Value - -| Type -| `deprecated_property` - - - -| Default -| -ifdef::env-cloud[] -Available in the Redpanda Cloud Console -endif::[] -ifndef::env-cloud[] -`null` -endif::[] - -| Nullable -| No - -| Unit -| Number of partitions per topic - -| Requires restart -| Yes - ifndef::env-cloud[] -| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] -| Yes +| Visibility +| Tunable endif::[] |=== -// end::deprecated[] === segment_appender_flush_timeout_ms @@ -17386,45 +17579,6 @@ endif::[] |=== -// tag::deprecated[] -=== seq_table_min_size - -No description available. - - -[cols="1s,2a"] -|=== -| Property | Value - -| Type -| `deprecated_property` - - - -| Default -| -ifdef::env-cloud[] -Available in the Redpanda Cloud Console -endif::[] -ifndef::env-cloud[] -`null` -endif::[] - -| Nullable -| No - -| Requires restart -| Yes - -ifndef::env-cloud[] -| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] -| Yes -endif::[] - -|=== - -// end::deprecated[] - === space_management_enable Option to explicitly disable automatic disk space management. If this property was explicitly disabled while using v23.2, it will remain disabled following an upgrade. @@ -18315,84 +18469,6 @@ endif::[] |=== -// tag::deprecated[] -=== target_fetch_quota_byte_rate - -No description available. - - -[cols="1s,2a"] -|=== -| Property | Value - -| Type -| `deprecated_property` - - - -| Default -| -ifdef::env-cloud[] -Available in the Redpanda Cloud Console -endif::[] -ifndef::env-cloud[] -`null` -endif::[] - -| Nullable -| No - -| Requires restart -| Yes - -ifndef::env-cloud[] -| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] -| Yes -endif::[] - -|=== - -// end::deprecated[] - -// tag::deprecated[] -=== target_quota_byte_rate - -No description available. - - -[cols="1s,2a"] -|=== -| Property | Value - -| Type -| `deprecated_property` - - - -| Default -| -ifdef::env-cloud[] -Available in the Redpanda Cloud Console -endif::[] -ifndef::env-cloud[] -`null` -endif::[] - -| Nullable -| No - -| Requires restart -| Yes - -ifndef::env-cloud[] -| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] -| Yes -endif::[] - -|=== - -// end::deprecated[] - === tls_certificate_name_format The format of the certificates's distinguished name to use for mTLS principal mapping. The `legacy` format would appear as 'C=US,ST=California,L=San Francisco,O=Redpanda,CN=redpanda', while the `rfc2253` format would appear as 'CN=redpanda,O=Redpanda,L=San Francisco,ST=California,C=US'. @@ -18417,108 +18493,17 @@ endif::[] | Default | ifdef::env-cloud[] -Available in the Redpanda Cloud Console -endif::[] -ifndef::env-cloud[] -`legacy` -endif::[] - -| Nullable -| No - -| Requires restart -| No - -ifndef::env-cloud[] -| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] -| Yes -endif::[] - -ifndef::env-cloud[] -| Visibility -| User -endif::[] - -|=== - - -=== tls_enable_renegotiation - -TLS client-initiated renegotiation is considered unsafe and is disabled by default . Only re-enable it if you are experiencing issues with your TLS-enabled client. This option has no effect on TLSv1.3 connections as client-initiated renegotiation was removed. - -[cols="1s,2a"] -|=== -| Property | Value - -| Type -| `boolean` - - - -| Default -| -ifdef::env-cloud[] -Available in the Redpanda Cloud Console -endif::[] -ifndef::env-cloud[] -`false` -endif::[] - -| Nullable -| No - -| Requires restart -| Yes - -ifndef::env-cloud[] -| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] -| Yes -endif::[] - -ifndef::env-cloud[] -| Visibility -| Tunable -endif::[] - -|=== - - -// tag::redpanda-cloud[] -=== tls_min_version - -The minimum TLS version that Redpanda clusters support. This property prevents client applications from negotiating a downgrade to the TLS version when they make a connection to a Redpanda cluster. - -[cols="1s,2a"] -|=== -| Property | Value - -| Type -| `string` (enum) - -| Accepted values -| -ifndef::env-cloud[] -`v1.0`, `v1.1`, `v1.2`, `v1.3` -endif::[] -ifdef::env-cloud[] -`v1.0`, `v1.1`, `v1.2`, `v1.3` -endif::[] - - -| Default -| -ifdef::env-cloud[] -Available in the Redpanda Cloud Console (read-only) +Available in the Redpanda Cloud Console endif::[] ifndef::env-cloud[] -`v1.2` +`legacy` endif::[] | Nullable | No | Requires restart -| Yes +| No ifndef::env-cloud[] | Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] @@ -18532,18 +18517,17 @@ endif::[] |=== -// end::redpanda-cloud[] -=== tls_v1_2_cipher_suites +=== tls_enable_renegotiation -The encryption algorithms available for TLS 1.2 client connections, specified as a colon-separated list in OpenSSL format. Use this to support older clients that require specific encryption methods. +TLS client-initiated renegotiation is considered unsafe and is disabled by default . Only re-enable it if you are experiencing issues with your TLS-enabled client. This option has no effect on TLSv1.3 connections as client-initiated renegotiation was removed. [cols="1s,2a"] |=== | Property | Value | Type -| `string` +| `boolean` @@ -18553,7 +18537,7 @@ ifdef::env-cloud[] Available in the Redpanda Cloud Console endif::[] ifndef::env-cloud[] -`ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:AES256-GCM-SHA384:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:AES128-SHA:AES128-CCM:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:AES256-SHA:AES256-CCM` +`false` endif::[] | Nullable @@ -18569,32 +18553,41 @@ endif::[] ifndef::env-cloud[] | Visibility -| User +| Tunable endif::[] |=== -=== tls_v1_3_cipher_suites +// tag::redpanda-cloud[] +=== tls_min_version -The encryption algorithms available for TLS 1.3 client connections, specified as a colon-separated list in OpenSSL format. Most deployments don't need to change this. Only modify it to meet specific organizational security requirements. +The minimum TLS version that Redpanda clusters support. This property prevents client applications from negotiating a downgrade to the TLS version when they make a connection to a Redpanda cluster. [cols="1s,2a"] |=== | Property | Value | Type -| `string` +| `string` (enum) +| Accepted values +| +ifndef::env-cloud[] +`v1.0`, `v1.1`, `v1.2`, `v1.3` +endif::[] +ifdef::env-cloud[] +`v1.0`, `v1.1`, `v1.2`, `v1.3` +endif::[] | Default | ifdef::env-cloud[] -Available in the Redpanda Cloud Console +Available in the Redpanda Cloud Console (read-only) endif::[] ifndef::env-cloud[] -`TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_CCM_SHA256` +`v1.2` endif::[] | Nullable @@ -18615,22 +18608,20 @@ endif::[] |=== +// end::redpanda-cloud[] -=== tm_sync_timeout_ms +=== tls_v1_2_cipher_suites -Transaction manager's synchronization timeout. Maximum time to wait for internal state machine to catch up before rejecting a request. +The encryption algorithms available for TLS 1.2 client connections, specified as a colon-separated list in OpenSSL format. Use this to support older clients that require specific encryption methods. [cols="1s,2a"] |=== | Property | Value | Type -| `integer` - +| `string` -| Range -| [`-17592186044416`, `17592186044415`] | Default | @@ -18638,15 +18629,12 @@ ifdef::env-cloud[] Available in the Redpanda Cloud Console endif::[] ifndef::env-cloud[] -`10000` (10 seconds) +`TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256:TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256:TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384:TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384:TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256:TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256` endif::[] | Nullable | No -| Unit -| Milliseconds - | Requires restart | Yes @@ -18663,18 +18651,16 @@ endif::[] |=== -// tag::deprecated[] -=== tm_violation_recovery_policy - -No description available. +=== tls_v1_3_cipher_suites +The encryption algorithms available for TLS 1.3 client connections, specified as a colon-separated list in OpenSSL format. Most deployments don't need to change this. Only modify it to meet specific organizational security requirements. [cols="1s,2a"] |=== | Property | Value | Type -| `deprecated_property` +| `string` @@ -18684,7 +18670,7 @@ ifdef::env-cloud[] Available in the Redpanda Cloud Console endif::[] ifndef::env-cloud[] -`null` +`TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_CCM_SHA256` endif::[] | Nullable @@ -18698,9 +18684,13 @@ ifndef::env-cloud[] | Yes endif::[] +ifndef::env-cloud[] +| Visibility +| User +endif::[] + |=== -// end::deprecated[] === tombstone_retention_ms @@ -19213,45 +19203,6 @@ endif::[] |=== -// tag::deprecated[] -=== transaction_coordinator_replication - -No description available. - - -[cols="1s,2a"] -|=== -| Property | Value - -| Type -| `deprecated_property` - - - -| Default -| -ifdef::env-cloud[] -Available in the Redpanda Cloud Console -endif::[] -ifndef::env-cloud[] -`null` -endif::[] - -| Nullable -| No - -| Requires restart -| Yes - -ifndef::env-cloud[] -| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] -| Yes -endif::[] - -|=== - -// end::deprecated[] - === transaction_max_timeout_ms The maximum allowed timeout for transactions. If a client-requested transaction timeout exceeds this configuration, the broker returns an error during transactional producer initialization. This guardrail prevents hanging transactions from blocking consumer progress. @@ -19400,87 +19351,6 @@ endif::[] // end::deprecated[] -// tag::deprecated[] -=== tx_registry_log_capacity - -No description available. - - -[cols="1s,2a"] -|=== -| Property | Value - -| Type -| `deprecated_property` - - - -| Default -| -ifdef::env-cloud[] -Available in the Redpanda Cloud Console -endif::[] -ifndef::env-cloud[] -`null` -endif::[] - -| Nullable -| No - -| Requires restart -| Yes - -ifndef::env-cloud[] -| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] -| Yes -endif::[] - -|=== - -// end::deprecated[] - -// tag::deprecated[] -=== tx_registry_sync_timeout_ms - -No description available. - - -[cols="1s,2a"] -|=== -| Property | Value - -| Type -| `deprecated_property` - - - -| Default -| -ifdef::env-cloud[] -Available in the Redpanda Cloud Console -endif::[] -ifndef::env-cloud[] -`null` -endif::[] - -| Nullable -| No - -| Unit -| Milliseconds - -| Requires restart -| Yes - -ifndef::env-cloud[] -| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] -| Yes -endif::[] - -|=== - -// end::deprecated[] - === tx_timeout_delay_ms Delay before scheduling the next check for timed out transactions. @@ -19839,45 +19709,6 @@ endif::[] |=== -// tag::deprecated[] -=== use_scheduling_groups - -No description available. - - -[cols="1s,2a"] -|=== -| Property | Value - -| Type -| `deprecated_property` - - - -| Default -| -ifdef::env-cloud[] -Available in the Redpanda Cloud Console -endif::[] -ifndef::env-cloud[] -`null` -endif::[] - -| Nullable -| No - -| Requires restart -| Yes - -ifndef::env-cloud[] -| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] -| Yes -endif::[] - -|=== - -// end::deprecated[] - === virtual_cluster_min_producer_ids Minimum number of active producers per virtual cluster. @@ -19922,53 +19753,6 @@ endif::[] |=== -=== wait_for_leader_timeout_ms - -Timeout to wait for leadership in metadata cache. - -[cols="1s,2a"] -|=== -| Property | Value - -| Type -| `integer` - - - -| Range -| [`-17592186044416`, `17592186044415`] - -| Default -| -ifdef::env-cloud[] -Available in the Redpanda Cloud Console -endif::[] -ifndef::env-cloud[] -`5000` (5 seconds) -endif::[] - -| Nullable -| No - -| Unit -| Milliseconds - -| Requires restart -| No - -ifndef::env-cloud[] -| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] -| Yes -endif::[] - -ifndef::env-cloud[] -| Visibility -| Tunable -endif::[] - -|=== - - === write_caching_default The default write caching mode to apply to user topics. Write caching acknowledges a message as soon as it is received and acknowledged on a majority of brokers, without waiting for it to be written to disk. With `acks=all`, this provides lower latency while still ensuring that a majority of brokers acknowledge the write. diff --git a/modules/reference/partials/properties/object-storage-properties.adoc b/modules/reference/partials/properties/object-storage-properties.adoc index a37097649d..ed37c029f1 100644 --- a/modules/reference/partials/properties/object-storage-properties.adoc +++ b/modules/reference/partials/properties/object-storage-properties.adoc @@ -1739,45 +1739,6 @@ endif::[] |=== -// tag::deprecated[] -=== cloud_storage_disable_metadata_consistency_checks - -No description available. - - -[cols="1s,2a"] -|=== -| Property | Value - -| Type -| `deprecated_property` - - - -| Default -| -ifdef::env-cloud[] -Available in the Redpanda Cloud Console -endif::[] -ifndef::env-cloud[] -`null` -endif::[] - -| Nullable -| No - -| Requires restart -| Yes - -ifndef::env-cloud[] -| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] -| Yes -endif::[] - -|=== - -// end::deprecated[] - === cloud_storage_disable_read_replica_loop_for_tests Begins the read replica sync loop in topic partitions with Tiered Storage enabled. The property exists to simplify testing and shouldn't be set in production. @@ -3773,9 +3734,13 @@ endif::[] |=== -=== cloud_storage_readreplica_manifest_sync_timeout_ms +=== cloud_storage_prefetch_segments_max -Timeout to check if new data is available for partitions in object storage for read replicas. +ifndef::env-cloud[] +*Introduced in v26.1.1* +endif::[] + +Maximum number of small segments (size <= chunk size) to prefetch ahead during sequential reads. Set to 0 to disable cross-segment prefetch. [cols="1s,2a"] |=== @@ -3786,8 +3751,8 @@ Timeout to check if new data is available for partitions in object storage for r -| Range -| [`-17592186044416`, `17592186044415`] +| Maximum +| `65535` | Default | @@ -3795,15 +3760,12 @@ ifdef::env-cloud[] Available in the Redpanda Cloud Console endif::[] ifndef::env-cloud[] -`30000` (30 seconds) +`3` endif::[] | Nullable | No -| Unit -| Milliseconds - | Requires restart | No @@ -3820,20 +3782,21 @@ endif::[] |=== -// tag::deprecated[] -=== cloud_storage_reconciliation_ms - -No description available. +=== cloud_storage_readreplica_manifest_sync_timeout_ms +Timeout to check if new data is available for partitions in object storage for read replicas. [cols="1s,2a"] |=== | Property | Value | Type -| `deprecated_property` +| `integer` + +| Range +| [`-17592186044416`, `17592186044415`] | Default | @@ -3841,7 +3804,7 @@ ifdef::env-cloud[] Available in the Redpanda Cloud Console endif::[] ifndef::env-cloud[] -`null` +`30000` (30 seconds) endif::[] | Nullable @@ -3851,16 +3814,20 @@ endif::[] | Milliseconds | Requires restart -| Yes +| No ifndef::env-cloud[] | Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] | Yes endif::[] +ifndef::env-cloud[] +| Visibility +| Tunable +endif::[] + |=== -// end::deprecated[] === cloud_storage_recovery_temporary_retention_bytes_default diff --git a/modules/reference/partials/properties/topic-properties.adoc b/modules/reference/partials/properties/topic-properties.adoc index 1463d30bfd..2f599a9cb3 100644 --- a/modules/reference/partials/properties/topic-properties.adoc +++ b/modules/reference/partials/properties/topic-properties.adoc @@ -945,11 +945,11 @@ endif::[] // end::category-retention-compaction[] // tag::category-tiered-storage[] -// tag::exclude-from-docs[] === redpanda.cloud_topic.enabled -No description available. +Enable Cloud Topic storage mode for this topic. When enabled, topic data is stored primarily in object storage with local storage used only as a write buffer. +TIP: To configure storage modes with more flexibility, use <> which supports `local`, `tiered`, `cloud`, and `unset` modes. [cols="1s,2a"] |=== @@ -977,9 +977,16 @@ ifndef::env-cloud[] | Yes endif::[] +| Related topics +| +ifndef::env-cloud[] +* xref:develop:manage-topics/cloud-topics.adoc[Cloud Topics] + +* xref:manage:tiered-storage.adoc[Tiered Storage] +endif::[] + |=== -// end::exclude-from-docs[] // end::category-tiered-storage[] // tag::category-iceberg-integration[] @@ -1550,6 +1557,77 @@ endif::[] // end::category-tiered-storage[] +// tag::category-tiered-storage[] +=== redpanda.storage.mode + +*Introduced in v26.1.1* + +The storage mode for a topic. Determines how topic data is stored and whether it is eligible for upload to object storage. + +Accepted values: + +* `local`: Topic data is stored only on the broker's local disk. Object storage upload is disabled for the topic, regardless of cluster-level Tiered Storage settings. +ifndef::env-cloud[] +* `tiered`: Topic data is stored on local disk and also uploaded to object storage. Enables xref:manage:tiered-storage.adoc[Tiered Storage] for the topic. +* `cloud`: Topic data is stored in object storage using the glossterm:Cloud Topic[,Cloud Topics] architecture. Local storage is used only as a write buffer. +endif::[] +ifdef::env-cloud[] +* `tiered`: Topic data is stored on local disk and also uploaded to object storage. Enables Tiered Storage for the topic. +* `cloud`: Topic data is stored in object storage using the Cloud Topics architecture. Local storage is used only as a write buffer. +endif::[] +* `unset`: Specifies that the topic's storage mode is unset, regardless of the cluster default. The topic may still have Tiered Storage enabled through the legacy properties `redpanda.remote.read` and `redpanda.remote.write`. + +This property overrides the cluster-wide config_ref:default_redpanda_storage_mode,true,properties/cluster-properties[] setting for individual topics. + +[cols="1s,2a"] +|=== +| Property | Value + +| Type +| `string` (enum) + +| Accepted Values +| +ifndef::env-cloud[] +`local`, `tiered`, `cloud`, `unset` +endif::[] +ifdef::env-cloud[] +`local`, `tiered`, `cloud`, `unset` +endif::[] + + +| Corresponding cluster property +| xref:reference:cluster-properties.adoc#default_redpanda_storage_mode[default_redpanda_storage_mode] + +| Default +| +ifdef::env-cloud[] +Available in the Redpanda Cloud Console +endif::[] +ifndef::env-cloud[] +`unset` +endif::[] + +| Nullable +| No + +ifndef::env-cloud[] +| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] +| Yes +endif::[] + +| Related topics +| +ifndef::env-cloud[] +* xref:manage:tiered-storage.adoc[Tiered Storage] + +* xref:develop:manage-topics/cloud-topics.adoc[Manage Cloud Topics] +endif::[] + +|=== + +// end::category-tiered-storage[] + // tag::category-schema-registry[] === redpanda.value.schema.id.validation @@ -1710,9 +1788,9 @@ endif::[] * xref:reference:properties/cluster-properties.adoc#default_topic_replication[`default_topic_replication`] -* xref:develop:config-topics.adoc#choose-the-replication-factor[Choose the replication factor] +* xref:develop:manage-topics/config-topics.adoc#choose-the-replication-factor[Choose the replication factor] -* xref:develop:config-topics.adoc#change-the-replication-factor[Change the replication factor] +* xref:develop:manage-topics/config-topics.adoc#change-the-replication-factor[Change the replication factor] * xref:reference:properties/cluster-properties.adoc#default_topic_replication[default_topic_replication] diff --git a/modules/reference/partials/properties/topic-property-mappings.adoc b/modules/reference/partials/properties/topic-property-mappings.adoc index 238d37a826..6843535cc9 100644 --- a/modules/reference/partials/properties/topic-property-mappings.adoc +++ b/modules/reference/partials/properties/topic-property-mappings.adoc @@ -63,6 +63,9 @@ | <> | xref:./object-storage-properties.adoc#cloud_storage_enable_remote_write[`cloud_storage_enable_remote_write`] +| <> +| xref:./cluster-properties.adoc#default_redpanda_storage_mode[`default_redpanda_storage_mode`] + | <> | xref:./cluster-properties.adoc#retention_bytes[`retention_bytes`] diff --git a/modules/shared/partials/tristate-behavior-change-25-3.adoc b/modules/shared/partials/tristate-behavior-change-25-3.adoc index e79d95b59a..cce6a25c22 100644 --- a/modules/shared/partials/tristate-behavior-change-25-3.adoc +++ b/modules/shared/partials/tristate-behavior-change-25-3.adoc @@ -1,5 +1,5 @@ Starting in Redpanda v25.3, several topic properties support enhanced tristate behavior. Properties like `retention.ms`, `retention.bytes`, `segment.ms`, and others now distinguish between zero values (immediate eligibility for cleanup/compaction) and negative values (disable the feature entirely). Previously, zero and negative values were treated the same way. ifndef::env-cloud[] -For the complete list of affected properties and detailed information, see xref:25.3@get-started:release-notes/redpanda.adoc#behavior-changes[Redpanda v25.3 behavior changes]. +For the complete list of affected properties and detailed information, see xref:get-started:release-notes/redpanda.adoc#behavior-changes[Redpanda v26.1 behavior changes]. endif::[] Review your topic configurations if you currently use zero values for these properties. \ No newline at end of file diff --git a/modules/upgrade/pages/k-compatibility.adoc b/modules/upgrade/pages/k-compatibility.adoc index b5c000c31c..4f755cfd26 100644 --- a/modules/upgrade/pages/k-compatibility.adoc +++ b/modules/upgrade/pages/k-compatibility.adoc @@ -19,7 +19,7 @@ Starting from version 25.1.1, the Redpanda Operator and Redpanda Helm chart foll NOTE: If a version includes `-beta`, it is a pre-release version of the Redpanda Operator and Helm chart. These versions are not supported and should not be used in production environments. Beta versions are available only for testing and feedback. To give feedback on beta releases, reach out to the Redpanda team in https://redpanda.com/slack[Redpanda Community Slack^]. -Each Redpanda Operator and Helm chart version supports the corresponding Redpanda core version plus one minor version above and one below. This approach ensures flexibility during upgrades. For example, Redpanda Operator version 25.1.1 supports Redpanda core versions 25.2.x, 25.1.x, and 24.3.x. +Each Redpanda Operator and Helm chart version supports the corresponding Redpanda core version plus one minor version above and one below. This approach ensures flexibility during upgrades. For example, Redpanda Operator version 25.3.1 supports Redpanda core versions 26.1.x, 25.3.x, and 25.2.x. Redpanda Operator and Helm chart versions are supported only while their associated Redpanda core version remains supported. If the core version reaches end of life (EoL), the corresponding versions of the Redpanda Operator and Helm chart also reach EoL. @@ -37,7 +37,30 @@ Redpanda Core has no direct dependency on Kubernetes. Compatibility is influence |=== |Redpanda Core / `rpk` |Helm Chart |Operator Helm Chart |Operator |Helm CLI |Kubernetes -.2+|25.3.x +.2+|26.1.x + +|26.1.x +|26.1.x +|26.1.x +|3.12+ +// d (default) on Kubernetes cells is required to render footnotes at the +// bottom of the page rather than inside the table cell. +// See https://github.com/asciidoctor/asciidoctor/issues/2350#issuecomment-546841684 +d|1.32.x - 1.35.x{fn-k8s-compatibility} + +|25.3.x +|25.3.x +|25.3.x +|3.12+ +d|1.30.x - 1.33.x{fn-k8s-compatibility} + +.3+|25.3.x + +|26.1.x +|26.1.x +|26.1.x +|3.12+ +d|1.30.x - 1.33.x{fn-k8s-compatibility} |25.3.x |25.3.x @@ -70,28 +93,6 @@ d|1.30.x - 1.33.x{fn-k8s-compatibility} |25.1.x |3.12+ d|1.28.x - 1.33.x{fn-k8s-compatibility} - -.3+|25.1.x -|25.1.x -|25.1.x -|25.1.x -|3.12+ -// d (default) here is required to get footnotes to appear at the bottom of the page -// instead of inside the table cell. -// See https://github.com/asciidoctor/asciidoctor/issues/2350#issuecomment-546841684 -d|1.28.x - 1.32.x{fn-k8s-compatibility} - -|5.10.x -|2.4.x -|2.4.x -|3.12+ -d|1.28.x - 1.32.x{fn-k8s-compatibility} - -|5.9.x -|0.4.36 -|2.3.x -|3.12+ -d|1.28.x - 1.32.x{fn-k8s-compatibility} |=== By default, the Redpanda Helm chart depends on cert-manager for enabling TLS. @@ -111,12 +112,8 @@ Upgrading the Helm chart may also upgrade Redpanda Console. Because of this buil |Redpanda Console |Helm Chart |Operator |v3.x.x -|25.3.x, 25.2.x, 25.1.x -|25.3.x, 25.2.x - -|v2.x.x -|5.10.1, 5.9.x, 5.8.x -|25.3.x, 25.2.x, 25.1.x, 2.4.x, 2.3.x, 2.2.x +|26.1.x, 25.3.x, 25.2.x +|26.1.x, 25.3.x, 25.2.x |=== diff --git a/modules/upgrade/partials/incompat-changes.adoc b/modules/upgrade/partials/incompat-changes.adoc index 9610dfe846..ba16ec587a 100644 --- a/modules/upgrade/partials/incompat-changes.adoc +++ b/modules/upgrade/partials/incompat-changes.adoc @@ -1,5 +1,8 @@ === Review incompatible changes +* *Breaking changes in Redpanda 26.1*: +** If FIPS mode is enabled, change any SASL/SCRAM user passwords shorter than 14 characters to at least 14 characters before upgrading. FIPS 140-3 enforces stricter HMAC key size requirements than FIPS 140-2. Because Redpanda stores passwords in encrypted form, it cannot check the length of existing passwords. Clients with passwords shorter than 14 characters will fail to authenticate after the upgrade. See xref:manage:security/fips-compliance.adoc[Configure Redpanda for FIPS]. + * *Breaking changes in Redpanda 25.3*: ** {empty} +