[CORE-149029] - Scaleops Update [WIP]

.gitignore

@@ -8,7 +8,7 @@
 bin
 charts/**/charts
 charts/koperator/requirements.lock
-
+charts/kafka-operator/ingress
 # Test binary, build with `go test -c`
 *.test
 

api/v1beta1/kafkacluster_types.go

@@ -157,9 +157,22 @@ type KafkaClusterSpec struct {
 	// This is default to be true; if set to false, the Kafka cluster is in ZooKeeper mode.
 	// +kubebuilder:default=false
 	// +optional
-	KRaftMode              bool            `json:"kRaft"`
-	HeadlessServiceEnabled bool            `json:"headlessServiceEnabled"`
-	ListenersConfig        ListenersConfig `json:"listenersConfig"`
+	KRaftMode              bool `json:"kRaft"`
+	HeadlessServiceEnabled bool `json:"headlessServiceEnabled"`
+	// localDebugEnabled is used to decide whether to create a separate loadbalancer services for the
+	// Kafka and Cruise Control Pods. These services will expose the internal listener ports of the Kafka
+	// cluster with LoadBalancer type, which can be used for running Koperator on a local machine against
+	// a kafkaCluster instance on a Kind Cluster.
+	// +kubebuilder:default=false
+	// +optional
+	LocalDebugEnabled bool `json:"localDebugEnabled"`
+	// Allows ScaleOps to manage Memory and CPU Resource Requests for Kafka Broker Pods.
+	// This Disables CPU and Memory request reconciliation from the desired state defined in
+	// the KafkaCluster to the current state in the Kubernetes Cluster
+	// +kubebuilder:default=false
+	// +optional
+	ScaleOpsEnabled bool            `json:"scaleOpsEnabled"`
+	ListenersConfig ListenersConfig `json:"listenersConfig"`
 	// Custom ports to expose in the container. Example use case: a custom kafka distribution, that includes an integrated metrics api endpoint
 	AdditionalPorts []corev1.ContainerPort `json:"additionalPorts,omitempty"`
 	// ZKAddresses specifies the ZooKeeper connection string

charts/kafka-operator/crds/kafkaclusters.yaml

@@ -23670,6 +23670,14 @@ spec:
                 required:
                 - internalListeners
                 type: object
+              localDebugEnabled:
+                default: false
+                description: |-
+                  localDebugEnabled is used to decide whether to create a separate loadbalancer services for the
+                  Kafka and Cruise Control Pods. These services will expose the internal listener ports of the Kafka
+                  cluster with LoadBalancer type, which can be used for running Koperator on a local machine against
+                  a kafkaCluster instance on a Kind Cluster.
+                type: boolean
               monitoringConfig:
                 description: MonitoringConfig defines the config for monitoring Kafka
                   and Cruise Control
@@ -23735,6 +23743,13 @@ spec:
                 required:
                 - failureThreshold
                 type: object
+              scaleOpsEnabled:
+                default: false
+                description: |-
+                  Allows ScaleOps to manage Memory and CPU Resource Requests for Kafka Broker Pods.
+                  This Disables CPU and Memory request reconciliation from the desired state defined in
+                  the KafkaCluster to the current state in the Kubernetes Cluster
+                type: boolean
               taintedBrokersSelector:
                 description: Selector for broker pods that need to be recycled/reconciled
                 properties:

config/base/crds/kafka.banzaicloud.io_kafkaclusters.yaml

@@ -23670,6 +23670,14 @@ spec:
                 required:
                 - internalListeners
                 type: object
+              localDebugEnabled:
+                default: false
+                description: |-
+                  localDebugEnabled is used to decide whether to create a separate loadbalancer services for the
+                  Kafka and Cruise Control Pods. These services will expose the internal listener ports of the Kafka
+                  cluster with LoadBalancer type, which can be used for running Koperator on a local machine against
+                  a kafkaCluster instance on a Kind Cluster.
+                type: boolean
               monitoringConfig:
                 description: MonitoringConfig defines the config for monitoring Kafka
                   and Cruise Control
@@ -23735,6 +23743,13 @@ spec:
                 required:
                 - failureThreshold
                 type: object
+              scaleOpsEnabled:
+                default: false
+                description: |-
+                  Allows ScaleOps to manage Memory and CPU Resource Requests for Kafka Broker Pods.
+                  This Disables CPU and Memory request reconciliation from the desired state defined in
+                  the KafkaCluster to the current state in the Kubernetes Cluster
+                type: boolean
               taintedBrokersSelector:
                 description: Selector for broker pods that need to be recycled/reconciled
                 properties:

config/samples/simpleZookeeper.yaml

@@ -0,0 +1,10 @@
+apiVersion: zookeeper.pravega.io/v1beta1
+kind: ZookeeperCluster
+metadata:
+    name: zookeeper-server
+    namespace: zookeeper
+spec:
+    replicas: 3
+    persistence:
+        reclaimPolicy: Delete
+

config/samples/simplekafkacluster.yaml

@@ -5,10 +5,11 @@ metadata:
     controller-tools.k8s.io: "1.0"
   name: kafka
 spec:
+  localDebugEnabled: true
   kRaft: false
   monitoringConfig:
     jmxImage: "ghcr.io/adobe/koperator/jmx-javaagent:1.4.0"
-  headlessServiceEnabled: true
+  headlessServiceEnabled: false
   zkAddresses:
     - "zookeeper-server-client.zookeeper:2181"
   propagateLabels: false

config/scaleops/CustomOwnerGrouping.yaml

@@ -0,0 +1,22 @@
+
+kind: CustomOwnerGrouping
+apiVersion: analysis.scaleops.sh/v1alpha1
+metadata:
+  name: kafkabroker
+  namespace: scaleops-system
+spec:
+  groupBy:
+    positiveRegexMatch: false
+  groupBys:
+    - labels:
+        - 'isBrokerNode: true'
+      positiveRegexMatch: false
+      topOwnerController:
+        apiVersion: kafka.banzaicloud.io/v1beta1
+        kind: KafkaCluster
+  displayOptions:
+    hideGeneratedSuffix: true
+    fields:
+      - ownerName
+  defaultPolicy: kafka-brokers
+  enabled: true