USHIFT-6797: C2CC Probe remote clusters

Makefile

@@ -346,6 +346,11 @@ generate-config:
 verify-config: generate-config
 	./scripts/verify/verify-config.sh
 
+.PHONY: generate-crds
+generate-crds:
+	./scripts/fetch_tools.sh controller-gen && \
+	./scripts/generate-crds.sh
+
 # Run all of the end to end tests
 .PHONY: e2e
 e2e:

assets/components/c2cc/clusterrole.yaml

@@ -0,0 +1,28 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: microshift-c2cc-probe
+rules:
+- apiGroups:
+  - microshift.io
+  resources:
+  - remoteclusters
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - microshift.io
+  resources:
+  - remoteclusters/status
+  verbs:
+  - update
+  - patch
+- apiGroups:
+  - security.openshift.io
+  resources:
+  - securitycontextconstraints
+  verbs:
+  - use
+  resourceNames:
+  - privileged

assets/components/c2cc/clusterrolebinding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: microshift-c2cc-probe
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: microshift-c2cc-probe
+subjects:
+- kind: ServiceAccount
+  namespace: microshift-c2cc
+  name: c2cc-probe

assets/components/c2cc/deployment.yaml

@@ -0,0 +1,68 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  namespace: microshift-c2cc
+  name: c2cc-probe
+  labels:
+    app: c2cc-probe
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: c2cc-probe
+  template:
+    metadata:
+      labels:
+        app: c2cc-probe
+      annotations:
+        target.workload.openshift.io/management: '{"effect": "PreferredDuringScheduling"}'
+        openshift.io/required-scc: privileged
+    spec:
+      serviceAccountName: c2cc-probe
+      containers:
+      - name: c2cc-probe
+        image: '{{ .ReleaseImage.cli }}'
+        imagePullPolicy: IfNotPresent
+        command:
+        - /host/usr/bin/microshift
+        - c2cc-probe
+        ports:
+        - containerPort: 8080
+          name: probe
+          protocol: TCP
+        livenessProbe:
+          httpGet:
+            path: /
+            port: 8080
+          initialDelaySeconds: 10
+          periodSeconds: 10
+        resources:
+          requests:
+            cpu: 50m
+            memory: 64Mi
+        volumeMounts:
+        - name: microshift-binary
+          mountPath: /host/usr/bin/microshift
+          readOnly: true
+      volumes:
+      - name: microshift-binary
+        hostPath:
+          path: /usr/bin/microshift
+          type: File
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      priorityClassName: system-cluster-critical
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+        effect: NoSchedule
+      - key: node.kubernetes.io/unreachable
+        operator: Exists
+        effect: NoExecute
+        tolerationSeconds: 120
+      - key: node.kubernetes.io/not-ready
+        operator: Exists
+        effect: NoExecute
+        tolerationSeconds: 120

assets/components/c2cc/namespace.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: microshift-c2cc
+  labels:
+    pod-security.kubernetes.io/enforce: privileged
+    pod-security.kubernetes.io/audit: privileged
+    pod-security.kubernetes.io/warn: privileged
+  annotations:
+    openshift.io/node-selector: ""
+    workload.openshift.io/allowed: "management"

assets/components/c2cc/service.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  namespace: microshift-c2cc
+  name: c2cc-probe
+spec:
+  clusterIP: '{{ .ProbeServiceClusterIP }}'
+  ports:
+  - name: probe
+    port: 8080
+    targetPort: 8080
+    protocol: TCP
+  selector:
+    app: c2cc-probe

assets/components/c2cc/serviceaccount.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  namespace: microshift-c2cc
+  name: c2cc-probe

assets/crd/microshift.io_remoteclusters.yaml

@@ -0,0 +1,85 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.18.0
+  name: remoteclusters.microshift.io
+spec:
+  group: microshift.io
+  names:
+    kind: RemoteCluster
+    listKind: RemoteClusterList
+    plural: remoteclusters
+    singular: remotecluster
+  scope: Cluster
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: |-
+          RemoteCluster represents a remote cluster's healthcheck probe target.
+          Created by the C2CC controller, read and updated by the probe pod.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            properties:
+              probeInterval:
+                default: 10s
+                description: Interval between probe attempts (e.g. "10s", "1m").
+                type: string
+              probeTarget:
+                description: IP:port of the remote cluster's probe service (11th IP
+                  in remote service CIDR, port 8080).
+                type: string
+            required:
+            - probeInterval
+            - probeTarget
+            type: object
+          status:
+            description: RemoteClusterStatus is populated by the probe pod with health
+              probe results.
+            properties:
+              errors:
+                items:
+                  type: string
+                type: array
+              lastProbeTime:
+                format: date-time
+                type: string
+              lastSuccessfulProbe:
+                format: date-time
+                type: string
+              state:
+                default: NeverProbed
+                enum:
+                - NeverProbed
+                - Healthy
+                - Unhealthy
+                type: string
+            required:
+            - state
+            type: object
+        required:
+        - spec
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}

cmd/generate-config/config/config-openapi-spec.json

@@ -203,6 +203,11 @@
             }
           }
         },
+        "probeInterval": {
+          "description": "Interval between healthcheck probe attempts to each remote cluster.\nParsed as a Go duration string (e.g. \"10s\", \"1m\"). Must be between 1s and 5m.",
+          "type": "string",
+          "default": "10s"
+        },
         "remoteClusters": {
           "description": "List of remote clusters to establish connectivity with.\nC2CC is disabled when this list is empty.",
           "type": "array",

cmd/microshift/main.go

@@ -42,5 +42,6 @@ func newCommand() *cobra.Command {
 	cmd.AddCommand(cmds.NewRestoreCommand())
 	cmd.AddCommand(cmds.NewHealthcheckCommand())
 	cmd.AddCommand(cmds.NewAddNodeCommand())
+	cmd.AddCommand(cmds.NewC2CCProbeCommand())
 	return cmd
 }

docs/user/howto_config.md

@@ -34,6 +34,7 @@ clusterToCluster:
     dns:
         cacheNegativeTTL: 0
         cacheTTL: 0
+    probeInterval: ""
     remoteClusters:
         - clusterNetwork: []
           domain: ""
@@ -196,6 +197,7 @@ clusterToCluster:
     dns:
         cacheNegativeTTL: 10
         cacheTTL: 10
+    probeInterval: 10s
     remoteClusters:
         - clusterNetwork: []
           domain: ""

packaging/microshift/config.yaml

@@ -50,6 +50,9 @@ clusterToCluster:
         # Maximum TTL (seconds) for positive DNS cache entries in CoreDNS server blocks
         # generated for remote clusters. Must be >= 0. Setting to 0 disables positive caching.
         cacheTTL: 10
+    # Interval between healthcheck probe attempts to each remote cluster.
+    # Parsed as a Go duration string (e.g. "10s", "1m"). Must be between 1s and 5m.
+    probeInterval: 10s
     # List of remote clusters to establish connectivity with.
     # C2CC is disabled when this list is empty.
     remoteClusters:

pkg/apis/microshift/v1alpha1/doc.go

@@ -0,0 +1,5 @@
+// +kubebuilder:object:generate=true
+// +groupName=microshift.io
+// +k8s:deepcopy-gen=package
+
+package v1alpha1

pkg/apis/microshift/v1alpha1/groupversion_info.go

@@ -0,0 +1,30 @@
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+var (
+	GroupName    = "microshift.io"
+	GroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha1"}
+
+	SchemeGroupVersion = GroupVersion
+
+	schemeBuilder = runtime.NewSchemeBuilder(addKnownTypes)
+	AddToScheme   = schemeBuilder.AddToScheme
+)
+
+func Resource(resource string) schema.GroupResource {
+	return schema.GroupResource{Group: GroupName, Resource: resource}
+}
+
+func addKnownTypes(scheme *runtime.Scheme) error {
+	scheme.AddKnownTypes(GroupVersion,
+		&RemoteCluster{},
+		&RemoteClusterList{},
+	)
+	metav1.AddToGroupVersion(scheme, GroupVersion)
+	return nil
+}

pkg/apis/microshift/v1alpha1/types.go

@@ -0,0 +1,56 @@
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// +genclient
+// +genclient:nonNamespaced
+// +kubebuilder:object:root=true
+// +kubebuilder:resource:scope=Cluster
+// +kubebuilder:subresource:status
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// RemoteCluster represents a remote cluster's healthcheck probe target.
+// Created by the C2CC controller, read and updated by the probe pod.
+type RemoteCluster struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   RemoteClusterSpec   `json:"spec"`
+	Status RemoteClusterStatus `json:"status,omitempty"`
+}
+
+type RemoteClusterSpec struct {
+	// IP:port of the remote cluster's probe service (11th IP in remote service CIDR, port 8080).
+	// +kubebuilder:validation:Required
+	ProbeTarget string `json:"probeTarget"`
+
+	// Interval between probe attempts (e.g. "10s", "1m").
+	// +kubebuilder:default="10s"
+	ProbeInterval metav1.Duration `json:"probeInterval"`
+}
+
+// RemoteClusterStatus is populated by the probe pod with health probe results.
+type RemoteClusterStatus struct {
+	// +kubebuilder:validation:Enum=NeverProbed;Healthy;Unhealthy
+	// +kubebuilder:default="NeverProbed"
+	State string `json:"state"`
+	// +optional
+	LastSuccessfulProbe *metav1.Time `json:"lastSuccessfulProbe,omitempty"`
+	// +optional
+	LastProbeTime *metav1.Time `json:"lastProbeTime,omitempty"`
+	// +optional
+	Errors []string `json:"errors,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// RemoteClusterList contains a list of RemoteCluster resources.
+type RemoteClusterList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+
+	Items []RemoteCluster `json:"items"`
+}