diff --git a/Makefile b/Makefile index 0286fef873..b4d9130b4e 100644 --- a/Makefile +++ b/Makefile @@ -346,6 +346,11 @@ generate-config: verify-config: generate-config ./scripts/verify/verify-config.sh +.PHONY: generate-crds +generate-crds: + ./scripts/fetch_tools.sh controller-gen && \ + ./scripts/generate-crds.sh + # Run all of the end to end tests .PHONY: e2e e2e: diff --git a/assets/components/c2cc/clusterrole.yaml b/assets/components/c2cc/clusterrole.yaml new file mode 100644 index 0000000000..affd57f978 --- /dev/null +++ b/assets/components/c2cc/clusterrole.yaml @@ -0,0 +1,28 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: microshift-c2cc-probe +rules: +- apiGroups: + - microshift.io + resources: + - remoteclusters + verbs: + - get + - list + - watch +- apiGroups: + - microshift.io + resources: + - remoteclusters/status + verbs: + - update + - patch +- apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + verbs: + - use + resourceNames: + - privileged diff --git a/assets/components/c2cc/clusterrolebinding.yaml b/assets/components/c2cc/clusterrolebinding.yaml new file mode 100644 index 0000000000..71c7b1ad48 --- /dev/null +++ b/assets/components/c2cc/clusterrolebinding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: microshift-c2cc-probe +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: microshift-c2cc-probe +subjects: +- kind: ServiceAccount + namespace: microshift-c2cc + name: c2cc-probe diff --git a/assets/components/c2cc/deployment.yaml b/assets/components/c2cc/deployment.yaml new file mode 100644 index 0000000000..5d960866ca --- /dev/null +++ b/assets/components/c2cc/deployment.yaml @@ -0,0 +1,68 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: microshift-c2cc + name: c2cc-probe + labels: + app: c2cc-probe +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: c2cc-probe + template: + metadata: + labels: + app: c2cc-probe + annotations: + target.workload.openshift.io/management: '{"effect": "PreferredDuringScheduling"}' + openshift.io/required-scc: privileged + spec: + serviceAccountName: c2cc-probe + containers: + - name: c2cc-probe + image: '{{ .ReleaseImage.cli }}' + imagePullPolicy: IfNotPresent + command: + - /host/usr/bin/microshift + - c2cc-probe + ports: + - containerPort: 8080 + name: probe + protocol: TCP + livenessProbe: + httpGet: + path: / + port: 8080 + initialDelaySeconds: 10 + periodSeconds: 10 + resources: + requests: + cpu: 50m + memory: 64Mi + volumeMounts: + - name: microshift-binary + mountPath: /host/usr/bin/microshift + readOnly: true + volumes: + - name: microshift-binary + hostPath: + path: /usr/bin/microshift + type: File + nodeSelector: + node-role.kubernetes.io/master: "" + priorityClassName: system-cluster-critical + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + - key: node.kubernetes.io/unreachable + operator: Exists + effect: NoExecute + tolerationSeconds: 120 + - key: node.kubernetes.io/not-ready + operator: Exists + effect: NoExecute + tolerationSeconds: 120 diff --git a/assets/components/c2cc/namespace.yaml b/assets/components/c2cc/namespace.yaml new file mode 100644 index 0000000000..b90dd12ab5 --- /dev/null +++ b/assets/components/c2cc/namespace.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: microshift-c2cc + labels: + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/audit: privileged + pod-security.kubernetes.io/warn: privileged + annotations: + openshift.io/node-selector: "" + workload.openshift.io/allowed: "management" diff --git a/assets/components/c2cc/service.yaml b/assets/components/c2cc/service.yaml new file mode 100644 index 0000000000..d820a21ba3 --- /dev/null +++ b/assets/components/c2cc/service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + namespace: microshift-c2cc + name: c2cc-probe +spec: + clusterIP: '{{ .ProbeServiceClusterIP }}' + ports: + - name: probe + port: 8080 + targetPort: 8080 + protocol: TCP + selector: + app: c2cc-probe diff --git a/assets/components/c2cc/serviceaccount.yaml b/assets/components/c2cc/serviceaccount.yaml new file mode 100644 index 0000000000..b1a1faea48 --- /dev/null +++ b/assets/components/c2cc/serviceaccount.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + namespace: microshift-c2cc + name: c2cc-probe diff --git a/assets/crd/microshift.io_remoteclusters.yaml b/assets/crd/microshift.io_remoteclusters.yaml new file mode 100644 index 0000000000..daf41edbed --- /dev/null +++ b/assets/crd/microshift.io_remoteclusters.yaml @@ -0,0 +1,85 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.18.0 + name: remoteclusters.microshift.io +spec: + group: microshift.io + names: + kind: RemoteCluster + listKind: RemoteClusterList + plural: remoteclusters + singular: remotecluster + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + RemoteCluster represents a remote cluster's healthcheck probe target. + Created by the C2CC controller, read and updated by the probe pod. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + probeInterval: + default: 10s + description: Interval between probe attempts (e.g. "10s", "1m"). + type: string + probeTarget: + description: IP:port of the remote cluster's probe service (11th IP + in remote service CIDR, port 8080). + type: string + required: + - probeInterval + - probeTarget + type: object + status: + description: RemoteClusterStatus is populated by the probe pod with health + probe results. + properties: + errors: + items: + type: string + type: array + lastProbeTime: + format: date-time + type: string + lastSuccessfulProbe: + format: date-time + type: string + state: + default: NeverProbed + enum: + - NeverProbed + - Healthy + - Unhealthy + type: string + required: + - state + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/cmd/generate-config/config/config-openapi-spec.json b/cmd/generate-config/config/config-openapi-spec.json index c2d98d1471..52ba7573cf 100755 --- a/cmd/generate-config/config/config-openapi-spec.json +++ b/cmd/generate-config/config/config-openapi-spec.json @@ -203,6 +203,11 @@ } } }, + "probeInterval": { + "description": "Interval between healthcheck probe attempts to each remote cluster.\nParsed as a Go duration string (e.g. \"10s\", \"1m\"). Must be between 1s and 5m.", + "type": "string", + "default": "10s" + }, "remoteClusters": { "description": "List of remote clusters to establish connectivity with.\nC2CC is disabled when this list is empty.", "type": "array", diff --git a/cmd/microshift/main.go b/cmd/microshift/main.go index bf275fb316..a35978c32b 100644 --- a/cmd/microshift/main.go +++ b/cmd/microshift/main.go @@ -42,5 +42,6 @@ func newCommand() *cobra.Command { cmd.AddCommand(cmds.NewRestoreCommand()) cmd.AddCommand(cmds.NewHealthcheckCommand()) cmd.AddCommand(cmds.NewAddNodeCommand()) + cmd.AddCommand(cmds.NewC2CCProbeCommand()) return cmd } diff --git a/docs/user/howto_config.md b/docs/user/howto_config.md index 636acf1ae7..dcb55fd367 100644 --- a/docs/user/howto_config.md +++ b/docs/user/howto_config.md @@ -34,6 +34,7 @@ clusterToCluster: dns: cacheNegativeTTL: 0 cacheTTL: 0 + probeInterval: "" remoteClusters: - clusterNetwork: [] domain: "" @@ -196,6 +197,7 @@ clusterToCluster: dns: cacheNegativeTTL: 10 cacheTTL: 10 + probeInterval: 10s remoteClusters: - clusterNetwork: [] domain: "" diff --git a/packaging/microshift/config.yaml b/packaging/microshift/config.yaml index 69ab22e68c..81106f224b 100644 --- a/packaging/microshift/config.yaml +++ b/packaging/microshift/config.yaml @@ -50,6 +50,9 @@ clusterToCluster: # Maximum TTL (seconds) for positive DNS cache entries in CoreDNS server blocks # generated for remote clusters. Must be >= 0. Setting to 0 disables positive caching. cacheTTL: 10 + # Interval between healthcheck probe attempts to each remote cluster. + # Parsed as a Go duration string (e.g. "10s", "1m"). Must be between 1s and 5m. + probeInterval: 10s # List of remote clusters to establish connectivity with. # C2CC is disabled when this list is empty. remoteClusters: diff --git a/pkg/apis/microshift/v1alpha1/doc.go b/pkg/apis/microshift/v1alpha1/doc.go new file mode 100644 index 0000000000..13ddd35673 --- /dev/null +++ b/pkg/apis/microshift/v1alpha1/doc.go @@ -0,0 +1,5 @@ +// +kubebuilder:object:generate=true +// +groupName=microshift.io +// +k8s:deepcopy-gen=package + +package v1alpha1 diff --git a/pkg/apis/microshift/v1alpha1/groupversion_info.go b/pkg/apis/microshift/v1alpha1/groupversion_info.go new file mode 100644 index 0000000000..4fce1bcb8b --- /dev/null +++ b/pkg/apis/microshift/v1alpha1/groupversion_info.go @@ -0,0 +1,30 @@ +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +var ( + GroupName = "microshift.io" + GroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha1"} + + SchemeGroupVersion = GroupVersion + + schemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) + AddToScheme = schemeBuilder.AddToScheme +) + +func Resource(resource string) schema.GroupResource { + return schema.GroupResource{Group: GroupName, Resource: resource} +} + +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(GroupVersion, + &RemoteCluster{}, + &RemoteClusterList{}, + ) + metav1.AddToGroupVersion(scheme, GroupVersion) + return nil +} diff --git a/pkg/apis/microshift/v1alpha1/types.go b/pkg/apis/microshift/v1alpha1/types.go new file mode 100644 index 0000000000..b3b68cbbd2 --- /dev/null +++ b/pkg/apis/microshift/v1alpha1/types.go @@ -0,0 +1,56 @@ +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// +genclient +// +genclient:nonNamespaced +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Cluster +// +kubebuilder:subresource:status +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// RemoteCluster represents a remote cluster's healthcheck probe target. +// Created by the C2CC controller, read and updated by the probe pod. +type RemoteCluster struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec RemoteClusterSpec `json:"spec"` + Status RemoteClusterStatus `json:"status,omitempty"` +} + +type RemoteClusterSpec struct { + // IP:port of the remote cluster's probe service (11th IP in remote service CIDR, port 8080). + // +kubebuilder:validation:Required + ProbeTarget string `json:"probeTarget"` + + // Interval between probe attempts (e.g. "10s", "1m"). + // +kubebuilder:default="10s" + ProbeInterval metav1.Duration `json:"probeInterval"` +} + +// RemoteClusterStatus is populated by the probe pod with health probe results. +type RemoteClusterStatus struct { + // +kubebuilder:validation:Enum=NeverProbed;Healthy;Unhealthy + // +kubebuilder:default="NeverProbed" + State string `json:"state"` + // +optional + LastSuccessfulProbe *metav1.Time `json:"lastSuccessfulProbe,omitempty"` + // +optional + LastProbeTime *metav1.Time `json:"lastProbeTime,omitempty"` + // +optional + Errors []string `json:"errors,omitempty"` +} + +// +kubebuilder:object:root=true +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// RemoteClusterList contains a list of RemoteCluster resources. +type RemoteClusterList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + + Items []RemoteCluster `json:"items"` +} diff --git a/pkg/apis/microshift/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/microshift/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 0000000000..0cdb32da4d --- /dev/null +++ b/pkg/apis/microshift/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,112 @@ +//go:build !ignore_autogenerated + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RemoteCluster) DeepCopyInto(out *RemoteCluster) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RemoteCluster. +func (in *RemoteCluster) DeepCopy() *RemoteCluster { + if in == nil { + return nil + } + out := new(RemoteCluster) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *RemoteCluster) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RemoteClusterList) DeepCopyInto(out *RemoteClusterList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]RemoteCluster, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RemoteClusterList. +func (in *RemoteClusterList) DeepCopy() *RemoteClusterList { + if in == nil { + return nil + } + out := new(RemoteClusterList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *RemoteClusterList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RemoteClusterSpec) DeepCopyInto(out *RemoteClusterSpec) { + *out = *in + out.ProbeInterval = in.ProbeInterval +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RemoteClusterSpec. +func (in *RemoteClusterSpec) DeepCopy() *RemoteClusterSpec { + if in == nil { + return nil + } + out := new(RemoteClusterSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RemoteClusterStatus) DeepCopyInto(out *RemoteClusterStatus) { + *out = *in + if in.LastSuccessfulProbe != nil { + in, out := &in.LastSuccessfulProbe, &out.LastSuccessfulProbe + *out = (*in).DeepCopy() + } + if in.LastProbeTime != nil { + in, out := &in.LastProbeTime, &out.LastProbeTime + *out = (*in).DeepCopy() + } + if in.Errors != nil { + in, out := &in.Errors, &out.Errors + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RemoteClusterStatus. +func (in *RemoteClusterStatus) DeepCopy() *RemoteClusterStatus { + if in == nil { + return nil + } + out := new(RemoteClusterStatus) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/assets/crd.go b/pkg/assets/crd.go index beb2cfb14d..4c4fcdf9dc 100644 --- a/pkg/assets/crd.go +++ b/pkg/assets/crd.go @@ -2,6 +2,7 @@ package assets import ( "context" + "errors" "fmt" "time" @@ -9,6 +10,7 @@ import ( apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextclientv1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" apiruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" @@ -107,13 +109,21 @@ func WaitForCrdsEstablished(ctx context.Context, cfg *config.Config) error { return nil } -func readCRDOrDie(objBytes []byte) *apiextv1.CustomResourceDefinition { +func readCRD(objBytes []byte) (*apiextv1.CustomResourceDefinition, error) { var crd apiextv1.CustomResourceDefinition err := apiruntime.DecodeInto(apiExtensionsCodecs.UniversalDecoder(apiextv1.SchemeGroupVersion), objBytes, &crd) + if err != nil { + return nil, err + } + return &crd, nil +} + +func readCRDOrDie(objBytes []byte) *apiextv1.CustomResourceDefinition { + crd, err := readCRD(objBytes) if err != nil { panic(err) } - return &crd + return crd } func applyCRD(ctx context.Context, client *apiextclientv1.ApiextensionsV1Client, crd *apiextv1.CustomResourceDefinition) error { @@ -182,6 +192,45 @@ func ApplyCRDs(ctx context.Context, cfg *config.Config) error { return nil } +func DeleteCRDs(ctx context.Context, crds []string, kubeconfigPath string) error { + lock.Lock() + defer lock.Unlock() + + restConfig, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath) + if err != nil { + return err + } + rest.AddUserAgent(restConfig, "crd-agent") + + client, err := apiextclientv1.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("failed to create client: %v", err) + } + + var errs []error + for _, crd := range crds { + crdBytes, err := embedded.Asset(crd) + if err != nil { + errs = append(errs, fmt.Errorf("error getting asset %s: %v", crd, err)) + continue + } + c, err := readCRD(crdBytes) + if err != nil { + errs = append(errs, fmt.Errorf("decoding CRD from asset %s: %w", crd, err)) + continue + } + klog.Infof("Deleting CRD %s", c.Name) + if err := client.CustomResourceDefinitions().Delete(ctx, c.Name, metav1.DeleteOptions{}); err != nil { + if !apierrors.IsNotFound(err) { + errs = append(errs, fmt.Errorf("deleting CRD %s: %w", c.Name, err)) + } + } else { + klog.Infof("Deleted CRD %s", c.Name) + } + } + return errors.Join(errs...) +} + func ApplyCRDAndWaitForEstablish(ctx context.Context, crds []string, kubeconfigPath string) error { lock.Lock() defer lock.Unlock() diff --git a/pkg/cmd/c2cc_probe.go b/pkg/cmd/c2cc_probe.go new file mode 100644 index 0000000000..d2e07c79ce --- /dev/null +++ b/pkg/cmd/c2cc_probe.go @@ -0,0 +1,17 @@ +package cmd + +import ( + "github.com/openshift/microshift/pkg/controllers/c2cc" + "github.com/spf13/cobra" +) + +func NewC2CCProbeCommand() *cobra.Command { + return &cobra.Command{ + Use: "c2cc-probe", + Short: "Run C2CC remote cluster probe (designed to run as a pod)", + Hidden: true, + RunE: func(cmd *cobra.Command, _ []string) error { + return c2cc.RunProbe(cmd.Context()) + }, + } +} diff --git a/pkg/config/c2cc.go b/pkg/config/c2cc.go index d82eb5aa59..c742b732bb 100644 --- a/pkg/config/c2cc.go +++ b/pkg/config/c2cc.go @@ -5,7 +5,9 @@ import ( "fmt" "net" "strings" + "time" + "github.com/apparentlymart/go-cidr/cidr" "github.com/vishvananda/netlink" "k8s.io/apimachinery/pkg/util/validation" netutils "k8s.io/utils/net" @@ -31,9 +33,15 @@ type C2CC struct { // C2CC is disabled when this list is empty. RemoteClusters []RemoteCluster `json:"remoteClusters,omitempty"` + // Interval between healthcheck probe attempts to each remote cluster. + // Parsed as a Go duration string (e.g. "10s", "1m"). Must be between 1s and 5m. + // +kubebuilder:default="10s" + ProbeInterval string `json:"probeInterval,omitempty"` + // Populated during validation with parsed network objects. - Resolved []ResolvedRemoteCluster `json:"-"` - ResolvedAllCIDRs []*net.IPNet `json:"-"` + Resolved []ResolvedRemoteCluster `json:"-"` + ResolvedAllCIDRs []*net.IPNet `json:"-"` + ResolvedProbeInterval time.Duration `json:"-"` } type RemoteCluster struct { @@ -55,6 +63,7 @@ type ResolvedRemoteCluster struct { ServiceNetwork []*net.IPNet Domain string DNSIP string // 10th IP of ServiceNetwork[0], computed during validation when Domain is set + ProbeIP string // 11th IP of ServiceNetwork[0], deterministic probe service ClusterIP } func (rc *ResolvedRemoteCluster) AllCIDRs() []*net.IPNet { @@ -148,6 +157,15 @@ func (c *C2CC) parseRemoteClusters() ([]ResolvedRemoteCluster, []error) { } resolved[i].Domain = rc.Domain + + if len(resolved[i].ServiceNetwork) > 0 { + probeIP, err := cidr.Host(resolved[i].ServiceNetwork[0], 11) + if err != nil { + errs = append(errs, fmt.Errorf("%s: failed to compute probe IP from serviceNetwork[0]: %w", label, err)) + } else { + resolved[i].ProbeIP = probeIP.String() + } + } } return resolved, errs @@ -236,11 +254,17 @@ func (c *C2CC) validate(cfg *Config) error { seenNextHops, seenRemoteDomains, &seenCIDRs)...) } + probeInterval, err := c.validateProbeInterval() + if err != nil { + errs = append(errs, err) + } + if err := errors.Join(errs...); err != nil { return err } c.Resolved = resolved + c.ResolvedProbeInterval = probeInterval var allCIDRs []*net.IPNet for i := range resolved { @@ -251,6 +275,17 @@ func (c *C2CC) validate(cfg *Config) error { return nil } +func (c *C2CC) validateProbeInterval() (time.Duration, error) { + d, err := time.ParseDuration(c.ProbeInterval) + if err != nil { + return 0, fmt.Errorf("probeInterval %q is not a valid duration: %w", c.ProbeInterval, err) + } + if d < 1*time.Second || d > 5*time.Minute { + return 0, fmt.Errorf("probeInterval must be between 1s and 5m, got %s", d) + } + return d, nil +} + func validateRemoteCluster( i int, rc *RemoteCluster, res *ResolvedRemoteCluster, nodeIP, nodeIPv6 net.IP, localV4, localV6 bool, hostIPs []net.IP, diff --git a/pkg/config/c2cc_test.go b/pkg/config/c2cc_test.go index 50a28984cf..9a87530392 100644 --- a/pkg/config/c2cc_test.go +++ b/pkg/config/c2cc_test.go @@ -4,6 +4,7 @@ import ( "net" "strings" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -69,6 +70,9 @@ func withDNSDefaults(c2cc C2CC) C2CC { } func mkC2CCConfig(c2cc C2CC) *Config { + if c2cc.ProbeInterval == "" { + c2cc.ProbeInterval = "10s" + } return &Config{ Network: Network{ CNIPlugin: CniPluginOVNK, @@ -83,6 +87,9 @@ func mkC2CCConfig(c2cc C2CC) *Config { } func mkDualStackC2CCConfig(c2cc C2CC) *Config { + if c2cc.ProbeInterval == "" { + c2cc.ProbeInterval = "10s" + } return &Config{ Network: Network{ CNIPlugin: CniPluginOVNK, @@ -98,6 +105,9 @@ func mkDualStackC2CCConfig(c2cc C2CC) *Config { } func mkIPv6OnlyC2CCConfig(c2cc C2CC) *Config { + if c2cc.ProbeInterval == "" { + c2cc.ProbeInterval = "10s" + } return &Config{ Network: Network{ CNIPlugin: CniPluginOVNK, @@ -592,6 +602,132 @@ func TestC2CC_ValidateDualStack(t *testing.T) { }) } +func TestC2CC_ProbeIntervalValidation(t *testing.T) { + stubHostIPs(t, nil) + + t.Run("too low", func(t *testing.T) { + cfg := mkC2CCConfig(C2CC{ + ProbeInterval: "500ms", + RemoteClusters: []RemoteCluster{{ + NextHop: "10.100.0.2", + ClusterNetwork: []string{"10.45.0.0/16"}, + ServiceNetwork: []string{"10.46.0.0/16"}, + }}, + }) + err := cfg.C2CC.validate(cfg) + require.Error(t, err) + assert.Contains(t, err.Error(), "probeInterval must be between 1s and 5m") + }) + + t.Run("too high", func(t *testing.T) { + cfg := mkC2CCConfig(C2CC{ + ProbeInterval: "6m", + RemoteClusters: []RemoteCluster{{ + NextHop: "10.100.0.2", + ClusterNetwork: []string{"10.45.0.0/16"}, + ServiceNetwork: []string{"10.46.0.0/16"}, + }}, + }) + err := cfg.C2CC.validate(cfg) + require.Error(t, err) + assert.Contains(t, err.Error(), "probeInterval must be between 1s and 5m") + }) + + t.Run("invalid duration string", func(t *testing.T) { + cfg := mkC2CCConfig(C2CC{ + ProbeInterval: "not-a-duration", + RemoteClusters: []RemoteCluster{{ + NextHop: "10.100.0.2", + ClusterNetwork: []string{"10.45.0.0/16"}, + ServiceNetwork: []string{"10.46.0.0/16"}, + }}, + }) + err := cfg.C2CC.validate(cfg) + require.Error(t, err) + assert.Contains(t, err.Error(), "not a valid duration") + }) + + t.Run("minimum boundary", func(t *testing.T) { + cfg := mkC2CCConfig(C2CC{ + ProbeInterval: "1s", + RemoteClusters: []RemoteCluster{{ + NextHop: "10.100.0.2", + ClusterNetwork: []string{"10.45.0.0/16"}, + ServiceNetwork: []string{"10.46.0.0/16"}, + }}, + }) + assert.NoError(t, cfg.C2CC.validate(cfg)) + }) + + t.Run("maximum boundary", func(t *testing.T) { + cfg := mkC2CCConfig(C2CC{ + ProbeInterval: "5m", + RemoteClusters: []RemoteCluster{{ + NextHop: "10.100.0.2", + ClusterNetwork: []string{"10.45.0.0/16"}, + ServiceNetwork: []string{"10.46.0.0/16"}, + }}, + }) + assert.NoError(t, cfg.C2CC.validate(cfg)) + }) + + t.Run("valid mid-range value", func(t *testing.T) { + cfg := mkC2CCConfig(C2CC{ + ProbeInterval: "30s", + RemoteClusters: []RemoteCluster{{ + NextHop: "10.100.0.2", + ClusterNetwork: []string{"10.45.0.0/16"}, + ServiceNetwork: []string{"10.46.0.0/16"}, + }}, + }) + require.NoError(t, cfg.C2CC.validate(cfg)) + assert.Equal(t, 30*time.Second, cfg.C2CC.ResolvedProbeInterval) + }) +} + +func TestC2CC_ProbeIP(t *testing.T) { + stubHostIPs(t, nil) + + t.Run("IPv4 service network", func(t *testing.T) { + cfg := mkC2CCConfig(C2CC{ + RemoteClusters: []RemoteCluster{{ + NextHop: "10.100.0.2", + ClusterNetwork: []string{"10.45.0.0/16"}, + ServiceNetwork: []string{"10.46.0.0/16"}, + }}, + }) + require.NoError(t, cfg.C2CC.validate(cfg)) + require.Len(t, cfg.C2CC.Resolved, 1) + assert.Equal(t, "10.46.0.11", cfg.C2CC.Resolved[0].ProbeIP) + }) + + t.Run("IPv6 service network", func(t *testing.T) { + cfg := mkIPv6OnlyC2CCConfig(C2CC{ + RemoteClusters: []RemoteCluster{{ + NextHop: "fd00::2", + ClusterNetwork: []string{"fd03::/48"}, + ServiceNetwork: []string{"fd04::/112"}, + }}, + }) + require.NoError(t, cfg.C2CC.validate(cfg)) + require.Len(t, cfg.C2CC.Resolved, 1) + assert.Equal(t, "fd04::b", cfg.C2CC.Resolved[0].ProbeIP) + }) + + t.Run("dual-stack uses first service network", func(t *testing.T) { + cfg := mkDualStackC2CCConfig(C2CC{ + RemoteClusters: []RemoteCluster{{ + NextHop: "10.100.0.2", + ClusterNetwork: []string{"10.45.0.0/16", "fd03::/48"}, + ServiceNetwork: []string{"10.46.0.0/16", "fd04::/112"}, + }}, + }) + require.NoError(t, cfg.C2CC.validate(cfg)) + require.Len(t, cfg.C2CC.Resolved, 1) + assert.Equal(t, "10.46.0.11", cfg.C2CC.Resolved[0].ProbeIP) + }) +} + func TestC2CC_DNSIP(t *testing.T) { stubHostIPs(t, nil) @@ -722,3 +858,42 @@ func TestRenderC2CCDNSBlocks(t *testing.T) { assert.NotContains(t, result, "cluster-c") }) } + +func TestC2CC_ProbeIntervalDefault(t *testing.T) { + cfg := &Config{} + require.NoError(t, cfg.fillDefaults()) + assert.Equal(t, "10s", cfg.C2CC.ProbeInterval) +} + +func TestC2CC_IncorporateUserSettings(t *testing.T) { + t.Run("user overrides probe interval", func(t *testing.T) { + cfg := &Config{} + require.NoError(t, cfg.fillDefaults()) + + user := &Config{ + C2CC: C2CC{ + ProbeInterval: "30s", + }, + } + cfg.incorporateUserSettings(user) + assert.Equal(t, "30s", cfg.C2CC.ProbeInterval) + }) + + t.Run("user sets remoteClusters without probeInterval preserves default", func(t *testing.T) { + cfg := &Config{} + require.NoError(t, cfg.fillDefaults()) + + user := &Config{ + C2CC: C2CC{ + RemoteClusters: []RemoteCluster{{ + NextHop: "10.100.0.2", + ClusterNetwork: []string{"10.45.0.0/16"}, + ServiceNetwork: []string{"10.46.0.0/16"}, + }}, + }, + } + cfg.incorporateUserSettings(user) + assert.Equal(t, "10s", cfg.C2CC.ProbeInterval) + assert.Len(t, cfg.C2CC.RemoteClusters, 1) + }) +} diff --git a/pkg/config/config.go b/pkg/config/config.go index bac66b96b7..28cd90d5e0 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -200,7 +200,8 @@ func (c *Config) fillDefaults() error { c.Telemetry = telemetryDefaults() c.DNS = dnsDefaults() c.C2CC = C2CC{ - DNS: C2CCDNS{CacheTTL: ptr.To(10), CacheNegativeTTL: ptr.To(10)}, + DNS: C2CCDNS{CacheTTL: ptr.To(10), CacheNegativeTTL: ptr.To(10)}, + ProbeInterval: "10s", } return nil } @@ -469,6 +470,9 @@ func (c *Config) incorporateUserSettings(u *Config) { if u.C2CC.DNS.CacheNegativeTTL != nil { c.C2CC.DNS.CacheNegativeTTL = u.C2CC.DNS.CacheNegativeTTL } + if u.C2CC.ProbeInterval != "" { + c.C2CC.ProbeInterval = u.C2CC.ProbeInterval + } } // updateComputedValues examins the existing settings and converts any diff --git a/pkg/controllers/c2cc/controller.go b/pkg/controllers/c2cc/controller.go index 8091817cec..397edd98f7 100644 --- a/pkg/controllers/c2cc/controller.go +++ b/pkg/controllers/c2cc/controller.go @@ -5,7 +5,9 @@ import ( "fmt" "time" + "github.com/openshift/microshift/pkg/assets" "github.com/openshift/microshift/pkg/config" + microshiftclient "github.com/openshift/microshift/pkg/generated/clientset/versioned/typed/microshift/v1alpha1" "github.com/ovn-kubernetes/libovsdb/client" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" @@ -16,17 +18,21 @@ const ( reconcileInterval = 2 * time.Second ) +var healthcheckCRD = []string{"crd/microshift.io_remoteclusters.yaml"} + type C2CCRouteManager struct { cfg *config.Config nodeName string kubeconfig string - kubeClient kubernetes.Interface - ovn *ovnRouteManager - annotation *annotationManager - nftMgr *nftablesManager - routes *linuxRouteManager - svcRoutes *serviceRouteManager + kubeClient kubernetes.Interface + microshiftClient microshiftclient.MicroshiftV1alpha1Interface + ovn *ovnRouteManager + annotation *annotationManager + nftMgr *nftablesManager + routes *linuxRouteManager + svcRoutes *serviceRouteManager + healthcheck *healthcheckCRManager } func NewC2CCRouteManager(cfg *config.Config) *C2CCRouteManager { @@ -96,6 +102,10 @@ func (c *C2CCRouteManager) Run(ctx context.Context, ready chan<- struct{}, stopp c.annotation.subscribe(ctx, reconcileCh) + if err := assets.ApplyCRDAndWaitForEstablish(ctx, healthcheckCRD, c.kubeconfig); err != nil { + return fmt.Errorf("failed to apply C2CC healthcheck CRD: %w", err) + } + c.fullReconcile(ctx) ticker := time.NewTicker(reconcileInterval) @@ -142,6 +152,13 @@ func (c *C2CCRouteManager) initKubeClient() error { return fmt.Errorf("failed to create kubernetes client: %w", err) } c.kubeClient = kClient + + msClient, err := microshiftclient.NewForConfig(restCfg) + if err != nil { + return fmt.Errorf("failed to create microshift client: %w", err) + } + c.microshiftClient = msClient + return nil } @@ -156,6 +173,7 @@ func (c *C2CCRouteManager) initSubsystems(nbClient client.Client) error { return fmt.Errorf("failed to init nftables manager: %w", err) } c.nftMgr = nftMgr + c.healthcheck = newHealthcheckCRManager(c.microshiftClient, c.cfg) return nil } @@ -206,6 +224,8 @@ func (c *C2CCRouteManager) fullReconcile(ctx context.Context) { {"linux-routes", c.routes.reconcile}, {"service-routes", c.svcRoutes.reconcile}, {"nftables", c.nftMgr.reconcile}, + {"healthcheck-crs", c.healthcheck.reconcile}, + {"probe-deployment", c.deployProbe}, } for _, s := range subsystems { if err := s.fn(ctx); err != nil { @@ -239,6 +259,18 @@ func (c *C2CCRouteManager) cleanupAll(ctx context.Context) { if c.nftMgr != nil { cleanups = append(cleanups, cleanable{"nftables", c.nftMgr.cleanup}) } + cleanups = append(cleanups, cleanable{"probe-namespace", func(ctx context.Context) error { + return assets.DeleteNamespaces(ctx, c2ccNamespace, c.kubeconfig) + }}) + cleanups = append(cleanups, cleanable{"probe-clusterrolebinding", func(ctx context.Context) error { + return assets.DeleteClusterRoleBindings(ctx, c2ccClusterRoleBinding, c.kubeconfig) + }}) + cleanups = append(cleanups, cleanable{"probe-clusterrole", func(ctx context.Context) error { + return assets.DeleteClusterRoles(ctx, c2ccClusterRole, c.kubeconfig) + }}) + cleanups = append(cleanups, cleanable{"healthcheck-crd", func(ctx context.Context) error { + return assets.DeleteCRDs(ctx, healthcheckCRD, c.kubeconfig) + }}) for _, cl := range cleanups { if err := cl.fn(ctx); err != nil { diff --git a/pkg/controllers/c2cc/deploy_probe.go b/pkg/controllers/c2cc/deploy_probe.go new file mode 100644 index 0000000000..ed5b664f32 --- /dev/null +++ b/pkg/controllers/c2cc/deploy_probe.go @@ -0,0 +1,73 @@ +package c2cc + +import ( + "bytes" + "context" + "fmt" + "net" + "text/template" + + "github.com/apparentlymart/go-cidr/cidr" + "github.com/openshift/microshift/pkg/assets" + "github.com/openshift/microshift/pkg/release" + "k8s.io/klog/v2" +) + +var ( + c2ccNamespace = []string{"components/c2cc/namespace.yaml"} + c2ccServiceAccount = []string{"components/c2cc/serviceaccount.yaml"} + c2ccClusterRole = []string{"components/c2cc/clusterrole.yaml"} + c2ccClusterRoleBinding = []string{"components/c2cc/clusterrolebinding.yaml"} + c2ccDeployment = []string{"components/c2cc/deployment.yaml"} + c2ccService = []string{"components/c2cc/service.yaml"} +) + +func (c *C2CCRouteManager) deployProbe(ctx context.Context) error { + _, svcNet, err := net.ParseCIDR(c.cfg.Network.ServiceNetwork[0]) + if err != nil { + return fmt.Errorf("failed to parse local service network: %w", err) + } + probeIP, err := cidr.Host(svcNet, 11) + if err != nil { + return fmt.Errorf("failed to compute probe service ClusterIP: %w", err) + } + + params := assets.RenderParams{ + "ReleaseImage": release.Image, + "ProbeServiceClusterIP": probeIP.String(), + } + + if err := assets.ApplyNamespaces(ctx, c2ccNamespace, c.kubeconfig); err != nil { + return fmt.Errorf("failed to apply c2cc namespace: %w", err) + } + if err := assets.ApplyServiceAccounts(ctx, c2ccServiceAccount, c.kubeconfig); err != nil { + return fmt.Errorf("failed to apply c2cc service account: %w", err) + } + if err := assets.ApplyClusterRoles(ctx, c2ccClusterRole, c.kubeconfig); err != nil { + return fmt.Errorf("failed to apply c2cc cluster role: %w", err) + } + if err := assets.ApplyClusterRoleBindings(ctx, c2ccClusterRoleBinding, c.kubeconfig); err != nil { + return fmt.Errorf("failed to apply c2cc cluster role binding: %w", err) + } + if err := assets.ApplyDeployments(ctx, c2ccDeployment, renderTemplate, params, c.kubeconfig); err != nil { + return fmt.Errorf("failed to apply c2cc deployment: %w", err) + } + if err := assets.ApplyServices(ctx, c2ccService, renderTemplate, params, c.kubeconfig); err != nil { + return fmt.Errorf("failed to apply c2cc service: %w", err) + } + + klog.Infof("C2CC probe assets deployed (probe ClusterIP=%s)", probeIP) + return nil +} + +func renderTemplate(tb []byte, data assets.RenderParams) ([]byte, error) { + tmpl, err := template.New("").Option("missingkey=error").Parse(string(tb)) + if err != nil { + return nil, err + } + var buf bytes.Buffer + if err := tmpl.Execute(&buf, data); err != nil { + return nil, err + } + return buf.Bytes(), nil +} diff --git a/pkg/controllers/c2cc/healthcheck.go b/pkg/controllers/c2cc/healthcheck.go new file mode 100644 index 0000000000..cb6ba5a6a2 --- /dev/null +++ b/pkg/controllers/c2cc/healthcheck.go @@ -0,0 +1,107 @@ +package c2cc + +import ( + "context" + "fmt" + "net" + "strings" + + microshiftv1alpha1 "github.com/openshift/microshift/pkg/apis/microshift/v1alpha1" + "github.com/openshift/microshift/pkg/config" + microshiftclient "github.com/openshift/microshift/pkg/generated/clientset/versioned/typed/microshift/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" +) + +const ( + probePort = 8080 + managedByLabel = "app.kubernetes.io/managed-by" + managerName = "c2cc-route-manager" +) + +type healthcheckCRManager struct { + client microshiftclient.MicroshiftV1alpha1Interface + cfg *config.Config +} + +func newHealthcheckCRManager(client microshiftclient.MicroshiftV1alpha1Interface, cfg *config.Config) *healthcheckCRManager { + return &healthcheckCRManager{ + client: client, + cfg: cfg, + } +} + +func (h *healthcheckCRManager) reconcile(ctx context.Context) error { + desired := h.buildDesiredCRs() + + existing, err := h.client.RemoteClusters().List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", managedByLabel, managerName), + }) + if err != nil { + return fmt.Errorf("listing RemoteCluster CRs: %w", err) + } + + existingByName := make(map[string]*microshiftv1alpha1.RemoteCluster, len(existing.Items)) + for i := range existing.Items { + existingByName[existing.Items[i].Name] = &existing.Items[i] + } + + for name, want := range desired { + got, ok := existingByName[name] + if !ok { + if _, err := h.client.RemoteClusters().Create(ctx, want, metav1.CreateOptions{}); err != nil { + return fmt.Errorf("creating RemoteCluster %q: %w", name, err) + } + klog.Infof("Created RemoteCluster CR %q", name) + continue + } + + delete(existingByName, name) + + if got.Spec.ProbeTarget == want.Spec.ProbeTarget && got.Spec.ProbeInterval == want.Spec.ProbeInterval { + continue + } + + got.Spec = want.Spec + if _, err := h.client.RemoteClusters().Update(ctx, got, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("updating RemoteCluster %q: %w", name, err) + } + klog.V(2).Infof("Updated RemoteCluster CR %q", name) + } + + for name := range existingByName { + if err := h.client.RemoteClusters().Delete(ctx, name, metav1.DeleteOptions{}); err != nil { + return fmt.Errorf("deleting stale RemoteCluster %q: %w", name, err) + } + klog.Infof("Deleted stale RemoteCluster CR %q", name) + } + + return nil +} + +func (h *healthcheckCRManager) buildDesiredCRs() map[string]*microshiftv1alpha1.RemoteCluster { + desired := make(map[string]*microshiftv1alpha1.RemoteCluster, len(h.cfg.C2CC.Resolved)) + for _, rc := range h.cfg.C2CC.Resolved { + name := crNameForRemote(rc.NextHop) + desired[name] = µshiftv1alpha1.RemoteCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + managedByLabel: managerName, + }, + }, + Spec: microshiftv1alpha1.RemoteClusterSpec{ + ProbeTarget: net.JoinHostPort(rc.ProbeIP, fmt.Sprintf("%d", probePort)), + ProbeInterval: metav1.Duration{Duration: h.cfg.C2CC.ResolvedProbeInterval}, + }, + } + } + return desired +} + +func crNameForRemote(nextHop net.IP) string { + s := nextHop.String() + s = strings.ReplaceAll(s, ".", "-") + s = strings.ReplaceAll(s, ":", "-") + return "c2cc-" + s +} diff --git a/pkg/controllers/c2cc/healthcheck_test.go b/pkg/controllers/c2cc/healthcheck_test.go new file mode 100644 index 0000000000..febf2389ba --- /dev/null +++ b/pkg/controllers/c2cc/healthcheck_test.go @@ -0,0 +1,177 @@ +package c2cc + +import ( + "context" + "net" + "testing" + "time" + + microshiftv1alpha1 "github.com/openshift/microshift/pkg/apis/microshift/v1alpha1" + "github.com/openshift/microshift/pkg/config" + fakeclientset "github.com/openshift/microshift/pkg/generated/clientset/versioned/fake" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ktesting "k8s.io/client-go/testing" +) + +func TestCrNameForRemote(t *testing.T) { + tests := []struct { + name string + nextHop string + expected string + }{ + {"IPv4", "10.100.0.2", "c2cc-10-100-0-2"}, + {"IPv6", "fd00::2", "c2cc-fd00--2"}, + {"IPv6 full", "2001:db8::1", "c2cc-2001-db8--1"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ip := net.ParseIP(tt.nextHop) + require.NotNil(t, ip) + assert.Equal(t, tt.expected, crNameForRemote(ip)) + }) + } +} + +func TestBuildDesiredCRs(t *testing.T) { + cfg := &config.Config{ + C2CC: config.C2CC{ + ResolvedProbeInterval: 15 * time.Second, + Resolved: []config.ResolvedRemoteCluster{ + { + NextHop: net.ParseIP("10.100.0.2"), + ProbeIP: "10.46.0.11", + }, + { + NextHop: net.ParseIP("10.100.0.3"), + ProbeIP: "10.47.0.11", + }, + }, + }, + } + + mgr := newHealthcheckCRManager(nil, cfg) + desired := mgr.buildDesiredCRs() + + assert.Len(t, desired, 2) + + cr1, ok := desired["c2cc-10-100-0-2"] + require.True(t, ok, "expected CR for 10.100.0.2") + assert.Equal(t, "10.46.0.11:8080", cr1.Spec.ProbeTarget) + assert.Equal(t, 15*time.Second, cr1.Spec.ProbeInterval.Duration) + assert.Equal(t, managerName, cr1.Labels[managedByLabel]) + + cr2, ok := desired["c2cc-10-100-0-3"] + require.True(t, ok, "expected CR for 10.100.0.3") + assert.Equal(t, "10.47.0.11:8080", cr2.Spec.ProbeTarget) + assert.Equal(t, 15*time.Second, cr2.Spec.ProbeInterval.Duration) +} + +func newFakeClientset(objects ...runtime.Object) *fakeclientset.Clientset { + return fakeclientset.NewSimpleClientset(objects...) +} + +func TestReconcileCreatesNewCRs(t *testing.T) { + cs := newFakeClientset() + cfg := &config.Config{ + C2CC: config.C2CC{ + ResolvedProbeInterval: 10 * time.Second, + Resolved: []config.ResolvedRemoteCluster{ + { + NextHop: net.ParseIP("10.100.0.2"), + ProbeIP: "10.46.0.11", + }, + }, + }, + } + + mgr := newHealthcheckCRManager(cs.MicroshiftV1alpha1(), cfg) + err := mgr.reconcile(context.Background()) + require.NoError(t, err) + + var creates int + for _, a := range cs.Actions() { + if a.GetVerb() == "create" { + creates++ + cr := a.(ktesting.CreateAction).GetObject().(*microshiftv1alpha1.RemoteCluster) + assert.Equal(t, "c2cc-10-100-0-2", cr.Name) + assert.Equal(t, "10.46.0.11:8080", cr.Spec.ProbeTarget) + } + } + assert.Equal(t, 1, creates) +} + +func TestReconcileDeletesStaleCRs(t *testing.T) { + staleCR := µshiftv1alpha1.RemoteCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "c2cc-10-100-0-99", + Labels: map[string]string{managedByLabel: managerName}, + }, + Spec: microshiftv1alpha1.RemoteClusterSpec{ + ProbeTarget: "10.99.0.11:8080", + ProbeInterval: metav1.Duration{Duration: 10 * time.Second}, + }, + } + cs := newFakeClientset(staleCR) + + cfg := &config.Config{ + C2CC: config.C2CC{ + ResolvedProbeInterval: 10 * time.Second, + Resolved: []config.ResolvedRemoteCluster{}, + }, + } + + mgr := newHealthcheckCRManager(cs.MicroshiftV1alpha1(), cfg) + err := mgr.reconcile(context.Background()) + require.NoError(t, err) + + var deletes int + for _, a := range cs.Actions() { + if a.GetVerb() == "delete" { + deletes++ + } + } + assert.Equal(t, 1, deletes) +} + +func TestReconcileUpdatesCR(t *testing.T) { + existingCR := µshiftv1alpha1.RemoteCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "c2cc-10-100-0-2", + Labels: map[string]string{managedByLabel: managerName}, + }, + Spec: microshiftv1alpha1.RemoteClusterSpec{ + ProbeTarget: "10.46.0.11:8080", + ProbeInterval: metav1.Duration{Duration: 30 * time.Second}, + }, + } + cs := newFakeClientset(existingCR) + + cfg := &config.Config{ + C2CC: config.C2CC{ + ResolvedProbeInterval: 15 * time.Second, + Resolved: []config.ResolvedRemoteCluster{ + { + NextHop: net.ParseIP("10.100.0.2"), + ProbeIP: "10.46.0.11", + }, + }, + }, + } + + mgr := newHealthcheckCRManager(cs.MicroshiftV1alpha1(), cfg) + err := mgr.reconcile(context.Background()) + require.NoError(t, err) + + var updates int + for _, a := range cs.Actions() { + if a.GetVerb() == "update" { + updates++ + cr := a.(ktesting.UpdateAction).GetObject().(*microshiftv1alpha1.RemoteCluster) + assert.Equal(t, 15*time.Second, cr.Spec.ProbeInterval.Duration) + } + } + assert.Equal(t, 1, updates) +} diff --git a/pkg/controllers/c2cc/probe.go b/pkg/controllers/c2cc/probe.go new file mode 100644 index 0000000000..1bb6d4e034 --- /dev/null +++ b/pkg/controllers/c2cc/probe.go @@ -0,0 +1,228 @@ +package c2cc + +import ( + "context" + "fmt" + "net/http" + "sync" + "time" + + microshiftv1alpha1 "github.com/openshift/microshift/pkg/apis/microshift/v1alpha1" + microshiftclientset "github.com/openshift/microshift/pkg/generated/clientset/versioned" + microshiftinformers "github.com/openshift/microshift/pkg/generated/informers/externalversions" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" +) + +const ( + unhealthyThreshold = 3 + probeHTTPTimeout = 5 * time.Second + informerResync = 30 * time.Second +) + +// RunProbe is the entrypoint for the healthcheck-probe subcommand. +// It runs inside a pod on the cluster network, serving as both a probe +// target (HTTP :8080) and an active prober of remote clusters. +func RunProbe(ctx context.Context) error { + restCfg, err := rest.InClusterConfig() + if err != nil { + return fmt.Errorf("failed to build in-cluster config: %w", err) + } + + msClient, err := microshiftclientset.NewForConfig(restCfg) + if err != nil { + return fmt.Errorf("failed to create microshift client: %w", err) + } + + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, "ok") + }) + server := &http.Server{Addr: ":8080", Handler: mux} + + go func() { + klog.Infof("Starting probe target HTTP server on :8080") + if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { + klog.Errorf("Probe HTTP server error: %v", err) + } + }() + + pm := &probeManager{ + client: msClient, + probes: make(map[string]context.CancelFunc), + } + + factory := microshiftinformers.NewSharedInformerFactory(msClient, informerResync) + informer := factory.Microshift().V1alpha1().RemoteClusters().Informer() + + informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + if rc, ok := obj.(*microshiftv1alpha1.RemoteCluster); ok { + pm.startProbe(ctx, rc) + } + }, + UpdateFunc: func(oldObj, newObj interface{}) { + oldRC, ok1 := oldObj.(*microshiftv1alpha1.RemoteCluster) + newRC, ok2 := newObj.(*microshiftv1alpha1.RemoteCluster) + if ok1 && ok2 && (oldRC.Spec.ProbeTarget != newRC.Spec.ProbeTarget || + oldRC.Spec.ProbeInterval != newRC.Spec.ProbeInterval) { + pm.restartProbe(ctx, newRC) + } + }, + DeleteFunc: func(obj interface{}) { + rc, ok := obj.(*microshiftv1alpha1.RemoteCluster) + if !ok { + if tombstone, ok := obj.(cache.DeletedFinalStateUnknown); ok { + rc, _ = tombstone.Obj.(*microshiftv1alpha1.RemoteCluster) + } + } + if rc != nil { + pm.stopProbe(rc.Name) + } + }, + }) + + factory.Start(ctx.Done()) + factory.WaitForCacheSync(ctx.Done()) + klog.Infof("Probe manager running, watching RemoteCluster CRs") + + <-ctx.Done() + pm.stopAll() + server.Shutdown(context.Background()) //nolint:errcheck + klog.Infof("Probe manager shut down") + return nil +} + +type probeManager struct { + client microshiftclientset.Interface + mu sync.Mutex + probes map[string]context.CancelFunc +} + +func (pm *probeManager) startProbe(ctx context.Context, rc *microshiftv1alpha1.RemoteCluster) { + pm.mu.Lock() + defer pm.mu.Unlock() + + if _, exists := pm.probes[rc.Name]; exists { + return + } + + probeCtx, cancel := context.WithCancel(ctx) + pm.probes[rc.Name] = cancel + + klog.Infof("Starting probe for %q (target=%s, interval=%s)", + rc.Name, rc.Spec.ProbeTarget, rc.Spec.ProbeInterval.Duration) + go pm.runProbeLoop(probeCtx, rc.Name, rc.Spec.ProbeTarget, rc.Spec.ProbeInterval.Duration) +} + +func (pm *probeManager) restartProbe(ctx context.Context, rc *microshiftv1alpha1.RemoteCluster) { + pm.mu.Lock() + if cancel, exists := pm.probes[rc.Name]; exists { + cancel() + delete(pm.probes, rc.Name) + } + pm.mu.Unlock() + + pm.startProbe(ctx, rc) +} + +func (pm *probeManager) stopProbe(name string) { + pm.mu.Lock() + defer pm.mu.Unlock() + + if cancel, exists := pm.probes[name]; exists { + cancel() + delete(pm.probes, name) + klog.Infof("Stopped probe for %q", name) + } +} + +func (pm *probeManager) stopAll() { + pm.mu.Lock() + defer pm.mu.Unlock() + + for name, cancel := range pm.probes { + cancel() + delete(pm.probes, name) + } +} + +func (pm *probeManager) runProbeLoop(ctx context.Context, name, target string, interval time.Duration) { + httpClient := &http.Client{Timeout: probeHTTPTimeout} + consecutiveFailures := 0 + url := "http://" + target + "/" + + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + probeErr := doProbe(ctx, httpClient, url) + now := metav1.Now() + + status := microshiftv1alpha1.RemoteClusterStatus{ + LastProbeTime: &now, + } + + if probeErr != nil { + consecutiveFailures++ + klog.V(2).Infof("Probe %q failed (%d consecutive): %v", name, consecutiveFailures, probeErr) + + if consecutiveFailures >= unhealthyThreshold { + status.State = "Unhealthy" + } else { + status.State = "Healthy" + } + status.Errors = []string{probeErr.Error()} + } else { + consecutiveFailures = 0 + status.State = "Healthy" + status.LastSuccessfulProbe = &now + } + + if err := pm.updateStatus(ctx, name, status); err != nil { + klog.Errorf("Failed to update status for %q: %v", name, err) + } + } + } +} + +func doProbe(ctx context.Context, client *http.Client, url string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("failed to execute probe request: %w", err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed with unexpected status %d", resp.StatusCode) + } + return nil +} + +func (pm *probeManager) updateStatus(ctx context.Context, name string, status microshiftv1alpha1.RemoteClusterStatus) error { + rcClient := pm.client.MicroshiftV1alpha1().RemoteClusters() + + rc, err := rcClient.Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get RemoteCluster %q: %w", name, err) + } + + // Preserve LastSuccessfulProbe from the existing status if this probe failed + if rc.Status.LastSuccessfulProbe != nil && status.LastSuccessfulProbe == nil { + status.LastSuccessfulProbe = rc.Status.LastSuccessfulProbe + } + + rc.Status = status + _, err = rcClient.UpdateStatus(ctx, rc, metav1.UpdateOptions{}) + return err +} diff --git a/pkg/generated/clientset/versioned/clientset.go b/pkg/generated/clientset/versioned/clientset.go new file mode 100644 index 0000000000..5bdc46c587 --- /dev/null +++ b/pkg/generated/clientset/versioned/clientset.go @@ -0,0 +1,104 @@ +// Code generated by client-gen. DO NOT EDIT. + +package versioned + +import ( + fmt "fmt" + http "net/http" + + microshiftv1alpha1 "github.com/openshift/microshift/pkg/generated/clientset/versioned/typed/microshift/v1alpha1" + discovery "k8s.io/client-go/discovery" + rest "k8s.io/client-go/rest" + flowcontrol "k8s.io/client-go/util/flowcontrol" +) + +type Interface interface { + Discovery() discovery.DiscoveryInterface + MicroshiftV1alpha1() microshiftv1alpha1.MicroshiftV1alpha1Interface +} + +// Clientset contains the clients for groups. +type Clientset struct { + *discovery.DiscoveryClient + microshiftV1alpha1 *microshiftv1alpha1.MicroshiftV1alpha1Client +} + +// MicroshiftV1alpha1 retrieves the MicroshiftV1alpha1Client +func (c *Clientset) MicroshiftV1alpha1() microshiftv1alpha1.MicroshiftV1alpha1Interface { + return c.microshiftV1alpha1 +} + +// Discovery retrieves the DiscoveryClient +func (c *Clientset) Discovery() discovery.DiscoveryInterface { + if c == nil { + return nil + } + return c.DiscoveryClient +} + +// NewForConfig creates a new Clientset for the given config. +// If config's RateLimiter is not set and QPS and Burst are acceptable, +// NewForConfig will generate a rate-limiter in configShallowCopy. +// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), +// where httpClient was generated with rest.HTTPClientFor(c). +func NewForConfig(c *rest.Config) (*Clientset, error) { + configShallowCopy := *c + + if configShallowCopy.UserAgent == "" { + configShallowCopy.UserAgent = rest.DefaultKubernetesUserAgent() + } + + // share the transport between all clients + httpClient, err := rest.HTTPClientFor(&configShallowCopy) + if err != nil { + return nil, err + } + + return NewForConfigAndClient(&configShallowCopy, httpClient) +} + +// NewForConfigAndClient creates a new Clientset for the given config and http client. +// Note the http client provided takes precedence over the configured transport values. +// If config's RateLimiter is not set and QPS and Burst are acceptable, +// NewForConfigAndClient will generate a rate-limiter in configShallowCopy. +func NewForConfigAndClient(c *rest.Config, httpClient *http.Client) (*Clientset, error) { + configShallowCopy := *c + if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 { + if configShallowCopy.Burst <= 0 { + return nil, fmt.Errorf("burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0") + } + configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst) + } + + var cs Clientset + var err error + cs.microshiftV1alpha1, err = microshiftv1alpha1.NewForConfigAndClient(&configShallowCopy, httpClient) + if err != nil { + return nil, err + } + + cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfigAndClient(&configShallowCopy, httpClient) + if err != nil { + return nil, err + } + return &cs, nil +} + +// NewForConfigOrDie creates a new Clientset for the given config and +// panics if there is an error in the config. +func NewForConfigOrDie(c *rest.Config) *Clientset { + cs, err := NewForConfig(c) + if err != nil { + panic(err) + } + return cs +} + +// New creates a new Clientset for the given RESTClient. +func New(c rest.Interface) *Clientset { + var cs Clientset + cs.microshiftV1alpha1 = microshiftv1alpha1.New(c) + + cs.DiscoveryClient = discovery.NewDiscoveryClient(c) + return &cs +} diff --git a/pkg/generated/clientset/versioned/fake/clientset_generated.go b/pkg/generated/clientset/versioned/fake/clientset_generated.go new file mode 100644 index 0000000000..9e7a616bca --- /dev/null +++ b/pkg/generated/clientset/versioned/fake/clientset_generated.go @@ -0,0 +1,85 @@ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + clientset "github.com/openshift/microshift/pkg/generated/clientset/versioned" + microshiftv1alpha1 "github.com/openshift/microshift/pkg/generated/clientset/versioned/typed/microshift/v1alpha1" + fakemicroshiftv1alpha1 "github.com/openshift/microshift/pkg/generated/clientset/versioned/typed/microshift/v1alpha1/fake" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/discovery" + fakediscovery "k8s.io/client-go/discovery/fake" + "k8s.io/client-go/testing" +) + +// NewSimpleClientset returns a clientset that will respond with the provided objects. +// It's backed by a very simple object tracker that processes creates, updates and deletions as-is, +// without applying any field management, validations and/or defaults. It shouldn't be considered a replacement +// for a real clientset and is mostly useful in simple unit tests. +func NewSimpleClientset(objects ...runtime.Object) *Clientset { + o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder()) + for _, obj := range objects { + if err := o.Add(obj); err != nil { + panic(err) + } + } + + cs := &Clientset{tracker: o} + cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake} + cs.AddReactor("*", "*", testing.ObjectReaction(o)) + cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) { + var opts metav1.ListOptions + if watchAction, ok := action.(testing.WatchActionImpl); ok { + opts = watchAction.ListOptions + } + gvr := action.GetResource() + ns := action.GetNamespace() + watch, err := o.Watch(gvr, ns, opts) + if err != nil { + return false, nil, err + } + return true, watch, nil + }) + + return cs +} + +// Clientset implements clientset.Interface. Meant to be embedded into a +// struct to get a default implementation. This makes faking out just the method +// you want to test easier. +type Clientset struct { + testing.Fake + discovery *fakediscovery.FakeDiscovery + tracker testing.ObjectTracker +} + +func (c *Clientset) Discovery() discovery.DiscoveryInterface { + return c.discovery +} + +func (c *Clientset) Tracker() testing.ObjectTracker { + return c.tracker +} + +// IsWatchListSemanticsSupported informs the reflector that this client +// doesn't support WatchList semantics. +// +// This is a synthetic method whose sole purpose is to satisfy the optional +// interface check performed by the reflector. +// Returning true signals that WatchList can NOT be used. +// No additional logic is implemented here. +func (c *Clientset) IsWatchListSemanticsUnSupported() bool { + return true +} + +var ( + _ clientset.Interface = &Clientset{} + _ testing.FakeClient = &Clientset{} +) + +// MicroshiftV1alpha1 retrieves the MicroshiftV1alpha1Client +func (c *Clientset) MicroshiftV1alpha1() microshiftv1alpha1.MicroshiftV1alpha1Interface { + return &fakemicroshiftv1alpha1.FakeMicroshiftV1alpha1{Fake: &c.Fake} +} diff --git a/pkg/generated/clientset/versioned/fake/doc.go b/pkg/generated/clientset/versioned/fake/doc.go new file mode 100644 index 0000000000..3630ed1cd1 --- /dev/null +++ b/pkg/generated/clientset/versioned/fake/doc.go @@ -0,0 +1,4 @@ +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated fake clientset. +package fake diff --git a/pkg/generated/clientset/versioned/fake/register.go b/pkg/generated/clientset/versioned/fake/register.go new file mode 100644 index 0000000000..639bdeb636 --- /dev/null +++ b/pkg/generated/clientset/versioned/fake/register.go @@ -0,0 +1,40 @@ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + microshiftv1alpha1 "github.com/openshift/microshift/pkg/apis/microshift/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + serializer "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" +) + +var scheme = runtime.NewScheme() +var codecs = serializer.NewCodecFactory(scheme) + +var localSchemeBuilder = runtime.SchemeBuilder{ + microshiftv1alpha1.AddToScheme, +} + +// AddToScheme adds all types of this clientset into the given scheme. This allows composition +// of clientsets, like in: +// +// import ( +// "k8s.io/client-go/kubernetes" +// clientsetscheme "k8s.io/client-go/kubernetes/scheme" +// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" +// ) +// +// kclientset, _ := kubernetes.NewForConfig(c) +// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) +// +// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types +// correctly. +var AddToScheme = localSchemeBuilder.AddToScheme + +func init() { + v1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"}) + utilruntime.Must(AddToScheme(scheme)) +} diff --git a/pkg/generated/clientset/versioned/scheme/doc.go b/pkg/generated/clientset/versioned/scheme/doc.go new file mode 100644 index 0000000000..14db57a58f --- /dev/null +++ b/pkg/generated/clientset/versioned/scheme/doc.go @@ -0,0 +1,4 @@ +// Code generated by client-gen. DO NOT EDIT. + +// This package contains the scheme of the automatically generated clientset. +package scheme diff --git a/pkg/generated/clientset/versioned/scheme/register.go b/pkg/generated/clientset/versioned/scheme/register.go new file mode 100644 index 0000000000..4a7dc57bb1 --- /dev/null +++ b/pkg/generated/clientset/versioned/scheme/register.go @@ -0,0 +1,40 @@ +// Code generated by client-gen. DO NOT EDIT. + +package scheme + +import ( + microshiftv1alpha1 "github.com/openshift/microshift/pkg/apis/microshift/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + serializer "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" +) + +var Scheme = runtime.NewScheme() +var Codecs = serializer.NewCodecFactory(Scheme) +var ParameterCodec = runtime.NewParameterCodec(Scheme) +var localSchemeBuilder = runtime.SchemeBuilder{ + microshiftv1alpha1.AddToScheme, +} + +// AddToScheme adds all types of this clientset into the given scheme. This allows composition +// of clientsets, like in: +// +// import ( +// "k8s.io/client-go/kubernetes" +// clientsetscheme "k8s.io/client-go/kubernetes/scheme" +// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" +// ) +// +// kclientset, _ := kubernetes.NewForConfig(c) +// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) +// +// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types +// correctly. +var AddToScheme = localSchemeBuilder.AddToScheme + +func init() { + v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) + utilruntime.Must(AddToScheme(Scheme)) +} diff --git a/pkg/generated/clientset/versioned/typed/microshift/v1alpha1/doc.go b/pkg/generated/clientset/versioned/typed/microshift/v1alpha1/doc.go new file mode 100644 index 0000000000..93a7ca4e0e --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/microshift/v1alpha1/doc.go @@ -0,0 +1,4 @@ +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated typed clients. +package v1alpha1 diff --git a/pkg/generated/clientset/versioned/typed/microshift/v1alpha1/fake/doc.go b/pkg/generated/clientset/versioned/typed/microshift/v1alpha1/fake/doc.go new file mode 100644 index 0000000000..2b5ba4c8e4 --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/microshift/v1alpha1/fake/doc.go @@ -0,0 +1,4 @@ +// Code generated by client-gen. DO NOT EDIT. + +// Package fake has the automatically generated clients. +package fake diff --git a/pkg/generated/clientset/versioned/typed/microshift/v1alpha1/fake/fake_microshift_client.go b/pkg/generated/clientset/versioned/typed/microshift/v1alpha1/fake/fake_microshift_client.go new file mode 100644 index 0000000000..08c59dd07c --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/microshift/v1alpha1/fake/fake_microshift_client.go @@ -0,0 +1,24 @@ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1alpha1 "github.com/openshift/microshift/pkg/generated/clientset/versioned/typed/microshift/v1alpha1" + rest "k8s.io/client-go/rest" + testing "k8s.io/client-go/testing" +) + +type FakeMicroshiftV1alpha1 struct { + *testing.Fake +} + +func (c *FakeMicroshiftV1alpha1) RemoteClusters() v1alpha1.RemoteClusterInterface { + return newFakeRemoteClusters(c) +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *FakeMicroshiftV1alpha1) RESTClient() rest.Interface { + var ret *rest.RESTClient + return ret +} diff --git a/pkg/generated/clientset/versioned/typed/microshift/v1alpha1/fake/fake_remotecluster.go b/pkg/generated/clientset/versioned/typed/microshift/v1alpha1/fake/fake_remotecluster.go new file mode 100644 index 0000000000..a5ff25a030 --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/microshift/v1alpha1/fake/fake_remotecluster.go @@ -0,0 +1,36 @@ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1alpha1 "github.com/openshift/microshift/pkg/apis/microshift/v1alpha1" + microshiftv1alpha1 "github.com/openshift/microshift/pkg/generated/clientset/versioned/typed/microshift/v1alpha1" + gentype "k8s.io/client-go/gentype" +) + +// fakeRemoteClusters implements RemoteClusterInterface +type fakeRemoteClusters struct { + *gentype.FakeClientWithList[*v1alpha1.RemoteCluster, *v1alpha1.RemoteClusterList] + Fake *FakeMicroshiftV1alpha1 +} + +func newFakeRemoteClusters(fake *FakeMicroshiftV1alpha1) microshiftv1alpha1.RemoteClusterInterface { + return &fakeRemoteClusters{ + gentype.NewFakeClientWithList[*v1alpha1.RemoteCluster, *v1alpha1.RemoteClusterList]( + fake.Fake, + "", + v1alpha1.SchemeGroupVersion.WithResource("remoteclusters"), + v1alpha1.SchemeGroupVersion.WithKind("RemoteCluster"), + func() *v1alpha1.RemoteCluster { return &v1alpha1.RemoteCluster{} }, + func() *v1alpha1.RemoteClusterList { return &v1alpha1.RemoteClusterList{} }, + func(dst, src *v1alpha1.RemoteClusterList) { dst.ListMeta = src.ListMeta }, + func(list *v1alpha1.RemoteClusterList) []*v1alpha1.RemoteCluster { + return gentype.ToPointerSlice(list.Items) + }, + func(list *v1alpha1.RemoteClusterList, items []*v1alpha1.RemoteCluster) { + list.Items = gentype.FromPointerSlice(items) + }, + ), + fake, + } +} diff --git a/pkg/generated/clientset/versioned/typed/microshift/v1alpha1/generated_expansion.go b/pkg/generated/clientset/versioned/typed/microshift/v1alpha1/generated_expansion.go new file mode 100644 index 0000000000..87c1860492 --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/microshift/v1alpha1/generated_expansion.go @@ -0,0 +1,5 @@ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +type RemoteClusterExpansion interface{} diff --git a/pkg/generated/clientset/versioned/typed/microshift/v1alpha1/microshift_client.go b/pkg/generated/clientset/versioned/typed/microshift/v1alpha1/microshift_client.go new file mode 100644 index 0000000000..54637da3e6 --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/microshift/v1alpha1/microshift_client.go @@ -0,0 +1,85 @@ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + http "net/http" + + microshiftv1alpha1 "github.com/openshift/microshift/pkg/apis/microshift/v1alpha1" + scheme "github.com/openshift/microshift/pkg/generated/clientset/versioned/scheme" + rest "k8s.io/client-go/rest" +) + +type MicroshiftV1alpha1Interface interface { + RESTClient() rest.Interface + RemoteClustersGetter +} + +// MicroshiftV1alpha1Client is used to interact with features provided by the microshift.io group. +type MicroshiftV1alpha1Client struct { + restClient rest.Interface +} + +func (c *MicroshiftV1alpha1Client) RemoteClusters() RemoteClusterInterface { + return newRemoteClusters(c) +} + +// NewForConfig creates a new MicroshiftV1alpha1Client for the given config. +// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), +// where httpClient was generated with rest.HTTPClientFor(c). +func NewForConfig(c *rest.Config) (*MicroshiftV1alpha1Client, error) { + config := *c + setConfigDefaults(&config) + httpClient, err := rest.HTTPClientFor(&config) + if err != nil { + return nil, err + } + return NewForConfigAndClient(&config, httpClient) +} + +// NewForConfigAndClient creates a new MicroshiftV1alpha1Client for the given config and http client. +// Note the http client provided takes precedence over the configured transport values. +func NewForConfigAndClient(c *rest.Config, h *http.Client) (*MicroshiftV1alpha1Client, error) { + config := *c + setConfigDefaults(&config) + client, err := rest.RESTClientForConfigAndClient(&config, h) + if err != nil { + return nil, err + } + return &MicroshiftV1alpha1Client{client}, nil +} + +// NewForConfigOrDie creates a new MicroshiftV1alpha1Client for the given config and +// panics if there is an error in the config. +func NewForConfigOrDie(c *rest.Config) *MicroshiftV1alpha1Client { + client, err := NewForConfig(c) + if err != nil { + panic(err) + } + return client +} + +// New creates a new MicroshiftV1alpha1Client for the given RESTClient. +func New(c rest.Interface) *MicroshiftV1alpha1Client { + return &MicroshiftV1alpha1Client{c} +} + +func setConfigDefaults(config *rest.Config) { + gv := microshiftv1alpha1.SchemeGroupVersion + config.GroupVersion = &gv + config.APIPath = "/apis" + config.NegotiatedSerializer = rest.CodecFactoryForGeneratedClient(scheme.Scheme, scheme.Codecs).WithoutConversion() + + if config.UserAgent == "" { + config.UserAgent = rest.DefaultKubernetesUserAgent() + } +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *MicroshiftV1alpha1Client) RESTClient() rest.Interface { + if c == nil { + return nil + } + return c.restClient +} diff --git a/pkg/generated/clientset/versioned/typed/microshift/v1alpha1/remotecluster.go b/pkg/generated/clientset/versioned/typed/microshift/v1alpha1/remotecluster.go new file mode 100644 index 0000000000..df3849ca17 --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/microshift/v1alpha1/remotecluster.go @@ -0,0 +1,54 @@ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + context "context" + + microshiftv1alpha1 "github.com/openshift/microshift/pkg/apis/microshift/v1alpha1" + scheme "github.com/openshift/microshift/pkg/generated/clientset/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" +) + +// RemoteClustersGetter has a method to return a RemoteClusterInterface. +// A group's client should implement this interface. +type RemoteClustersGetter interface { + RemoteClusters() RemoteClusterInterface +} + +// RemoteClusterInterface has methods to work with RemoteCluster resources. +type RemoteClusterInterface interface { + Create(ctx context.Context, remoteCluster *microshiftv1alpha1.RemoteCluster, opts v1.CreateOptions) (*microshiftv1alpha1.RemoteCluster, error) + Update(ctx context.Context, remoteCluster *microshiftv1alpha1.RemoteCluster, opts v1.UpdateOptions) (*microshiftv1alpha1.RemoteCluster, error) + // Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). + UpdateStatus(ctx context.Context, remoteCluster *microshiftv1alpha1.RemoteCluster, opts v1.UpdateOptions) (*microshiftv1alpha1.RemoteCluster, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*microshiftv1alpha1.RemoteCluster, error) + List(ctx context.Context, opts v1.ListOptions) (*microshiftv1alpha1.RemoteClusterList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *microshiftv1alpha1.RemoteCluster, err error) + RemoteClusterExpansion +} + +// remoteClusters implements RemoteClusterInterface +type remoteClusters struct { + *gentype.ClientWithList[*microshiftv1alpha1.RemoteCluster, *microshiftv1alpha1.RemoteClusterList] +} + +// newRemoteClusters returns a RemoteClusters +func newRemoteClusters(c *MicroshiftV1alpha1Client) *remoteClusters { + return &remoteClusters{ + gentype.NewClientWithList[*microshiftv1alpha1.RemoteCluster, *microshiftv1alpha1.RemoteClusterList]( + "remoteclusters", + c.RESTClient(), + scheme.ParameterCodec, + "", + func() *microshiftv1alpha1.RemoteCluster { return µshiftv1alpha1.RemoteCluster{} }, + func() *microshiftv1alpha1.RemoteClusterList { return µshiftv1alpha1.RemoteClusterList{} }, + ), + } +} diff --git a/pkg/generated/informers/externalversions/factory.go b/pkg/generated/informers/externalversions/factory.go new file mode 100644 index 0000000000..0c2e8e7c5a --- /dev/null +++ b/pkg/generated/informers/externalversions/factory.go @@ -0,0 +1,247 @@ +// Code generated by informer-gen. DO NOT EDIT. + +package externalversions + +import ( + reflect "reflect" + sync "sync" + time "time" + + versioned "github.com/openshift/microshift/pkg/generated/clientset/versioned" + internalinterfaces "github.com/openshift/microshift/pkg/generated/informers/externalversions/internalinterfaces" + microshift "github.com/openshift/microshift/pkg/generated/informers/externalversions/microshift" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + cache "k8s.io/client-go/tools/cache" +) + +// SharedInformerOption defines the functional option type for SharedInformerFactory. +type SharedInformerOption func(*sharedInformerFactory) *sharedInformerFactory + +type sharedInformerFactory struct { + client versioned.Interface + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc + lock sync.Mutex + defaultResync time.Duration + customResync map[reflect.Type]time.Duration + transform cache.TransformFunc + + informers map[reflect.Type]cache.SharedIndexInformer + // startedInformers is used for tracking which informers have been started. + // This allows Start() to be called multiple times safely. + startedInformers map[reflect.Type]bool + // wg tracks how many goroutines were started. + wg sync.WaitGroup + // shuttingDown is true when Shutdown has been called. It may still be running + // because it needs to wait for goroutines. + shuttingDown bool +} + +// WithCustomResyncConfig sets a custom resync period for the specified informer types. +func WithCustomResyncConfig(resyncConfig map[v1.Object]time.Duration) SharedInformerOption { + return func(factory *sharedInformerFactory) *sharedInformerFactory { + for k, v := range resyncConfig { + factory.customResync[reflect.TypeOf(k)] = v + } + return factory + } +} + +// WithTweakListOptions sets a custom filter on all listers of the configured SharedInformerFactory. +func WithTweakListOptions(tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerOption { + return func(factory *sharedInformerFactory) *sharedInformerFactory { + factory.tweakListOptions = tweakListOptions + return factory + } +} + +// WithNamespace limits the SharedInformerFactory to the specified namespace. +func WithNamespace(namespace string) SharedInformerOption { + return func(factory *sharedInformerFactory) *sharedInformerFactory { + factory.namespace = namespace + return factory + } +} + +// WithTransform sets a transform on all informers. +func WithTransform(transform cache.TransformFunc) SharedInformerOption { + return func(factory *sharedInformerFactory) *sharedInformerFactory { + factory.transform = transform + return factory + } +} + +// NewSharedInformerFactory constructs a new instance of sharedInformerFactory for all namespaces. +func NewSharedInformerFactory(client versioned.Interface, defaultResync time.Duration) SharedInformerFactory { + return NewSharedInformerFactoryWithOptions(client, defaultResync) +} + +// NewFilteredSharedInformerFactory constructs a new instance of sharedInformerFactory. +// Listers obtained via this SharedInformerFactory will be subject to the same filters +// as specified here. +// +// Deprecated: Please use NewSharedInformerFactoryWithOptions instead +func NewFilteredSharedInformerFactory(client versioned.Interface, defaultResync time.Duration, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerFactory { + return NewSharedInformerFactoryWithOptions(client, defaultResync, WithNamespace(namespace), WithTweakListOptions(tweakListOptions)) +} + +// NewSharedInformerFactoryWithOptions constructs a new instance of a SharedInformerFactory with additional options. +func NewSharedInformerFactoryWithOptions(client versioned.Interface, defaultResync time.Duration, options ...SharedInformerOption) SharedInformerFactory { + factory := &sharedInformerFactory{ + client: client, + namespace: v1.NamespaceAll, + defaultResync: defaultResync, + informers: make(map[reflect.Type]cache.SharedIndexInformer), + startedInformers: make(map[reflect.Type]bool), + customResync: make(map[reflect.Type]time.Duration), + } + + // Apply all options + for _, opt := range options { + factory = opt(factory) + } + + return factory +} + +func (f *sharedInformerFactory) Start(stopCh <-chan struct{}) { + f.lock.Lock() + defer f.lock.Unlock() + + if f.shuttingDown { + return + } + + for informerType, informer := range f.informers { + if !f.startedInformers[informerType] { + f.wg.Add(1) + // We need a new variable in each loop iteration, + // otherwise the goroutine would use the loop variable + // and that keeps changing. + informer := informer + go func() { + defer f.wg.Done() + informer.Run(stopCh) + }() + f.startedInformers[informerType] = true + } + } +} + +func (f *sharedInformerFactory) Shutdown() { + f.lock.Lock() + f.shuttingDown = true + f.lock.Unlock() + + // Will return immediately if there is nothing to wait for. + f.wg.Wait() +} + +func (f *sharedInformerFactory) WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool { + informers := func() map[reflect.Type]cache.SharedIndexInformer { + f.lock.Lock() + defer f.lock.Unlock() + + informers := map[reflect.Type]cache.SharedIndexInformer{} + for informerType, informer := range f.informers { + if f.startedInformers[informerType] { + informers[informerType] = informer + } + } + return informers + }() + + res := map[reflect.Type]bool{} + for informType, informer := range informers { + res[informType] = cache.WaitForCacheSync(stopCh, informer.HasSynced) + } + return res +} + +// InformerFor returns the SharedIndexInformer for obj using an internal +// client. +func (f *sharedInformerFactory) InformerFor(obj runtime.Object, newFunc internalinterfaces.NewInformerFunc) cache.SharedIndexInformer { + f.lock.Lock() + defer f.lock.Unlock() + + informerType := reflect.TypeOf(obj) + informer, exists := f.informers[informerType] + if exists { + return informer + } + + resyncPeriod, exists := f.customResync[informerType] + if !exists { + resyncPeriod = f.defaultResync + } + + informer = newFunc(f.client, resyncPeriod) + informer.SetTransform(f.transform) + f.informers[informerType] = informer + + return informer +} + +// SharedInformerFactory provides shared informers for resources in all known +// API group versions. +// +// It is typically used like this: +// +// ctx, cancel := context.WithCancel(context.Background()) +// defer cancel() +// factory := NewSharedInformerFactory(client, resyncPeriod) +// defer factory.WaitForStop() // Returns immediately if nothing was started. +// genericInformer := factory.ForResource(resource) +// typedInformer := factory.SomeAPIGroup().V1().SomeType() +// factory.Start(ctx.Done()) // Start processing these informers. +// synced := factory.WaitForCacheSync(ctx.Done()) +// for v, ok := range synced { +// if !ok { +// fmt.Fprintf(os.Stderr, "caches failed to sync: %v", v) +// return +// } +// } +// +// // Creating informers can also be created after Start, but then +// // Start must be called again: +// anotherGenericInformer := factory.ForResource(resource) +// factory.Start(ctx.Done()) +type SharedInformerFactory interface { + internalinterfaces.SharedInformerFactory + + // Start initializes all requested informers. They are handled in goroutines + // which run until the stop channel gets closed. + // Warning: Start does not block. When run in a go-routine, it will race with a later WaitForCacheSync. + Start(stopCh <-chan struct{}) + + // Shutdown marks a factory as shutting down. At that point no new + // informers can be started anymore and Start will return without + // doing anything. + // + // In addition, Shutdown blocks until all goroutines have terminated. For that + // to happen, the close channel(s) that they were started with must be closed, + // either before Shutdown gets called or while it is waiting. + // + // Shutdown may be called multiple times, even concurrently. All such calls will + // block until all goroutines have terminated. + Shutdown() + + // WaitForCacheSync blocks until all started informers' caches were synced + // or the stop channel gets closed. + WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool + + // ForResource gives generic access to a shared informer of the matching type. + ForResource(resource schema.GroupVersionResource) (GenericInformer, error) + + // InformerFor returns the SharedIndexInformer for obj using an internal + // client. + InformerFor(obj runtime.Object, newFunc internalinterfaces.NewInformerFunc) cache.SharedIndexInformer + + Microshift() microshift.Interface +} + +func (f *sharedInformerFactory) Microshift() microshift.Interface { + return microshift.New(f, f.namespace, f.tweakListOptions) +} diff --git a/pkg/generated/informers/externalversions/generic.go b/pkg/generated/informers/externalversions/generic.go new file mode 100644 index 0000000000..346b9935ce --- /dev/null +++ b/pkg/generated/informers/externalversions/generic.go @@ -0,0 +1,46 @@ +// Code generated by informer-gen. DO NOT EDIT. + +package externalversions + +import ( + fmt "fmt" + + v1alpha1 "github.com/openshift/microshift/pkg/apis/microshift/v1alpha1" + schema "k8s.io/apimachinery/pkg/runtime/schema" + cache "k8s.io/client-go/tools/cache" +) + +// GenericInformer is type of SharedIndexInformer which will locate and delegate to other +// sharedInformers based on type +type GenericInformer interface { + Informer() cache.SharedIndexInformer + Lister() cache.GenericLister +} + +type genericInformer struct { + informer cache.SharedIndexInformer + resource schema.GroupResource +} + +// Informer returns the SharedIndexInformer. +func (f *genericInformer) Informer() cache.SharedIndexInformer { + return f.informer +} + +// Lister returns the GenericLister. +func (f *genericInformer) Lister() cache.GenericLister { + return cache.NewGenericLister(f.Informer().GetIndexer(), f.resource) +} + +// ForResource gives generic access to a shared informer of the matching type +// TODO extend this to unknown resources with a client pool +func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource) (GenericInformer, error) { + switch resource { + // Group=microshift.io, Version=v1alpha1 + case v1alpha1.SchemeGroupVersion.WithResource("remoteclusters"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Microshift().V1alpha1().RemoteClusters().Informer()}, nil + + } + + return nil, fmt.Errorf("no informer found for %v", resource) +} diff --git a/pkg/generated/informers/externalversions/internalinterfaces/factory_interfaces.go b/pkg/generated/informers/externalversions/internalinterfaces/factory_interfaces.go new file mode 100644 index 0000000000..3b1079b95b --- /dev/null +++ b/pkg/generated/informers/externalversions/internalinterfaces/factory_interfaces.go @@ -0,0 +1,24 @@ +// Code generated by informer-gen. DO NOT EDIT. + +package internalinterfaces + +import ( + time "time" + + versioned "github.com/openshift/microshift/pkg/generated/clientset/versioned" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + cache "k8s.io/client-go/tools/cache" +) + +// NewInformerFunc takes versioned.Interface and time.Duration to return a SharedIndexInformer. +type NewInformerFunc func(versioned.Interface, time.Duration) cache.SharedIndexInformer + +// SharedInformerFactory a small interface to allow for adding an informer without an import cycle +type SharedInformerFactory interface { + Start(stopCh <-chan struct{}) + InformerFor(obj runtime.Object, newFunc NewInformerFunc) cache.SharedIndexInformer +} + +// TweakListOptionsFunc is a function that transforms a v1.ListOptions. +type TweakListOptionsFunc func(*v1.ListOptions) diff --git a/pkg/generated/informers/externalversions/microshift/interface.go b/pkg/generated/informers/externalversions/microshift/interface.go new file mode 100644 index 0000000000..6524ea6695 --- /dev/null +++ b/pkg/generated/informers/externalversions/microshift/interface.go @@ -0,0 +1,30 @@ +// Code generated by informer-gen. DO NOT EDIT. + +package microshift + +import ( + internalinterfaces "github.com/openshift/microshift/pkg/generated/informers/externalversions/internalinterfaces" + v1alpha1 "github.com/openshift/microshift/pkg/generated/informers/externalversions/microshift/v1alpha1" +) + +// Interface provides access to each of this group's versions. +type Interface interface { + // V1alpha1 provides access to shared informers for resources in V1alpha1. + V1alpha1() v1alpha1.Interface +} + +type group struct { + factory internalinterfaces.SharedInformerFactory + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// New returns a new Interface. +func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { + return &group{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} +} + +// V1alpha1 returns a new v1alpha1.Interface. +func (g *group) V1alpha1() v1alpha1.Interface { + return v1alpha1.New(g.factory, g.namespace, g.tweakListOptions) +} diff --git a/pkg/generated/informers/externalversions/microshift/v1alpha1/interface.go b/pkg/generated/informers/externalversions/microshift/v1alpha1/interface.go new file mode 100644 index 0000000000..e2c3ba6acd --- /dev/null +++ b/pkg/generated/informers/externalversions/microshift/v1alpha1/interface.go @@ -0,0 +1,29 @@ +// Code generated by informer-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + internalinterfaces "github.com/openshift/microshift/pkg/generated/informers/externalversions/internalinterfaces" +) + +// Interface provides access to all the informers in this group version. +type Interface interface { + // RemoteClusters returns a RemoteClusterInformer. + RemoteClusters() RemoteClusterInformer +} + +type version struct { + factory internalinterfaces.SharedInformerFactory + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// New returns a new Interface. +func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { + return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} +} + +// RemoteClusters returns a RemoteClusterInformer. +func (v *version) RemoteClusters() RemoteClusterInformer { + return &remoteClusterInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} +} diff --git a/pkg/generated/informers/externalversions/microshift/v1alpha1/remotecluster.go b/pkg/generated/informers/externalversions/microshift/v1alpha1/remotecluster.go new file mode 100644 index 0000000000..6f14bc5c67 --- /dev/null +++ b/pkg/generated/informers/externalversions/microshift/v1alpha1/remotecluster.go @@ -0,0 +1,85 @@ +// Code generated by informer-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + context "context" + time "time" + + apismicroshiftv1alpha1 "github.com/openshift/microshift/pkg/apis/microshift/v1alpha1" + versioned "github.com/openshift/microshift/pkg/generated/clientset/versioned" + internalinterfaces "github.com/openshift/microshift/pkg/generated/informers/externalversions/internalinterfaces" + microshiftv1alpha1 "github.com/openshift/microshift/pkg/generated/listers/microshift/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// RemoteClusterInformer provides access to a shared informer and lister for +// RemoteClusters. +type RemoteClusterInformer interface { + Informer() cache.SharedIndexInformer + Lister() microshiftv1alpha1.RemoteClusterLister +} + +type remoteClusterInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// NewRemoteClusterInformer constructs a new informer for RemoteCluster type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewRemoteClusterInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredRemoteClusterInformer(client, resyncPeriod, indexers, nil) +} + +// NewFilteredRemoteClusterInformer constructs a new informer for RemoteCluster type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredRemoteClusterInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + cache.ToListWatcherWithWatchListSemantics(&cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.MicroshiftV1alpha1().RemoteClusters().List(context.Background(), options) + }, + WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.MicroshiftV1alpha1().RemoteClusters().Watch(context.Background(), options) + }, + ListWithContextFunc: func(ctx context.Context, options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.MicroshiftV1alpha1().RemoteClusters().List(ctx, options) + }, + WatchFuncWithContext: func(ctx context.Context, options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.MicroshiftV1alpha1().RemoteClusters().Watch(ctx, options) + }, + }, client), + &apismicroshiftv1alpha1.RemoteCluster{}, + resyncPeriod, + indexers, + ) +} + +func (f *remoteClusterInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredRemoteClusterInformer(client, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *remoteClusterInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&apismicroshiftv1alpha1.RemoteCluster{}, f.defaultInformer) +} + +func (f *remoteClusterInformer) Lister() microshiftv1alpha1.RemoteClusterLister { + return microshiftv1alpha1.NewRemoteClusterLister(f.Informer().GetIndexer()) +} diff --git a/pkg/generated/listers/microshift/v1alpha1/expansion_generated.go b/pkg/generated/listers/microshift/v1alpha1/expansion_generated.go new file mode 100644 index 0000000000..b4e677bfcb --- /dev/null +++ b/pkg/generated/listers/microshift/v1alpha1/expansion_generated.go @@ -0,0 +1,7 @@ +// Code generated by lister-gen. DO NOT EDIT. + +package v1alpha1 + +// RemoteClusterListerExpansion allows custom methods to be added to +// RemoteClusterLister. +type RemoteClusterListerExpansion interface{} diff --git a/pkg/generated/listers/microshift/v1alpha1/remotecluster.go b/pkg/generated/listers/microshift/v1alpha1/remotecluster.go new file mode 100644 index 0000000000..d217908a58 --- /dev/null +++ b/pkg/generated/listers/microshift/v1alpha1/remotecluster.go @@ -0,0 +1,32 @@ +// Code generated by lister-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + microshiftv1alpha1 "github.com/openshift/microshift/pkg/apis/microshift/v1alpha1" + labels "k8s.io/apimachinery/pkg/labels" + listers "k8s.io/client-go/listers" + cache "k8s.io/client-go/tools/cache" +) + +// RemoteClusterLister helps list RemoteClusters. +// All objects returned here must be treated as read-only. +type RemoteClusterLister interface { + // List lists all RemoteClusters in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*microshiftv1alpha1.RemoteCluster, err error) + // Get retrieves the RemoteCluster from the index for a given name. + // Objects returned here must be treated as read-only. + Get(name string) (*microshiftv1alpha1.RemoteCluster, error) + RemoteClusterListerExpansion +} + +// remoteClusterLister implements the RemoteClusterLister interface. +type remoteClusterLister struct { + listers.ResourceIndexer[*microshiftv1alpha1.RemoteCluster] +} + +// NewRemoteClusterLister returns a new RemoteClusterLister. +func NewRemoteClusterLister(indexer cache.Indexer) RemoteClusterLister { + return &remoteClusterLister{listers.New[*microshiftv1alpha1.RemoteCluster](indexer, microshiftv1alpha1.Resource("remotecluster"))} +} diff --git a/scripts/auto-rebase/assets.yaml b/scripts/auto-rebase/assets.yaml index bc5a6ffb87..b4f34d3f6c 100644 --- a/scripts/auto-rebase/assets.yaml +++ b/scripts/auto-rebase/assets.yaml @@ -218,6 +218,8 @@ assets: - file: 0000_03_config-operator_02_rangeallocations.crd.yaml - file: storage_version_migration.crd.yaml src: 0000_50_cluster-kube-storage-version-migrator-operator_01_storage_migration_crd.yaml + - file: microshift.io_remoteclusters.yaml + git_restore: True - file: route.crd.yaml src: /api/route/v1/zz_generated.crd-manifests/routes.crd.yaml @@ -289,6 +291,16 @@ assets: - file: release-multus-aarch64.json - file: release-multus-x86_64.json + - dir: components/c2cc/ + ignore: "C2CC probe pod assets - MicroShift specific" + files: + - file: clusterrole.yaml + - file: clusterrolebinding.yaml + - file: deployment.yaml + - file: namespace.yaml + - file: service.yaml + - file: serviceaccount.yaml + - dir: optional/observability/ ignore: "they don't exist in upstream repository - only in microshift" files: diff --git a/scripts/boilerplate.go.txt b/scripts/boilerplate.go.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/generate-crds.sh b/scripts/generate-crds.sh new file mode 100755 index 0000000000..cb687c83ff --- /dev/null +++ b/scripts/generate-crds.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -euo pipefail + +ROOTDIR=$(git rev-parse --show-toplevel) +CONTROLLER_BIN="${ROOTDIR}/_output/bin/controller-gen" +CODEGEN_DIR="${ROOTDIR}/deps/github.com/openshift/kubernetes/staging/src/k8s.io/code-generator" + +OUTPUT_PKG="github.com/openshift/microshift/pkg/generated" + +pushd "${ROOTDIR}" &>/dev/null + +echo "Generating deepcopy methods" +${CONTROLLER_BIN} object paths=./pkg/apis/microshift/v1alpha1/ + +echo "Generating CRD YAML" +${CONTROLLER_BIN} crd paths=./pkg/apis/microshift/v1alpha1/ output:crd:artifacts:config=assets/crd/ + +echo "Generating typed clientset, listers, informers" +# shellcheck source=/dev/null +source "${CODEGEN_DIR}/kube_codegen.sh" + +kube::codegen::gen_client \ + --output-dir "${ROOTDIR}/pkg/generated" \ + --output-pkg "${OUTPUT_PKG}" \ + --boilerplate "${ROOTDIR}/scripts/boilerplate.go.txt" \ + --with-watch \ + "${ROOTDIR}/pkg/apis" + +popd &>/dev/null + +echo "Done" diff --git a/test/resources/c2cc.resource b/test/resources/c2cc.resource index badd015f7e..7e5b0f63e1 100644 --- a/test/resources/c2cc.resource +++ b/test/resources/c2cc.resource @@ -283,6 +283,15 @@ Service Endpoints Should Exist ... oc get endpoints hello-microshift -n ${ns} -o jsonpath='{.subsets[0].addresses[0].ip}' Should Not Be Empty ${stdout} +Compute 11th IP + [Documentation] Return the 11th host address in a CIDR (e.g. 10.43.0.0/16 -> 10.43.0.11). + [Arguments] ${cidr} + VAR ${cmd}= import ipaddress; n=ipaddress.ip_network('${cidr}', strict=False); print(n[11]) + ${result}= Process.Run Process python3 -c ${cmd} + Should Be Equal As Integers ${result.rc} 0 + ${ip}= Strip String ${result.stdout} + RETURN ${ip} + Cleanup Test Workloads [Documentation] Delete test namespace on both clusters. Ignores errors. FOR ${alias} IN cluster-a cluster-b diff --git a/test/suites/c2cc/healthcheck.robot b/test/suites/c2cc/healthcheck.robot new file mode 100644 index 0000000000..d507669398 --- /dev/null +++ b/test/suites/c2cc/healthcheck.robot @@ -0,0 +1,103 @@ +*** Settings *** +Documentation Verify C2CC RemoteCluster CRD and CR lifecycle. +... Checks that the CRD is registered, CRs are created per remote cluster, +... and CR specs match the expected probe targets. + +Resource ../../resources/microshift-process.resource +Resource ../../resources/kubeconfig.resource +Resource ../../resources/oc.resource +Resource ../../resources/c2cc.resource + +Suite Setup Setup +Suite Teardown Teardown + +Test Tags c2cc + + +*** Test Cases *** +RemoteCluster CRD Exists + [Documentation] Verify the remoteclusters.microshift.io CRD is registered on all clusters. + Wait Until Keyword Succeeds 2m 15s + ... Verify RemoteCluster CRD Exists cluster-a + Wait Until Keyword Succeeds 2m 15s + ... Verify RemoteCluster CRD Exists cluster-b + +RemoteCluster CR Created + [Documentation] Verify exactly one RemoteCluster CR exists on all clusters. + Wait Until Keyword Succeeds 2m 15s + ... Verify RemoteCluster CR Count cluster-a 1 + Wait Until Keyword Succeeds 2m 15s + ... Verify RemoteCluster CR Count cluster-b 1 + +Correct RemoteCluster CR Spec + [Documentation] Verify the RemoteCluster CR has the correct probe target and interval. + Verify RemoteCluster CR Spec cluster-a ${CLUSTER_B_SVC_CIDR} + Verify RemoteCluster CR Spec cluster-b ${CLUSTER_A_SVC_CIDR} + +RemoteCluster CR Has Managed-By Label + [Documentation] Verify the RemoteCluster CR has the expected managed-by label. + Verify RemoteCluster CR Label cluster-a + Verify RemoteCluster CR Label cluster-b + + +*** Keywords *** +Setup + [Documentation] Set up SSH connections and kubeconfigs for all clusters. + Check Required Env Variables + Login MicroShift Host + Setup Kubeconfig + Register Local Cluster cluster-a + Register Remote Cluster cluster-b ${HOST2_IP} ${HOST2_SSH_PORT} ${KUBECONFIG_B} + +Teardown + [Documentation] Close all connections and clean up kubeconfigs. + Teardown All Remote Clusters + Remove Kubeconfig + Logout MicroShift Host + +Verify RemoteCluster CRD Exists + [Documentation] Verify that the remoteclusters.microshift.io CRD is registered. + [Arguments] ${alias} + ${stdout}= Oc On Cluster ${alias} oc get crd remoteclusters.microshift.io -o name + Should Be Equal As Strings + ... ${stdout} + ... customresourcedefinition.apiextensions.k8s.io/remoteclusters.microshift.io + ... strip_spaces=True + +Verify RemoteCluster CR Count + [Documentation] Verify the number of RemoteCluster CRs matches the expected count. + [Arguments] ${alias} ${expected_count} + ${stdout}= Oc On Cluster ${alias} + ... oc get remoteclusters.microshift.io -l app.kubernetes.io/managed-by=c2cc-route-manager -o name + @{lines}= Split To Lines ${stdout} + ${count}= Get Length ${lines} + Should Be Equal As Integers ${count} ${expected_count} + +Verify RemoteCluster CR Spec + [Documentation] Verify the RemoteCluster CR spec has the correct probe target + ... (11th IP in the remote service CIDR on port 8080) + ... and a non-empty probe interval duration string. + [Arguments] ${alias} ${remote_svc_cidr} + ${expected_ip}= Compute 11th IP ${remote_svc_cidr} + ${stdout}= Oc On Cluster + ... ${alias} + ... oc get remoteclusters.microshift.io -l app.kubernetes.io/managed-by=c2cc-route-manager -o jsonpath='{.items[0].spec.probeTarget}' + IF ":" in """${expected_ip}""" + VAR ${expected_target}= [${expected_ip}]:8080 + ELSE + VAR ${expected_target}= ${expected_ip}:8080 + END + Should Be Equal As Strings ${stdout} ${expected_target} strip_spaces=True + ${interval}= Oc On Cluster + ... ${alias} + ... oc get remoteclusters.microshift.io -l app.kubernetes.io/managed-by=c2cc-route-manager -o jsonpath='{.items[0].spec.probeInterval}' + Should Not Be Empty ${interval} + Should Match Regexp ${interval} ^[0-9]+(s|m|h)$ + +Verify RemoteCluster CR Label + [Documentation] Verify the RemoteCluster CR has the app.kubernetes.io/managed-by=c2cc-route-manager label. + [Arguments] ${alias} + ${stdout}= Oc On Cluster + ... ${alias} + ... oc get remoteclusters.microshift.io -l app.kubernetes.io/managed-by=c2cc-route-manager -o jsonpath='{.items[0].metadata.labels.app\\.kubernetes\\.io/managed-by}' + Should Be Equal As Strings ${stdout} c2cc-route-manager strip_spaces=True diff --git a/test/suites/c2cc/probe.robot b/test/suites/c2cc/probe.robot new file mode 100644 index 0000000000..57d82aa806 --- /dev/null +++ b/test/suites/c2cc/probe.robot @@ -0,0 +1,121 @@ +*** Settings *** +Documentation Verify C2CC probe pod deployment and health status reporting. +... Checks that the probe pod is deployed, the Service has the correct +... ClusterIP, RemoteCluster CRs transition to Healthy, and the +... deployment self-heals after deletion. + +Resource ../../resources/microshift-process.resource +Resource ../../resources/kubeconfig.resource +Resource ../../resources/oc.resource +Resource ../../resources/c2cc.resource + +Suite Setup Setup +Suite Teardown Teardown + +Test Tags c2cc + + +*** Variables *** +${C2CC_NAMESPACE} microshift-c2cc +${PROBE_DEPLOYMENT} c2cc-probe +${PROBE_PORT} 8080 + + +*** Test Cases *** +Probe Namespace Exists + [Documentation] Verify the microshift-c2cc namespace exists on both clusters. + FOR ${alias} IN cluster-a cluster-b + ${stdout}= Oc On Cluster ${alias} oc get namespace ${C2CC_NAMESPACE} -o name + Should Contain ${stdout} namespace/${C2CC_NAMESPACE} + END + +Probe Deployment Running + [Documentation] Verify the c2cc-probe deployment is running with 1 ready replica. + FOR ${alias} IN cluster-a cluster-b + Wait Until Keyword Succeeds 2m 10s + ... Verify Probe Pod Is Ready ${alias} + END + +Probe Service Has Correct ClusterIP + [Documentation] Verify the probe service has the 11th IP of the local service CIDR. + Verify Probe Service ClusterIP cluster-a ${CLUSTER_A_SVC_CIDR} + Verify Probe Service ClusterIP cluster-b ${CLUSTER_B_SVC_CIDR} + +RemoteCluster Status Becomes Healthy + [Documentation] Wait for RemoteCluster CRs to transition to Healthy on both clusters. + Wait Until Keyword Succeeds 3m 10s + ... Verify RemoteCluster State cluster-a Healthy + Wait Until Keyword Succeeds 3m 10s + ... Verify RemoteCluster State cluster-b Healthy + +RemoteCluster Status Has LastProbeTime + [Documentation] Verify that LastProbeTime is populated after probing starts. + FOR ${alias} IN cluster-a cluster-b + ${stdout}= Oc On Cluster ${alias} + ... oc get remoteclusters.microshift.io -o jsonpath='{.items[0].status.lastProbeTime}' + Should Not Be Empty ${stdout} + END + +RemoteCluster Status Has LastSuccessfulProbe + [Documentation] Verify that LastSuccessfulProbe is populated when state is Healthy. + FOR ${alias} IN cluster-a cluster-b + ${stdout}= Oc On Cluster ${alias} + ... oc get remoteclusters.microshift.io -o jsonpath='{.items[0].status.lastSuccessfulProbe}' + Should Not Be Empty ${stdout} + END + +Probe Deployment Self-Heals After Deletion + [Documentation] Delete the probe deployment and verify it is recreated by the controller. + Oc On Cluster cluster-a + ... oc delete deployment ${PROBE_DEPLOYMENT} -n ${C2CC_NAMESPACE} + Wait Until Keyword Succeeds 2m 10s + ... Verify Probe Pod Is Ready cluster-a + +Probe Deployment Self-Heals After Scale Down + [Documentation] Scale down the probe deployment to 0 and verify it is restored to 1. + Oc On Cluster cluster-a + ... oc scale deployment ${PROBE_DEPLOYMENT} -n ${C2CC_NAMESPACE} --replicas=0 + Wait Until Keyword Succeeds 2m 10s + ... Verify Probe Pod Is Ready cluster-a + + +*** Keywords *** +Setup + [Documentation] Set up SSH connections and kubeconfigs for all clusters. + Check Required Env Variables + Login MicroShift Host + Setup Kubeconfig + Register Local Cluster cluster-a + Register Remote Cluster cluster-b ${HOST2_IP} ${HOST2_SSH_PORT} ${KUBECONFIG_B} + +Teardown + [Documentation] Close all connections and clean up kubeconfigs. + Teardown All Remote Clusters + Remove Kubeconfig + Logout MicroShift Host + +Verify Probe Pod Is Ready + [Documentation] Check that the probe deployment has 1 available replica. + [Arguments] ${alias} + ${stdout}= Oc On Cluster ${alias} + ... oc get deployment ${PROBE_DEPLOYMENT} -n ${C2CC_NAMESPACE} -o jsonpath='{.status.availableReplicas}' + Should Be Equal As Strings ${stdout} 1 + +Verify Probe Service ClusterIP + [Documentation] Verify that the probe service ClusterIP matches the 11th IP of the given CIDR. + [Arguments] ${alias} ${svc_cidr} + ${expected_ip}= Compute 11th IP ${svc_cidr} + ${actual_ip}= Oc On Cluster ${alias} + ... oc get service ${PROBE_DEPLOYMENT} -n ${C2CC_NAMESPACE} -o jsonpath='{.spec.clusterIP}' + Should Be Equal As Strings ${actual_ip} ${expected_ip} strip_spaces=True + +Verify RemoteCluster State + [Documentation] Check that all RemoteCluster CRs on this cluster have the expected state. + [Arguments] ${alias} ${expected_state} + ${stdout}= Oc On Cluster ${alias} + ... oc get remoteclusters.microshift.io -o jsonpath='{.items[*].status.state}' + Should Not Be Empty ${stdout} + @{states}= Split String ${stdout} + FOR ${state} IN @{states} + Should Be Equal As Strings ${state} ${expected_state} + END