WIP: Add Application Load Balancer Controller Manager

Makefile

@@ -2,7 +2,7 @@
 # Options are set to exit when a recipe line exits non-zero or a piped command fails.
 SHELL = /usr/bin/env bash -o pipefail
 .SHELLFLAGS = -ec
-BUILD_IMAGES ?= stackit-csi-plugin cloud-controller-manager
+BUILD_IMAGES ?= stackit-csi-plugin cloud-controller-manager application-load-balancer-controller-manager
 SOURCES := Makefile go.mod go.sum $(shell find $(DEST) -name '*.go' 2>/dev/null)
 VERSION ?= $(shell git describe --dirty --tags --match='v*' 2>/dev/null || git rev-parse --short HEAD)
 REGISTRY ?= ghcr.io
@@ -60,7 +60,7 @@ modules: ## Runs go mod to ensure modules are up to date.
 	go mod tidy
 
 .PHONY: test
-test: ## Run tests.
+test: $(ENVTEST) ## Run tests.
 	./hack/test.sh ./cmd/... ./pkg/...
 
 .PHONY: test-cover
@@ -141,6 +141,9 @@ mocks: $(MOCKGEN)
 	@$(MOCKGEN) -destination ./pkg/stackit/server_mock.go -package stackit ./pkg/stackit NodeClient
 	@$(MOCKGEN) -destination ./pkg/stackit/metadata/metadata_mock.go -package metadata ./pkg/stackit/metadata IMetadata
 	@$(MOCKGEN) -destination ./pkg/csi/util/mount/mount_mock.go -package mount ./pkg/csi/util/mount IMount
+	@$(MOCKGEN) -destination ./pkg/stackit/applicationloadbalancercertificates_mock.go -package stackit ./pkg/stackit CertificatesClient
+	@$(MOCKGEN) -destination ./pkg/stackit/applicationloadbalancer_mock.go -package stackit ./pkg/stackit ApplicationLoadBalancerClient
+
 
 .PHONY: generate
 generate: mocks

cmd/application-load-balancer-controller-manager/main.go

@@ -0,0 +1,140 @@
+package main
+
+import (
+	"flag"
+	"os"
+
+	"github.com/stackitcloud/cloud-provider-stackit/pkg/alb/ingress"
+	albclient "github.com/stackitcloud/cloud-provider-stackit/pkg/stackit"
+	stackitconfig "github.com/stackitcloud/cloud-provider-stackit/pkg/stackit/config"
+	sdkconfig "github.com/stackitcloud/stackit-sdk-go/core/config"
+	albsdk "github.com/stackitcloud/stackit-sdk-go/services/alb/v2api"
+	certsdk "github.com/stackitcloud/stackit-sdk-go/services/certificates/v2api"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	_ "k8s.io/client-go/plugin/pkg/client/auth"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
+)
+
+var (
+	scheme   = runtime.NewScheme()
+	setupLog = ctrl.Log.WithName("setup")
+)
+
+func init() {
+	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
+}
+
+// nolint:funlen // TODO: Refactor into smaller functions.
+func main() {
+	var metricsAddr string
+	var enableLeaderElection bool
+	var leaderElectionNamespace string
+	var leaderElectionID string
+	var probeAddr string
+	var cloudConfig string
+	flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
+		"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
+	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
+	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
+		"Enable leader election for controller manager. "+
+			"Enabling this will ensure there is only one active controller manager.")
+	flag.StringVar(&leaderElectionNamespace, "leader-election-namespace", "default", "The namespace in which the leader "+
+		"election resource will be created.")
+	flag.StringVar(&leaderElectionID, "leader-election-id", "d0fbe9c4.stackit.cloud", "The name of the resource that "+
+		"leader election will use for holding the leader lock.")
+	flag.StringVar(&cloudConfig, "cloud-config", "cloud.yaml", "The path to the cloud config file.")
+	opts := zap.Options{
+		Development: true,
+	}
+	opts.BindFlags(flag.CommandLine)
+	flag.Parse()
+
+	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
+
+	config, err := stackitconfig.ReadALBConfigFromFile(cloudConfig)
+	if err != nil {
+		setupLog.Error(err, "Failed to read cloud config")
+		os.Exit(1)
+	}
+
+	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+		Scheme: scheme,
+		Metrics: metricsserver.Options{
+			BindAddress: metricsAddr,
+		},
+		HealthProbeBindAddress:        probeAddr,
+		LeaderElection:                enableLeaderElection,
+		LeaderElectionID:              leaderElectionID,
+		LeaderElectionNamespace:       leaderElectionNamespace,
+		LeaderElectionReleaseOnCancel: true,
+	})
+	if err != nil {
+		setupLog.Error(err, "unable to start manager")
+		os.Exit(1)
+	}
+	albOpts := []sdkconfig.ConfigurationOption{}
+	if config.Global.APIEndpoints.ApplicationLoadBalancerAPI != "" {
+		albOpts = append(albOpts, sdkconfig.WithEndpoint(config.Global.APIEndpoints.ApplicationLoadBalancerAPI))
+	}
+
+	certOpts := []sdkconfig.ConfigurationOption{}
+	if config.Global.APIEndpoints.ApplicationLoadBalancerCertificateAPI != "" {
+		certOpts = append(certOpts, sdkconfig.WithEndpoint(config.Global.APIEndpoints.ApplicationLoadBalancerCertificateAPI))
+	}
+
+	// Setup ALB API client
+	sdkClient, err := albsdk.NewAPIClient(albOpts...)
+	if err != nil {
+		setupLog.Error(err, "unable to create ALB SDK client", "controller", "IngressClass")
+		os.Exit(1)
+	}
+	albClient, err := albclient.NewApplicationLoadBalancerClient(sdkClient)
+	if err != nil {
+		setupLog.Error(err, "unable to create ALB client", "controller", "IngressClass")
+		os.Exit(1)
+	}
+
+	// Setup Certificates API client
+	certificateAPI, err := certsdk.NewAPIClient(certOpts...)
+	if err != nil {
+		setupLog.Error(err, "unable to create certificate SDK client", "controller", "IngressClass")
+		os.Exit(1)
+	}
+	certificateClient, err := albclient.NewCertClient(certificateAPI)
+	if err != nil {
+		setupLog.Error(err, "unable to create Certificates client", "controller", "IngressClass")
+		os.Exit(1)
+	}
+
+	if err = (&ingress.IngressClassReconciler{
+		Client:            mgr.GetClient(),
+		ALBClient:         albClient,
+		CertificateClient: certificateClient,
+		Scheme:            mgr.GetScheme(),
+		ALBConfig:         config,
+	}).SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "IngressClass")
+		os.Exit(1)
+	}
+
+	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
+		setupLog.Error(err, "unable to set up health check")
+		os.Exit(1)
+	}
+	if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
+		setupLog.Error(err, "unable to set up ready check")
+		os.Exit(1)
+	}
+
+	setupLog.Info("starting manager")
+	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
+		setupLog.Error(err, "problem running manager")
+		os.Exit(1)
+	}
+}

deploy/application-load-balancer-controller-manager/deployment.yaml

@@ -0,0 +1,59 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  namespace: kube-system
+  name: stackit-application-load-balancer-contoller-manager
+  labels:
+    app: stackit-application-load-balancer-contoller-manager
+spec:
+  replicas: 2
+  strategy:
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app: stackit-application-load-balancer-contoller-manager
+  template:
+    metadata:
+      labels:
+        app: stackit-application-load-balancer-contoller-manager
+    spec:
+      serviceAccountName: stackit-application-load-balancer-contoller-manager
+      terminationGracePeriodSeconds: 30
+      containers:
+      - name: stackit-application-load-balancer-contoller-manager
+        # TODO(jamand): Adapt image tag
+        image: ghcr.io/stackitcloud/cloud-provider-stackit/stackit-application-load-balancer-contoller-manager:XXX
+        args:
+        - "--authorization-always-allow-paths=/metrics"
+        - "--leader-elect=true"
+        - "--leader-elect-resource-name=stackit-application-load-balancer-contoller-manager"
+        - "--enable-http2"
+        - "--metrics-bind-address=8080"
+        - "--secureMetrics=false"
+        # TODO(jamand): Check webhook cert + enableHTTP2 flag
+        env:
+        - name: STACKIT_SERVICE_ACCOUNT_KEY_PATH
+          value: /etc/serviceaccount/sa_key.json
+        ports:
+        - containerPort: 8080
+          hostPort: 8080
+          name: metrics
+          protocol: TCP
+        - containerPort: 8081
+          hostPort: 8081
+          name: probe
+          protocol: TCP
+        resources:
+          limits:
+            cpu: "0.5"
+            memory: 500Mi
+          requests:
+            cpu: "0.1"
+            memory: 100Mi
+        volumeMounts:
+        - mountPath: /etc/serviceaccount
+          name: cloud-secret
+      volumes:
+      - name: cloud-secret
+        secret:
+          secretName: stackit-cloud-secret

deploy/application-load-balancer-controller-manager/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- deployment.yaml
+- rbac.yaml
+

deploy/application-load-balancer-controller-manager/rbac.yaml

@@ -0,0 +1,60 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  namespace: kube-system
+  name: stackit-application-load-balancer-contoller-manager
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: stackit-application-load-balancer-contoller-manager
+rules:
+  # TODO(jamand): Go through rules again
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - list
+- apiGroups:
+  - "networking.k8s.io"
+  resources:
+  - ingress
+  verbs:
+  - get
+- apiGroups:
+  - "networking.k8s.io"
+  resources:
+  - ingress/status
+  verbs:
+  - patch
+- apiGroups:
+  - "networking.k8s.io"
+  resources:
+  - ingressclass
+  verbs:
+  - list
+  - patch
+  - update
+  - watch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: stackit-application-load-balancer-contoller-manager
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: stackit-application-load-balancer-contoller-manager
+subjects:
+- kind: ServiceAccount
+  name: stackit-application-load-balancer-contoller-manager
+  namespace: kube-system

deploy/application-load-balancer-controller-manager/service.yaml

@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: stackit-application-load-balancer-contoller-manager
+  namespace: kube-system
+  name: stackit-application-load-balancer-contoller-manager
+spec:
+  selector:
+    app: stackit-application-load-balancer-contoller-manager
+  ports:
+  - name: probe
+    port: 8081
+    targetPort: probe
+    protocol: TCP
+  - name: metrics
+    port: 8080
+    targetPort: metrics
+    protocol: TCP
+  type: ClusterIP