feat(instances): cache server lookups to reduce API calls #1252
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,4 +1,4 @@ | ||
| FROM golang:1.26 AS builder | ||
|
|
||
| WORKDIR /build | ||
|
|
||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| # Instance Cache | ||
|
|
||
| > **Experimental:** Instance caching is experimental, breaking changes may occur within minor releases. We believe the implementation is safe in practice — that is why it ships enabled by default (`all-server`). Set `HCLOUD_CACHE_MODE=off` to opt out. | ||
|
|
||
| The instance cache reduces calls to the Hetzner Cloud API made by the `InstancesV2` controller, which looks up Servers by ID or name to reconcile Node state. The cache sits between the controller and the Hetzner Cloud API; behavior is controlled by the environment variables below. | ||
|
|
||
| ## Environment Variables | ||
|
|
||
| | Name | Type | Default | Description | | ||
| | ------------------- | ------------------- | ------- | ------------------------------------------------------------------------------------- | | ||
| | `HCLOUD_CACHE_MODE` | `all \| one \| off` | `all` | Selects the caching strategy. See [Modes](#modes) below. | | ||
| | `HCLOUD_CACHE_TTL` | `duration` | `10s` | Lifetime of cached entries. Accepts any Go `time.Duration` string (e.g. `30s`, `2m`). | | ||
|
|
||
| ## Modes | ||
|
|
||
| ### `all` | ||
|
|
||
| Fetches every Server in the project with a single `GET /servers` call and serves all subsequent `ByID` / `ByName` lookups from the resulting snapshot until the TTL expires. The snapshot is refreshed on the next lookup after expiry. On a cache miss within the TTL (e.g. a freshly created Server), one rate-limited refresh per TTL window is allowed to pick up the new Server; further misses in the same window return without an API call. | ||
|
|
||
| ### `one` | ||
|
|
||
| Caches each Server individually with its own expiration. A `ByID` / `ByName` lookup either returns a non-expired entry or issues a `GET /servers/{id}` (or `GET /servers?name=`) call and stores the result. Expired entries are evicted lazily when other entries are inserted. | ||
|
|
||
| ### `off` | ||
|
|
||
| Disables caching entirely. Every lookup goes directly to the API. | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -31,6 +31,7 @@ import ( | |
| "k8s.io/client-go/tools/record" | ||
|
|
||
| "github.com/hetznercloud/hcloud-cloud-controller-manager/internal/config" | ||
| "github.com/hetznercloud/hcloud-cloud-controller-manager/internal/servercache" | ||
| "github.com/hetznercloud/hcloud-cloud-controller-manager/internal/testsupport" | ||
| "github.com/hetznercloud/hcloud-go/v2/hcloud" | ||
| "github.com/hetznercloud/hcloud-go/v2/hcloud/schema" | ||
|
|
@@ -41,6 +42,7 @@ type testEnv struct { | |
| Mux *http.ServeMux | ||
| Client *hcloud.Client | ||
| RobotClient hrobot.RobotClient | ||
| ServerCache *servercache.Cache[hcloud.Server] | ||
| Recorder record.EventRecorder | ||
| Cfg config.HCCMConfiguration | ||
| } | ||
|
|
@@ -51,6 +53,7 @@ func (env *testEnv) Teardown() { | |
| env.Mux = nil | ||
| env.Client = nil | ||
| env.RobotClient = nil | ||
| env.ServerCache = nil | ||
| env.Recorder = nil | ||
| } | ||
|
|
||
|
|
@@ -66,6 +69,7 @@ func newTestEnv() testEnv { | |
| ) | ||
| robotClient := hrobot.NewBasicAuthClient("", "") | ||
| robotClient.SetBaseURL(server.URL + "/robot") | ||
| serverCache := servercache.NewServerCache(client, servercache.ModeOne, 10*time.Second) | ||
| recorder := record.NewBroadcaster().NewRecorder(scheme.Scheme, corev1.EventSource{Component: "hcloud-cloud-controller-manager"}) | ||
|
|
||
| cfg := config.HCCMConfiguration{} | ||
|
|
@@ -76,6 +80,7 @@ func newTestEnv() testEnv { | |
| Mux: mux, | ||
| Client: client, | ||
| RobotClient: robotClient, | ||
| ServerCache: serverCache, | ||
| Recorder: recorder, | ||
| Cfg: cfg, | ||
| } | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -33,17 +33,20 @@ import ( | |
| "github.com/hetznercloud/hcloud-cloud-controller-manager/internal/legacydatacenter" | ||
| "github.com/hetznercloud/hcloud-cloud-controller-manager/internal/metrics" | ||
| "github.com/hetznercloud/hcloud-cloud-controller-manager/internal/providerid" | ||
| "github.com/hetznercloud/hcloud-cloud-controller-manager/internal/servercache" | ||
| "github.com/hetznercloud/hcloud-go/v2/hcloud" | ||
| ) | ||
|
|
||
| const ( | ||
| ProvidedBy = "instance.hetzner.cloud/provided-by" | ||
| MisconfiguredInternalIP = "MisconfiguredInternalIP" | ||
| instancesV2Subsystem = "instances_v2" | ||
| ) | ||
|
|
||
| type instances struct { | ||
| client *hcloud.Client | ||
| robotClient hrobot.RobotClient | ||
| serverCache *servercache.Cache[hcloud.Server] | ||
| recorder record.EventRecorder | ||
| networkID int64 | ||
| cfg config.HCCMConfiguration | ||
|
|
@@ -57,13 +60,15 @@ var ( | |
| func newInstances( | ||
| client *hcloud.Client, | ||
| robotClient hrobot.RobotClient, | ||
| serverCache *servercache.Cache[hcloud.Server], | ||
| recorder record.EventRecorder, | ||
| networkID int64, | ||
| cfg config.HCCMConfiguration, | ||
| ) *instances { | ||
| return &instances{ | ||
| client, | ||
| robotClient, | ||
| serverCache, | ||
| recorder, | ||
| networkID, | ||
| cfg, | ||
|
|
@@ -77,16 +82,17 @@ func (i *instances) lookupServer( | |
| ctx context.Context, | ||
| node *corev1.Node, | ||
| ) (genericServer, error) { | ||
| ctx = servercache.SetSubsystem(ctx, instancesV2Subsystem) | ||
|
|
||
| if node.Spec.ProviderID != "" { | ||
| var serverID int64 | ||
| serverID, isCloudServer, err := providerid.ToServerID(node.Spec.ProviderID) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to convert provider id to server id: %w", err) | ||
| } | ||
|
|
||
| if isCloudServer { | ||
| server, err := i.serverCache.ByID(ctx, serverID) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to get hcloud server \"%d\": %w", serverID, err) | ||
| } | ||
|
|
@@ -115,7 +121,7 @@ func (i *instances) lookupServer( | |
|
|
||
| // If the node has no provider ID we try to find the server by name from | ||
| // both sources. In case we find two servers, we return an error. | ||
| cloudServer, err := i.serverCache.ByName(ctx, node.Name) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to get hcloud server %q: %w", node.Name, err) | ||
| } | ||
|
|
@@ -153,6 +159,7 @@ func (i *instances) lookupServer( | |
| func (i *instances) InstanceExists(ctx context.Context, node *corev1.Node) (bool, error) { | ||
| const op = "hcloud/instancesv2.InstanceExists" | ||
| metrics.OperationCalled.WithLabelValues(op).Inc() | ||
| klog.V(4).InfoS("InstanceExists called", "node", node.Name, "providerID", node.Spec.ProviderID) | ||
|
|
||
| server, err := i.lookupServer(ctx, node) | ||
| if err != nil { | ||
|
|
@@ -165,6 +172,7 @@ func (i *instances) InstanceExists(ctx context.Context, node *corev1.Node) (bool | |
| func (i *instances) InstanceShutdown(ctx context.Context, node *corev1.Node) (bool, error) { | ||
| const op = "hcloud/instancesv2.InstanceShutdown" | ||
| metrics.OperationCalled.WithLabelValues(op).Inc() | ||
| klog.V(4).InfoS("InstanceShutdown called", "node", node.Name, "providerID", node.Spec.ProviderID) | ||
|
|
||
| server, err := i.lookupServer(ctx, node) | ||
| if err != nil { | ||
|
|
@@ -174,7 +182,8 @@ func (i *instances) InstanceShutdown(ctx context.Context, node *corev1.Node) (bo | |
| if server == nil { | ||
| return false, fmt.Errorf( | ||
| "%s: failed to get instance metadata: no matching server found for node '%s': %w", | ||
| op, node.Name, errServerNotFound, | ||
| ) | ||
| } | ||
|
|
||
| isShutdown, err := server.IsShutdown() | ||
|
|
@@ -188,6 +197,7 @@ func (i *instances) InstanceShutdown(ctx context.Context, node *corev1.Node) (bo | |
| func (i *instances) InstanceMetadata(ctx context.Context, node *corev1.Node) (*cloudprovider.InstanceMetadata, error) { | ||
| const op = "hcloud/instancesv2.InstanceMetadata" | ||
| metrics.OperationCalled.WithLabelValues(op).Inc() | ||
| klog.V(4).InfoS("InstanceMetadata called", "node", node.Name, "providerID", node.Spec.ProviderID) | ||
|
|
||
| server, err := i.lookupServer(ctx, node) | ||
| if err != nil { | ||
|
|
@@ -197,7 +207,8 @@ func (i *instances) InstanceMetadata(ctx context.Context, node *corev1.Node) (*c | |
| if server == nil { | ||
| return nil, fmt.Errorf( | ||
| "%s: failed to get instance metadata: no matching server found for node '%s': %w", | ||
| op, node.Name, errServerNotFound, | ||
| ) | ||
| } | ||
|
|
||
| metadata, err := server.Metadata(i.networkID, node, i.cfg) | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -91,7 +91,7 @@ func TestInstances_InstanceExists(t *testing.T) { | |
| }) | ||
| }) | ||
|
|
||
| instances := newInstances(env.Client, env.RobotClient, env.ServerCache, env.Recorder, 0, env.Cfg) | ||
|
|
||
| tests := []struct { | ||
| name string | ||
|
|
@@ -104,7 +104,8 @@ func TestInstances_InstanceExists(t *testing.T) { | |
| Spec: corev1.NodeSpec{ProviderID: "hcloud://1"}, | ||
| }, | ||
| expected: true, | ||
| }, | ||
| { | ||
| name: "existing robot server by id", | ||
| node: &corev1.Node{ | ||
| ObjectMeta: metav1.ObjectMeta{ | ||
|
|
@@ -123,49 +124,56 @@ func TestInstances_InstanceExists(t *testing.T) { | |
| Spec: corev1.NodeSpec{ProviderID: "hcloud://bm-321"}, | ||
| }, | ||
| expected: true, | ||
| }, | ||
| { | ||
| name: "missing server by id", | ||
| node: &corev1.Node{ | ||
| Spec: corev1.NodeSpec{ProviderID: "hcloud://2"}, | ||
| }, | ||
| expected: false, | ||
| }, | ||
| { | ||
| name: "missing robot server by id", | ||
| node: &corev1.Node{ | ||
| Spec: corev1.NodeSpec{ProviderID: "hrobot://322"}, | ||
| }, | ||
| expected: false, | ||
| }, | ||
| { | ||
| name: "missing robot server by (legacy) id", | ||
| node: &corev1.Node{ | ||
| Spec: corev1.NodeSpec{ProviderID: "hcloud://bm-322"}, | ||
| }, | ||
| expected: false, | ||
| }, | ||
| { | ||
| name: "existing server by name", | ||
| node: &corev1.Node{ | ||
| ObjectMeta: metav1.ObjectMeta{ | ||
| Name: "foobar", | ||
| }, | ||
| }, | ||
| expected: true, | ||
| }, | ||
| { | ||
| name: "existing robot server by name", | ||
| node: &corev1.Node{ | ||
| ObjectMeta: metav1.ObjectMeta{ | ||
| Name: "robot-server1", | ||
| }, | ||
| }, | ||
| expected: true, | ||
| }, | ||
| { | ||
| name: "missing server by name", | ||
| node: &corev1.Node{ | ||
| ObjectMeta: metav1.ObjectMeta{ | ||
| Name: "barfoo", | ||
| }, | ||
| }, | ||
| expected: false, | ||
| }, | ||
| { | ||
| name: "missing robot server by name", | ||
| node: &corev1.Node{ | ||
| ObjectMeta: metav1.ObjectMeta{ | ||
|
|
@@ -211,7 +219,7 @@ func TestInstances_InstanceShutdown(t *testing.T) { | |
| }) | ||
| }) | ||
|
|
||
| instances := newInstances(env.Client, env.RobotClient, env.ServerCache, env.Recorder, 0, env.Cfg) | ||
| env.Mux.HandleFunc("/robot/server/3", func(w http.ResponseWriter, _ *http.Request) { | ||
| json.NewEncoder(w).Encode(hrobotmodels.ServerResponse{ | ||
| Server: hrobotmodels.Server{ | ||
|
|
@@ -274,13 +282,15 @@ func TestInstances_InstanceShutdown(t *testing.T) { | |
| Spec: corev1.NodeSpec{ProviderID: "hcloud://1"}, | ||
| }, | ||
| expected: false, | ||
| }, | ||
| { | ||
| name: "[cloud] shutdown", | ||
| node: &corev1.Node{ | ||
| Spec: corev1.NodeSpec{ProviderID: "hcloud://2"}, | ||
| }, | ||
| expected: true, | ||
| }, | ||
| { | ||
| name: "[robot] running", | ||
| node: &corev1.Node{ | ||
| ObjectMeta: metav1.ObjectMeta{ | ||
|
|
@@ -289,7 +299,8 @@ func TestInstances_InstanceShutdown(t *testing.T) { | |
| Spec: corev1.NodeSpec{ProviderID: "hrobot://3"}, | ||
| }, | ||
| expected: false, | ||
| }, | ||
| { | ||
| name: "[robot] shutdown", | ||
| node: &corev1.Node{ | ||
| ObjectMeta: metav1.ObjectMeta{ | ||
|
|
@@ -346,7 +357,7 @@ func TestInstances_InstanceMetadata(t *testing.T) { | |
| }) | ||
| }) | ||
|
|
||
| instances := newInstances(env.Client, env.RobotClient, env.ServerCache, env.Recorder, 0, env.Cfg) | ||
|
|
||
| metadata, err := instances.InstanceMetadata(context.TODO(), &corev1.Node{ | ||
| Spec: corev1.NodeSpec{ProviderID: "hcloud://1"}, | ||
|
|
@@ -390,7 +401,7 @@ func TestInstances_InstanceMetadataRobotServer(t *testing.T) { | |
| }) | ||
| }) | ||
|
|
||
| instances := newInstances(env.Client, env.RobotClient, env.ServerCache, env.Recorder, 0, env.Cfg) | ||
|
|
||
| metadata, err := instances.InstanceMetadata(context.TODO(), &corev1.Node{ | ||
| ObjectMeta: metav1.ObjectMeta{ | ||
|
|
||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.