diff --git a/.gitignore b/.gitignore index 0eee18f..9c650b3 100644 --- a/.gitignore +++ b/.gitignore @@ -51,3 +51,7 @@ terraform.tfvars # POC poc/ + +# Go build outputs +go/**/*.exe +go/**/*.test diff --git a/go/cluster_setup_basic/main.go b/go/cluster_setup_basic/main.go new file mode 100644 index 0000000..2241bf8 --- /dev/null +++ b/go/cluster_setup_basic/main.go @@ -0,0 +1,305 @@ +// © 2026 NetApp, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// See the NOTICE file in the repo root for trademark and attribution details. + +// Cluster Setup — create a storage cluster from two pre-cluster nodes (ONTAP 9 unified). +// +// Steps: +// +// 1 discoverNodes — GET /cluster/nodes (membership=available, retry 3x/30s) +// 2 discoverLocal — isolate the local node (management_interfaces != null) +// 3 discoverPartner — isolate the partner node (exclude local node UUID) +// 4 createCluster — POST /cluster +// 5 trackJob — switch to cluster credentials, poll job until complete +// +// Prerequisites: +// 1. Two ONTAP 9 nodes in pre-cluster state (factory default or freshly wiped) +// 2. Both nodes reachable at their management IPs +// 3. Node 1 (ONTAP_HOST) must have at least one cluster interface already configured +// +// Usage: +// +// export ONTAP_HOST=10.x.x.x ONTAP_USER=admin ONTAP_PASS= +// export CLUSTER_NAME=mycluster CLUSTER_PASS=secret +// export CLUSTER_MGMT_IP=10.x.x.x CLUSTER_NETMASK=255.255.192.0 CLUSTER_GATEWAY=10.x.x.1 +// export PARTNER_MGMT_IP=10.x.x.y +// go run . +package main + +import ( + "errors" + "fmt" + "log" + "os" + "strings" + "time" + + ontapclient "github.com/netapp/pace/go/ontapclient" +) + +// --------------------------------------------------------------------------- + +const nodeFields = "name,uuid,model,state,ha,version,serial_number,membership," + + "cluster_interfaces,management_interfaces,metrocluster" + +const clusterNodesPath = "/cluster/nodes" + +func main() { + log.SetFlags(log.LstdFlags) + loadDotEnv() + + host := mustEnv("ONTAP_HOST") + user := envOrDefault("ONTAP_USER", "admin") + pass := envOrDefault("ONTAP_PASS", "") // empty on pre-cluster nodes + + log.Printf("Cluster setup starting — connecting to %s", host) + + client := ontapclient.New(host, user, pass, false) + defer client.Close() + + // Step 1: Discover available nodes (retry 3x) + log.Println("=== Step 1: Discover nodes ===") + discoverNodes(client, 3, 30) + + // Step 2: Find local node + log.Println("=== Step 2: Discover local node ===") + localNode := discoverLocal(client) + localUUID := ontapclient.NestedStr(localNode, "uuid") + + // Step 3: Find partner node + log.Println("=== Step 3: Discover partner node ===") + partnerNode := discoverPartner(client, localUUID) + + // Step 4: Create cluster + log.Println("=== Step 4: Create cluster ===") + jobUUID := createCluster(client, localNode, partnerNode) + + // Step 5: Track job — switch to cluster credentials first + log.Println("=== Step 5: Track cluster creation job ===") + clusterPass := mustEnv("CLUSTER_PASS") + clusterMgmtIP := mustEnv("CLUSTER_MGMT_IP") + trackJob(host, user, clusterPass, jobUUID) + + log.Printf("=== CLUSTER CREATED ===\n"+ + " Name : %s\n"+ + " UI : https://%s\n"+ + " Login : %s / %s", + mustEnv("CLUSTER_NAME"), clusterMgmtIP, user, clusterPass) +} + +// discoverNodes GETs /cluster/nodes with membership=available, retrying up to maxAttempts times. +func discoverNodes(client *ontapclient.Client, maxAttempts, delaySecs int) { + var lastErr error + for attempt := 1; attempt <= maxAttempts; attempt++ { + resp, err := client.Get(clusterNodesPath, map[string]string{ + "fields": nodeFields, + "membership": "available", + }) + if err == nil { + log.Printf("discover_nodes — %d node(s) found", ontapclient.NumRecords(resp)) + return + } + lastErr = err + if attempt < maxAttempts { + log.Printf("discover_nodes failed (attempt %d/%d), retrying in %ds — %v", + attempt, maxAttempts, delaySecs, err) + time.Sleep(time.Duration(delaySecs) * time.Second) + } + } + log.Fatalf("discover_nodes failed after %d attempts: %v", maxAttempts, lastErr) +} + +// discoverLocal finds the local node (the one with management_interfaces set). +// Returns the first matching node record. +func discoverLocal(client *ontapclient.Client) map[string]interface{} { + resp, err := client.Get(clusterNodesPath, map[string]string{ + "fields": nodeFields, + "membership": "available", + "management_interfaces": "!null", + }) + dieOnErr("discover_local", err) + nodes := ontapclient.Records(resp) + if len(nodes) == 0 { + log.Fatal("discover_local: no local node returned") + } + log.Printf("discover_local — %s", ontapclient.NestedStr(nodes[0], "name")) + return nodes[0] +} + +// discoverPartner finds the partner node by excluding the local node UUID. +// Returns the first matching node record. +func discoverPartner(client *ontapclient.Client, localUUID string) map[string]interface{} { + resp, err := client.Get(clusterNodesPath, map[string]string{ + "fields": nodeFields, + "membership": "available", + "uuid": "!" + localUUID, + }) + dieOnErr("discover_partner", err) + nodes := ontapclient.Records(resp) + if len(nodes) == 0 { + log.Fatal("discover_partner: no partner node returned") + } + log.Printf("discover_partner — %s", ontapclient.NestedStr(nodes[0], "name")) + return nodes[0] +} + +// createCluster POSTs /cluster to create the cluster; returns the job UUID. +func createCluster(client *ontapclient.Client, localNode, partnerNode map[string]interface{}) string { + clusterName := mustEnv("CLUSTER_NAME") + clusterPass := mustEnv("CLUSTER_PASS") + clusterMgmtIP := mustEnv("CLUSTER_MGMT_IP") + clusterNetmask := mustEnv("CLUSTER_NETMASK") + clusterGateway := mustEnv("CLUSTER_GATEWAY") + ontapHost := mustEnv("ONTAP_HOST") + partnerMgmtIP := mustEnv("PARTNER_MGMT_IP") + + localClusterIP := clusterIfaceIP(localNode) + partnerClusterIP := clusterIfaceIP(partnerNode) + + if localClusterIP == "" { + log.Fatal("ABORTED — local node has no cluster interface IP") + } + if partnerClusterIP == "" { + log.Fatal("ABORTED — partner node has no cluster interface IP") + } + + body := map[string]interface{}{ + "name": clusterName, + "password": clusterPass, + "management_interface": map[string]interface{}{ + "ip": map[string]string{ + "address": clusterMgmtIP, + "netmask": clusterNetmask, + "gateway": clusterGateway, + }, + }, + "nodes": []map[string]interface{}{ + { + "name": fmt.Sprintf("%s-01", clusterName), + "management_interface": map[string]interface{}{ + "ip": map[string]string{"address": ontapHost}, + }, + "cluster_interface": map[string]interface{}{ + "ip": map[string]string{"address": localClusterIP}, + }, + }, + { + "name": fmt.Sprintf("%s-02", clusterName), + "management_interface": map[string]interface{}{ + "ip": map[string]string{"address": partnerMgmtIP}, + }, + "cluster_interface": map[string]interface{}{ + "ip": map[string]string{"address": partnerClusterIP}, + }, + }, + }, + "name_servers": map[string]interface{}{}, + "ntp_servers": map[string]interface{}{}, + "dns_domains": map[string]interface{}{}, + "configuration_backup": map[string]interface{}{}, + } + + resp, err := client.Post("/cluster?keep_precluster_config=true", body) + dieOnErr("create_cluster", err) + + jobUUID := ontapclient.JobUUID(resp) + log.Printf("create_cluster — job %s", jobUUID) + return jobUUID +} + +// trackJob switches to cluster credentials then polls the job until complete. +// After POST /cluster the node reboots its management stack — network errors +// are expected and retried until the deadline. HTTP-level errors (4xx/5xx) are fatal. +func trackJob(host, user, clusterPass, jobUUID string) { + clusterClient := ontapclient.New(host, user, clusterPass, false) + defer clusterClient.Close() + + deadline := time.Now().Add(10 * time.Minute) + jobPath := fmt.Sprintf("/cluster/jobs/%s", jobUUID) + + for { + if time.Now().After(deadline) { + log.Fatal("track_job: timed out waiting for cluster creation") + } + + result, err := clusterClient.Get(jobPath, + map[string]string{"fields": "state,message,error,code"}) + if err != nil { + var apiErr *ontapclient.OntapApiError + if errors.As(err, &apiErr) { + // HTTP-level error (e.g. 401, 500) — something is wrong. + log.Fatalf("track_job: %v", err) + } + // Network error: node is rebooting as part of cluster creation. + log.Printf(" node rebooting (network error), retrying in 15s — %v", err) + time.Sleep(15 * time.Second) + continue + } + + state, _ := result["state"].(string) + log.Printf(" job %s — state=%s", jobUUID, state) + switch state { + case "running", "queued", "paused": + time.Sleep(10 * time.Second) + case "success": + return + default: + msg, _ := result["message"].(string) + log.Fatalf("job %s ended with state=%s: %s", jobUUID, state, msg) + } + } +} + +// clusterIfaceIP extracts the IP address of the first cluster interface from a node record. +func clusterIfaceIP(node map[string]interface{}) string { + ifaces, _ := node["cluster_interfaces"].([]interface{}) + if len(ifaces) == 0 { + return "" + } + iface, _ := ifaces[0].(map[string]interface{}) + return ontapclient.NestedStr(iface, "ip", "address") +} + +func mustEnv(key string) string { + if v := os.Getenv(key); v != "" { + return v + } + log.Fatalf("'%s' is required — set it in go/.env or as an environment variable", key) + return "" +} + +func envOrDefault(key, defaultVal string) string { + if v := os.Getenv(key); v != "" { + return v + } + return defaultVal +} + +func dieOnErr(context string, err error) { + if err != nil { + log.Fatalf("%s: %v", context, err) + } +} + +// loadDotEnv reads a .env file from the current directory and exports each +// KEY=VALUE pair as an environment variable (only if not already set). +// The file is gitignored — safe to store credentials there for local testing. +func loadDotEnv() { + data, err := os.ReadFile(".env") + if err != nil { + return + } + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + k, v, ok := strings.Cut(line, "=") + if !ok { + continue + } + if os.Getenv(strings.TrimSpace(k)) == "" { + _ = os.Setenv(strings.TrimSpace(k), strings.TrimSpace(v)) + } + } +} diff --git a/go/go.mod b/go/go.mod new file mode 100644 index 0000000..f8778af --- /dev/null +++ b/go/go.mod @@ -0,0 +1,3 @@ +module github.com/netapp/pace/go + +go 1.22 diff --git a/go/ontapclient/ontap_client.go b/go/ontapclient/ontap_client.go new file mode 100644 index 0000000..9897e61 --- /dev/null +++ b/go/ontapclient/ontap_client.go @@ -0,0 +1,301 @@ +// © 2026 NetApp, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// See the NOTICE file in the repo root for trademark and attribution details. + +// Package ontapclient provides a lightweight ONTAP REST API client. +// +// Usage: +// +// client := ontapclient.New("10.x.x.x", "admin", "secret", false) +// defer client.Close() +// cluster, err := client.Get("/cluster", map[string]string{"fields": "name,version"}) +package ontapclient + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/url" + "os" + "time" +) + +const ( + defaultTimeout = 30 * time.Second + clientAppHdr = "pace-example" + maxJobWait = 10 * time.Minute +) + +// OntapApiError is returned when the ONTAP REST API responds with a non-2xx status. +type OntapApiError struct { + StatusCode int + Detail interface{} +} + +func (e *OntapApiError) Error() string { + return fmt.Sprintf("HTTP %d: %v", e.StatusCode, e.Detail) +} + +// Client is a thin HTTP client for the ONTAP REST API. +type Client struct { + baseURL string + username string + password string + httpClient *http.Client +} + +// New creates a new Client. +// Set verifySSL=false to allow self-signed certificates (common in lab environments). +func New(host, username, password string, verifySSL bool) *Client { + transport := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: !verifySSL}, // #nosec G402 — intentional for lab certs + } + return &Client{ + baseURL: fmt.Sprintf("https://%s/api", host), + username: username, + password: password, + httpClient: &http.Client{ + Timeout: defaultTimeout, + Transport: transport, + }, + } +} + +// FromEnv creates a Client from standard ONTAP_* environment variables. +// Required: ONTAP_HOST, ONTAP_PASS. Optional: ONTAP_USER (default "admin"). +func FromEnv() *Client { + host := os.Getenv("ONTAP_HOST") + if host == "" { + log.Fatal("ONTAP_HOST environment variable is required") + } + password := os.Getenv("ONTAP_PASS") + if password == "" { + log.Fatal("ONTAP_PASS environment variable is required") + } + user := os.Getenv("ONTAP_USER") + if user == "" { + user = "admin" + } + return New(host, user, password, false) +} + +// Close is a no-op provided for symmetry with connection-pooling patterns. +func (c *Client) Close() { + c.httpClient.CloseIdleConnections() +} + +// buildURL constructs a full API URL with query parameters. +func (c *Client) buildURL(path string, params map[string]string) string { + u := c.baseURL + path + if len(params) == 0 { + return u + } + q := url.Values{} + for k, v := range params { + q.Set(k, v) + } + return u + "?" + q.Encode() +} + +// do executes an HTTP request and decodes the JSON response body. +func (c *Client) do(method, rawURL string, body interface{}) (map[string]interface{}, error) { + var bodyReader io.Reader + if body != nil { + b, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("marshal request body: %w", err) + } + bodyReader = bytes.NewReader(b) + } + + req, err := http.NewRequest(method, rawURL, bodyReader) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + req.SetBasicAuth(c.username, c.password) + req.Header.Set("Accept", "application/hal+json") + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Dot-Client-App", clientAppHdr) + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("execute request: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + respBytes, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("read response body: %w", err) + } + + var result map[string]interface{} + if len(respBytes) > 0 { + if err := json.Unmarshal(respBytes, &result); err != nil { + result = map[string]interface{}{"_raw": string(respBytes)} + } + } + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return result, &OntapApiError{StatusCode: resp.StatusCode, Detail: result} + } + return result, nil +} + +// Get sends a GET request to the given API path with optional query params. +func (c *Client) Get(path string, params map[string]string) (map[string]interface{}, error) { + return c.do(http.MethodGet, c.buildURL(path, params), nil) +} + +// Post sends a POST request with a JSON body. +func (c *Client) Post(path string, body interface{}) (map[string]interface{}, error) { + return c.do(http.MethodPost, c.baseURL+path, body) +} + +// Patch sends a PATCH request with a JSON body. +func (c *Client) Patch(path string, body interface{}) (map[string]interface{}, error) { + return c.do(http.MethodPatch, c.baseURL+path, body) +} + +// Delete sends a DELETE request. +func (c *Client) Delete(path string) (map[string]interface{}, error) { + return c.do(http.MethodDelete, c.baseURL+path, nil) +} + +// PollJob polls /cluster/jobs/{uuid} until the job reaches a terminal state. +// Returns an error if the job ends in any state other than "success". +func (c *Client) PollJob(jobUUID string, intervalSecs int) (map[string]interface{}, error) { + if intervalSecs <= 0 { + intervalSecs = 10 + } + deadline := time.Now().Add(maxJobWait) + for { + if time.Now().After(deadline) { + return nil, fmt.Errorf("poll job %s: timed out after %s", jobUUID, maxJobWait) + } + result, err := c.Get(fmt.Sprintf("/cluster/jobs/%s", jobUUID), + map[string]string{"fields": "state,message,error,code"}) + if err != nil { + return nil, fmt.Errorf("poll job %s: %w", jobUUID, err) + } + state, _ := result["state"].(string) + log.Printf(" job %s — state=%s", jobUUID, state) + switch state { + case "running", "queued", "paused": + time.Sleep(time.Duration(intervalSecs) * time.Second) + case "success": + return result, nil + default: + msg, _ := result["message"].(string) + return nil, fmt.Errorf("job %s ended with state=%s: %s", jobUUID, state, msg) + } + } +} + +// WaitSnapmirrored polls a SnapMirror relationship until state == "snapmirrored". +// maxWaitSecs defaults to 1800 if <= 0. +func (c *Client) WaitSnapmirrored(relUUID string, intervalSecs, maxWaitSecs int) (map[string]interface{}, error) { + if intervalSecs <= 0 { + intervalSecs = 15 + } + if maxWaitSecs <= 0 { + maxWaitSecs = 1800 + } + elapsed := 0 + for elapsed < maxWaitSecs { + result, err := c.Get(fmt.Sprintf("/snapmirror/relationships/%s", relUUID), + map[string]string{"fields": "state,lag_time,healthy"}) + if err != nil { + return nil, fmt.Errorf("poll relationship %s: %w", relUUID, err) + } + state, _ := result["state"].(string) + log.Printf(" relationship %s — state=%s", relUUID, state) + if state == "snapmirrored" { + return result, nil + } + time.Sleep(time.Duration(intervalSecs) * time.Second) + elapsed += intervalSecs + } + return nil, fmt.Errorf("timed out waiting for relationship %s to reach snapmirrored", relUUID) +} + +// NestedStr safely extracts a nested string value from a map[string]interface{}. +// Keys are applied in order: NestedStr(m, "a", "b") => m["a"].(map)["b"].(string). +func NestedStr(m map[string]interface{}, keys ...string) string { + cur := m + for i, k := range keys { + v, ok := cur[k] + if !ok { + return "" + } + if i == len(keys)-1 { + s, _ := v.(string) + return s + } + cur, ok = v.(map[string]interface{}) + if !ok { + return "" + } + } + return "" +} + +// NestedFloat safely extracts a float64 from a nested map. +func NestedFloat(m map[string]interface{}, keys ...string) float64 { + cur := m + for i, k := range keys { + v, ok := cur[k] + if !ok { + return 0 + } + if i == len(keys)-1 { + f, _ := v.(float64) + return f + } + cur, ok = v.(map[string]interface{}) + if !ok { + return 0 + } + } + return 0 +} + +// Records returns the "records" slice from a collection response. +func Records(resp map[string]interface{}) []map[string]interface{} { + raw, ok := resp["records"] + if !ok { + return nil + } + slice, ok := raw.([]interface{}) + if !ok { + return nil + } + out := make([]map[string]interface{}, 0, len(slice)) + for _, item := range slice { + if m, ok := item.(map[string]interface{}); ok { + out = append(out, m) + } + } + return out +} + +// NumRecords returns the num_records integer from a collection response. +func NumRecords(resp map[string]interface{}) int { + v, ok := resp["num_records"] + if !ok { + return 0 + } + f, ok := v.(float64) + if !ok { + return 0 + } + return int(f) +} + +// JobUUID extracts job.uuid from a response. +func JobUUID(resp map[string]interface{}) string { + return NestedStr(resp, "job", "uuid") +} diff --git a/go/snapmirror_cleanup_test_failover/main.go b/go/snapmirror_cleanup_test_failover/main.go new file mode 100644 index 0000000..8c8cde2 --- /dev/null +++ b/go/snapmirror_cleanup_test_failover/main.go @@ -0,0 +1,284 @@ +// © 2026 NetApp, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// See the NOTICE file in the repo root for trademark and attribution details. + +// SnapMirror Test Failover Cleanup — deletes the FlexClone created by test_failover. +// +// Finds the clone via SnapMirror relationship UUID tag (":test"). +// Only clones tagged by the snapmirror_test_failover workflow are touched — manually +// created volumes are never matched or deleted. +// +// Phases: +// +// 0 Relationship-pick — find SM relationship on correct cluster +// A Tag-based find — locate clone tagged with ":test" +// B SMAS removal — delete any SMAS relationship on the clone (releases lock) +// C Unmount — remove NAS junction path (with retry) +// D Offline — set volume state to offline +// E Delete — delete the clone and confirm removal +// +// Prerequisites: +// 1. ONTAP 9.8+ on both clusters +// 2. snapmirror_test_failover.go must have been run first +// 3. The SnapMirror relationship must still be accessible on one of the clusters +// 4. Admin credentials for both clusters +// +// Usage: +// +// export CLUSTER_A=10.x.x.x CLUSTER_B=10.y.y.y +// export DEST_USER=admin DEST_PASS=secret +// export SOURCE_VOLUME=vol_rw_01 +// export SOURCE_SVM=vs0 +// go run . +package main + +import ( + "fmt" + "log" + "os" + "strings" + "time" + + ontapclient "github.com/netapp/pace/go/ontapclient" +) + +const volumePatchPath = "/storage/volumes/%s?return_timeout=120" + +// --------------------------------------------------------------------------- + +func main() { + log.SetFlags(log.LstdFlags) + loadDotEnv() + + clusterA := mustEnv("CLUSTER_A") + clusterB := mustEnv("CLUSTER_B") + destUser := envOrDefault("DEST_USER", "admin") + destPass := mustEnv("DEST_PASS") + sourceVolume := mustEnv("SOURCE_VOLUME") + sourceSVM := mustEnv("SOURCE_SVM") + + // === Phase 0: Find SnapMirror relationship === + log.Println("=== Phase 0: Find SnapMirror relationship ===") + destHost, rel := pickClusterByRelationship(clusterA, clusterB, destUser, destPass, sourceSVM, sourceVolume) + relUUID := ontapclient.NestedStr(rel, "uuid") + log.Printf("RELATIONSHIP FOUND | cluster=%s | uuid=%s | source=%s | dest=%s | state=%s | healthy=%v", + destHost, + relUUID, + ontapclient.NestedStr(rel, "source", "path"), + ontapclient.NestedStr(rel, "destination", "path"), + ontapclient.NestedStr(rel, "state"), + rel["healthy"]) + + if ontapclient.NestedStr(rel, "state") != "snapmirrored" { + log.Printf("Relationship state=%s healthy=%v — proceeding with cleanup anyway", + ontapclient.NestedStr(rel, "state"), rel["healthy"]) + } + + client := ontapclient.New(destHost, destUser, destPass, false) + defer client.Close() + + // === Phase A: Find tagged clone === + log.Println("=== Phase A: Find tagged clone ===") + clone := findTaggedClone(client, relUUID) + if clone == nil { + log.Printf("NO TAGGED CLONE FOUND for %s:%s on %s — nothing to clean up", + sourceSVM, sourceVolume, destHost) + return + } + log.Printf("CLONE FOUND | name=%s | uuid=%s | svm=%s | cluster=%s", + clone["name"], clone["uuid"], clone["svm"], destHost) + + cloneUUID, _ := clone["uuid"].(string) + cloneSVM, _ := clone["svm"].(string) + cloneName, _ := clone["name"].(string) + + removeSMASAndBringOnline(client, cloneUUID, cloneSVM, cloneName) + unmountClone(client, cloneUUID) + offlineClone(client, cloneUUID) + deleteAndConfirmClone(client, cloneUUID, cloneName, destHost) +} + +// pickClusterByRelationship returns (clusterIP, relationshipRecord) for the cluster owning this SM rel. +func pickClusterByRelationship(clusterA, clusterB, user, passwd, sourceSVM, sourceVolume string) (string, map[string]interface{}) { + sourcePath := sourceSVM + ":" + sourceVolume + for _, host := range []string{clusterA, clusterB} { + client := ontapclient.New(host, user, passwd, false) + resp, err := client.Get("/snapmirror/relationships", map[string]string{ + "fields": "uuid,source.path,destination.path,state,healthy", + "source.path": sourcePath, + "max_records": "1", + }) + client.Close() + if err != nil { + log.Printf(" cluster %s — %v", host, err) + continue + } + if ontapclient.NumRecords(resp) >= 1 { + return host, ontapclient.Records(resp)[0] + } + } + log.Fatalf("No SM relationship found for %s on either cluster (%s, %s)", sourcePath, clusterA, clusterB) + return "", nil +} + +// findTaggedClone returns the clone tagged ':test', or nil if not found. +func findTaggedClone(client *ontapclient.Client, relUUID string) map[string]interface{} { + resp, err := client.Get("/storage/volumes", map[string]string{ + "fields": "name,uuid,svm.name,state,nas.path", + "_tags": relUUID + ":test", + "max_records": "1", + }) + if err != nil || ontapclient.NumRecords(resp) == 0 { + return nil + } + rec := ontapclient.Records(resp)[0] + return map[string]interface{}{ + "uuid": ontapclient.NestedStr(rec, "uuid"), + "name": ontapclient.NestedStr(rec, "name"), + "svm": ontapclient.NestedStr(rec, "svm", "name"), + } +} + +// removeSMASAndBringOnline deletes any SMAS relationship on the clone, then ensures it is online. +func removeSMASAndBringOnline(client *ontapclient.Client, cloneUUID, cloneSVM, cloneName string) { + log.Println("=== Phase B: Remove SMAS relationship on clone (if any) ===") + smasResp, err := client.Get("/snapmirror/relationships", map[string]string{ + "fields": "uuid,state", + "destination.path": cloneSVM + ":" + cloneName, + "max_records": "10", + }) + if err != nil { + log.Printf("list smas relationships: %v (continuing)", err) + } + smasRels := ontapclient.Records(smasResp) + for _, r := range smasRels { + smasUUID := ontapclient.NestedStr(r, "uuid") + log.Printf(" Deleting SMAS relationship %s on clone", smasUUID) + resp, err := client.Delete(fmt.Sprintf("/snapmirror/relationships/%s?return_timeout=120&force=true", smasUUID)) + if err != nil { + log.Printf("delete_smas_rel %s — %v (continuing)", smasUUID, err) + continue + } + if jobUUID := ontapclient.JobUUID(resp); jobUUID != "" { + if _, err := client.PollJob(jobUUID, 10); err != nil { + log.Printf("poll delete smas job — %v", err) + } + } + } + if len(smasRels) == 0 { + log.Println(" No SMAS relationships found on clone — continuing") + } + + resp, err := client.Patch(fmt.Sprintf(volumePatchPath, cloneUUID), + map[string]interface{}{"state": "online"}) + if err != nil { + log.Printf("bring_online — %v (continuing)", err) + return + } + if jobUUID := ontapclient.JobUUID(resp); jobUUID != "" { + if _, err := client.PollJob(jobUUID, 10); err != nil { + log.Printf("poll bring-online job — %v", err) + } + } +} + +// unmountClone removes the NAS junction path; retries up to 6 times before aborting. +func unmountClone(client *ontapclient.Client, cloneUUID string) { + log.Println("=== Phase C: Unmount clone ===") + for attempt := 1; attempt <= 6; attempt++ { + resp, err := client.Patch(fmt.Sprintf(volumePatchPath, cloneUUID), + map[string]interface{}{"nas": map[string]string{"path": ""}}) + if err != nil { + log.Printf("unmount_clone attempt %d/6 — %v", attempt, err) + if attempt < 6 { + time.Sleep(10 * time.Second) + } + continue + } + if jobUUID := ontapclient.JobUUID(resp); jobUUID != "" { + if _, err := client.PollJob(jobUUID, 10); err != nil { + log.Printf("poll unmount job — %v", err) + } + } + return + } + log.Fatal("Failed to unmount clone after 6 attempts — aborting") +} + +// offlineClone sets the volume state to offline (required before delete). +func offlineClone(client *ontapclient.Client, cloneUUID string) { + log.Println("=== Phase D: Offline clone ===") + resp, err := client.Patch(fmt.Sprintf(volumePatchPath, cloneUUID), + map[string]interface{}{"state": "offline"}) + if err != nil { + log.Printf("offline_clone — %v", err) + return + } + if jobUUID := ontapclient.JobUUID(resp); jobUUID != "" { + if _, err := client.PollJob(jobUUID, 10); err != nil { + log.Printf("poll offline job — %v", err) + } + } +} + +// deleteAndConfirmClone deletes the clone volume and confirms it is gone. +func deleteAndConfirmClone(client *ontapclient.Client, cloneUUID, cloneName, destHost string) { + log.Println("=== Phase E: Delete clone ===") + resp, err := client.Delete(fmt.Sprintf(volumePatchPath, cloneUUID)) + if err != nil { + log.Printf("delete_clone — %v", err) + } else if jobUUID := ontapclient.JobUUID(resp); jobUUID != "" { + if _, err := client.PollJob(jobUUID, 10); err != nil { + log.Printf("poll delete job — %v", err) + } + } + + confirm, err := client.Get("/storage/volumes", map[string]string{ + "fields": "name,uuid", + "uuid": cloneUUID, + "max_records": "1", + }) + if err != nil || ontapclient.NumRecords(confirm) == 0 { + log.Printf("=== CLEANUP COMPLETE — clone '%s' deleted from cluster %s ===", cloneName, destHost) + } else { + log.Fatalf("Clone '%s' still exists after delete attempt", cloneName) + } +} + +// loadDotEnv reads a .env file from the current directory and exports each +// KEY=VALUE pair as an environment variable (only if not already set). +// The file is gitignored — safe to store credentials there for local testing. +func loadDotEnv() { + data, err := os.ReadFile(".env") + if err != nil { + return + } + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + k, v, ok := strings.Cut(line, "=") + if !ok { + continue + } + if os.Getenv(strings.TrimSpace(k)) == "" { + _ = os.Setenv(strings.TrimSpace(k), strings.TrimSpace(v)) + } + } +} + +func mustEnv(key string) string { + if v := os.Getenv(key); v != "" { + return v + } + log.Fatalf("'%s' is required — set it in go/.env or as an environment variable", key) + return "" +} + +func envOrDefault(key, defaultVal string) string { + if v := os.Getenv(key); v != "" { + return v + } + return defaultVal +} diff --git a/go/snapmirror_provision_dest_managed/main.go b/go/snapmirror_provision_dest_managed/main.go new file mode 100644 index 0000000..68007bc --- /dev/null +++ b/go/snapmirror_provision_dest_managed/main.go @@ -0,0 +1,635 @@ +// © 2026 NetApp, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// See the NOTICE file in the repo root for trademark and attribution details. + +// SnapMirror Provision — Destination-Managed view. +// +// All SnapMirror API calls driven from the DESTINATION cluster. +// Source RW volume must already exist; dest DP volume is auto-created. +// +// Steps: +// 1. Verify source cluster connectivity +// 2. Verify dest cluster connectivity +// 3. Setup cluster peer (auto-create if missing) +// 4. Validate source volume exists and is RW +// 5. Get dest aggregate +// 6. Setup SVM peer (auto-create if missing) +// 7. Auto-create dest DP volume (skip if already exists) +// 8. Validate dest DP volume exists +// 9. Check if relationship already exists +// 10. Create + initialize SnapMirror relationship +// 11. Poll create/init job +// 12. Fetch relationship UUID +// 13. Initialize relationship (trigger baseline transfer) +// 14. Wait for state = snapmirrored +// 15. Validate health + print final report +// +// Prerequisites: +// 1. ONTAP 9.8+ on both clusters +// 2. SnapMirror licence installed on both clusters +// 3. At least one intercluster LIF on each cluster +// 4. Admin credentials for both clusters +// +// Usage: +// +// export SOURCE_HOST=10.x.x.x SOURCE_USER=admin SOURCE_PASS=secret +// export SOURCE_SVM=vs0 SOURCE_VOLUME=vol_rw_01 +// export DEST_HOST=10.y.y.y DEST_USER=admin DEST_PASS=secret +// export DEST_SVM=vs1 +// export SM_POLICY=Asynchronous +// go run . +package main + +import ( + "fmt" + "log" + "os" + "strings" + "time" + + ontapclient "github.com/netapp/pace/go/ontapclient" +) + +const ( + pathStorageVolumes = "/storage/volumes" // NOSONAR + pathClusterPeers = "/cluster/peers" + pathSVMPeers = "/svm/peers" + keySVMName = "svm.name" + peerFields = "name,uuid,status.state" +) + +// smRelConfig groups SnapMirror relationship parameters to keep function signatures compact. +type smRelConfig struct { + destSVM, destVolume, sourceSVMAlias, sourceVolume, peerName, smPolicy string +} + +// --------------------------------------------------------------------------- +// USER INPUTS — fill in your values here before running +// --------------------------------------------------------------------------- +var inputs = map[string]string{ + "SOURCE_HOST": "", // set via SOURCE_HOST in go/.env or env var + "SOURCE_USER": "admin", + "SOURCE_PASS": "", // set via SOURCE_PASS in go/.env or env var + "SOURCE_SVM": "", // set via SOURCE_SVM in go/.env or env var + "SOURCE_VOLUME": "", // set via SOURCE_VOLUME in go/.env or env var + "DEST_HOST": "", // set via DEST_HOST in go/.env or env var + "DEST_USER": "admin", + "DEST_PASS": "", // set via DEST_PASS in go/.env or env var + "DEST_SVM": "", // set via DEST_SVM in go/.env or env var + "SM_POLICY": "Asynchronous", +} + +// --------------------------------------------------------------------------- + +func main() { + log.SetFlags(log.LstdFlags) + loadDotEnv() + + sourceHost := mustEnv("SOURCE_HOST") + sourceUser := envOrDefault("SOURCE_USER", "admin") + sourcePass := mustEnv("SOURCE_PASS") + sourceSVM := mustEnv("SOURCE_SVM") + sourceVolume := mustEnv("SOURCE_VOLUME") + + destHost := mustEnv("DEST_HOST") + destUser := envOrDefault("DEST_USER", "admin") + destPass := mustEnv("DEST_PASS") + destSVM := mustEnv("DEST_SVM") + smPolicy := envOrDefault("SM_POLICY", "Asynchronous") + + destVolume := sourceVolume + "_dest" + + src := ontapclient.New(sourceHost, sourceUser, sourcePass, false) + defer src.Close() + dst := ontapclient.New(destHost, destUser, destPass, false) + defer dst.Close() + + // === Phase A: Source pre-flight === + log.Println("=== Phase A: Source pre-flight ===") + srcVol := phaseASourcePreflight(src, sourceSVM, sourceVolume, sourceHost) + srcVolSize := fmt.Sprintf("%.0f", ontapclient.NestedFloat(srcVol, "space", "size")) + + // === Phase B: Dest pre-flight === + log.Println("=== Phase B: Dest pre-flight ===") + dstCluster, err := dst.Get("/cluster", map[string]string{"fields": "name,version"}) + dieOnErr("get dest cluster", err) + log.Printf("DEST CLUSTER | name=%s | ontap=%s", + ontapclient.NestedStr(dstCluster, "name"), + ontapclient.NestedStr(dstCluster, "version", "full")) + + // === Phase B0: Cluster peer setup === + log.Println("=== Phase B0: Cluster peer setup ===") + srcPeerName, peerName, dstPeerUUID := setupClusterPeer(src, dst, sourceSVM, destSVM) + + aggrResp, err := dst.Get("/storage/aggregates", map[string]string{ + "fields": "name,state", + "max_records": "1", + }) + dieOnErr("get dest aggregate", err) + aggrName := "" + aggrs := ontapclient.Records(aggrResp) + if len(aggrs) > 0 { + aggrName = ontapclient.NestedStr(aggrs[0], "name") + } + if aggrName == "" { + log.Fatal("ABORTED — no aggregates found on destination cluster") + } + log.Printf("DEST AGGREGATE | name=%s", aggrName) + + // === Phase B1: SVM peer setup === + log.Println("=== Phase B1: SVM peer setup ===") + sourceSVMAlias := setupSVMPeer(src, dst, sourceSVM, destSVM, srcPeerName, peerName, dstPeerUUID) + + // === Phase C: Dest volume setup === + log.Println("=== Phase C: Dest volume setup ===") + _, err = dst.Post("/storage/volumes?return_timeout=120", map[string]interface{}{ + "name": destVolume, + "type": "dp", + "svm": map[string]string{"name": destSVM}, + "aggregates": []map[string]string{ + {"name": aggrName}, + }, + "space": map[string]string{"size": srcVolSize}, + }) + if err != nil { + log.Printf("create_dest_volume — %v (skipped — may already exist)", err) + } else { + log.Printf("DEST VOLUME | created '%s' on aggregate '%s'", destVolume, aggrName) + } + + dstVolResp, err := dst.Get("/storage/volumes", map[string]string{ + "fields": "name,uuid,state,type", + "max_records": "1", + "name": destVolume, + keySVMName: destSVM, + }) + dieOnErr("verify dest volume", err) + dstVols := ontapclient.Records(dstVolResp) + if len(dstVols) == 0 { + log.Fatalf("ABORTED — dest volume '%s' not found on SVM '%s' after create", destVolume, destSVM) + } + dstVol := dstVols[0] + log.Printf("DEST VOLUME | svm=%s | name=%s | uuid=%s | state=%s | type=%s", + destSVM, + ontapclient.NestedStr(dstVol, "name"), + ontapclient.NestedStr(dstVol, "uuid"), + ontapclient.NestedStr(dstVol, "state"), + ontapclient.NestedStr(dstVol, "type")) + + // === Phase D: Relationship setup === + log.Println("=== Phase D: Relationship setup ===") + relUUID := phaseDSetupRelationship(src, dst, smRelConfig{ + destSVM: destSVM, destVolume: destVolume, + sourceSVMAlias: sourceSVMAlias, sourceVolume: sourceVolume, + peerName: peerName, smPolicy: smPolicy, + }) + + // === Phase E: Convergence polling === + log.Println("=== Phase E: Convergence polling ===") + if _, err := dst.WaitSnapmirrored(relUUID, 15, 1800); err != nil { + log.Fatalf("wait snapmirrored: %v", err) + } + + // === Phase F: Final validation === + log.Println("=== Phase F: Final validation ===") + final, err := dst.Get(fmt.Sprintf("/snapmirror/relationships/%s", relUUID), + map[string]string{"fields": "uuid,source.path,destination.path,state,lag_time,healthy,policy.name"}) + dieOnErr("final validation", err) + log.Printf("=== SNAPMIRROR PROVISION COMPLETE ===\n"+ + " source : %s:%s\n"+ + " destination : %s:%s\n"+ + " state : %s\n"+ + " healthy : %v\n"+ + " policy : %s\n"+ + " lag_time : %v", + sourceSVM, sourceVolume, + destSVM, destVolume, + ontapclient.NestedStr(final, "state"), + final["healthy"], + ontapclient.NestedStr(final, "policy", "name"), + final["lag_time"]) +} + +// phaseASourcePreflight verifies source cluster connectivity and validates the source volume. +func phaseASourcePreflight(src *ontapclient.Client, sourceSVM, sourceVolume, sourceHost string) map[string]interface{} { + srcCluster, err := src.Get("/cluster", map[string]string{"fields": "name,version"}) + dieOnErr("get source cluster", err) + log.Printf("SOURCE CLUSTER | name=%s | ontap=%s", + ontapclient.NestedStr(srcCluster, "name"), + ontapclient.NestedStr(srcCluster, "version", "full")) + + srcVolResp, err := src.Get("/storage/volumes", map[string]string{ + "fields": "name,uuid,state,type,space.size", + "max_records": "1", + "name": sourceVolume, + keySVMName: sourceSVM, + }) + dieOnErr("get source volume", err) + if ontapclient.NumRecords(srcVolResp) == 0 { + log.Fatalf("ABORTED — source volume '%s' not found on %s", sourceVolume, sourceHost) + } + srcVol := ontapclient.Records(srcVolResp)[0] + if ontapclient.NestedStr(srcVol, "type") == "dp" { + log.Fatal("ABORTED — source volume is type=dp; specify the RW volume") + } + log.Printf("SOURCE VOLUME | svm=%s | name=%s | uuid=%s | state=%s | type=%s | size=%.0f", + sourceSVM, + ontapclient.NestedStr(srcVol, "name"), + ontapclient.NestedStr(srcVol, "uuid"), + ontapclient.NestedStr(srcVol, "state"), + ontapclient.NestedStr(srcVol, "type"), + ontapclient.NestedFloat(srcVol, "space", "size")) + return srcVol +} + +// getICLIFIPs returns intercluster LIF IP addresses from a cluster. +func getICLIFIPs(client *ontapclient.Client) []string { + resp, err := client.Get("/network/ip/interfaces", map[string]string{ + "fields": "name,ip.address,services", + "max_records": "50", + }) + if err != nil { + return nil + } + var ips []string + for _, r := range ontapclient.Records(resp) { + services, _ := r["services"].([]interface{}) + for _, s := range services { + if strings.Contains(fmt.Sprintf("%v", s), "intercluster") { + if ip := ontapclient.NestedStr(r, "ip", "address"); ip != "" { + ips = append(ips, ip) + break + } + } + } + } + return ips +} + +// checkICLIFPreconditions validates intercluster LIFs exist on both clusters. +func checkICLIFPreconditions(srcIPs, dstIPs []string) { + if len(srcIPs) == 0 { + log.Fatal("PRE-CONDITION FAILED | Source cluster has no intercluster LIFs.\n" + + " SnapMirror requires at least one IC LIF on each cluster.") + } + if len(dstIPs) == 0 { + log.Fatal("PRE-CONDITION FAILED | Dest cluster has no intercluster LIFs.\n" + + " SnapMirror requires at least one IC LIF on each cluster.") + } + subnet24 := func(ip string) string { + parts := strings.SplitN(ip, ".", 4) + if len(parts) >= 3 { + return parts[0] + "." + parts[1] + "." + parts[2] + } + return ip + } + srcSubnets := map[string]bool{} + for _, ip := range srcIPs { + srcSubnets[subnet24(ip)] = true + } + dstSubnets := map[string]bool{} + for _, ip := range dstIPs { + dstSubnets[subnet24(ip)] = true + } + shared := false + for s := range srcSubnets { + if dstSubnets[s] { + shared = true + break + } + } + if !shared { + log.Printf("PRE-CONDITION WARNING | IC LIFs are on different subnets.\n"+ + " src IPs : %v\n dst IPs : %v\n"+ + " SnapMirror data transfers require TCP 11104 and 11105 to be open between these subnets.", + srcIPs, dstIPs) + } else { + log.Printf("PRE-CONDITION OK | IC LIFs share a common subnet — transfers should work") + } +} + +// setupClusterPeer ensures a cluster peer exists; auto-creates if missing. +// Returns (srcPeerName, dstPeerName, dstPeerUUID). +func setupClusterPeer(src, dst *ontapclient.Client, sourceSVM, destSVM string) (string, string, string) { + okStates := map[string]bool{"available": true, "partial": true, "pending": true} + + dstCP, err := dst.Get(pathClusterPeers, map[string]string{ + "fields": peerFields, + "max_records": "10", + }) + dieOnErr("get dest cluster peers", err) + + for _, p := range ontapclient.Records(dstCP) { + state := ontapclient.NestedStr(p, "status", "state") + if !okStates[state] { + continue + } + // Peer already exists + srcCP, err2 := src.Get(pathClusterPeers, map[string]string{ + "fields": peerFields, + "max_records": "10", + }) + if err2 != nil { + log.Printf("get src cluster peers: %v", err2) + } + srcPeerName := "" + for _, q := range ontapclient.Records(srcCP) { + if okStates[ontapclient.NestedStr(q, "status", "state")] { + srcPeerName = ontapclient.NestedStr(q, "name") + break + } + } + srcIPs := getICLIFIPs(src) + dstIPs := getICLIFIPs(dst) + log.Printf("CLUSTER PEER | already peered — dst sees src as '%s' (state=%s) — skipping", + ontapclient.NestedStr(p, "name"), state) + log.Printf("IC LIFs | src=%v dst=%v", srcIPs, dstIPs) + checkICLIFPreconditions(srcIPs, dstIPs) + return srcPeerName, ontapclient.NestedStr(p, "name"), ontapclient.NestedStr(p, "uuid") + } + + // No existing peer — auto-create + log.Println("CLUSTER PEER | no existing peer found — auto-creating") + srcIPs := getICLIFIPs(src) + dstIPs := getICLIFIPs(dst) + log.Printf("CLUSTER PEER | src IC LIFs=%v dst IC LIFs=%v", srcIPs, dstIPs) + checkICLIFPreconditions(srcIPs, dstIPs) + return createNewClusterPeer(src, dst, srcIPs, dstIPs, sourceSVM, destSVM) +} + +// createNewClusterPeer posts a new cluster peer on both sides. +// Returns (srcPeerName, dstPeerName, dstPeerUUID). +func createNewClusterPeer(src, dst *ontapclient.Client, srcIPs, dstIPs []string, sourceSVM, destSVM string) (string, string, string) { + if len(srcIPs) == 0 { + log.Fatal("ABORTED — no intercluster LIFs found on source cluster.") + } + if len(dstIPs) == 0 { + log.Fatal("ABORTED — no intercluster LIFs found on dest cluster.") + } + + peerAddrs := make([]string, len(dstIPs)) + copy(peerAddrs, dstIPs) + srcResp, err := src.Post(pathClusterPeers, map[string]interface{}{ + "peer_addresses": peerAddrs, + "generate_passphrase": true, + "encryption": map[string]string{"proposed": "tls-psk"}, + "initial_allowed_svms": []map[string]string{ + {"name": sourceSVM}, + }, + }) + dieOnErr("create cluster peer on source", err) + passphrase, _ := srcResp["passphrase"].(string) + log.Println("CLUSTER PEER | created on source") + + dstPeerAddrs := make([]string, len(srcIPs)) + copy(dstPeerAddrs, srcIPs) + _, err = dst.Post(pathClusterPeers, map[string]interface{}{ + "peer_addresses": dstPeerAddrs, + "passphrase": passphrase, + "initial_allowed_svms": []map[string]string{ + {"name": destSVM}, + }, + }) + dieOnErr("accept cluster peer on dest", err) + log.Println("CLUSTER PEER | accepted on dest") + + time.Sleep(5 * time.Second) + return fetchCreatedPeerNames(src, dst) +} + +// fetchCreatedPeerNames retrieves peer names from both clusters after creation. +func fetchCreatedPeerNames(src, dst *ontapclient.Client) (string, string, string) { + okStates := map[string]bool{"available": true, "partial": true, "pending": true} + dstCP, err := dst.Get(pathClusterPeers, map[string]string{"fields": peerFields, "max_records": "10"}) + if err != nil { + log.Fatalf("ABORTED — could not query cluster peers on destination: %v", err) + } + dstPeer := map[string]interface{}{} + for _, p := range ontapclient.Records(dstCP) { + if okStates[ontapclient.NestedStr(p, "status", "state")] { + dstPeer = p + break + } + } + if len(dstPeer) == 0 { + log.Fatal("ABORTED — no usable cluster peer found on destination after creation") + } + srcCP, err := src.Get(pathClusterPeers, map[string]string{"fields": peerFields, "max_records": "10"}) + if err != nil { + log.Fatalf("ABORTED — could not query cluster peers on source: %v", err) + } + srcPeer := map[string]interface{}{} + for _, p := range ontapclient.Records(srcCP) { + if okStates[ontapclient.NestedStr(p, "status", "state")] { + srcPeer = p + break + } + } + if len(srcPeer) == 0 { + log.Fatal("ABORTED — no usable cluster peer found on source after creation") + } + log.Printf("CLUSTER PEER | dst sees src as '%s'", ontapclient.NestedStr(dstPeer, "name")) + return ontapclient.NestedStr(srcPeer, "name"), + ontapclient.NestedStr(dstPeer, "name"), + ontapclient.NestedStr(dstPeer, "uuid") +} + +// grantSVMPeerPermission grants SnapMirror peer-permission on the source SVM. +func grantSVMPeerPermission(src *ontapclient.Client, sourceSVM, srcPeerName string) { + _, err := src.Post("/svm/peer-permissions", map[string]interface{}{ + "svm": map[string]string{"name": sourceSVM}, + "cluster_peer": map[string]string{"name": srcPeerName}, + "applications": []string{"snapmirror"}, + }) + if err != nil { + s := err.Error() + if strings.Contains(s, "already exists") || strings.Contains(strings.ToLower(s), "duplicate") || strings.Contains(s, "13001") { + log.Println("SVM PEER | peer-permission already exists — skipping") + return + } + log.Fatalf("SVM PEER | peer-permission failed: %v", err) + } + log.Println("SVM PEER | peer-permission granted on source") +} + +// createSVMPeerRelationship creates the SVM peer relationship on the destination. +func createSVMPeerRelationship(dst *ontapclient.Client, destSVM, sourceSVM, dstPeerName string) { + resp, err := dst.Post(pathSVMPeers, map[string]interface{}{ + "svm": map[string]string{"name": destSVM}, + "peer": map[string]interface{}{ + "svm": map[string]string{"name": sourceSVM}, + "cluster": map[string]string{"name": dstPeerName}, + }, + "applications": []string{"snapmirror"}, + }) + if err != nil { + s := err.Error() + if strings.Contains(s, "already exists") || strings.Contains(strings.ToLower(s), "duplicate") || strings.Contains(s, "13001") { + log.Println("SVM PEER | already exists — skipping") + return + } + log.Fatalf("SVM PEER | create failed: %v", err) + } + if jobUUID := ontapclient.JobUUID(resp); jobUUID != "" { + if _, err := dst.PollJob(jobUUID, 10); err != nil { + log.Printf("poll svm peer job: %v", err) + } + } + log.Printf("SVM PEER | created '%s' <-> '%s'", destSVM, sourceSVM) +} + +// setupSVMPeer ensures SVM peer exists; returns the source SVM alias used in SnapMirror paths. +func setupSVMPeer(src, dst *ontapclient.Client, sourceSVM, destSVM, srcPeerName, dstPeerName, srcClusterPeerUUID string) string { + svmResp, err := dst.Get(pathSVMPeers, map[string]string{ + "fields": "uuid,name,state,peer", + keySVMName: destSVM, + }) + dieOnErr("get svm peers", err) + + for _, p := range ontapclient.Records(svmResp) { + state := ontapclient.NestedStr(p, "state") + if state != "peered" && state != "initiated" { + continue + } + if ontapclient.NestedStr(p, "peer", "cluster", "uuid") != srcClusterPeerUUID { + continue + } + alias := ontapclient.NestedStr(p, "peer", "svm", "name") + if alias == "" { + alias = sourceSVM + } + log.Printf("SVM PEER | already peered '%s' <-> '%s' (alias='%s', state=%s) — skipping", + destSVM, sourceSVM, alias, state) + return alias + } + + grantSVMPeerPermission(src, sourceSVM, srcPeerName) + createSVMPeerRelationship(dst, destSVM, sourceSVM, dstPeerName) + + svmResp2, err := dst.Get(pathSVMPeers, map[string]string{ + "fields": "uuid,name,state,peer", + keySVMName: destSVM, + }) + if err != nil { + return sourceSVM + } + for _, p := range ontapclient.Records(svmResp2) { + if ontapclient.NestedStr(p, "peer", "cluster", "uuid") == srcClusterPeerUUID { + alias := ontapclient.NestedStr(p, "peer", "svm", "name") + if alias == "" { + alias = sourceSVM + } + return alias + } + } + return sourceSVM +} + +// phaseDSetupRelationship creates and initializes the SnapMirror relationship; returns its UUID. +func phaseDSetupRelationship(src, dst *ontapclient.Client, cfg smRelConfig) string { + existing, err := dst.Get("/snapmirror/relationships", map[string]string{ + "fields": "uuid,state,healthy", + "destination.path": cfg.destSVM + ":" + cfg.destVolume, + "max_records": "1", + }) + dieOnErr("check existing relationship", err) + log.Printf("RELATIONSHIP CHECK | existing=%d", ontapclient.NumRecords(existing)) + + createResp, err := dst.Post("/snapmirror/relationships?return_timeout=120", map[string]interface{}{ + "source": map[string]interface{}{ + "path": cfg.sourceSVMAlias + ":" + cfg.sourceVolume, + "cluster": map[string]string{"name": cfg.peerName}, + }, + "destination": map[string]string{"path": cfg.destSVM + ":" + cfg.destVolume}, + "policy": map[string]string{"name": cfg.smPolicy}, + }) + if err != nil { + log.Printf("create_and_initialize_relationship — %v (may already exist)", err) + } else if jobUUID := ontapclient.JobUUID(createResp); jobUUID != "" { + if _, err := dst.PollJob(jobUUID, 10); err != nil { + log.Printf("poll create job — %v", err) + } + } + + relResp, err := dst.Get("/snapmirror/relationships", map[string]string{ + "fields": "uuid,source.path,destination.path,state,lag_time,healthy,policy.name", + "destination.path": cfg.destSVM + ":" + cfg.destVolume, + "max_records": "1", + }) + dieOnErr("get relationship", err) + relRecords := ontapclient.Records(relResp) + if len(relRecords) == 0 { + log.Fatalf("ABORTED — SnapMirror relationship not found for '%s:%s'", cfg.destSVM, cfg.destVolume) + } + rel := relRecords[0] + relUUID := ontapclient.NestedStr(rel, "uuid") + log.Printf("RELATIONSHIP | uuid=%s | state=%s | healthy=%v | policy=%s", + relUUID, + ontapclient.NestedStr(rel, "state"), + rel["healthy"], + ontapclient.NestedStr(rel, "policy", "name")) + + _, err = dst.Post(fmt.Sprintf("/snapmirror/relationships/%s/transfers?return_timeout=120", relUUID), map[string]interface{}{}) + if err != nil { + s := err.Error() + if strings.Contains(s, "13303812") { + srcIPs := getICLIFIPs(src) + dstIPs := getICLIFIPs(dst) + log.Fatalf("ABORTED — SnapMirror initialize failed: intercluster LIF connectivity issue.\n"+ + " Error : %s\n src IC : %v\n dst IC : %v\n"+ + " Cause : TCP ports 11104/11105 are likely blocked between these IPs.", + s, srcIPs, dstIPs) + } + log.Printf("initialize_relationship — %v (may already be initialized)", err) + } + return relUUID +} + +func mustEnv(key string) string { + if v := inputs[key]; v != "" { + return v + } + if v := os.Getenv(key); v != "" { + return v + } + log.Fatalf("'%s' is required — set it in the INPUTS block at the top of this file", key) + return "" +} + +func envOrDefault(key, defaultVal string) string { + if v := inputs[key]; v != "" { + return v + } + if v := os.Getenv(key); v != "" { + return v + } + return defaultVal +} + +func dieOnErr(context string, err error) { + if err != nil { + log.Fatalf("%s: %v", context, err) + } +} + +// loadDotEnv reads go/.env and sets each KEY=VALUE as an env var (if not already set). +// Equivalent to Python's os.environ — credentials stay out of source code. +func loadDotEnv() { + data, err := os.ReadFile(".env") + if err != nil { + return + } + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + k, v, ok := strings.Cut(line, "=") + if !ok { + continue + } + if os.Getenv(strings.TrimSpace(k)) == "" { + _ = os.Setenv(strings.TrimSpace(k), strings.TrimSpace(v)) + } + } +} diff --git a/go/snapmirror_provision_src_managed/main.go b/go/snapmirror_provision_src_managed/main.go new file mode 100644 index 0000000..cc8d013 --- /dev/null +++ b/go/snapmirror_provision_src_managed/main.go @@ -0,0 +1,334 @@ +// © 2026 NetApp, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// See the NOTICE file in the repo root for trademark and attribution details. + +// SnapMirror Provision — Source-Managed view. +// +// Connects to BOTH clusters for pre-flight verification, then drives all +// relationship/volume API calls from the DESTINATION cluster (ONTAP requirement). +// +// Phases: +// +// A Source pre-flight — verify source cluster + volume +// B Dest pre-flight — verify dest cluster + aggregate +// C Dest volume — auto-create DP volume if missing +// D Relationship — create + initialize SnapMirror +// E Convergence — poll until state=snapmirrored +// F Validation — health check + final report +// +// Prerequisites: +// 1. ONTAP 9.8+ on both clusters +// 2. SnapMirror licence installed on both clusters +// 3. At least one intercluster LIF on each cluster +// 4. Cluster peer relationship already exists between source and dest clusters +// 5. SVM peer relationship already exists (source SVM <-> dest SVM) +// 6. Source RW volume (SOURCE_VOLUME) already exists on SOURCE_SVM +// 7. At least one online aggregate on the destination cluster +// 8. Admin credentials for both clusters +// +// Usage: +// +// export SOURCE_HOST=10.x.x.x SOURCE_USER=admin SOURCE_PASS=secret +// export SOURCE_SVM=vs0 SOURCE_VOLUME=vol_rw_01 +// export DEST_HOST=10.y.y.y DEST_USER=admin DEST_PASS=secret +// export DEST_SVM=vs1 +// export SM_POLICY=Asynchronous +// go run . +package main + +import ( + "fmt" + "log" + "os" + "strings" + + ontapclient "github.com/netapp/pace/go/ontapclient" +) + +const ( + pathStorageVolumes = "/storage/volumes" // NOSONAR + keySVMName = "svm.name" +) + +// --------------------------------------------------------------------------- + +func main() { + log.SetFlags(log.LstdFlags) + loadDotEnv() + + sourceHost := mustEnv("SOURCE_HOST") + sourceUser := envOrDefault("SOURCE_USER", "admin") + sourcePass := mustEnv("SOURCE_PASS") + sourceSVM := mustEnv("SOURCE_SVM") + sourceVolume := mustEnv("SOURCE_VOLUME") + + destHost := mustEnv("DEST_HOST") + destUser := envOrDefault("DEST_USER", "admin") + destPass := mustEnv("DEST_PASS") + destSVM := mustEnv("DEST_SVM") + smPolicy := envOrDefault("SM_POLICY", "Asynchronous") + + destVolume := sourceVolume + "_dest" + + src := ontapclient.New(sourceHost, sourceUser, sourcePass, false) + defer src.Close() + dst := ontapclient.New(destHost, destUser, destPass, false) + defer dst.Close() + + log.Println("=== Phase A: Source pre-flight ===") + srcVolSize, srcVol := smSrcPhaseA(src, sourceSVM, sourceVolume, sourceHost) + + log.Println("=== Phase B: Dest pre-flight ===") + peerName, aggrName := smSrcPhaseB(dst) + + log.Println("=== Phase C: Dest volume setup ===") + smSrcPhaseC(dst, destSVM, destVolume, aggrName, srcVolSize) + + log.Println("=== Phase D: Relationship setup ===") + relUUID := smSrcPhaseD(dst, sourceSVM, sourceVolume, destSVM, destVolume, peerName, smPolicy) + + log.Println("=== Phase E: Convergence polling ===") + if _, err := dst.WaitSnapmirrored(relUUID, 15, 1800); err != nil { + log.Fatalf("wait snapmirrored: %v", err) + } + + log.Println("=== Phase F: Final validation ===") + smSrcPhaseF(dst, relUUID, sourceSVM, sourceVolume, destSVM, destVolume) + + _ = srcVol // used via srcVolSize +} + +// smSrcPhaseA verifies the source cluster and validates the source volume. +// Returns (srcVolSize string, srcVol record). +func smSrcPhaseA(src *ontapclient.Client, sourceSVM, sourceVolume, sourceHost string) (string, map[string]interface{}) { + srcCluster, err := src.Get("/cluster", map[string]string{"fields": "name,version"}) + dieOnErr("get source cluster", err) + log.Printf("SOURCE CLUSTER | name=%s | ontap=%s", + ontapclient.NestedStr(srcCluster, "name"), + ontapclient.NestedStr(srcCluster, "version", "full")) + + srcVolResp, err := src.Get(pathStorageVolumes, map[string]string{ + "fields": "name,uuid,state,type,space.size", + "max_records": "1", + "name": sourceVolume, + keySVMName: sourceSVM, + }) + dieOnErr("get source volume", err) + if ontapclient.NumRecords(srcVolResp) == 0 { + log.Fatalf("ABORTED — source volume '%s' not found on %s", sourceVolume, sourceHost) + } + srcVol := ontapclient.Records(srcVolResp)[0] + if ontapclient.NestedStr(srcVol, "type") == "dp" { + log.Fatal("ABORTED — source volume is type=dp; specify the RW volume") + } + srcVolSize := fmt.Sprintf("%.0f", ontapclient.NestedFloat(srcVol, "space", "size")) + log.Printf("SOURCE VOLUME | name=%s | uuid=%s | state=%s | type=%s | size=%s", + ontapclient.NestedStr(srcVol, "name"), + ontapclient.NestedStr(srcVol, "uuid"), + ontapclient.NestedStr(srcVol, "state"), + ontapclient.NestedStr(srcVol, "type"), + srcVolSize) + return srcVolSize, srcVol +} + +// smSrcPhaseB verifies the dest cluster, fetches peer name and best aggregate. +// Returns (peerName, aggrName). +func smSrcPhaseB(dst *ontapclient.Client) (string, string) { + dstCluster, err := dst.Get("/cluster", map[string]string{"fields": "name,version"}) + dieOnErr("get dest cluster", err) + log.Printf("DEST CLUSTER | name=%s | ontap=%s", + ontapclient.NestedStr(dstCluster, "name"), + ontapclient.NestedStr(dstCluster, "version", "full")) + + peerResp, err := dst.Get("/cluster/peers", map[string]string{ + "fields": "name,status.state", + "max_records": "1", + }) + dieOnErr("get cluster peers", err) + peerName := "" + if peers := ontapclient.Records(peerResp); len(peers) > 0 { + peerName = ontapclient.NestedStr(peers[0], "name") + } + if peerName == "" { + log.Fatal("ABORTED — no cluster peer found on destination cluster; run snapmirror_peer_setup first") + } + log.Printf("CLUSTER PEER | name=%s", peerName) + + aggrResp, err := dst.Get("/storage/aggregates", map[string]string{ + "fields": "name,space.block_storage.available", + "state": "online", + "max_records": "1", + "order_by": "space.block_storage.available desc", + }) + dieOnErr("get dest aggregates", err) + aggrName := "" + if aggrs := ontapclient.Records(aggrResp); len(aggrs) > 0 { + aggrName = ontapclient.NestedStr(aggrs[0], "name") + } + if aggrName == "" { + log.Fatal("ABORTED — no online aggregates found on destination cluster") + } + log.Printf("DEST AGGREGATE | name=%s", aggrName) + return peerName, aggrName +} + +// smSrcPhaseC ensures the dest DP volume exists, creating it if needed. +func smSrcPhaseC(dst *ontapclient.Client, destSVM, destVolume, aggrName, srcVolSize string) { + checkDest, err := dst.Get(pathStorageVolumes, map[string]string{ + "fields": "name,uuid,state,type", + "max_records": "1", + "name": destVolume, + keySVMName: destSVM, + }) + dieOnErr("check dest volume", err) + if ontapclient.NumRecords(checkDest) == 0 { + log.Printf("Creating dest DP volume '%s' on '%s'…", destVolume, aggrName) + _, err = dst.Post(pathStorageVolumes+"?return_timeout=120", map[string]interface{}{ + "name": destVolume, + "type": "dp", + "svm": map[string]string{"name": destSVM}, + "aggregates": []map[string]string{ + {"name": aggrName}, + }, + "size": srcVolSize, + }) + if err != nil { + log.Printf("create_dest_volume — %v (may already exist)", err) + } + } else { + log.Printf("Dest volume '%s' already exists — skipping create", destVolume) + } + + dstVolResp, err := dst.Get(pathStorageVolumes, map[string]string{ + "fields": "name,uuid,state,type", + "max_records": "1", + "name": destVolume, + keySVMName: destSVM, + }) + dieOnErr("verify dest volume", err) + vols := ontapclient.Records(dstVolResp) + if len(vols) == 0 { + log.Fatalf("ABORTED — dest volume '%s' not found on SVM '%s' after create", destVolume, destSVM) + } + dstVol := vols[0] + log.Printf("DEST VOLUME | name=%s | uuid=%s | state=%s | type=%s", + ontapclient.NestedStr(dstVol, "name"), + ontapclient.NestedStr(dstVol, "uuid"), + ontapclient.NestedStr(dstVol, "state"), + ontapclient.NestedStr(dstVol, "type")) +} + +// smSrcPhaseD creates and initializes the SnapMirror relationship; returns the relationship UUID. +func smSrcPhaseD(dst *ontapclient.Client, sourceSVM, sourceVolume, destSVM, destVolume, peerName, smPolicy string) string { + existing, err := dst.Get("/snapmirror/relationships", map[string]string{ + "fields": "uuid,state,healthy", + "destination.path": destSVM + ":" + destVolume, + "max_records": "1", + }) + dieOnErr("check existing relationship", err) + log.Printf("RELATIONSHIP CHECK | existing=%d", ontapclient.NumRecords(existing)) + + createResp, err := dst.Post("/snapmirror/relationships?return_timeout=120", map[string]interface{}{ + "source": map[string]interface{}{ + "path": sourceSVM + ":" + sourceVolume, + "cluster": map[string]string{"name": peerName}, + }, + "destination": map[string]string{"path": destSVM + ":" + destVolume}, + "policy": map[string]string{"name": smPolicy}, + }) + if err != nil { + log.Printf("create_and_initialize_relationship — %v (may already exist)", err) + } else if jobUUID := ontapclient.JobUUID(createResp); jobUUID != "" { + if _, err := dst.PollJob(jobUUID, 10); err != nil { + log.Printf("poll create job — %v", err) + } + } + + relResp, err := dst.Get("/snapmirror/relationships", map[string]string{ + "fields": "uuid,source.path,destination.path,state,lag_time,healthy,policy.name", + "destination.path": destSVM + ":" + destVolume, + "max_records": "1", + }) + dieOnErr("get relationship", err) + rels := ontapclient.Records(relResp) + if len(rels) == 0 { + log.Fatalf("ABORTED — SnapMirror relationship not found for '%s:%s'", destSVM, destVolume) + } + rel := rels[0] + relUUID := ontapclient.NestedStr(rel, "uuid") + log.Printf("RELATIONSHIP FOUND | uuid=%s | state=%s | healthy=%v", + relUUID, ontapclient.NestedStr(rel, "state"), rel["healthy"]) + + _, err = dst.Post(fmt.Sprintf("/snapmirror/relationships/%s/transfers?return_timeout=120", relUUID), map[string]interface{}{}) + if err != nil { + log.Printf("initialize_relationship — %v (may already be initialized)", err) + } + return relUUID +} + +// smSrcPhaseF prints the final validation report. +func smSrcPhaseF(dst *ontapclient.Client, relUUID, sourceSVM, sourceVolume, destSVM, destVolume string) { + final, err := dst.Get(fmt.Sprintf("/snapmirror/relationships/%s", relUUID), + map[string]string{"fields": "uuid,source.path,destination.path,state,lag_time,healthy,policy.name"}) + dieOnErr("final validation", err) + log.Printf("=== SNAPMIRROR PROVISION COMPLETE ===\n"+ + " source : %s:%s\n"+ + " destination : %s:%s\n"+ + " state : %s\n"+ + " healthy : %v\n"+ + " policy : %s\n"+ + " lag_time : %v", + sourceSVM, sourceVolume, + destSVM, destVolume, + ontapclient.NestedStr(final, "state"), + final["healthy"], + ontapclient.NestedStr(final, "policy", "name"), + final["lag_time"]) +} + +// mustEnv reads an environment variable and exits if it is not set. +func mustEnv(key string) string { + if v := os.Getenv(key); v != "" { + return v + } + log.Fatalf("'%s' is required — set it in go/.env or as an environment variable", key) + return "" +} + +// envOrDefault reads an environment variable, returning defaultVal if unset. +func envOrDefault(key, defaultVal string) string { + if v := os.Getenv(key); v != "" { + return v + } + return defaultVal +} + +// dieOnErr logs a fatal error if err is non-nil. +func dieOnErr(context string, err error) { + if err != nil { + log.Fatalf("%s: %v", context, err) + } +} + +// loadDotEnv reads a .env file from the current directory and exports each +// KEY=VALUE pair as an environment variable (only if not already set). +// The file is gitignored — safe to store credentials there for local testing. +func loadDotEnv() { + data, err := os.ReadFile(".env") + if err != nil { + return + } + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + k, v, ok := strings.Cut(line, "=") + if !ok { + continue + } + if os.Getenv(strings.TrimSpace(k)) == "" { + _ = os.Setenv(strings.TrimSpace(k), strings.TrimSpace(v)) + } + } +} diff --git a/go/snapmirror_test_failover/main.go b/go/snapmirror_test_failover/main.go new file mode 100644 index 0000000..316a536 --- /dev/null +++ b/go/snapmirror_test_failover/main.go @@ -0,0 +1,303 @@ +// © 2026 NetApp, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// See the NOTICE file in the repo root for trademark and attribution details. + +// SnapMirror Test Failover — creates a writable FlexClone of a SnapMirror dest volume. +// +// AUTO mode (SOURCE_VOLUME=* or unset): +// +// Queries clusters A then B and selects the first cluster that has a matching DP volume +// (the newest matching DP volume within that cluster). +// +// TARGETED mode (SOURCE_VOLUME=vol_rw_01): +// +// Finds vol_rw_01_dest on either cluster. +// +// Phases: +// +// 0 Auto-detect which cluster has the target DP volume +// A Pre-flight — verify cluster + relationship health +// B Snapshot — get latest SnapMirror snapshot on dest volume +// C Clone — create writable FlexClone +// D Verify — confirm clone online + tag with SM relationship UUID +// E Resync — resync SnapMirror + validate healthy state +// +// Prerequisites: +// 1. ONTAP 9.8+ on both clusters +// 2. A healthy SnapMirror relationship must already exist +// 3. Relationship state must be 'snapmirrored' (baseline transfer complete) +// 4. At least one SnapMirror snapshot on the destination volume +// 5. Admin credentials for both clusters +// +// Usage: +// +// export CLUSTER_A=10.x.x.x CLUSTER_B=10.y.y.y +// export DEST_USER=admin DEST_PASS=secret +// export SOURCE_VOLUME=* # or a specific volume name e.g. "vol_rw_01" +// go run . +package main + +import ( + "fmt" + "log" + "os" + "strings" + + ontapclient "github.com/netapp/pace/go/ontapclient" +) + +const pathStorageVolumes = "/storage/volumes" // NOSONAR + +// --------------------------------------------------------------------------- + +func main() { + log.SetFlags(log.LstdFlags) + loadDotEnv() + + clusterA := mustEnv("CLUSTER_A") + clusterB := mustEnv("CLUSTER_B") + destUser := envOrDefault("DEST_USER", "admin") + destPass := mustEnv("DEST_PASS") + sourceVolume := envOrDefault("SOURCE_VOLUME", "*") + + log.Println("=== Phase 0: Auto-detect target cluster ===") + destHost, dpVol := pickCluster(clusterA, clusterB, destUser, destPass, sourceVolume) + dpVolName := ontapclient.NestedStr(dpVol, "name") + dpSVMName := ontapclient.NestedStr(dpVol, "svm", "name") + dpVolUUID := ontapclient.NestedStr(dpVol, "uuid") + log.Printf("SELECTED | cluster=%s | volume=%s | svm=%s | uuid=%s | state=%s | size=%.0f", + destHost, dpVolName, dpSVMName, dpVolUUID, + ontapclient.NestedStr(dpVol, "state"), + ontapclient.NestedFloat(dpVol, "space", "size")) + + client := ontapclient.New(destHost, destUser, destPass, false) + defer client.Close() + + log.Println("=== Phase A: Pre-flight ===") + relUUID := tfPhaseA(client, dpSVMName, dpVolName) + + log.Println("=== Phase B: Get latest SnapMirror snapshot ===") + snapshotName := tfPhaseB(client, dpVolUUID, dpVolName) + + log.Println("=== Phase C: Create FlexClone ===") + cloneName, cloneUUID := tfPhaseC(client, dpVolName, dpSVMName, snapshotName) + + log.Println("=== Phase D: Verify clone + tag ===") + tfPhaseD(client, cloneName, cloneUUID, relUUID, dpSVMName, snapshotName) + + log.Println("=== Phase E: Resync SnapMirror ===") + tfPhaseE(client, relUUID) +} + +// tfPhaseA verifies cluster connectivity and fetches the SnapMirror relationship UUID. +func tfPhaseA(client *ontapclient.Client, dpSVMName, dpVolName string) string { + cluster, err := client.Get("/cluster", map[string]string{"fields": "name,version"}) + dieOnErr("get cluster", err) + log.Printf("DEST CLUSTER | name=%s | ontap=%s", + ontapclient.NestedStr(cluster, "name"), + ontapclient.NestedStr(cluster, "version", "full")) + + relResp, err := client.Get("/snapmirror/relationships", map[string]string{ + "fields": "uuid,source.path,destination.path,state,lag_time,healthy,policy.name", + "destination.path": dpSVMName + ":" + dpVolName, + "max_records": "1", + }) + dieOnErr("get snapmirror relationship", err) + rels := ontapclient.Records(relResp) + if len(rels) == 0 { + log.Fatalf("No SnapMirror relationship found for %s:%s", dpSVMName, dpVolName) + } + rel := rels[0] + relUUID := ontapclient.NestedStr(rel, "uuid") + log.Printf("RELATIONSHIP | uuid=%s | source=%s | dest=%s | state=%s | healthy=%v | lag=%v", + relUUID, + ontapclient.NestedStr(rel, "source", "path"), + ontapclient.NestedStr(rel, "destination", "path"), + ontapclient.NestedStr(rel, "state"), + rel["healthy"], rel["lag_time"]) + return relUUID +} + +// tfPhaseB fetches the latest SnapMirror snapshot name from the DP volume. +func tfPhaseB(client *ontapclient.Client, dpVolUUID, dpVolName string) string { + snapResp, err := client.Get(fmt.Sprintf("/storage/volumes/%s/snapshots", dpVolUUID), map[string]string{ + "fields": "name,create_time", + "max_records": "1", + "order_by": "create_time desc", + }) + dieOnErr("get snapshots", err) + if ontapclient.NumRecords(snapResp) == 0 { + log.Fatalf("No SnapMirror snapshots on %s — run provision workflow first", dpVolName) + } + snap := ontapclient.Records(snapResp)[0] + snapshotName := ontapclient.NestedStr(snap, "name") + log.Printf("LATEST SM SNAPSHOT | name=%s | created=%v", snapshotName, snap["create_time"]) + return snapshotName +} + +// tfPhaseC creates the writable FlexClone; returns (cloneName, cloneUUID). +func tfPhaseC(client *ontapclient.Client, dpVolName, dpSVMName, snapshotName string) (string, string) { + cloneName := dpVolName + "_clone" + cloneResp, err := client.Post("/storage/volumes?return_timeout=120", map[string]interface{}{ + "name": cloneName, + "svm": map[string]string{"name": dpSVMName}, + "nas": map[string]string{"path": "/" + cloneName}, + "clone": map[string]interface{}{ + "is_flexclone": true, + "parent_volume": map[string]string{"name": dpVolName}, + "parent_snapshot": map[string]string{"name": snapshotName}, + }, + }) + if err != nil { + log.Printf("create_test_clone — %v (may already exist)", err) + } else if jobUUID := ontapclient.JobUUID(cloneResp); jobUUID != "" { + if _, err := client.PollJob(jobUUID, 10); err != nil { + log.Printf("poll clone job — %v", err) + } + } + + cloneVolResp, err := client.Get(pathStorageVolumes, map[string]string{ + "fields": "name,uuid,state,nas.path,space.size", + "max_records": "1", + "name": cloneName, + "svm.name": dpSVMName, + }) + dieOnErr("get clone volume", err) + cloneVol := map[string]interface{}{} + if vols := ontapclient.Records(cloneVolResp); len(vols) > 0 { + cloneVol = vols[0] + } + cloneUUID := ontapclient.NestedStr(cloneVol, "uuid") + if cloneUUID == "" { + log.Fatalf("ABORTED — FlexClone '%s' not found after create (create may have failed)", cloneName) + } + log.Printf("CLONE | name=%s | uuid=%s | state=%s | junction=%s", + ontapclient.NestedStr(cloneVol, "name"), cloneUUID, + ontapclient.NestedStr(cloneVol, "state"), + ontapclient.NestedStr(cloneVol, "nas", "path")) + return cloneName, cloneUUID +} + +// tfPhaseD tags the clone and prints the test-failover-ready message. +func tfPhaseD(client *ontapclient.Client, cloneName, cloneUUID, relUUID, dpSVMName, snapshotName string) { + _, err := client.Patch(fmt.Sprintf("/storage/volumes/%s?return_timeout=120", cloneUUID), + map[string]interface{}{"_tags": []string{relUUID + ":test"}}) + if err != nil { + log.Printf("tag_clone_volume — %v", err) + } else { + log.Printf("TAG APPLIED | clone=%s | tag=%s:test", cloneName, relUUID) + } + + cloneVolResp, err := client.Get(pathStorageVolumes, map[string]string{ + "fields": "name,uuid,state,nas.path", + "max_records": "1", + "name": cloneName, + "svm.name": dpSVMName, + }) + if err != nil { + log.Printf("re-fetch clone — %v", err) + return + } + cloneVol := map[string]interface{}{} + if vols := ontapclient.Records(cloneVolResp); len(vols) > 0 { + cloneVol = vols[0] + } + junctionPath := ontapclient.NestedStr(cloneVol, "nas", "path") + log.Printf("=== TEST FAILOVER READY ===\n"+ + " Clone : %s\n UUID : %s\n State : %s\n"+ + " Junction : %s\n SVM : %s\n Snapshot : %s\n\n"+ + " ACTION: Mount %s from SVM %s on a test client.", + ontapclient.NestedStr(cloneVol, "name"), cloneUUID, + ontapclient.NestedStr(cloneVol, "state"), + junctionPath, dpSVMName, snapshotName, + junctionPath, dpSVMName) +} + +// tfPhaseE resyncs the SnapMirror relationship and waits for snapmirrored state. +func tfPhaseE(client *ontapclient.Client, relUUID string) { + resyncResp, err := client.Patch(fmt.Sprintf("/snapmirror/relationships/%s?return_timeout=120", relUUID), + map[string]interface{}{"state": "snapmirrored"}) + if err != nil { + log.Printf("resync_sm_relationship — %v", err) + } else if jobUUID := ontapclient.JobUUID(resyncResp); jobUUID != "" { + if _, err := client.PollJob(jobUUID, 10); err != nil { + log.Printf("poll resync job — %v", err) + } + } + if _, err := client.WaitSnapmirrored(relUUID, 15, 1800); err != nil { + log.Fatalf("wait snapmirrored: %v", err) + } + log.Println("=== TEST FAILOVER COMPLETE — SnapMirror resynced ===") +} + +// pickCluster finds which cluster has the target DP volume; returns (clusterIP, volRecord). +func pickCluster(clusterA, clusterB, user, passwd, volNameFilter string) (string, map[string]interface{}) { + destFilter := volNameFilter + "_dest" + if volNameFilter == "*" { + destFilter = "*_dest" + } + for _, host := range []string{clusterA, clusterB} { + client := ontapclient.New(host, user, passwd, false) + resp, err := client.Get(pathStorageVolumes, map[string]string{ + "fields": "name,create_time,uuid,svm.name,state,space.size", + "type": "dp", + "name": destFilter, + "order_by": "create_time desc", + "max_records": "1", + }) + client.Close() + if err != nil { + log.Printf(" cluster %s — %v", host, err) + continue + } + if ontapclient.NumRecords(resp) >= 1 { + return host, ontapclient.Records(resp)[0] + } + } + log.Fatalf("No DP volumes found on either cluster (%s, %s)", clusterA, clusterB) + return "", nil +} + +func mustEnv(key string) string { + if v := os.Getenv(key); v != "" { + return v + } + log.Fatalf("'%s' is required — set it in go/.env or as an environment variable", key) + return "" +} + +func envOrDefault(key, defaultVal string) string { + if v := os.Getenv(key); v != "" { + return v + } + return defaultVal +} + +func dieOnErr(context string, err error) { + if err != nil { + log.Fatalf("%s: %v", context, err) + } +} + +// loadDotEnv reads a .env file from the current directory and exports each +// KEY=VALUE pair as an environment variable (only if not already set). +// The file is gitignored — safe to store credentials there for local testing. +func loadDotEnv() { + data, err := os.ReadFile(".env") + if err != nil { + return + } + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + k, v, ok := strings.Cut(line, "=") + if !ok { + continue + } + if os.Getenv(strings.TrimSpace(k)) == "" { + _ = os.Setenv(strings.TrimSpace(k), strings.TrimSpace(v)) + } + } +}