diff --git a/cmd/mapt/cmd/ibmcloud/hosts/ibm-power.go b/cmd/mapt/cmd/ibmcloud/hosts/ibm-power.go index 1878d4985..3173b552e 100644 --- a/cmd/mapt/cmd/ibmcloud/hosts/ibm-power.go +++ b/cmd/mapt/cmd/ibmcloud/hosts/ibm-power.go @@ -101,12 +101,14 @@ func ibmPowerDestroy() *cobra.Command { DebugLevel: viper.GetUint(params.DebugLevel), Serverless: viper.IsSet(params.Serverless), ForceDestroy: viper.IsSet(params.ForceDestroy), + KeepState: viper.IsSet(params.KeepState), }) }, } flagSet := pflag.NewFlagSet(params.DestroyCmdName, pflag.ExitOnError) flagSet.Bool(params.Serverless, false, params.ServerlessDesc) flagSet.Bool(params.ForceDestroy, false, params.ForceDestroyDesc) + flagSet.Bool(params.KeepState, false, params.KeepStateDesc) c.PersistentFlags().AddFlagSet(flagSet) return c } diff --git a/cmd/mapt/cmd/ibmcloud/hosts/ibm-z.go b/cmd/mapt/cmd/ibmcloud/hosts/ibm-z.go index 9e8305ded..4c5c08ffe 100644 --- a/cmd/mapt/cmd/ibmcloud/hosts/ibm-z.go +++ b/cmd/mapt/cmd/ibmcloud/hosts/ibm-z.go @@ -95,12 +95,14 @@ func ibmZDestroy() *cobra.Command { DebugLevel: viper.GetUint(params.DebugLevel), Serverless: viper.IsSet(params.Serverless), ForceDestroy: viper.IsSet(params.ForceDestroy), + KeepState: viper.IsSet(params.KeepState), }) }, } flagSet := pflag.NewFlagSet(params.DestroyCmdName, pflag.ExitOnError) flagSet.Bool(params.Serverless, false, params.ServerlessDesc) flagSet.Bool(params.ForceDestroy, false, params.ForceDestroyDesc) + flagSet.Bool(params.KeepState, false, params.KeepStateDesc) c.PersistentFlags().AddFlagSet(flagSet) return c } diff --git a/docs/ibmcloud/ibm-power.md b/docs/ibmcloud/ibm-power.md index 325803514..cfff6033d 100644 --- a/docs/ibmcloud/ibm-power.md +++ b/docs/ibmcloud/ibm-power.md @@ -24,6 +24,9 @@ On first boot, cloud-init automatically configures the PowerVS instance for on-p | `IBMCLOUD_ACCOUNT` | yes | IBM Cloud account ID | | `IBMCLOUD_API_KEY` | yes | IBM Cloud API key | | `IC_REGION` | yes | IBM Cloud region (e.g. `us-south`, `us-east`) | +| `IBMCLOUD_COS_ACCESS_KEY_ID` | only with S3 `--backed-url` | HMAC access key for IBM Cloud Object Storage | +| `IBMCLOUD_COS_SECRET_ACCESS_KEY` | only with S3 `--backed-url` | HMAC secret key for IBM Cloud Object Storage | +| `IBMCLOUD_COS_ENDPOINT` | no | COS S3 endpoint (defaults to `s3..cloud-object-storage.appdomain.cloud`) | ## Create @@ -134,6 +137,34 @@ podman run -d --name ibm-power \ --otel-auth-token ``` +## Using IBM Cloud Object Storage as S3 backend + +To store Pulumi state in IBM COS instead of a local file, create [HMAC credentials](https://cloud.ibm.com/docs/cloud-object-storage?topic=cloud-object-storage-uhc-hmac-credentials-main) for your COS instance and pass an `s3://` backed URL: + +```bash +podman run -d --name ibm-power \ + -v ${PWD}:/workspace:z \ + -e IBMCLOUD_API_KEY=XXX \ + -e IBMCLOUD_ACCOUNT=XXX \ + -e IC_REGION=us-south \ + -e IBMCLOUD_COS_ACCESS_KEY_ID=XXX \ + -e IBMCLOUD_COS_SECRET_ACCESS_KEY=XXX \ + quay.io/redhat-developer/mapt:v0.8.0 ibmcloud ibm-power create \ + --project-name ibm-power \ + --backed-url s3://my-cos-bucket \ + --conn-details-output /workspace \ + --workspace-id \ + --pi-private-subnet-id +``` + +An HTTPS endpoint URL is also supported as `--backed-url`, with the bucket name in the path: + +``` +--backed-url https://s3.us-south.cloud-object-storage.appdomain.cloud/my-cos-bucket +``` + +The COS endpoint and `PULUMI_BACKEND_URL` are constructed automatically from the region and bucket name. + ## Destroy ```bash @@ -144,4 +175,20 @@ podman run -d --name ibm-power \ quay.io/redhat-developer/mapt:v0.8.0 ibmcloud ibm-power destroy \ --project-name ibm-power \ --backed-url file:///workspace +``` + +By default, destroy removes the Pulumi state files from the backend after a successful destroy. Use `--keep-state` to preserve them: + +```bash +podman run -d --name ibm-power \ + -v ${PWD}:/workspace:z \ + -e IBMCLOUD_API_KEY=XXX \ + -e IBMCLOUD_ACCOUNT=XXX \ + -e IC_REGION=us-south \ + -e IBMCLOUD_COS_ACCESS_KEY_ID=XXX \ + -e IBMCLOUD_COS_SECRET_ACCESS_KEY=XXX \ + quay.io/redhat-developer/mapt:v0.8.0 ibmcloud ibm-power destroy \ + --project-name ibm-power \ + --backed-url s3://my-cos-bucket \ + --keep-state ``` \ No newline at end of file diff --git a/docs/ibmcloud/ibm-z.md b/docs/ibmcloud/ibm-z.md index 6ccc3f2ae..f8ce07f80 100644 --- a/docs/ibmcloud/ibm-z.md +++ b/docs/ibmcloud/ibm-z.md @@ -15,6 +15,9 @@ Two networking modes are supported: | `IBMCLOUD_API_KEY` | yes | IBM Cloud API key | | `IC_REGION` | yes | IBM Cloud region (e.g. `us-south`, `us-east`) | | `IC_ZONE` | only without `--subnet-id` | Availability zone (e.g. `us-south-2`) | +| `IBMCLOUD_COS_ACCESS_KEY_ID` | only with S3 `--backed-url` | HMAC access key for IBM Cloud Object Storage | +| `IBMCLOUD_COS_SECRET_ACCESS_KEY` | only with S3 `--backed-url` | HMAC secret key for IBM Cloud Object Storage | +| `IBMCLOUD_COS_ENDPOINT` | no | COS S3 endpoint (defaults to `s3..cloud-object-storage.appdomain.cloud`) | ## Create @@ -112,6 +115,33 @@ podman run -d --name ibm-z \ --otel-auth-token ``` +## Using IBM Cloud Object Storage as S3 backend + +To store Pulumi state in IBM COS instead of a local file, create [HMAC credentials](https://cloud.ibm.com/docs/cloud-object-storage?topic=cloud-object-storage-uhc-hmac-credentials-main) for your COS instance and pass an `s3://` backed URL: + +```bash +podman run -d --name ibm-z \ + -v ${PWD}:/workspace:z \ + -e IBMCLOUD_API_KEY=XXX \ + -e IBMCLOUD_ACCOUNT=XXX \ + -e IC_REGION=us-south \ + -e IC_ZONE=us-south-2 \ + -e IBMCLOUD_COS_ACCESS_KEY_ID=XXX \ + -e IBMCLOUD_COS_SECRET_ACCESS_KEY=XXX \ + quay.io/redhat-developer/mapt:v0.8.0 ibmcloud ibm-z create \ + --project-name ibm-z \ + --backed-url s3://my-cos-bucket \ + --conn-details-output /workspace +``` + +An HTTPS endpoint URL is also supported as `--backed-url`, with the bucket name in the path: + +``` +--backed-url https://s3.us-south.cloud-object-storage.appdomain.cloud/my-cos-bucket +``` + +The COS endpoint and `PULUMI_BACKEND_URL` are constructed automatically from the region and bucket name. + ## Destroy ```bash @@ -123,3 +153,19 @@ podman run -d --name ibm-z \ --project-name ibm-z \ --backed-url file:///workspace ``` + +By default, destroy removes the Pulumi state files from the backend after a successful destroy. Use `--keep-state` to preserve them: + +```bash +podman run -d --name ibm-z \ + -v ${PWD}:/workspace:z \ + -e IBMCLOUD_API_KEY=XXX \ + -e IBMCLOUD_ACCOUNT=XXX \ + -e IC_REGION=us-south \ + -e IBMCLOUD_COS_ACCESS_KEY_ID=XXX \ + -e IBMCLOUD_COS_SECRET_ACCESS_KEY=XXX \ + quay.io/redhat-developer/mapt:v0.8.0 ibmcloud ibm-z destroy \ + --project-name ibm-z \ + --backed-url s3://my-cos-bucket \ + --keep-state +``` diff --git a/pkg/manager/context/context.go b/pkg/manager/context/context.go index bb74bb301..bf21c5e33 100644 --- a/pkg/manager/context/context.go +++ b/pkg/manager/context/context.go @@ -74,7 +74,7 @@ type Context struct { } type Provider interface { - Init(ctx context.Context, backedURL string) error + Init(ctx context.Context, backedURL string) (string, error) DefaultHostingPlace() (*string, error) } @@ -110,9 +110,13 @@ func Init(ca *ContextArgs, provider Provider) (*Context, error) { c.targetHostingPlace = *hp } // Manage - if err := provider.Init(ctx, ca.BackedURL); err != nil { + resolvedURL, err := provider.Init(ctx, ca.BackedURL) + if err != nil { return nil, err } + if resolvedURL != "" { + c.backedURL = resolvedURL + } // Manage integrations if err := manageIntegration(c, ca); err != nil { return nil, err diff --git a/pkg/provider/aws/aws.go b/pkg/provider/aws/aws.go index dbe88bbff..8d1eb9333 100644 --- a/pkg/provider/aws/aws.go +++ b/pkg/provider/aws/aws.go @@ -25,11 +25,11 @@ const pulumiLocksPath = ".pulumi/locks" type AWS struct{} -func (a *AWS) Init(ctx context.Context, backedURL string) error { +func (a *AWS) Init(ctx context.Context, backedURL string) (string, error) { // Manage remote state requirements, if backedURL // is on a different region we need to change to that region // in order to interact with the state - return manageRemoteState(ctx, backedURL) + return "", manageRemoteState(ctx, backedURL) } func (a *AWS) DefaultHostingPlace() (*string, error) { @@ -217,6 +217,9 @@ func parseS3BackedURL(mCtx *mc.Context) (*string, *string, error) { return nil, nil, fmt.Errorf("failed to parse S3 URI: %w", err) } key := strings.TrimPrefix(u.Path, "/") + if key == "" { + return nil, nil, fmt.Errorf("invalid S3 URI %q: missing object key after bucket name", mCtx.BackedURL()) + } return &u.Host, &key, nil } diff --git a/pkg/provider/azure/azure.go b/pkg/provider/azure/azure.go index 1f7d5b922..5faab2a8e 100644 --- a/pkg/provider/azure/azure.go +++ b/pkg/provider/azure/azure.go @@ -28,9 +28,9 @@ func Provider() *Azure { return &Azure{} } -func (a *Azure) Init(ctx context.Context, backedURL string) error { +func (a *Azure) Init(ctx context.Context, backedURL string) (string, error) { setAZIdentityEnvs() - return nil + return "", nil } func (a *Azure) DefaultHostingPlace() (*string, error) { diff --git a/pkg/provider/ibmcloud/action/ibm-power/ibm-power.go b/pkg/provider/ibmcloud/action/ibm-power/ibm-power.go index 37a11d6b3..c46e31c12 100644 --- a/pkg/provider/ibmcloud/action/ibm-power/ibm-power.go +++ b/pkg/provider/ibmcloud/action/ibm-power/ibm-power.go @@ -142,7 +142,10 @@ func Destroy(mCtxArgs *mc.ContextArgs) (err error) { if err != nil { return err } - return ibmcloudp.Destroy(mCtx, stackIBMPowerVS) + if err := ibmcloudp.DestroyStack(mCtx, stackIBMPowerVS); err != nil { + return err + } + return ibmcloudp.CleanupState(mCtx) } func (r *pwRequest) deploy(ctx *pulumi.Context) error { diff --git a/pkg/provider/ibmcloud/action/ibm-z/ibm-z.go b/pkg/provider/ibmcloud/action/ibm-z/ibm-z.go index 37fbb1abb..3106a946c 100644 --- a/pkg/provider/ibmcloud/action/ibm-z/ibm-z.go +++ b/pkg/provider/ibmcloud/action/ibm-z/ibm-z.go @@ -136,7 +136,10 @@ func Destroy(mCtxArgs *mc.ContextArgs) (err error) { if err != nil { return err } - return ibmcloudp.Destroy(mCtx, stackIBMS390) + if err := ibmcloudp.DestroyStack(mCtx, stackIBMS390); err != nil { + return err + } + return ibmcloudp.CleanupState(mCtx) } func (r *zRequest) deploy(ctx *pulumi.Context) error { diff --git a/pkg/provider/ibmcloud/constants/constants.go b/pkg/provider/ibmcloud/constants/constants.go new file mode 100644 index 000000000..052fc8335 --- /dev/null +++ b/pkg/provider/ibmcloud/constants/constants.go @@ -0,0 +1,9 @@ +package constants + +const ( + EnvIBMCloudAccount = "IBMCLOUD_ACCOUNT" + EnvIBMCloudAPIKey = "IBMCLOUD_API_KEY" + EnvIBMCosAccessKeyID = "IBMCLOUD_COS_ACCESS_KEY_ID" + EnvIBMCosSecretAccessKey = "IBMCLOUD_COS_SECRET_ACCESS_KEY" + EnvIBMCosEndpoint = "IBMCLOUD_COS_ENDPOINT" +) diff --git a/pkg/provider/ibmcloud/data/piimages.go b/pkg/provider/ibmcloud/data/piimages.go index b72257ccf..91690f84a 100644 --- a/pkg/provider/ibmcloud/data/piimages.go +++ b/pkg/provider/ibmcloud/data/piimages.go @@ -11,6 +11,7 @@ import ( "github.com/IBM/go-sdk-core/v5/core" mc "github.com/redhat-developer/mapt/pkg/manager/context" + icConstants "github.com/redhat-developer/mapt/pkg/provider/ibmcloud/constants" ) const powerURLRegex = "%s.power-iaas.cloud.ibm.com" @@ -45,9 +46,9 @@ func GetImage(mCtx *mc.Context, args *PiImageArgs) (*string, error) { func piImagesClient(mCtx *mc.Context, cloudInstanceId string) (*v.IBMPIImageClient, error) { options := &ps.IBMPIOptions{ Authenticator: &core.IamAuthenticator{ - ApiKey: os.Getenv("IBMCLOUD_API_KEY"), + ApiKey: os.Getenv(icConstants.EnvIBMCloudAPIKey), }, - UserAccount: os.Getenv("IBMCLOUD_ACCOUNT"), + UserAccount: os.Getenv(icConstants.EnvIBMCloudAccount), Zone: os.Getenv("IC_ZONE"), URL: powerURL(os.Getenv("IC_REGION")), Debug: mCtx.Debug(), diff --git a/pkg/provider/ibmcloud/data/vpcimages.go b/pkg/provider/ibmcloud/data/vpcimages.go index 745e6cb9c..212ba5d43 100644 --- a/pkg/provider/ibmcloud/data/vpcimages.go +++ b/pkg/provider/ibmcloud/data/vpcimages.go @@ -9,6 +9,7 @@ import ( "github.com/IBM/vpc-go-sdk/vpcv1" "github.com/IBM/go-sdk-core/v5/core" + icConstants "github.com/redhat-developer/mapt/pkg/provider/ibmcloud/constants" ) const ( @@ -64,7 +65,7 @@ func vpcService() (*vpcv1.VpcV1, error) { } return vpcv1.NewVpcV1(&vpcv1.VpcV1Options{ Authenticator: &core.IamAuthenticator{ - ApiKey: os.Getenv("IBMCLOUD_API_KEY"), + ApiKey: os.Getenv(icConstants.EnvIBMCloudAPIKey), }, URL: serviceURL, }) diff --git a/pkg/provider/ibmcloud/ibmcloud.go b/pkg/provider/ibmcloud/ibmcloud.go index a001805ad..7250453f7 100644 --- a/pkg/provider/ibmcloud/ibmcloud.go +++ b/pkg/provider/ibmcloud/ibmcloud.go @@ -3,21 +3,30 @@ package ibmcloud import ( "context" "fmt" + "net/url" "os" + "strings" "github.com/redhat-developer/mapt/pkg/manager" mc "github.com/redhat-developer/mapt/pkg/manager/context" "github.com/redhat-developer/mapt/pkg/manager/credentials" + "github.com/redhat-developer/mapt/pkg/provider/aws/services/s3" + icConstants "github.com/redhat-developer/mapt/pkg/provider/ibmcloud/constants" + "github.com/redhat-developer/mapt/pkg/util/logging" ) const ( - LOCATION_ENV = "IC_REGION" + LOCATION_ENV = "IC_REGION" + pulumiLocksPath = ".pulumi/locks" ) type IBMCloud struct{} -func (i *IBMCloud) Init(ctx context.Context, backedURL string) error { - return nil +func (i *IBMCloud) Init(ctx context.Context, backedURL string) (string, error) { + if isCOSBackend(backedURL) { + return initCOSBackend(backedURL) + } + return "", nil } func (a *IBMCloud) DefaultHostingPlace() (*string, error) { @@ -50,6 +59,159 @@ var ( DefaultCredentials = GetClouProviderCredentials(nil) ) +const cosHostSuffix = "cloud-object-storage.appdomain.cloud" + +func isCOSBackend(backedURL string) bool { + return strings.HasPrefix(backedURL, "s3://") || + strings.Contains(backedURL, cosHostSuffix) +} + +func ensureHTTPS(endpoint string) string { + endpoint = strings.TrimSpace(endpoint) + if strings.HasPrefix(endpoint, "https://") { + return endpoint + } + if strings.HasPrefix(endpoint, "http://") { + return "https://" + strings.TrimPrefix(endpoint, "http://") + } + return "https://" + endpoint +} + +func requireEnv(name string) (string, error) { + v, ok := os.LookupEnv(name) + if !ok || v == "" { + return "", fmt.Errorf("%s is required when using S3-compatible backend", name) + } + return v, nil +} + +func extractBucket(backedURL string) (string, error) { + u, err := url.Parse(backedURL) + if err != nil { + return "", fmt.Errorf("failed to parse backed URL %q: %w", backedURL, err) + } + if strings.HasPrefix(backedURL, "s3://") { + if u.Host == "" { + return "", fmt.Errorf("backed URL %q missing bucket name (expected s3://bucket-name)", backedURL) + } + return u.Host, nil + } + bucket := strings.TrimPrefix(u.Path, "/") + if bucket == "" { + return "", fmt.Errorf("backed URL %q missing bucket name in path (expected https:///)", backedURL) + } + return strings.SplitN(bucket, "/", 2)[0], nil +} + +func initCOSBackend(backedURL string) (string, error) { + accessKey, err := requireEnv(icConstants.EnvIBMCosAccessKeyID) + if err != nil { + return "", err + } + secretKey, err := requireEnv(icConstants.EnvIBMCosSecretAccessKey) + if err != nil { + return "", err + } + region, err := requireEnv(LOCATION_ENV) + if err != nil { + return "", err + } + + endpoint, _ := os.LookupEnv(icConstants.EnvIBMCosEndpoint) + if endpoint == "" { + endpoint = fmt.Sprintf("s3.%s.cloud-object-storage.appdomain.cloud", region) + } + + bucket, err := extractBucket(backedURL) + if err != nil { + return "", err + } + + resolvedURL := fmt.Sprintf("s3://%s?endpoint=%s&s3ForcePathStyle=true", + bucket, endpoint) + + for k, v := range map[string]string{ + "AWS_ACCESS_KEY_ID": accessKey, + "AWS_SECRET_ACCESS_KEY": secretKey, + "AWS_ENDPOINT_URL": ensureHTTPS(endpoint), + "AWS_REGION": region, + "AWS_DEFAULT_REGION": region, + "AWS_S3_USE_PATH_STYLE": "true", + "PULUMI_BACKEND_URL": resolvedURL, + } { + if err := os.Setenv(k, v); err != nil { + return "", err + } + } + logging.Debugf("COS backend configured: %s", resolvedURL) + return resolvedURL, nil +} + +func parseCOSBackedURL(mCtx *mc.Context) (*string, *string, error) { + backendURL := os.Getenv("PULUMI_BACKEND_URL") + if backendURL == "" { + backendURL = mCtx.BackedURL() + } + if !strings.HasPrefix(backendURL, "s3://") { + return nil, nil, fmt.Errorf("invalid S3 URI %q: must start with s3://", backendURL) + } + u, err := url.Parse(backendURL) + if err != nil { + return nil, nil, fmt.Errorf("failed to parse S3 URI: %w", err) + } + key := strings.TrimPrefix(u.Path, "/") + if key == "" { + return nil, nil, fmt.Errorf("invalid S3 URI %q: missing object key after bucket name", backendURL) + } + return &u.Host, &key, nil +} + +func DestroyStack(mCtx *mc.Context, stackName string) error { + logging.Debug("Running destroy operation") + if len(stackName) == 0 { + return fmt.Errorf("stackname is required") + } + if mCtx.IsForceDestroy() { + bucket, key, err := parseCOSBackedURL(mCtx) + if err != nil { + logging.Error(err) + } else { + lockPathKey := fmt.Sprintf("%s/%s", *key, pulumiLocksPath) + if err := s3.Delete(mCtx.Context(), bucket, &lockPathKey); err != nil { + logging.Error(err) + } + } + } + stack := manager.Stack{ + StackName: mCtx.StackNameByProject(stackName), + ProjectName: mCtx.ProjectName(), + BackedURL: mCtx.BackedURL(), + ProviderCredentials: DefaultCredentials, + } + return manager.DestroyStack(mCtx, stack) +} + +func CleanupState(mCtx *mc.Context) error { + if mCtx.IsKeepState() { + return nil + } + + bucket, key, parseErr := parseCOSBackedURL(mCtx) + if parseErr != nil { + logging.Warnf("Failed to parse S3 backend URL, skipping state cleanup: %v", parseErr) + return nil + } + + logging.Infof("Cleaning up Pulumi state from s3://%s/%s", *bucket, *key) + if deleteErr := s3.Delete(mCtx.Context(), bucket, key); deleteErr != nil { + logging.Warnf("Failed to cleanup S3 state: %v", deleteErr) + } else { + logging.Info("Successfully cleaned up Pulumi state from S3") + } + + return nil +} + func Destroy(mCtx *mc.Context, stackName string) error { stack := manager.Stack{ StackName: mCtx.StackNameByProject(stackName), diff --git a/pkg/provider/ibmcloud/services/power/power.go b/pkg/provider/ibmcloud/services/power/power.go index e210f6645..487b39ca0 100644 --- a/pkg/provider/ibmcloud/services/power/power.go +++ b/pkg/provider/ibmcloud/services/power/power.go @@ -10,6 +10,7 @@ import ( "github.com/IBM-Cloud/power-go-client/power/models" "github.com/IBM/go-sdk-core/v5/core" mc "github.com/redhat-developer/mapt/pkg/manager/context" + icConstants "github.com/redhat-developer/mapt/pkg/provider/ibmcloud/constants" "github.com/redhat-developer/mapt/pkg/util/logging" ) @@ -70,9 +71,9 @@ func waitForInstance(mCtx *mc.Context, pc *v.IBMPIInstanceClient, instanceId str func client(mCtx *mc.Context, cloudInstanceId string) (*v.IBMPIInstanceClient, error) { options := &ps.IBMPIOptions{ Authenticator: &core.IamAuthenticator{ - ApiKey: os.Getenv("IBMCLOUD_API_KEY"), + ApiKey: os.Getenv(icConstants.EnvIBMCloudAPIKey), }, - UserAccount: os.Getenv("IBMCLOUD_ACCOUNT"), + UserAccount: os.Getenv(icConstants.EnvIBMCloudAccount), Zone: os.Getenv("IC_ZONE"), URL: powerURL(os.Getenv("IC_REGION")), Debug: mCtx.Debug(),