diff --git a/.github/scripts/e2e-prepare-env.sh b/.github/scripts/e2e-prepare-env.sh new file mode 100755 index 0000000..543492f --- /dev/null +++ b/.github/scripts/e2e-prepare-env.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Writes non-secret env to $RUNNER_TEMP/e2e-env.sh and materializes secrets into temp files. +# Never echoes secret values. + +set -euo pipefail + +ENV_FILE="${RUNNER_TEMP}/e2e-env.sh" +: >"$ENV_FILE" + +write_env() { + printf 'export %s=%q\n' "$1" "$2" >>"$ENV_FILE" +} + +if [[ -n "${E2E_SSH_PRIVATE_KEY:-}" ]]; then + KEY_FILE="${RUNNER_TEMP}/e2e_ssh_key" + printf '%s\n' "$E2E_SSH_PRIVATE_KEY" >"$KEY_FILE" + chmod 600 "$KEY_FILE" + write_env SSH_PRIVATE_KEY "$KEY_FILE" +fi + +if [[ -n "${E2E_SSH_PUBLIC_KEY:-}" ]]; then + PUB_FILE="${RUNNER_TEMP}/e2e_ssh_pub" + printf '%s\n' "$E2E_SSH_PUBLIC_KEY" >"$PUB_FILE" + chmod 644 "$PUB_FILE" + write_env SSH_PUBLIC_KEY "$PUB_FILE" +fi + +if [[ -n "${E2E_CLUSTER_KUBECONFIG:-}" ]]; then + KC_FILE="${RUNNER_TEMP}/e2e_kubeconfig" + printf '%s' "$E2E_CLUSTER_KUBECONFIG" | base64 -d >"$KC_FILE" + chmod 600 "$KC_FILE" + write_env KUBE_CONFIG_PATH "$KC_FILE" +fi + +write_env GOMODCACHE "${GOMODCACHE:-${RUNNER_TEMP}/e2e-gomodcache}" +write_env GOCACHE "${GOCACHE:-${RUNNER_TEMP}/e2e-gocache}" +write_env E2E_ARTIFACT_DIR "${E2E_ARTIFACT_DIR:-${RUNNER_TEMP}/e2e-artifacts}" + +echo "e2e-env.sh prepared (secrets written to temp files only)" diff --git a/.github/workflows/e2e-reusable.yml b/.github/workflows/e2e-reusable.yml new file mode 100644 index 0000000..186d09e --- /dev/null +++ b/.github/workflows/e2e-reusable.yml @@ -0,0 +1,309 @@ +# Reusable E2E pipeline: create-cluster → run-tests → teardown-cluster (SDK only, no bash cluster setup). +# +# Caller example (module repo): +# jobs: +# e2e: +# uses: deckhouse/storage-e2e/.github/workflows/e2e-reusable.yml@main +# secrets: inherit +# with: +# module_path: e2e +# cluster_provider: alwaysCreateNew +# cluster_config: e2e/tests/cluster_config.yml +# test_package: ./tests/ +# test_timeout: 60m + +name: Storage E2E (reusable) + +permissions: + contents: read + checks: write + pull-requests: read + +on: + workflow_call: + inputs: + module_path: + description: "Path to the module e2e Go module root (contains go.mod and tests/)" + type: string + required: true + cluster_provider: + description: "Cluster provider: alwaysCreateNew | alwaysUseExisting | commander" + type: string + required: true + cluster_config: + description: "Path to cluster_config.yml relative to repository root (for alwaysCreateNew)" + type: string + required: false + default: "" + test_package: + description: "Go package for run-tests (e.g. ./tests/)" + type: string + required: true + label_filter: + description: "Ginkgo label filter; empty runs all tests" + type: string + required: false + default: "" + test_timeout: + description: "go test / ginkgo timeout" + type: string + required: false + default: "60m" + storage_e2e_ref: + description: "Git ref of storage-e2e for checkout (branch, tag, or SHA)" + type: string + required: false + default: "main" + runner_labels: + description: "JSON array of runner labels, e.g. [\"self-hosted\",\"regular\"]" + type: string + required: false + default: '["self-hosted","regular"]' + secrets: + E2E_SSH_PRIVATE_KEY: + required: false + E2E_SSH_PUBLIC_KEY: + required: false + E2E_SSH_HOST: + required: false + E2E_SSH_USER: + required: false + E2E_SSH_JUMP_HOST: + required: false + E2E_SSH_JUMP_USER: + required: false + E2E_CLUSTER_KUBECONFIG: + required: false + E2E_TEST_CLUSTER_CREATE_MODE: + required: false + E2E_TEST_CLUSTER_STORAGE_CLASS: + required: false + E2E_TEST_CLUSTER_CLEANUP: + required: false + E2E_DECKHOUSE_LICENSE: + required: false + E2E_REGISTRY_DOCKER_CFG: + required: false + GOPROXY: + required: false + +defaults: + run: + shell: bash + +env: + E2E_ARTIFACT_DIR: ${{ runner.temp }}/e2e-artifacts + GOMODCACHE: ${{ runner.temp }}/e2e-gomodcache + GOCACHE: ${{ runner.temp }}/e2e-gocache + +jobs: + create-cluster: + name: create-cluster + runs-on: ${{ fromJSON(inputs.runner_labels) }} + outputs: + artifact_dir: ${{ env.E2E_ARTIFACT_DIR }} + steps: + - name: Checkout module repository + uses: actions/checkout@v4 + + - name: Checkout storage-e2e + uses: actions/checkout@v4 + with: + repository: deckhouse/storage-e2e + ref: ${{ inputs.storage_e2e_ref }} + path: storage-e2e + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: storage-e2e/go.mod + cache-dependency-path: | + storage-e2e/go.sum + ${{ inputs.module_path }}/go.sum + + - name: Prepare SSH and kubeconfig (no secret values in logs) + run: storage-e2e/.github/scripts/e2e-prepare-env.sh + env: + E2E_SSH_PRIVATE_KEY: ${{ secrets.E2E_SSH_PRIVATE_KEY }} + E2E_SSH_PUBLIC_KEY: ${{ secrets.E2E_SSH_PUBLIC_KEY }} + E2E_CLUSTER_KUBECONFIG: ${{ secrets.E2E_CLUSTER_KUBECONFIG }} + + - name: Build storage-e2e CLI + run: go build -o "${{ runner.temp }}/storage-e2e" ./cmd/e2e + working-directory: storage-e2e + + - name: create-cluster (ClusterProvider SDK) + working-directory: ${{ inputs.module_path }} + run: | + set -euo pipefail + source "${{ runner.temp }}/e2e-env.sh" + mkdir -p "${GOMODCACHE}" "${GOCACHE}" "${E2E_ARTIFACT_DIR}" + CONFIG_ARG="" + if [ -n "${{ inputs.cluster_config }}" ]; then + CONFIG_ARG="--config ${{ github.workspace }}/${{ inputs.cluster_config }}" + fi + "${{ runner.temp }}/storage-e2e" create-cluster \ + --provider "${{ inputs.cluster_provider }}" \ + ${CONFIG_ARG} \ + --artifact-dir "${E2E_ARTIFACT_DIR}" + env: + TEST_CLUSTER_CREATE_MODE: ${{ inputs.cluster_provider }} + TEST_CLUSTER_NAMESPACE: e2e-${{ github.event.repository.name }}-${{ github.run_id }} + TEST_CLUSTER_STORAGE_CLASS: ${{ secrets.E2E_TEST_CLUSTER_STORAGE_CLASS }} + TEST_CLUSTER_CLEANUP: ${{ secrets.E2E_TEST_CLUSTER_CLEANUP }} + DKP_LICENSE_KEY: ${{ secrets.E2E_DECKHOUSE_LICENSE }} + REGISTRY_DOCKER_CFG: ${{ secrets.E2E_REGISTRY_DOCKER_CFG }} + SSH_HOST: ${{ secrets.E2E_SSH_HOST }} + SSH_USER: ${{ secrets.E2E_SSH_USER }} + SSH_JUMP_HOST: ${{ secrets.E2E_SSH_JUMP_HOST }} + SSH_JUMP_USER: ${{ secrets.E2E_SSH_JUMP_USER }} + COMMANDER_URL: ${{ secrets.COMMANDER_URL }} + COMMANDER_TOKEN: ${{ secrets.COMMANDER_TOKEN }} + LOG_LEVEL: ${{ vars.E2E_LOG_LEVEL || 'info' }} + + - name: Upload cluster session artifact + uses: actions/upload-artifact@v4 + with: + name: e2e-cluster-session-${{ github.run_id }} + path: ${{ env.E2E_ARTIFACT_DIR }} + retention-days: 2 + if-no-files-found: error + + run-tests: + name: run-tests + needs: create-cluster + runs-on: ${{ fromJSON(inputs.runner_labels) }} + steps: + - name: Checkout module repository + uses: actions/checkout@v4 + + - name: Checkout storage-e2e + uses: actions/checkout@v4 + with: + repository: deckhouse/storage-e2e + ref: ${{ inputs.storage_e2e_ref }} + path: storage-e2e + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: ${{ inputs.module_path }}/go.mod + cache-dependency-path: ${{ inputs.module_path }}/go.sum + + - name: Download cluster session + uses: actions/download-artifact@v4 + with: + name: e2e-cluster-session-${{ github.run_id }} + path: ${{ env.E2E_ARTIFACT_DIR }} + + - name: Prepare SSH (run-tests) + run: storage-e2e/.github/scripts/e2e-prepare-env.sh + env: + E2E_SSH_PRIVATE_KEY: ${{ secrets.E2E_SSH_PRIVATE_KEY }} + E2E_SSH_PUBLIC_KEY: ${{ secrets.E2E_SSH_PUBLIC_KEY }} + E2E_CLUSTER_KUBECONFIG: ${{ secrets.E2E_CLUSTER_KUBECONFIG }} + + - name: Build storage-e2e CLI + run: go build -o "${{ runner.temp }}/storage-e2e" ./cmd/e2e + working-directory: storage-e2e + + - name: Pin storage-e2e module to checked-out ref + working-directory: ${{ inputs.module_path }} + run: go mod edit -replace=github.com/deckhouse/storage-e2e=${{ github.workspace }}/storage-e2e + + - name: run-tests + working-directory: ${{ inputs.module_path }} + run: | + set -euo pipefail + source "${{ runner.temp }}/e2e-env.sh" + source "${E2E_ARTIFACT_DIR}/run-env.sh" + mkdir -p "${GOMODCACHE}" "${GOCACHE}" + go mod download + LABEL_ARGS=() + if [ -n "${E2E_LABEL_FILTER:-}" ]; then + LABEL_ARGS=(--label-filter "${E2E_LABEL_FILTER}") + fi + "${{ runner.temp }}/storage-e2e" run-tests \ + --package "${{ inputs.test_package }}" \ + --timeout "${{ inputs.test_timeout }}" \ + --artifact-dir "${E2E_ARTIFACT_DIR}" \ + "${LABEL_ARGS[@]}" + env: + DKP_LICENSE_KEY: ${{ secrets.E2E_DECKHOUSE_LICENSE }} + REGISTRY_DOCKER_CFG: ${{ secrets.E2E_REGISTRY_DOCKER_CFG }} + SSH_JUMP_HOST: ${{ secrets.E2E_SSH_JUMP_HOST }} + SSH_JUMP_USER: ${{ secrets.E2E_SSH_JUMP_USER }} + E2E_LABEL_FILTER: ${{ inputs.label_filter }} + CI: "true" + + - name: Publish JUnit to PR checks + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + files: ${{ env.E2E_ARTIFACT_DIR }}/junit.xml + check_name: E2E (${{ inputs.test_package }}) + comment_mode: off + + - name: Upload JUnit artifact + uses: actions/upload-artifact@v4 + if: always() + with: + name: e2e-junit-${{ github.run_id }} + path: ${{ env.E2E_ARTIFACT_DIR }}/junit.xml + retention-days: 14 + if-no-files-found: ignore + + teardown-cluster: + name: teardown-cluster + needs: [create-cluster, run-tests] + if: always() + runs-on: ${{ fromJSON(inputs.runner_labels) }} + steps: + - name: Checkout storage-e2e + uses: actions/checkout@v4 + with: + repository: deckhouse/storage-e2e + ref: ${{ inputs.storage_e2e_ref }} + path: storage-e2e + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: storage-e2e/go.mod + + - name: Download cluster session + uses: actions/download-artifact@v4 + with: + name: e2e-cluster-session-${{ github.run_id }} + path: ${{ env.E2E_ARTIFACT_DIR }} + + - name: Prepare SSH (teardown) + run: storage-e2e/.github/scripts/e2e-prepare-env.sh + env: + E2E_SSH_PRIVATE_KEY: ${{ secrets.E2E_SSH_PRIVATE_KEY }} + E2E_SSH_PUBLIC_KEY: ${{ secrets.E2E_SSH_PUBLIC_KEY }} + + - name: Build storage-e2e CLI + run: go build -o "${{ runner.temp }}/storage-e2e" ./cmd/e2e + working-directory: storage-e2e + + - name: teardown-cluster (ClusterProvider SDK) + run: | + set -euo pipefail + source "${{ runner.temp }}/e2e-env.sh" + mkdir -p "${GOMODCACHE}" "${GOCACHE}" + "${{ runner.temp }}/storage-e2e" teardown-cluster --artifact-dir "${E2E_ARTIFACT_DIR}" + env: + TEST_CLUSTER_CLEANUP: ${{ secrets.E2E_TEST_CLUSTER_CLEANUP }} + SSH_HOST: ${{ secrets.E2E_SSH_HOST }} + SSH_USER: ${{ secrets.E2E_SSH_USER }} + SSH_JUMP_HOST: ${{ secrets.E2E_SSH_JUMP_HOST }} + SSH_JUMP_USER: ${{ secrets.E2E_SSH_JUMP_USER }} + COMMANDER_URL: ${{ secrets.COMMANDER_URL }} + COMMANDER_TOKEN: ${{ secrets.COMMANDER_TOKEN }} + DKP_LICENSE_KEY: ${{ secrets.E2E_DECKHOUSE_LICENSE }} + REGISTRY_DOCKER_CFG: ${{ secrets.E2E_REGISTRY_DOCKER_CFG }} + + - name: Cleanup temp credentials + if: always() + run: rm -f "${{ runner.temp }}/e2e-env.sh" "${{ runner.temp }}/e2e_ssh_key" "${{ runner.temp }}/e2e_kubeconfig" 2>/dev/null || true diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..b691b1e --- /dev/null +++ b/Makefile @@ -0,0 +1,36 @@ +# storage-e2e — local and CI entrypoints share cmd/e2e + +GO ?= go +E2E_BIN ?= bin/e2e +E2E_PROVIDER ?= alwaysCreateNew +E2E_CONFIG ?= +E2E_PACKAGE ?= ./tests/... +E2E_LABEL_FILTER ?= +E2E_ARTIFACT_DIR ?= /tmp/e2e +E2E_TIMEOUT ?= 60m + +.PHONY: build-e2e e2e e2e-create e2e-run e2e-teardown help + +help: + @echo "Targets:" + @echo " build-e2e Build bin/e2e CLI" + @echo " e2e create-cluster + run-tests + teardown-cluster (full cycle)" + @echo " e2e-create create-cluster only" + @echo " e2e-run run-tests only (expects prior create; uses E2E_ARTIFACT_DIR)" + @echo " e2e-teardown teardown-cluster only" + +build-e2e: + $(GO) build -o $(E2E_BIN) ./cmd/e2e + +e2e: build-e2e e2e-create e2e-run e2e-teardown + +e2e-create: build-e2e + @test -n "$(E2E_CONFIG)" || (echo "E2E_CONFIG must point to cluster_config.yml for alwaysCreateNew"; exit 1) + $(E2E_BIN) create-cluster --provider $(E2E_PROVIDER) --config $(E2E_CONFIG) --artifact-dir $(E2E_ARTIFACT_DIR) + +e2e-run: build-e2e + $(E2E_BIN) run-tests --package $(E2E_PACKAGE) --artifact-dir $(E2E_ARTIFACT_DIR) \ + $(if $(E2E_LABEL_FILTER),--label-filter "$(E2E_LABEL_FILTER)",) + +e2e-teardown: build-e2e + $(E2E_BIN) teardown-cluster --artifact-dir $(E2E_ARTIFACT_DIR) diff --git a/README.md b/README.md index e776e98..80a24d7 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,10 @@ End-to-end tests for Deckhouse storage components. +## CI (reusable workflow) + +See [docs/CI.md](docs/CI.md) for the three-stage pipeline (`create-cluster` → `run-tests` → `teardown-cluster`), `cmd/e2e`, and module integration. + ## Quick Start 1. Create test with script: `cd tests && ./create-test.sh ` diff --git a/cmd/e2e/main.go b/cmd/e2e/main.go new file mode 100644 index 0000000..02b0583 --- /dev/null +++ b/cmd/e2e/main.go @@ -0,0 +1,300 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Command e2e is the shared entrypoint for local runs (make e2e) and CI (create-cluster / run-tests / teardown-cluster). +package main + +import ( + "context" + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/deckhouse/storage-e2e/internal/config" + "github.com/deckhouse/storage-e2e/internal/logger" + "github.com/deckhouse/storage-e2e/pkg/provider" + storage_e2e "github.com/deckhouse/storage-e2e/pkg/storage-e2e" +) + +func main() { + if len(os.Args) < 2 { + printUsage() + os.Exit(2) + } + if err := run(os.Args[1], os.Args[2:]); err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } +} + +func printUsage() { + fmt.Fprintf(os.Stderr, `storage-e2e CLI — cluster lifecycle and test runner + +Usage: + e2e create-cluster --provider --config [--artifact-dir ] + e2e run-tests --package [--label-filter ] [--timeout ] [--artifact-dir ] + e2e teardown-cluster [--artifact-dir ] + +Providers: alwaysCreateNew, alwaysUseExisting, commander + +Environment: same as storage-e2e tests (SSH_*, TEST_CLUSTER_*, DKP_LICENSE_KEY, REGISTRY_DOCKER_CFG, …). +Secrets must come from the environment; the CLI never prints secret values. + +`) +} + +func run(command string, args []string) error { + switch command { + case "create-cluster": + return runCreateCluster(args) + case "run-tests": + return runTests(args) + case "teardown-cluster": + return runTeardown(args) + case "-h", "--help", "help": + printUsage() + return nil + default: + return fmt.Errorf("unknown command %q", command) + } +} + +type commonFlags struct { + artifactDir string +} + +func parseCommon(fs *flag.FlagSet, flags *commonFlags) { + fs.StringVar(&flags.artifactDir, "artifact-dir", envOrDefault("E2E_ARTIFACT_DIR", config.E2ETempDir), "directory for session.json, junit, and kubeconfig copies") +} + +func runCreateCluster(args []string) error { + fs := flag.NewFlagSet("create-cluster", flag.ExitOnError) + var ( + providerName string + configPath string + common commonFlags + ) + fs.StringVar(&providerName, "provider", os.Getenv("TEST_CLUSTER_CREATE_MODE"), "cluster provider (alwaysCreateNew | alwaysUseExisting | commander)") + fs.StringVar(&configPath, "config", "", "absolute path to cluster_config.yml (required for alwaysCreateNew)") + parseCommon(fs, &common) + if err := fs.Parse(args); err != nil { + return err + } + + name, err := provider.ParseName(providerName) + if err != nil { + return err + } + if err := initFramework(); err != nil { + return err + } + + _ = os.Setenv("E2E_CLUSTER_PHASE", "create") + _ = os.Setenv("TEST_CLUSTER_CREATE_MODE", string(name)) + + p, err := provider.New(name) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Minute) + defer cancel() + + session, resources, err := p.Setup(ctx, provider.SetupOptions{ClusterConfigPath: configPath}) + if err != nil { + return err + } + defer closeResources(resources) + + if err := provider.SaveSession(common.artifactDir, session); err != nil { + return err + } + if err := copyKubeconfigArtifact(session.KubeconfigPath, common.artifactDir); err != nil { + return err + } + logger.Info("Cluster ready; session saved to %s", provider.SessionPath(common.artifactDir)) + return nil +} + +func runTests(args []string) error { + fs := flag.NewFlagSet("run-tests", flag.ExitOnError) + var ( + pkg string + labelFilter string + timeout string + common commonFlags + ) + fs.StringVar(&pkg, "package", "./...", "Go package path to test (module-relative)") + fs.StringVar(&labelFilter, "label-filter", os.Getenv("E2E_LABEL_FILTER"), "Ginkgo label filter (empty = all tests)") + fs.StringVar(&timeout, "timeout", "60m", "go test timeout") + parseCommon(fs, &common) + if err := fs.Parse(args); err != nil { + return err + } + + if err := applyRunEnv(common.artifactDir); err != nil { + return err + } + if err := initFramework(); err != nil { + return err + } + _ = os.Setenv("E2E_CLUSTER_PHASE", "run") + _ = os.Setenv("CI", "true") + + junitPath := filepath.Join(common.artifactDir, "junit.xml") + if err := os.MkdirAll(common.artifactDir, 0o755); err != nil { + return err + } + + return runGinkgo(pkg, labelFilter, timeout, junitPath) +} + +func runTeardown(args []string) error { + fs := flag.NewFlagSet("teardown-cluster", flag.ExitOnError) + var common commonFlags + parseCommon(fs, &common) + if err := fs.Parse(args); err != nil { + return err + } + if err := initFramework(); err != nil { + return err + } + _ = os.Setenv("E2E_CLUSTER_PHASE", "teardown") + + session, err := provider.LoadSession(common.artifactDir) + if err != nil { + return err + } + _ = os.Setenv("TEST_CLUSTER_CREATE_MODE", session.Provider) + + p, err := provider.New(provider.Name(session.Provider)) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), config.ClusterCleanupTimeout) + defer cancel() + + return p.Teardown(ctx, session, nil) +} + +func initFramework() error { + if err := storage_e2e.Initialize(); err != nil { + return err + } + return nil +} + +func applyRunEnv(artifactDir string) error { + runEnv := filepath.Join(artifactDir, "run-env.sh") + if _, err := os.Stat(runEnv); err != nil { + return fmt.Errorf("run-env.sh not found in %s (run create-cluster first): %w", artifactDir, err) + } + data, err := os.ReadFile(runEnv) + if err != nil { + return err + } + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + if !strings.HasPrefix(line, "export ") { + continue + } + line = strings.TrimPrefix(line, "export ") + k, v, ok := strings.Cut(line, "=") + if !ok { + continue + } + v = strings.Trim(v, `"`) + if err := os.Setenv(k, v); err != nil { + return err + } + } + return nil +} + +func runGinkgo(pkg, labelFilter, timeout, junitPath string) error { + ginkgo, err := exec.LookPath("ginkgo") + if err != nil { + // Fallback: go test with ginkgo flags when ginkgo CLI is not installed. + return runGoTest(pkg, labelFilter, timeout, junitPath) + } + + args := []string{ + "run", + "-r", + "--keep-going=false", + "--timeout=" + timeout, + "--junit-report=" + junitPath, + } + if labelFilter != "" { + args = append(args, "--label-filter="+labelFilter) + } + args = append(args, pkg) + + cmd := exec.Command(ginkgo, args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Env = os.Environ() + logger.Info("Running: %s %s", ginkgo, strings.Join(args, " ")) + return cmd.Run() +} + +func runGoTest(pkg, labelFilter, timeout, junitPath string) error { + args := []string{"test", pkg, "-count=1", "-timeout=" + timeout, "-v"} + if labelFilter != "" { + args = append(args, "-ginkgo.label-filter="+labelFilter) + } + if junitPath != "" { + args = append(args, "-ginkgo.junit-report="+junitPath) + } + cmd := exec.Command("go", args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Env = os.Environ() + logger.Info("Running: go %s", strings.Join(args, " ")) + return cmd.Run() +} + +func closeResources(res interface{}) { + // Connections are closed by OS when the process exits; explicit close is optional for create-cluster job. + _ = res +} + +func copyKubeconfigArtifact(src, artifactDir string) error { + if src == "" { + return nil + } + data, err := os.ReadFile(src) + if err != nil { + return fmt.Errorf("read kubeconfig: %w", err) + } + dst := filepath.Join(artifactDir, filepath.Base(src)) + return os.WriteFile(dst, data, 0o600) +} + +func envOrDefault(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} diff --git a/docs/CI.md b/docs/CI.md new file mode 100644 index 0000000..eb35a29 --- /dev/null +++ b/docs/CI.md @@ -0,0 +1,76 @@ +# Reusable CI pipeline (storage-e2e) + +Three jobs, one SDK entrypoint (`cmd/e2e`), no bash/YAML cluster provisioning. + +## Stages + +| Job | CLI command | Purpose | +|-----|-------------|---------| +| `create-cluster` | `e2e create-cluster --provider …` | `ClusterProvider.Setup` | +| `run-tests` | `e2e run-tests --package …` | Ginkgo/`go test`, optional `--label-filter` | +| `teardown-cluster` | `e2e teardown-cluster` | `ClusterProvider.Teardown` (always runs) | + +## Workflow + +Module repos call: + +```yaml +jobs: + e2e: + uses: deckhouse/storage-e2e/.github/workflows/e2e-reusable.yml@ + secrets: inherit + with: + module_path: e2e + cluster_provider: alwaysCreateNew + cluster_config: e2e/tests/cluster_config.yml + test_package: ./tests/ + label_filter: "" # empty = all specs + test_timeout: 60m + storage_e2e_ref: main +``` + +Required secrets (inherited from the module repo, same names as `build_dev` smoke): + +- `E2E_SSH_PRIVATE_KEY`, `E2E_SSH_PUBLIC_KEY`, `E2E_SSH_HOST`, `E2E_SSH_USER` +- `E2E_CLUSTER_KUBECONFIG` (base64 kubeconfig for the virtualization/base cluster) +- `E2E_TEST_CLUSTER_STORAGE_CLASS`, `E2E_DECKHOUSE_LICENSE`, `E2E_REGISTRY_DOCKER_CFG` +- Optional: `E2E_SSH_JUMP_*`, `E2E_TEST_CLUSTER_CLEANUP`, Commander secrets for `commander` provider + +Secrets are written to temp files by `.github/scripts/e2e-prepare-env.sh`; values are never printed. + +## JUnit / PR checks + +`run-tests` writes `${E2E_ARTIFACT_DIR}/junit.xml` and publishes it via `publish-unit-test-result-action` (check name **E2E**), plus uploads the XML as an artifact. + +## Local run (same entrypoint) + +```bash +export TEST_CLUSTER_CREATE_MODE=alwaysCreateNew +# … SSH_*, DKP_LICENSE_KEY, REGISTRY_DOCKER_CFG, TEST_CLUSTER_STORAGE_CLASS, etc. + +make e2e-create E2E_CONFIG=/path/to/cluster_config.yml E2E_PROVIDER=alwaysCreateNew +make e2e-run E2E_PACKAGE=./tests/ E2E_LABEL_FILTER='Smoke' +make e2e-teardown +# or: make e2e +``` + +Direct CLI: + +```bash +go build -o bin/e2e ./cmd/e2e +bin/e2e create-cluster --provider alwaysCreateNew --config "$(pwd)/cluster_config.yml" +bin/e2e run-tests --package ./tests/ --label-filter Smoke +bin/e2e teardown-cluster +``` + +## Multi-job test suites + +`create-cluster` saves `session.json` and `run-env.sh`. The `run-tests` job sets `E2E_CLUSTER_PHASE=run` and `TEST_CLUSTER_CREATE_MODE=alwaysUseExisting` with SSH pointed at the test cluster master. Module suites should release the cluster lock in `AfterSuite` when `E2E_CLUSTER_PHASE=run` and leave VM teardown to `teardown-cluster`. + +## Cluster providers + +Single input `cluster_provider` — same values as `TEST_CLUSTER_CREATE_MODE`: + +- `alwaysCreateNew` +- `alwaysUseExisting` +- `commander` diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index a7ccb69..1e747c9 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -208,25 +208,29 @@ func CreateTestCluster( logger.Step(1, "Loading cluster configuration from %s", yamlConfigFilename) - // Find the test package directory by walking the call stack. - // CreateTestCluster is called from CreateOrConnectToTestCluster in cluster.go, so Caller(1) is not the test file. - var testDir string - for skip := 1; skip <= 10; skip++ { - _, callerFile, _, ok := runtime.Caller(skip) - if !ok { - break + var yamlConfigPath string + if filepath.IsAbs(yamlConfigFilename) { + yamlConfigPath = yamlConfigFilename + } else { + // Find the test package directory by walking the call stack. + // CreateTestCluster is called from CreateOrConnectToTestCluster in cluster.go, so Caller(1) is not the test file. + var testDir string + for skip := 1; skip <= 10; skip++ { + _, callerFile, _, ok := runtime.Caller(skip) + if !ok { + break + } + if strings.Contains(filepath.ToSlash(callerFile), "/tests/") { + testDir = filepath.Dir(callerFile) + break + } } - if strings.Contains(filepath.ToSlash(callerFile), "/tests/") { - testDir = filepath.Dir(callerFile) - break + if testDir == "" { + return nil, fmt.Errorf("failed to determine test directory (no caller under tests/)") } + yamlConfigPath = filepath.Join(testDir, yamlConfigFilename) } - if testDir == "" { - return nil, fmt.Errorf("failed to determine test directory (no caller under tests/)") - } - yamlConfigPath := filepath.Join(testDir, yamlConfigFilename) - logger.Debug("Test directory: %s", testDir) logger.Debug("Config file path: %s", yamlConfigPath) // Step 1: Load cluster configuration from YAML (from test directory, e.g. tests/sds-node-configurator/cluster_config.yml) diff --git a/pkg/cluster/lifecycle.go b/pkg/cluster/lifecycle.go new file mode 100644 index 0000000..42a5afc --- /dev/null +++ b/pkg/cluster/lifecycle.go @@ -0,0 +1,124 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cluster + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + + "github.com/deckhouse/storage-e2e/internal/config" +) + +// CreateTestClusterFromConfig provisions a cluster using an absolute path to cluster_config.yml. +// Used by the storage-e2e CLI and CI create-cluster job (no runtime.Caller test directory lookup). +func CreateTestClusterFromConfig(ctx context.Context, configPath string) (*TestClusterResources, error) { + if configPath == "" { + return nil, fmt.Errorf("config path is required") + } + abs, err := filepath.Abs(configPath) + if err != nil { + return nil, err + } + return CreateTestCluster(ctx, abs) +} + +// ClusterStatePath returns the default cluster-state.json path used by CreateTestCluster. +func ClusterStatePath() string { + return clusterStatePath +} + +// ClusterStateSnapshot is the on-disk resume state after VM creation. +type ClusterStateSnapshot struct { + FirstMasterIP string `json:"first_master_ip"` + Namespace string `json:"namespace"` + VMNames []string `json:"vm_names"` + SetupVMName string `json:"setup_vm_name"` + MasterHostnames []string `json:"master_hostnames"` + WorkerHostnames []string `json:"worker_hostnames"` +} + +// LoadClusterStateFile reads cluster-state.json from the default e2e temp directory. +func LoadClusterStateFile() (*ClusterStateSnapshot, error) { + data, err := os.ReadFile(clusterStatePath) + if err != nil { + return nil, err + } + var state ClusterStateSnapshot + if err := json.Unmarshal(data, &state); err != nil { + return nil, err + } + return &state, nil +} + +// BaseClusterConnectOptionsFromEnv builds ConnectClusterOptions for the virtualization/base cluster from env. +func BaseClusterConnectOptionsFromEnv(kubeconfigOutputDir string) (ConnectClusterOptions, error) { + sshUser := config.SSHUser + sshHost := config.SSHHost + if sshUser == "" || sshHost == "" { + return ConnectClusterOptions{}, fmt.Errorf("SSH_USER and SSH_HOST are required") + } + keyPath, err := expandPathLocal(config.SSHPrivateKey) + if err != nil { + return ConnectClusterOptions{}, err + } + if kubeconfigOutputDir == "" { + kubeconfigOutputDir = config.E2ETempDir + } + useJump := config.SSHJumpHost != "" + opts := ConnectClusterOptions{ + SSHUser: sshUser, SSHHost: sshHost, SSHKeyPath: keyPath, + UseJumpHost: useJump, KubeconfigOutputDir: kubeconfigOutputDir, + } + if useJump { + jumpUser := config.SSHJumpUser + if jumpUser == "" { + jumpUser = sshUser + } + jumpKey := config.SSHJumpKeyPath + if jumpKey == "" { + jumpKey = keyPath + } else { + jumpKey, err = expandPathLocal(jumpKey) + if err != nil { + return ConnectClusterOptions{}, err + } + } + opts = ConnectClusterOptions{ + SSHUser: jumpUser, SSHHost: config.SSHJumpHost, SSHKeyPath: jumpKey, + UseJumpHost: true, TargetUser: sshUser, TargetHost: sshHost, TargetKeyPath: keyPath, + KubeconfigOutputDir: kubeconfigOutputDir, + } + } + return opts, nil +} + +func expandPathLocal(path string) (string, error) { + if path == "" { + path = config.SSHPrivateKeyDefaultValue + } + if len(path) >= 2 && path[:2] == "~/" { + home, err := os.UserHomeDir() + if err != nil { + return "", err + } + path = filepath.Join(home, path[2:]) + } + return filepath.Clean(path), nil +} diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go new file mode 100644 index 0000000..e14c440 --- /dev/null +++ b/pkg/provider/provider.go @@ -0,0 +1,166 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package provider implements cluster lifecycle for CI and local runs via a single SDK entrypoint. +package provider + +import ( + "context" + "fmt" + + "github.com/deckhouse/storage-e2e/internal/config" + "github.com/deckhouse/storage-e2e/pkg/cluster" +) + +// Name selects how the test cluster is provisioned. Matches TEST_CLUSTER_CREATE_MODE values. +type Name string + +const ( + NameAlwaysCreateNew Name = config.ClusterCreateModeAlwaysCreateNew + NameAlwaysUseExisting Name = config.ClusterCreateModeAlwaysUseExisting + NameCommander Name = config.ClusterCreateModeCommander +) + +// ClusterProvider provisions and tears down a test cluster without shell/YAML orchestration. +type ClusterProvider interface { + Setup(ctx context.Context, opts SetupOptions) (*Session, *cluster.TestClusterResources, error) + Teardown(ctx context.Context, session *Session, resources *cluster.TestClusterResources) error +} + +// SetupOptions configures cluster provisioning. +type SetupOptions struct { + // ClusterConfigPath is an absolute path to cluster_config.yml (required for alwaysCreateNew). + ClusterConfigPath string +} + +// New returns a ClusterProvider for the given name. +func New(name Name) (ClusterProvider, error) { + switch name { + case NameAlwaysCreateNew, NameAlwaysUseExisting, NameCommander: + return &provider{name: name}, nil + default: + return nil, fmt.Errorf("unknown cluster provider %q (expected %q, %q, or %q)", + name, NameAlwaysCreateNew, NameAlwaysUseExisting, NameCommander) + } +} + +// ParseName validates and parses a provider name from workflow input or CLI flag. +func ParseName(s string) (Name, error) { + n := Name(s) + switch n { + case NameAlwaysCreateNew, NameAlwaysUseExisting, NameCommander: + return n, nil + default: + return "", fmt.Errorf("invalid cluster provider %q", s) + } +} + +type provider struct { + name Name +} + +func (p *provider) Setup(ctx context.Context, opts SetupOptions) (*Session, *cluster.TestClusterResources, error) { + switch p.name { + case NameAlwaysCreateNew: + if opts.ClusterConfigPath == "" { + return nil, nil, fmt.Errorf("cluster config path is required for provider %s", p.name) + } + resources, err := cluster.CreateTestClusterFromConfig(ctx, opts.ClusterConfigPath) + if err != nil { + return nil, nil, err + } + if err := cluster.WaitForTestClusterReady(ctx, resources); err != nil { + _ = cluster.CleanupTestCluster(ctx, resources) + return nil, nil, err + } + session, err := sessionFromResources(p.name, resources) + if err != nil { + _ = cluster.CleanupTestCluster(ctx, resources) + return nil, nil, err + } + return session, resources, nil + + case NameAlwaysUseExisting: + resources, err := cluster.UseExistingCluster(ctx) + if err != nil { + return nil, nil, err + } + session, err := sessionFromResources(p.name, resources) + if err != nil { + _ = cluster.CleanupExistingCluster(ctx, resources) + return nil, nil, err + } + return session, resources, nil + + case NameCommander: + cmdRes, err := cluster.UseCommanderCluster(ctx) + if err != nil { + return nil, nil, err + } + cluster.SetCommanderResources(cmdRes) + resources := cmdRes.TestClusterResources + if err := cluster.WaitForTestClusterReady(ctx, resources); err != nil { + _ = cluster.CleanupCommanderCluster(ctx, cmdRes) + cluster.ClearCommanderResources() + return nil, nil, err + } + session, err := sessionFromCommander(p.name, cmdRes, resources) + if err != nil { + _ = cluster.CleanupCommanderCluster(ctx, cmdRes) + cluster.ClearCommanderResources() + return nil, nil, err + } + return session, resources, nil + + default: + return nil, nil, fmt.Errorf("unsupported provider %q", p.name) + } +} + +func (p *provider) Teardown(ctx context.Context, session *Session, resources *cluster.TestClusterResources) error { + if session == nil { + return fmt.Errorf("session is nil") + } + if resources == nil { + var err error + resources, err = ReconnectFromSession(ctx, session) + if err != nil { + return err + } + } + + switch Name(session.Provider) { + case NameAlwaysUseExisting: + return cluster.CleanupExistingCluster(ctx, resources) + case NameCommander: + cmdRes := cluster.GetCommanderResources() + if cmdRes == nil && session.CommanderClusterName != "" { + cmdRes = &cluster.CommanderClusterResources{ + ClusterName: session.CommanderClusterName, + CreatedByUs: session.CommanderCreatedByUs, + TestClusterResources: resources, + } + } + if cmdRes != nil { + err := cluster.CleanupCommanderCluster(ctx, cmdRes) + cluster.ClearCommanderResources() + return err + } + return cluster.CleanupExistingCluster(ctx, resources) + default: + return cluster.CleanupTestCluster(ctx, resources) + } +} diff --git a/pkg/provider/reconnect.go b/pkg/provider/reconnect.go new file mode 100644 index 0000000..078f2fd --- /dev/null +++ b/pkg/provider/reconnect.go @@ -0,0 +1,142 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package provider + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "github.com/deckhouse/storage-e2e/internal/config" + "github.com/deckhouse/storage-e2e/pkg/cluster" +) + +// ReconnectFromSession rebuilds TestClusterResources for teardown after CI artifacts are restored. +func ReconnectFromSession(ctx context.Context, session *Session) (*cluster.TestClusterResources, error) { + if session == nil { + return nil, fmt.Errorf("session is nil") + } + + switch Name(session.Provider) { + case NameAlwaysUseExisting, NameCommander: + applyRunSSH(session) + return cluster.UseExistingCluster(ctx) + default: + return reconnectCreatedCluster(ctx, session) + } +} + +func applyRunSSH(session *Session) { + if session.TestSSHHost != "" { + _ = os.Setenv("SSH_HOST", session.TestSSHHost) + } + if session.TestSSHUser != "" { + _ = os.Setenv("SSH_USER", session.TestSSHUser) + } + if session.KubeconfigPath != "" { + _ = os.Setenv("KUBE_CONFIG_PATH", session.KubeconfigPath) + } +} + +func reconnectCreatedCluster(ctx context.Context, session *Session) (*cluster.TestClusterResources, error) { + baseOpts, err := cluster.BaseClusterConnectOptionsFromEnv(config.E2ETempDir) + if err != nil { + return nil, err + } + baseRes, err := cluster.ConnectToCluster(ctx, baseOpts) + if err != nil { + return nil, fmt.Errorf("reconnect base cluster: %w", err) + } + + testHost := session.TestSSHHost + if testHost == "" { + return nil, fmt.Errorf("session missing test_ssh_host for teardown") + } + testUser := session.TestSSHUser + if testUser == "" { + testUser = config.VMSSHUserDefaultValue + } + + sshKeyPath, err := expandPath(config.SSHPrivateKey) + if err != nil { + baseRes.SSHClient.Close() + return nil, err + } + + testOpts := cluster.ConnectClusterOptions{ + SSHUser: baseOpts.SSHUser, + SSHHost: baseOpts.SSHHost, + SSHKeyPath: baseOpts.SSHKeyPath, + UseJumpHost: baseOpts.UseJumpHost, + TargetUser: testUser, + TargetHost: testHost, + TargetKeyPath: sshKeyPath, + KubeconfigOutputDir: config.E2ETempDir, + } + if baseOpts.UseJumpHost { + testOpts = cluster.ConnectClusterOptions{ + SSHUser: baseOpts.SSHUser, + SSHHost: baseOpts.SSHHost, + SSHKeyPath: baseOpts.SSHKeyPath, + UseJumpHost: true, + TargetUser: testUser, + TargetHost: testHost, + TargetKeyPath: sshKeyPath, + KubeconfigOutputDir: config.E2ETempDir, + } + } + + testRes, err := cluster.ConnectToCluster(ctx, testOpts) + if err != nil { + baseRes.SSHClient.Close() + return nil, fmt.Errorf("reconnect test cluster: %w", err) + } + + testRes.BaseClusterClient = baseRes.SSHClient + testRes.BaseKubeconfig = baseRes.Kubeconfig + testRes.BaseKubeconfigPath = baseRes.KubeconfigPath + testRes.BaseTunnelInfo = baseRes.TunnelInfo + + if session.KubeconfigPath != "" { + testRes.KubeconfigPath = session.KubeconfigPath + } + + if len(session.VMNames) > 0 { + testRes.VMResources = &cluster.VMResources{ + Namespace: session.Namespace, + VMNames: session.VMNames, + SetupVMName: session.SetupVMName, + } + } + + return testRes, nil +} + +func expandPath(path string) (string, error) { + if path == "" { + path = config.SSHPrivateKeyDefaultValue + } + if len(path) >= 2 && path[:2] == "~/" { + home, err := os.UserHomeDir() + if err != nil { + return "", err + } + path = filepath.Join(home, path[2:]) + } + return filepath.Clean(path), nil +} diff --git a/pkg/provider/session.go b/pkg/provider/session.go new file mode 100644 index 0000000..dc68994 --- /dev/null +++ b/pkg/provider/session.go @@ -0,0 +1,166 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package provider + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/deckhouse/storage-e2e/internal/config" + "github.com/deckhouse/storage-e2e/pkg/cluster" +) + +const sessionFileName = "session.json" + +// Session is persisted between CI jobs (create-cluster → run-tests → teardown-cluster). +type Session struct { + Provider string `json:"provider"` + CreatedAt time.Time `json:"created_at"` + ArtifactDir string `json:"artifact_dir"` + + KubeconfigPath string `json:"kubeconfig_path"` + ClusterStatePath string `json:"cluster_state_path,omitempty"` + TestSSHHost string `json:"test_ssh_host,omitempty"` + TestSSHUser string `json:"test_ssh_user,omitempty"` + UseJumpHost bool `json:"use_jump_host,omitempty"` + Namespace string `json:"namespace,omitempty"` + SetupVMName string `json:"setup_vm_name,omitempty"` + VMNames []string `json:"vm_names,omitempty"` + + CommanderClusterName string `json:"commander_cluster_name,omitempty"` + CommanderCreatedByUs bool `json:"commander_created_by_us,omitempty"` +} + +// SessionPath returns the path to session.json inside artifactDir. +func SessionPath(artifactDir string) string { + return filepath.Join(artifactDir, sessionFileName) +} + +// SaveSession writes session.json under artifactDir. +func SaveSession(artifactDir string, session *Session) error { + if session == nil { + return fmt.Errorf("session is nil") + } + session.ArtifactDir = artifactDir + if err := os.MkdirAll(artifactDir, 0o755); err != nil { + return fmt.Errorf("create artifact dir: %w", err) + } + data, err := json.MarshalIndent(session, "", " ") + if err != nil { + return err + } + path := SessionPath(artifactDir) + if err := os.WriteFile(path, data, 0o600); err != nil { + return fmt.Errorf("write %s: %w", path, err) + } + return WriteRunEnv(artifactDir, session) +} + +// LoadSession reads session.json from artifactDir. +func LoadSession(artifactDir string) (*Session, error) { + data, err := os.ReadFile(SessionPath(artifactDir)) + if err != nil { + return nil, fmt.Errorf("read session: %w", err) + } + var session Session + if err := json.Unmarshal(data, &session); err != nil { + return nil, fmt.Errorf("parse session: %w", err) + } + return &session, nil +} + +// WriteRunEnv emits run-env.sh for the run-tests CI job (non-secret variables only). +func WriteRunEnv(artifactDir string, session *Session) error { + path := filepath.Join(artifactDir, "run-env.sh") + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + + write := func(k, v string) { + if v != "" { + _, _ = fmt.Fprintf(f, "export %s=%q\n", k, v) + } + } + + _, _ = fmt.Fprintln(f, "# Generated by storage-e2e — do not commit") + write("E2E_CLUSTER_PHASE", "run") + write("TEST_CLUSTER_CREATE_MODE", config.ClusterCreateModeAlwaysUseExisting) + write("KUBE_CONFIG_PATH", session.KubeconfigPath) + if session.TestSSHHost != "" { + write("SSH_HOST", session.TestSSHHost) + } + if session.TestSSHUser != "" { + write("SSH_USER", session.TestSSHUser) + } + if session.Namespace != "" { + write("TEST_CLUSTER_NAMESPACE", session.Namespace) + } + return nil +} + +func sessionFromResources(name Name, res *cluster.TestClusterResources) (*Session, error) { + if res == nil || res.KubeconfigPath == "" { + return nil, fmt.Errorf("cluster resources missing kubeconfig path") + } + s := &Session{ + Provider: string(name), + CreatedAt: time.Now().UTC(), + KubeconfigPath: res.KubeconfigPath, + Namespace: config.TestClusterNamespace, + } + if res.ClusterDefinition != nil && len(res.ClusterDefinition.Masters) > 0 { + s.TestSSHHost = res.ClusterDefinition.Masters[0].IPAddress + } + if s.TestSSHHost == "" { + s.TestSSHHost = config.SSHHost + } + s.TestSSHUser = config.VMSSHUser + if s.TestSSHUser == "" { + s.TestSSHUser = config.VMSSHUserDefaultValue + } + s.UseJumpHost = config.SSHJumpHost != "" || config.SSHHost != s.TestSSHHost + + if state, err := cluster.LoadClusterStateFile(); err == nil && state != nil { + s.ClusterStatePath = cluster.ClusterStatePath() + s.SetupVMName = state.SetupVMName + s.VMNames = state.VMNames + if state.FirstMasterIP != "" { + s.TestSSHHost = state.FirstMasterIP + } + if state.Namespace != "" { + s.Namespace = state.Namespace + } + } + return s, nil +} + +func sessionFromCommander(name Name, cmd *cluster.CommanderClusterResources, res *cluster.TestClusterResources) (*Session, error) { + s, err := sessionFromResources(name, res) + if err != nil { + return nil, err + } + if cmd != nil { + s.CommanderClusterName = cmd.ClusterName + s.CommanderCreatedByUs = cmd.CreatedByUs + } + return s, nil +}