Skip to content

USHIFT-7019: C2CC - Increase amount of interconnected VMs in from 2 to 3 #6706

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
vanhalenar wants to merge 12 commits into
base: main
Choose a base branch
from
Open

USHIFT-7019: C2CC - Increase amount of interconnected VMs in from 2 to 3 #6706

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
349ec25
Up C2CC test VMs from 2 to 3
vanhalenar May 20, 2026
fcc1a18
Appease Robocop
vanhalenar May 20, 2026
ca2fd3b
Change CIDRs
vanhalenar May 20, 2026
e9e715f
Appease the rabbit
vanhalenar May 20, 2026
e76057a
Add host3 to el9 c2cc scenario
vanhalenar May 21, 2026
a4c6ff3
Fix overwriting configs
vanhalenar May 21, 2026
7f46373
Use 3 VMs in cleanup.robot
vanhalenar May 21, 2026
ed2bb76
Fix variable expansion
vanhalenar May 21, 2026
8c8b068
Template IP preservation test case
vanhalenar May 25, 2026
909b959
Remove duplicate keywords, fix c2cc.resource to account for 3 VMs
vanhalenar May 25, 2026
b59b7c6
Edit c2cc DNS suite to use 3 VMs
vanhalenar May 25, 2026
7a47af4
Appease Robocop
vanhalenar May 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 15 additions & 8 deletions test/resources/c2cc.resource
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,13 @@ ${HOST2_IP} ${EMPTY}
${HOST2_SSH_PORT} ${EMPTY}
${HOST2_API_PORT} ${EMPTY}
${KUBECONFIG_B} ${EMPTY}
${CLUSTER_C_POD_CIDR} ${EMPTY}
${CLUSTER_C_SVC_CIDR} ${EMPTY}
${CLUSTER_C_DOMAIN} ${EMPTY}
${HOST3_IP} ${EMPTY}
${HOST3_SSH_PORT} ${EMPTY}
${HOST3_API_PORT} ${EMPTY}
${KUBECONFIG_C} ${EMPTY}
&{C2CC_KUBECONFIGS} &{EMPTY}
&{C2CC_SSH_IDS} &{EMPTY}
@{C2CC_REMOTE_ALIASES} @{EMPTY}
Expand Down Expand Up @@ -249,9 +256,9 @@ Verify Corefile Does Not Contain C2CC Server Block
Should Not Contain ${stdout} ${domain}:5353

Deploy Test Workloads
[Documentation] Create namespace and deploy hello-microshift + curl-pod on both clusters.
[Documentation] Create namespace and deploy hello-microshift + curl-pod on all clusters.
VAR ${assets}= ${EXECDIR}/assets/c2cc
FOR ${alias} IN cluster-a cluster-b
FOR ${alias} IN cluster-a cluster-b cluster-c
${ns}= Create Unique Namespace On Cluster ${alias}
Set To Dictionary ${NAMESPACES} ${alias} ${ns}
Oc On Cluster ${alias} oc apply -n ${ns} -f ${assets}/hello-microshift.yaml
Expand All @@ -261,17 +268,17 @@ Deploy Test Workloads
Wait For Service Endpoints

Wait For Test Pods
[Documentation] Wait for all test pods to be Ready on both clusters.
FOR ${alias} IN cluster-a cluster-b
[Documentation] Wait for all test pods to be Ready on all clusters.
FOR ${alias} IN cluster-a cluster-b cluster-c
Oc On Cluster
... ${alias}
... oc wait pod/hello-microshift pod/curl-pod -n ${NAMESPACES}[${alias}] --for=condition=Ready --timeout=120s
END

Wait For Service Endpoints
[Documentation] Wait for hello-microshift service to have endpoints on both clusters.
[Documentation] Wait for hello-microshift service to have endpoints on all clusters.
... Ensures OVN-K has synced the EndpointSlice and programmed the OVN load balancer.
FOR ${alias} IN cluster-a cluster-b
FOR ${alias} IN cluster-a cluster-b cluster-c
Wait Until Keyword Succeeds 120s 5s
... Service Endpoints Should Exist ${alias} ${NAMESPACES}[${alias}]
END
Expand All @@ -284,7 +291,7 @@ Service Endpoints Should Exist
Should Not Be Empty ${stdout}

Cleanup Test Workloads
[Documentation] Delete test namespace on both clusters. Ignores errors.
FOR ${alias} IN cluster-a cluster-b
[Documentation] Delete test namespace on all clusters. Ignores errors.
FOR ${alias} IN cluster-a cluster-b cluster-c
Oc On Cluster ${alias} oc delete namespace ${NAMESPACES}[${alias}] --timeout=60s
END
93 changes: 75 additions & 18 deletions test/scenarios-bootc/el10/presubmits/el102-src@c2cc.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,15 @@ CLUSTER_B_POD_CIDR="10.45.0.0/16"
CLUSTER_B_SVC_CIDR="10.46.0.0/16"
CLUSTER_B_DOMAIN="cluster-b.remote"

# Cluster C (host3): non-overlapping CIDRs
CLUSTER_C_POD_CIDR="10.48.0.0/16"
CLUSTER_C_SVC_CIDR="10.49.0.0/16"
CLUSTER_C_DOMAIN="cluster-c.remote"

wait_for_greenboot_on_hosts() {
local junit_label=$1
local host
for host in host1 host2; do
for host in host1 host2 host3; do
local host_ip full_host
host_ip=$(get_vm_property "${host}" ip)
full_host=$(full_vm_name "${host}")
Expand All @@ -29,45 +34,74 @@ wait_for_greenboot_on_hosts() {
}

configure_c2cc_host() {
local host=$1 remote_ip=$2 remote_pod_cidr=$3 remote_svc_cidr=$4 remote_domain=$5
local host=$1
shift
# Remaining args are sets of 4: remote_ip remote_pod_cidr remote_svc_cidr remote_domain (repeat)

run_command_on_vm "${host}" "sudo mkdir -p /etc/microshift/config.d"
run_command_on_vm "${host}" "sudo tee /etc/microshift/config.d/50-c2cc.yaml > /dev/null << EOF
clusterToCluster:
remoteClusters:
- nextHop: ${remote_ip}
clusterNetwork:
- ${remote_pod_cidr}
serviceNetwork:
- ${remote_svc_cidr}
domain: ${remote_domain}

# Build the YAML config with all remote clusters
local yaml_content
yaml_content="clusterToCluster:"$'\n'" remoteClusters:"
local firewall_cidrs=()

while [ $# -gt 0 ]; do
local remote_ip=$1
local remote_pod_cidr=$2
local remote_svc_cidr=$3
local remote_domain=$4
shift 4

yaml_content+=$'\n'" - nextHop: ${remote_ip}"
yaml_content+=$'\n'" clusterNetwork:"
yaml_content+=$'\n'" - ${remote_pod_cidr}"
yaml_content+=$'\n'" serviceNetwork:"
yaml_content+=$'\n'" - ${remote_svc_cidr}"
yaml_content+=$'\n'" domain: ${remote_domain}"

firewall_cidrs+=("${remote_pod_cidr}" "${remote_svc_cidr}")
done

run_command_on_vm "${host}" "sudo tee /etc/microshift/config.d/50-c2cc.yaml > /dev/null <<EOF
${yaml_content}
EOF"

configure_vm_firewall "${host}"
run_command_on_vm "${host}" "sudo firewall-cmd --permanent --zone=trusted --add-source=${remote_pod_cidr}"
run_command_on_vm "${host}" "sudo firewall-cmd --permanent --zone=trusted --add-source=${remote_svc_cidr}"
for cidr in "${firewall_cidrs[@]}"; do
run_command_on_vm "${host}" "sudo firewall-cmd --permanent --zone=trusted --add-source=${cidr}"
done
run_command_on_vm "${host}" "sudo firewall-cmd --reload"

run_command_on_vm "${host}" "sudo systemctl restart microshift"
}

configure_c2cc_hosts() {
local -r host1_ip=$(get_vm_property host1 ip)
local -r host2_ip=$(get_vm_property host2 ip)
local -r host3_ip=$(get_vm_property host3 ip)

wait_for_greenboot_on_hosts "c2cc_pre_greenboot"

configure_c2cc_host host1 "${host2_ip}" "${CLUSTER_B_POD_CIDR}" "${CLUSTER_B_SVC_CIDR}" "${CLUSTER_B_DOMAIN}"
configure_c2cc_host host2 "${host1_ip}" "${CLUSTER_A_POD_CIDR}" "${CLUSTER_A_SVC_CIDR}" "${CLUSTER_A_DOMAIN}"
configure_c2cc_host host1 \
"${host2_ip}" "${CLUSTER_B_POD_CIDR}" "${CLUSTER_B_SVC_CIDR}" "${CLUSTER_B_DOMAIN}" \
"${host3_ip}" "${CLUSTER_C_POD_CIDR}" "${CLUSTER_C_SVC_CIDR}" "${CLUSTER_C_DOMAIN}"

configure_c2cc_host host2 \
"${host1_ip}" "${CLUSTER_A_POD_CIDR}" "${CLUSTER_A_SVC_CIDR}" "${CLUSTER_A_DOMAIN}" \
"${host3_ip}" "${CLUSTER_C_POD_CIDR}" "${CLUSTER_C_SVC_CIDR}" "${CLUSTER_C_DOMAIN}"

configure_c2cc_host host3 \
"${host1_ip}" "${CLUSTER_A_POD_CIDR}" "${CLUSTER_A_SVC_CIDR}" "${CLUSTER_A_DOMAIN}" \
"${host2_ip}" "${CLUSTER_B_POD_CIDR}" "${CLUSTER_B_SVC_CIDR}" "${CLUSTER_B_DOMAIN}"

wait_for_greenboot_on_hosts "c2cc_greenboot"
}

scenario_create_vms() {
prepare_kickstart host1 kickstart-bootc.ks.template rhel102-bootc-source
prepare_kickstart host2 kickstart-bootc.ks.template rhel102-bootc-source
prepare_kickstart host3 kickstart-bootc.ks.template rhel102-bootc-source

# Inject host2's non-default CIDRs into its kickstart config so MicroShift
# Inject host2's and host3's non-default CIDRs into its kickstart config so MicroShift
# boots with the correct network from the start (no cleanup-data needed).
local -r host2_ks_dir="${SCENARIO_INFO_DIR}/${SCENARIO}/vms/host2"
cat >> "${host2_ks_dir}/post-microshift.cfg" <<EOF
Expand All @@ -78,15 +112,27 @@ network:
serviceNetwork:
- ${CLUSTER_B_SVC_CIDR}
IEOF
EOF
local -r host3_ks_dir="${SCENARIO_INFO_DIR}/${SCENARIO}/vms/host3"
cat >> "${host3_ks_dir}/post-microshift.cfg" <<EOF
cat - >>/etc/microshift/config.yaml <<IEOF
network:
clusterNetwork:
- ${CLUSTER_C_POD_CIDR}
serviceNetwork:
- ${CLUSTER_C_SVC_CIDR}
IEOF
EOF

launch_vm rhel102-bootc --vmname host1
launch_vm rhel102-bootc --vmname host2
launch_vm rhel102-bootc --vmname host3
}

scenario_remove_vms() {
remove_vm host1
remove_vm host2
remove_vm host3
}

scenario_run_tests() {
Expand All @@ -97,12 +143,19 @@ scenario_run_tests() {
# Retrieve host2's kubeconfig
local -r host2_ip=$(get_vm_property host2 ip)
local -r kubeconfig_b="${SCENARIO_INFO_DIR}/${SCENARIO}/kubeconfig-b"

# Retrieve host3's kubeconfig
local -r host3_ip=$(get_vm_property host3 ip)
local -r kubeconfig_c="${SCENARIO_INFO_DIR}/${SCENARIO}/kubeconfig-c"

# Wait for host2 to be fully ready (run_tests only waits for host1)
# Wait for host2 and host3 to be fully ready (run_tests only waits for host1)
wait_for_microshift_to_be_ready host2
wait_for_microshift_to_be_ready host3

run_command_on_vm host2 "sudo cp /var/lib/microshift/resources/kubeadmin/${host2_ip}/kubeconfig /tmp/kubeconfig-b && sudo chmod 644 /tmp/kubeconfig-b"
run_command_on_vm host3 "sudo cp /var/lib/microshift/resources/kubeadmin/${host3_ip}/kubeconfig /tmp/kubeconfig-c && sudo chmod 644 /tmp/kubeconfig-c"
copy_file_from_vm host2 "/tmp/kubeconfig-b" "${kubeconfig_b}"
copy_file_from_vm host3 "/tmp/kubeconfig-c" "${kubeconfig_c}"

run_tests host1 \
--variable "CLUSTER_A_POD_CIDR:${CLUSTER_A_POD_CIDR}" \
Expand All @@ -112,5 +165,9 @@ scenario_run_tests() {
--variable "CLUSTER_B_SVC_CIDR:${CLUSTER_B_SVC_CIDR}" \
--variable "CLUSTER_B_DOMAIN:${CLUSTER_B_DOMAIN}" \
--variable "KUBECONFIG_B:${kubeconfig_b}" \
--variable "CLUSTER_C_POD_CIDR:${CLUSTER_C_POD_CIDR}" \
--variable "CLUSTER_C_SVC_CIDR:${CLUSTER_C_SVC_CIDR}" \
--variable "CLUSTER_C_DOMAIN:${CLUSTER_C_DOMAIN}" \
--variable "KUBECONFIG_C:${kubeconfig_c}" \
suites/c2cc/
}
93 changes: 75 additions & 18 deletions test/scenarios-bootc/el9/presubmits/el98-src@c2cc.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,15 @@ CLUSTER_B_POD_CIDR="10.45.0.0/16"
CLUSTER_B_SVC_CIDR="10.46.0.0/16"
CLUSTER_B_DOMAIN="cluster-b.remote"

# Cluster C (host3): non-overlapping CIDRs
CLUSTER_C_POD_CIDR="10.48.0.0/16"
CLUSTER_C_SVC_CIDR="10.49.0.0/16"
CLUSTER_C_DOMAIN="cluster-c.remote"

wait_for_greenboot_on_hosts() {
local junit_label=$1
local host
for host in host1 host2; do
for host in host1 host2 host3; do
local host_ip full_host
host_ip=$(get_vm_property "${host}" ip)
full_host=$(full_vm_name "${host}")
Expand All @@ -29,45 +34,74 @@ wait_for_greenboot_on_hosts() {
}

configure_c2cc_host() {
local host=$1 remote_ip=$2 remote_pod_cidr=$3 remote_svc_cidr=$4 remote_domain=$5
local host=$1
shift
# Remaining args are sets of 4: remote_ip remote_pod_cidr remote_svc_cidr remote_domain (repeat)

run_command_on_vm "${host}" "sudo mkdir -p /etc/microshift/config.d"
run_command_on_vm "${host}" "sudo tee /etc/microshift/config.d/50-c2cc.yaml > /dev/null << EOF
clusterToCluster:
remoteClusters:
- nextHop: ${remote_ip}
clusterNetwork:
- ${remote_pod_cidr}
serviceNetwork:
- ${remote_svc_cidr}
domain: ${remote_domain}

# Build the YAML config with all remote clusters
local yaml_content
yaml_content="clusterToCluster:"$'\n'" remoteClusters:"
local firewall_cidrs=()

while [ $# -gt 0 ]; do
local remote_ip=$1
local remote_pod_cidr=$2
local remote_svc_cidr=$3
local remote_domain=$4
shift 4

yaml_content+=$'\n'" - nextHop: ${remote_ip}"
yaml_content+=$'\n'" clusterNetwork:"
yaml_content+=$'\n'" - ${remote_pod_cidr}"
yaml_content+=$'\n'" serviceNetwork:"
yaml_content+=$'\n'" - ${remote_svc_cidr}"
yaml_content+=$'\n'" domain: ${remote_domain}"

firewall_cidrs+=("${remote_pod_cidr}" "${remote_svc_cidr}")
done

run_command_on_vm "${host}" "sudo tee /etc/microshift/config.d/50-c2cc.yaml > /dev/null <<EOF
${yaml_content}
EOF"

configure_vm_firewall "${host}"
run_command_on_vm "${host}" "sudo firewall-cmd --permanent --zone=trusted --add-source=${remote_pod_cidr}"
run_command_on_vm "${host}" "sudo firewall-cmd --permanent --zone=trusted --add-source=${remote_svc_cidr}"
for cidr in "${firewall_cidrs[@]}"; do
run_command_on_vm "${host}" "sudo firewall-cmd --permanent --zone=trusted --add-source=${cidr}"
done
run_command_on_vm "${host}" "sudo firewall-cmd --reload"

run_command_on_vm "${host}" "sudo systemctl restart microshift"
}

configure_c2cc_hosts() {
local -r host1_ip=$(get_vm_property host1 ip)
local -r host2_ip=$(get_vm_property host2 ip)
local -r host3_ip=$(get_vm_property host3 ip)

wait_for_greenboot_on_hosts "c2cc_pre_greenboot"

configure_c2cc_host host1 "${host2_ip}" "${CLUSTER_B_POD_CIDR}" "${CLUSTER_B_SVC_CIDR}" "${CLUSTER_B_DOMAIN}"
configure_c2cc_host host2 "${host1_ip}" "${CLUSTER_A_POD_CIDR}" "${CLUSTER_A_SVC_CIDR}" "${CLUSTER_A_DOMAIN}"
configure_c2cc_host host1 \
"${host2_ip}" "${CLUSTER_B_POD_CIDR}" "${CLUSTER_B_SVC_CIDR}" "${CLUSTER_B_DOMAIN}" \
"${host3_ip}" "${CLUSTER_C_POD_CIDR}" "${CLUSTER_C_SVC_CIDR}" "${CLUSTER_C_DOMAIN}"

configure_c2cc_host host2 \
"${host1_ip}" "${CLUSTER_A_POD_CIDR}" "${CLUSTER_A_SVC_CIDR}" "${CLUSTER_A_DOMAIN}" \
"${host3_ip}" "${CLUSTER_C_POD_CIDR}" "${CLUSTER_C_SVC_CIDR}" "${CLUSTER_C_DOMAIN}"

configure_c2cc_host host3 \
"${host1_ip}" "${CLUSTER_A_POD_CIDR}" "${CLUSTER_A_SVC_CIDR}" "${CLUSTER_A_DOMAIN}" \
"${host2_ip}" "${CLUSTER_B_POD_CIDR}" "${CLUSTER_B_SVC_CIDR}" "${CLUSTER_B_DOMAIN}"

wait_for_greenboot_on_hosts "c2cc_greenboot"
}

scenario_create_vms() {
prepare_kickstart host1 kickstart-bootc.ks.template rhel98-bootc-source
prepare_kickstart host2 kickstart-bootc.ks.template rhel98-bootc-source
prepare_kickstart host3 kickstart-bootc.ks.template rhel98-bootc-source

# Inject host2's non-default CIDRs into its kickstart config so MicroShift
# Inject host2's and host3's non-default CIDRs into its kickstart config so MicroShift
# boots with the correct network from the start (no cleanup-data needed).
local -r host2_ks_dir="${SCENARIO_INFO_DIR}/${SCENARIO}/vms/host2"
cat >> "${host2_ks_dir}/post-microshift.cfg" <<EOF
Expand All @@ -78,15 +112,27 @@ network:
serviceNetwork:
- ${CLUSTER_B_SVC_CIDR}
IEOF
EOF
local -r host3_ks_dir="${SCENARIO_INFO_DIR}/${SCENARIO}/vms/host3"
cat >> "${host3_ks_dir}/post-microshift.cfg" <<EOF
cat - >>/etc/microshift/config.yaml <<IEOF
network:
clusterNetwork:
- ${CLUSTER_C_POD_CIDR}
serviceNetwork:
- ${CLUSTER_C_SVC_CIDR}
IEOF
EOF

launch_vm rhel98-bootc --vmname host1
launch_vm rhel98-bootc --vmname host2
launch_vm rhel98-bootc --vmname host3
}

scenario_remove_vms() {
remove_vm host1
remove_vm host2
remove_vm host3
}

scenario_run_tests() {
Expand All @@ -97,12 +143,19 @@ scenario_run_tests() {
# Retrieve host2's kubeconfig
local -r host2_ip=$(get_vm_property host2 ip)
local -r kubeconfig_b="${SCENARIO_INFO_DIR}/${SCENARIO}/kubeconfig-b"

# Retrieve host3's kubeconfig
local -r host3_ip=$(get_vm_property host3 ip)
local -r kubeconfig_c="${SCENARIO_INFO_DIR}/${SCENARIO}/kubeconfig-c"

# Wait for host2 to be fully ready (run_tests only waits for host1)
# Wait for host2 and host3 to be fully ready (run_tests only waits for host1)
wait_for_microshift_to_be_ready host2
wait_for_microshift_to_be_ready host3

run_command_on_vm host2 "sudo cp /var/lib/microshift/resources/kubeadmin/${host2_ip}/kubeconfig /tmp/kubeconfig-b && sudo chmod 644 /tmp/kubeconfig-b"
run_command_on_vm host3 "sudo cp /var/lib/microshift/resources/kubeadmin/${host3_ip}/kubeconfig /tmp/kubeconfig-c && sudo chmod 644 /tmp/kubeconfig-c"
copy_file_from_vm host2 "/tmp/kubeconfig-b" "${kubeconfig_b}"
copy_file_from_vm host3 "/tmp/kubeconfig-c" "${kubeconfig_c}"

run_tests host1 \
--variable "CLUSTER_A_POD_CIDR:${CLUSTER_A_POD_CIDR}" \
Expand All @@ -112,5 +165,9 @@ scenario_run_tests() {
--variable "CLUSTER_B_SVC_CIDR:${CLUSTER_B_SVC_CIDR}" \
--variable "CLUSTER_B_DOMAIN:${CLUSTER_B_DOMAIN}" \
--variable "KUBECONFIG_B:${kubeconfig_b}" \
--variable "CLUSTER_C_POD_CIDR:${CLUSTER_C_POD_CIDR}" \
--variable "CLUSTER_C_SVC_CIDR:${CLUSTER_C_SVC_CIDR}" \
--variable "CLUSTER_C_DOMAIN:${CLUSTER_C_DOMAIN}" \
--variable "KUBECONFIG_C:${kubeconfig_c}" \
suites/c2cc/
}
Loading