Custom error page:The requested document was not found.
+ + diff --git a/test/assets/router/error-page-503.http b/test/assets/router/error-page-503.http new file mode 100644 index 0000000000..90c349904e --- /dev/null +++ b/test/assets/router/error-page-503.http @@ -0,0 +1,10 @@ +HTTP/1.0 503 Service Unavailable +Connection: close +Content-Type: text/html + + +Custom error page:The requested application is not available.
+ + diff --git a/test/assets/router/microshift-ingress-destca.yaml b/test/assets/router/microshift-ingress-destca.yaml new file mode 100644 index 0000000000..ac3452ff65 --- /dev/null +++ b/test/assets/router/microshift-ingress-destca.yaml @@ -0,0 +1,19 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress-ms-reen + annotations: + route.openshift.io/destination-ca-certificate-secret: service-secret + route.openshift.io/termination: reencrypt +spec: + rules: + - host: service-secure-test.example.com + http: + paths: + - backend: + service: + name: service-secure + port: + number: 27443 + path: "/" + pathType: Prefix diff --git a/test/assets/router/microshift-ingress-http.yaml b/test/assets/router/microshift-ingress-http.yaml new file mode 100644 index 0000000000..16fdb304d9 --- /dev/null +++ b/test/assets/router/microshift-ingress-http.yaml @@ -0,0 +1,16 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress-on-microshift +spec: + rules: + - host: service-unsecure-test.example.com + http: + paths: + - backend: + service: + name: service-unsecure + port: + number: 27017 + path: "/" + pathType: Prefix diff --git a/test/assets/router/rsyslogd-pod.yaml b/test/assets/router/rsyslogd-pod.yaml new file mode 100644 index 0000000000..07ba1425bc --- /dev/null +++ b/test/assets/router/rsyslogd-pod.yaml @@ -0,0 +1,20 @@ +kind: Pod +apiVersion: v1 +metadata: + name: rsyslogd-pod + labels: + name: rsyslogd +spec: + containers: + - image: quay.io/openshifttest/rsyslogd-container@sha256:e806eb41f05d7cc6eec96bf09c7bcb692f97562d4a983cb019289bd048d9aee2 + name: rsyslogd-container + securityContext: + privileged: true + ports: + - containerPort: 514 + protocol: TCP + - containerPort: 514 + protocol: UDP + resources: + limits: + memory: 340Mi diff --git a/test/assets/router/test-client-pod.yaml b/test/assets/router/test-client-pod.yaml new file mode 100644 index 0000000000..05c2e6638d --- /dev/null +++ b/test/assets/router/test-client-pod.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: hello-pod + name: hello-pod +spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - image: quay.io/openshifttest/nginx-alpine@sha256:cee6930776b92dc1e93b73f9e5965925d49cff3d2e91e1d071c2f0ff72cbca29 + name: hello-pod + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + ports: + - containerPort: 8080 + - containerPort: 8443 diff --git a/test/assets/router/web-server-deploy.yaml b/test/assets/router/web-server-deploy.yaml new file mode 100644 index 0000000000..9c31424f20 --- /dev/null +++ b/test/assets/router/web-server-deploy.yaml @@ -0,0 +1,57 @@ +apiVersion: v1 +kind: List +items: +- apiVersion: apps/v1 + kind: Deployment + metadata: + name: web-server-deploy + labels: + app: web-server-deploy + spec: + replicas: 1 + selector: + matchLabels: + name: web-server-deploy + template: + metadata: + labels: + name: web-server-deploy + spec: + containers: + - name: nginx + image: quay.io/openshifttest/nginx-alpine@sha256:cee6930776b92dc1e93b73f9e5965925d49cff3d2e91e1d071c2f0ff72cbca29 + ports: + - containerPort: 8080 + name: http + protocol: TCP + - containerPort: 8443 + name: https + protocol: TCP +- kind: Service + apiVersion: v1 + metadata: + labels: + name: service-secure + name: service-secure + spec: + ports: + - name: https + protocol: TCP + port: 27443 + targetPort: 8443 + selector: + name: web-server-deploy +- apiVersion: v1 + kind: Service + metadata: + labels: + name: service-unsecure + name: service-unsecure + spec: + ports: + - name: http + port: 27017 + protocol: TCP + targetPort: 8080 + selector: + name: web-server-deploy \ No newline at end of file diff --git a/test/assets/router/web-server-signed-deploy.yaml b/test/assets/router/web-server-signed-deploy.yaml new file mode 100644 index 0000000000..1a77bb7c09 --- /dev/null +++ b/test/assets/router/web-server-signed-deploy.yaml @@ -0,0 +1,94 @@ +apiVersion: v1 +kind: List +items: +- apiVersion: v1 + kind: ConfigMap + metadata: + name: nginx-config + data: + nginx.conf: | + events { + worker_connections 1024; + } + + http { + server { + listen 8080; + listen [::]:8080; + location / { + root /data/http; + } + } + + server { + listen 8443 ssl http2 default; + listen [::]:8443 ssl http2 default; + server_name _; + ssl_certificate certs/tls.crt; + ssl_certificate_key certs/tls.key; + location / { + root /data/https-default; + } + } + } +- apiVersion: apps/v1 + kind: Deployment + metadata: + name: web-server-deploy + labels: + name: web-server-deploy + spec: + replicas: 1 + selector: + matchExpressions: + - {key: name, operator: In, values: [web-server-deploy]} + template: + metadata: + labels: + name: web-server-deploy + spec: + containers: + - name: nginx + image: quay.io/openshifttest/nginx-alpine@sha256:cee6930776b92dc1e93b73f9e5965925d49cff3d2e91e1d071c2f0ff72cbca29 + volumeMounts: + - name: service-secret + mountPath: /etc/nginx/certs/ + - name: nginx-config + mountPath: /etc/nginx/ + volumes: + - name: service-secret + secret: + secretName: service-secret + - name: nginx-config + configMap: + name: nginx-config +- kind: Service + apiVersion: v1 + metadata: + annotations: + service.beta.openshift.io/serving-cert-secret-name: service-secret + labels: + name: service-secure + name: service-secure + spec: + ports: + - name: https + protocol: TCP + port: 27443 + targetPort: 8443 + selector: + name: web-server-deploy +- apiVersion: v1 + kind: Service + metadata: + labels: + name: service-unsecure + name: service-unsecure + spec: + ports: + - name: http + port: 27017 + protocol: TCP + targetPort: 8080 + selector: + name: web-server-deploy diff --git a/test/resources/router.resource b/test/resources/router.resource new file mode 100644 index 0000000000..e65a543c03 --- /dev/null +++ b/test/resources/router.resource @@ -0,0 +1,404 @@ +*** Settings *** +Documentation Keywords for router end-to-end testing + +Library OperatingSystem +Library Process +Library String +Library SSHLibrary +Resource oc.resource +Resource common.resource +Resource kubeconfig.resource +Resource microshift-config.resource +Resource microshift-process.resource + + +*** Variables *** +${WEB_SERVER_DEPLOY} ./assets/router/web-server-deploy.yaml +${WEB_SERVER_SIGNED_DEPLOY} ./assets/router/web-server-signed-deploy.yaml +${TEST_CLIENT_POD} ./assets/router/test-client-pod.yaml +${INGRESS_HTTP} ./assets/router/microshift-ingress-http.yaml +${INGRESS_DESTCA} ./assets/router/microshift-ingress-destca.yaml +${RSYSLOGD_POD} ./assets/router/rsyslogd-pod.yaml +${ROUTER_NS} openshift-ingress +${CLIENT_POD_NAME} hello-pod +${CLIENT_POD_LABEL} app\=hello-pod +${WEB_SERVER_LABEL} name\=web-server-deploy + + +*** Keywords *** +Deploy Web Server + [Documentation] Deploy the nginx web server (unsecured) with services. + [Arguments] ${ns}=${NAMESPACE} + Oc Create -f ${WEB_SERVER_DEPLOY} -n ${ns} + Labeled Pod Should Be Ready ${WEB_SERVER_LABEL} ns=${ns} timeout=120s + +Deploy Web Server Signed + [Documentation] Deploy the nginx web server with TLS (service-serving-cert) and services. + [Arguments] ${ns}=${NAMESPACE} + Oc Create -f ${WEB_SERVER_SIGNED_DEPLOY} -n ${ns} + Labeled Pod Should Be Ready ${WEB_SERVER_LABEL} ns=${ns} timeout=120s + +Deploy Test Client Pod + [Documentation] Deploy the test client pod for running curl commands. + [Arguments] ${ns}=${NAMESPACE} + Oc Create -f ${TEST_CLIENT_POD} -n ${ns} + Labeled Pod Should Be Ready ${CLIENT_POD_LABEL} ns=${ns} timeout=120s + +Deploy Rsyslogd Pod + [Documentation] Deploy the rsyslogd pod for syslog testing. + [Arguments] ${ns}=${NAMESPACE} + Oc Create -f ${RSYSLOGD_POD} -n ${ns} + Labeled Pod Should Be Ready name\=rsyslogd ns=${ns} timeout=120s + +Get Web Server Pod Name + [Documentation] Get the name of the first web-server-deploy pod. + [Arguments] ${ns}=${NAMESPACE} + ${name}= Run With Kubeconfig + ... oc get pod -l name=web-server-deploy -n ${ns} -o jsonpath='{.items[0].metadata.name}' + RETURN ${name} + +Create OC Route + [Documentation] Create an OpenShift route of the given type. + ... route_type: http (uses oc expose), or edge/passthrough/reencrypt (uses oc create route) + [Arguments] ${ns} ${route_type} ${route_name} ${service_name} @{extra_args} + IF "${route_type}" == "http" + ${extra}= Catenate SEPARATOR=${SPACE} @{extra_args} + Run With Kubeconfig oc expose -n ${ns} service ${service_name} --name=${route_name} ${extra} + ELSE + ${extra}= Catenate SEPARATOR=${SPACE} @{extra_args} + Run With Kubeconfig oc create route ${route_type} ${route_name} --service=${service_name} -n ${ns} ${extra} + END + +Route Should Be Admitted + [Documentation] Verify the route is admitted by the default ingress controller. + [Arguments] ${route_name} ${ns}=${NAMESPACE} ${timeout}=120s + Wait Until Keyword Succeeds ${timeout} 2s + ... Route Admission Status Should Be ${route_name} ${ns} True + +Route Should Not Be Admitted + [Documentation] Verify the route is NOT admitted by the default ingress controller. + [Arguments] ${route_name} ${ns}=${NAMESPACE} + Wait Until Keyword Succeeds 60s 2s + ... Route Admission Status Should Be ${route_name} ${ns} False + +Route Admission Status Should Be + [Documentation] Helper: assert route admission status equals expected. + [Arguments] ${route_name} ${ns} ${expected} + ${status}= Oc Get JsonPath route ${ns} ${route_name} + ... .status.ingress[0].conditions[0].status + Should Be Equal As Strings ${status} ${expected} + +Create And Admit Four Route Types + [Documentation] Create HTTP, edge, passthrough, and reencrypt routes and wait for reencrypt to be admitted. + [Arguments] ${http_host} ${edge_host} ${pass_host} ${reen_host} + Create OC Route ${NAMESPACE} http route-http service-unsecure --hostname=${http_host} + Create OC Route ${NAMESPACE} edge route-edge service-unsecure --hostname=${edge_host} + Create OC Route ${NAMESPACE} passthrough route-passth service-secure --hostname=${pass_host} + Create OC Route ${NAMESPACE} reencrypt route-reen service-secure --hostname=${reen_host} + Route Should Be Admitted route-http + Route Should Be Admitted route-edge + Route Should Be Admitted route-passth + Route Should Be Admitted route-reen + +Curl From Pod + [Documentation] Run curl from a pod via oc exec. Returns stdout. + [Arguments] ${pod_name} ${ns} ${url} ${resolve} ${flags}=${EMPTY} + ${output}= Run With Kubeconfig + ... oc exec -n ${ns} ${pod_name} -- curl ${url} -sI --resolve ${resolve} --connect-timeout 10 ${flags} + RETURN ${output} + +Curl From Pod Should Contain + [Documentation] Curl from a pod and assert response contains expected string (no extra flags). + [Arguments] ${pod_name} ${ns} ${url} ${resolve} ${expected}=200 + ${output}= Curl From Pod ${pod_name} ${ns} ${url} ${resolve} + Should Contain ${output} ${expected} + +Wait Until Curl Succeeds From Pod + [Documentation] Retry curl from a pod (no extra flags) until response contains expected string. + [Arguments] ${pod_name} ${ns} ${url} ${resolve} ${expected}=200 + Wait Until Keyword Succeeds 120s 2s + ... Curl From Pod Should Contain ${pod_name} ${ns} ${url} ${resolve} ${expected} + +Curl HTTPS From Pod Should Contain + [Documentation] Curl HTTPS from a pod using -k, assert response contains expected string. + [Arguments] ${pod_name} ${ns} ${url} ${resolve} ${expected}=200 + ${output}= Curl From Pod ${pod_name} ${ns} ${url} ${resolve} -k + Should Contain ${output} ${expected} + +Wait Until HTTPS Curl Succeeds From Pod + [Documentation] Retry HTTPS curl (with -k) from a pod until response contains expected string. + [Arguments] ${pod_name} ${ns} ${url} ${resolve} ${expected}=200 + Wait Until Keyword Succeeds 120s 2s + ... Curl HTTPS From Pod Should Contain ${pod_name} ${ns} ${url} ${resolve} ${expected} + +Curl With Cookie Should Return 200 + [Documentation] Curl from a pod with a cookie and assert 200 response. + [Arguments] ${pod_name} ${ns} ${url} ${resolve} ${cookie} + ${output}= Curl From Pod ${pod_name} ${ns} ${url} ${resolve} -b ${cookie} + Should Contain ${output} 200 + +Wait Until Curl With Cookie Succeeds From Pod + [Documentation] Retry curl with a cookie until 200 response. + [Arguments] ${pod_name} ${ns} ${url} ${resolve} ${cookie} + Wait Until Keyword Succeeds 120s 2s + ... Curl With Cookie Should Return 200 ${pod_name} ${ns} ${url} ${resolve} ${cookie} + +HTTPS Curl With Cookie Should Return 200 + [Documentation] Curl HTTPS from a pod with a cookie (using -k) and assert 200 response. + [Arguments] ${pod_name} ${ns} ${url} ${resolve} ${cookie} + ${output}= Curl From Pod ${pod_name} ${ns} ${url} ${resolve} -k -b ${cookie} + Should Contain ${output} 200 + +Wait Until HTTPS Curl With Cookie Succeeds From Pod + [Documentation] Retry HTTPS curl with a cookie until 200 response. + [Arguments] ${pod_name} ${ns} ${url} ${resolve} ${cookie} + Wait Until Keyword Succeeds 120s 2s + ... HTTPS Curl With Cookie Should Return 200 ${pod_name} ${ns} ${url} ${resolve} ${cookie} + +Curl With Client Cert Should Return + [Documentation] Curl with client cert from /data/certs/ and assert expected response. + [Arguments] ${pod_name} ${ns} ${url} ${resolve} ${expected}=200 OK + ${output}= Curl From Pod ${pod_name} ${ns} ${url} ${resolve} + ... --cacert /data/certs/ca.crt --cert /data/certs/usr.crt --key /data/certs/usr.key + Should Contain ${output} ${expected} + +Wait Until Curl With Client Cert Succeeds + [Documentation] Retry curl with client cert until expected response. + [Arguments] ${pod_name} ${ns} ${url} ${resolve} ${expected}=200 OK + Wait Until Keyword Succeeds 120s 2s + ... Curl With Client Cert Should Return ${pod_name} ${ns} ${url} ${resolve} ${expected} + +Curl With Cert File Should Return 200 + [Documentation] Curl with a specific cert file prefix from /data/certs/ and assert 200 response. + [Arguments] ${pod_name} ${ns} ${url} ${resolve} ${cert_prefix} + ${output}= Curl From Pod ${pod_name} ${ns} ${url} ${resolve} + ... --cacert /data/certs/ca.crt --cert /data/certs/${cert_prefix}.crt --key /data/certs/${cert_prefix}.key + Should Contain ${output} 200 OK + +Wait Until Curl With Cert File Succeeds + [Documentation] Retry curl with specific cert file until 200 response. + [Arguments] ${pod_name} ${ns} ${url} ${resolve} ${cert_prefix} + Wait Until Keyword Succeeds 120s 2s + ... Curl With Cert File Should Return 200 ${pod_name} ${ns} ${url} ${resolve} ${cert_prefix} + +Curl With Cert File Should Return 403 + [Documentation] Curl with a specific cert file prefix from /data/certs/ and assert 403 response. + [Arguments] ${pod_name} ${ns} ${url} ${resolve} ${cert_prefix} + ${output}= Curl From Pod ${pod_name} ${ns} ${url} ${resolve} + ... --cacert /data/certs/ca.crt --cert /data/certs/${cert_prefix}.crt --key /data/certs/${cert_prefix}.key + Should Contain ${output} 403 + +Wait Until Curl With Cert File Returns 403 + [Documentation] Retry curl with specific cert file until 403 response. + [Arguments] ${pod_name} ${ns} ${url} ${resolve} ${cert_prefix} + Wait Until Keyword Succeeds 120s 2s + ... Curl With Cert File Should Return 403 ${pod_name} ${ns} ${url} ${resolve} ${cert_prefix} + +Curl Without Cert Should Return SSL Error + [Documentation] Curl without a client cert and assert SSL handshake error (for mTLS Required). + [Arguments] ${pod_name} ${ns} ${url} ${resolve} + ${output}= Curl From Pod ${pod_name} ${ns} ${url} ${resolve} -skv + Should Contain ${output} SSL_read + +Get Router Pod Name + [Documentation] Get the name of the router-default pod. + ${name}= Run With Kubeconfig + ... oc get pod -l ingresscontroller.operator.openshift.io/deployment-ingresscontroller=default -n ${ROUTER_NS} -o jsonpath='{.items[0].metadata.name}' + RETURN ${name} + +Get Router Pod IP + [Documentation] Get the pod IP of the router pod. + ${pod_name}= Get Router Pod Name + ${ip}= Oc Get JsonPath pod ${ROUTER_NS} ${pod_name} .status.podIP + RETURN ${ip} + +Router Pod Env Should Have Value + [Documentation] Check an env var value in the router pod. + [Arguments] ${env_name} ${expected_value} + ${actual}= Oc Get JsonPath + ... pod ${ROUTER_NS} ${EMPTY} + ... .items[*].spec.containers[*].env[?(@.name=="${env_name}")].value + Should Be Equal As Strings ${actual} ${expected_value} + +Read Haproxy Config + [Documentation] Read haproxy.config from the router pod. + ${pod_name}= Get Router Pod Name + ${output}= Run With Kubeconfig + ... oc exec -n ${ROUTER_NS} ${pod_name} -- cat haproxy.config + RETURN ${output} + +Get Router Access Logs + [Documentation] Get access logs from the router pod's logs container. + ${output}= Oc Logs deployment/router-default -c logs ${ROUTER_NS} + RETURN ${output} + +Wait For Router Logs To Contain + [Documentation] Wait for the router access logs to contain a specific pattern. + [Arguments] ${pattern} ${timeout}=120s + Wait Until Keyword Succeeds ${timeout} 5s + ... Router Logs Should Contain ${pattern} + +Router Logs Should Contain + [Documentation] Check that router access logs contain a pattern. + [Arguments] ${pattern} + ${logs}= Get Router Access Logs + Should Contain ${logs} ${pattern} + +Wait For Router Ready + [Documentation] Wait for the default router deployment to be available. + Oc Wait namespace/openshift-ingress --for jsonpath='{.status.phase}=Active' --timeout=5m + Named Deployment Should Be Available router-default ${ROUTER_NS} 5m + +Setup Router Config And Restart + [Documentation] Apply drop-in config and restart MicroShift, then wait for router ready. + [Arguments] ${config_content} + Drop In MicroShift Config ${config_content} 10-router + Restart MicroShift + Wait For Router Ready + +Remove Router Config And Restart + [Documentation] Remove the drop-in config and restart MicroShift, then wait for router ready. + Remove Drop In MicroShift Config 10-router + Restart MicroShift + Wait For Router Ready + +Get LB IPs + [Documentation] Get all load balancer IPs of the router-default service as a space-separated string. + ${ips}= Oc Get JsonPath + ... service ${ROUTER_NS} router-default + ... .status.loadBalancer.ingress[*].ip + RETURN ${ips} + +Get LB Port + [Documentation] Get a specific named port of the router-default service. + [Arguments] ${port_name} + ${port}= Oc Get JsonPath + ... service ${ROUTER_NS} router-default + ... .spec.ports[?(@.name=="${port_name}")].port + RETURN ${port} + +Create Configmap From Files + [Documentation] Create a configmap from one or more local file paths. + [Arguments] ${ns} ${name} @{from_files} + ${file_args}= Catenate SEPARATOR=${SPACE} @{from_files} + Run With Kubeconfig oc create configmap ${name} ${file_args} -n ${ns} + +Scale Deployment + [Documentation] Scale a deployment to a given number of replicas. + [Arguments] ${ns} ${deploy_name} ${replicas} + Run With Kubeconfig oc scale deployment/${deploy_name} --replicas=${replicas} -n ${ns} + +Copy Files To Pod + [Documentation] Copy a local path to a pod path via oc cp. + [Arguments] ${ns} ${pod_name} ${src} ${dest} + Run With Kubeconfig oc cp -n ${ns} ${src} ${ns}/${pod_name}:${dest} + +Generate CA Certificate + [Documentation] Generate a self-signed CA key and certificate. + [Arguments] ${ca_key} ${ca_crt} ${ca_subj} + ${result}= Run Process openssl genrsa -out ${ca_key} 2048 shell=True + Should Be Equal As Integers ${result.rc} 0 + ${result}= Run Process + ... openssl req -x509 -new -nodes -key ${ca_key} -sha256 -days 30 -subj "${ca_subj}" -out ${ca_crt} + ... shell=True + Should Be Equal As Integers ${result.rc} 0 + +Generate CSR And Key + [Documentation] Generate a private key and a CSR for the given subject. + [Arguments] ${key_file} ${csr_file} ${subj} + ${result}= Run Process openssl genrsa -out ${key_file} 2048 shell=True + Should Be Equal As Integers ${result.rc} 0 + ${result}= Run Process + ... openssl req -new -key ${key_file} -subj "${subj}" -out ${csr_file} + ... shell=True + Should Be Equal As Integers ${result.rc} 0 + +Sign CSR With CA + [Documentation] Sign a CSR with a CA, writing SAN config to a temp file when provided. + [Arguments] ${csr_file} ${ca_crt} ${ca_key} ${signed_crt} ${san}=${EMPTY} + IF "${san}" != "${EMPTY}" + VAR ${ext_file}= /tmp/router-san-${signed_crt.replace('/', '-')}.cnf + Create File ${ext_file} ${san}\n + ${result}= Run Process + ... openssl x509 -req -in ${csr_file} -CA ${ca_crt} -CAkey ${ca_key} -CAcreateserial -out ${signed_crt} -days 30 -sha256 -extfile ${ext_file} + ... shell=True + Remove File ${ext_file} + ELSE + ${result}= Run Process + ... openssl x509 -req -in ${csr_file} -CA ${ca_crt} -CAkey ${ca_key} -CAcreateserial -out ${signed_crt} -days 30 -sha256 + ... shell=True + END + Should Be Equal As Integers ${result.rc} 0 + +Generate MTLS Client Cert + [Documentation] Generate a CA and a single client cert in the given directory. + [Arguments] ${tmpdir} ${edge_host} + Generate CA Certificate ${tmpdir}/ca.key ${tmpdir}/ca.crt /CN=MS-Test-Root-CA + Generate CSR And Key ${tmpdir}/usr.key ${tmpdir}/usr.csr /CN=example-test.com + VAR ${san}= subjectAltName = DNS.1:*.${BASE_DOMAIN},DNS.2:${edge_host} + Sign CSR With CA ${tmpdir}/usr.csr ${tmpdir}/ca.crt ${tmpdir}/ca.key ${tmpdir}/usr.crt ${san} + +Generate Client Cert File In Dir + [Documentation] Generate a CSR, key, and signed cert in the given directory using a name prefix. + [Arguments] ${tmpdir} ${host} ${cn} ${cert_prefix} + Generate CSR And Key ${tmpdir}/${cert_prefix}.key ${tmpdir}/${cert_prefix}.csr /CN=${cn} + VAR ${san}= subjectAltName = DNS.1:*.${BASE_DOMAIN},DNS.2:${host} + Sign CSR With CA + ... ${tmpdir}/${cert_prefix}.csr + ... ${tmpdir}/ca.crt + ... ${tmpdir}/ca.key + ... ${tmpdir}/${cert_prefix}.crt + ... ${san} + +Get First LB IP + [Documentation] Get the first load balancer IP of the router-default service. + ${ip}= Oc Get JsonPath + ... service ${ROUTER_NS} router-default + ... .status.loadBalancer.ingress[0].ip + RETURN ${ip} + +Get Router Deployment Generation + [Documentation] Get the current generation of the router-default deployment. + ${gen}= Oc Get JsonPath + ... deployment ${ROUTER_NS} router-default + ... .metadata.generation + RETURN ${gen} + +Get Host IPs Via SSH + [Documentation] Get all global IPv4 addresses from the MicroShift host. + ${stdout} ${stderr} ${rc}= SSHLibrary.Execute Command + ... ip -4 addr show scope global | grep -oP 'inet \\K[\\d.]+' + ... sudo=False return_rc=True return_stderr=True return_stdout=True + Should Be Equal As Integers 0 ${rc} + @{ips}= Split String ${stdout} \n + RETURN @{ips} + +Get First Host Interface And IP Via SSH + [Documentation] Get the first non-loopback IPv4 interface name and its IP from the host. + ${stdout} ${stderr} ${rc}= SSHLibrary.Execute Command + ... ip -4 addr show scope global | awk '/^[0-9]+:/{iface=$2; gsub(":","",iface)} /inet /{print iface, gensub(/\\/.*/, "", 1, $2); exit}' + ... sudo=False + ... return_rc=True + ... return_stderr=True + ... return_stdout=True + Should Be Equal As Integers 0 ${rc} + @{parts}= Split String ${stdout} + ${iface}= Get From List ${parts} 0 + ${ip}= Get From List ${parts} 1 + RETURN ${iface} ${ip} + +Show Invalid Drop In Config Should Fail With + [Documentation] Write an invalid drop-in config, run show-config, verify it fails with expected message, then remove the drop-in. + [Arguments] ${config_content} ${expected_error} + Drop In MicroShift Config ${config_content} 10-router-invalid + ${stdout} ${stderr} ${rc}= SSHLibrary.Execute Command + ... microshift show-config --mode effective + ... sudo=True return_rc=True return_stderr=True return_stdout=True + Remove Drop In MicroShift Config 10-router-invalid + Should Not Be Equal As Integers ${rc} 0 + Should Contain ${stderr}${stdout} ${expected_error} + +Privileged Namespace + [Documentation] Grant privileged SCC to the default service account in a namespace. + [Arguments] ${ns}=${NAMESPACE} + Run With Kubeconfig oc adm policy add-scc-to-user privileged -z default -n ${ns} diff --git a/test/scenarios-bootc/el10/presubmits/el102-src@router.sh b/test/scenarios-bootc/el10/presubmits/el102-src@router.sh index 5185849f03..85fe4c9b73 100644 --- a/test/scenarios-bootc/el10/presubmits/el102-src@router.sh +++ b/test/scenarios-bootc/el10/presubmits/el102-src@router.sh @@ -13,5 +13,5 @@ scenario_remove_vms() { scenario_run_tests() { run_tests host1 \ - suites/router + suites/router/router.robot } diff --git a/test/scenarios-bootc/el10/releases/el102-lrel@osconfig-router.sh b/test/scenarios-bootc/el10/releases/el102-lrel@osconfig-router.sh index 177e9ad3e6..6fdb06b3d6 100644 --- a/test/scenarios-bootc/el10/releases/el102-lrel@osconfig-router.sh +++ b/test/scenarios-bootc/el10/releases/el102-lrel@osconfig-router.sh @@ -23,7 +23,7 @@ scenario_run_tests() { run_tests host1 \ suites/osconfig/clusterid.robot \ suites/osconfig/systemd-resolved.robot \ - suites/router/ \ + suites/router/router.robot \ suites/otp-workloads/oc-cli.robot \ suites/otp-workloads/statefulset-pvc.robot } diff --git a/test/scenarios-bootc/el10/releases/el102-lrel@router-extended.sh b/test/scenarios-bootc/el10/releases/el102-lrel@router-extended.sh new file mode 100644 index 0000000000..5e7f13f12e --- /dev/null +++ b/test/scenarios-bootc/el10/releases/el102-lrel@router-extended.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Sourced from scenario.sh and uses functions defined there. + +start_image="rhel102-bootc-brew-lrel-optional" + +scenario_create_vms() { + exit_if_image_not_found "${start_image}" + + prepare_kickstart host1 kickstart-bootc.ks.template "${start_image}" + launch_vm rhel102-bootc --vm_vcpus 4 +} + +scenario_remove_vms() { + exit_if_image_not_found "${start_image}" + + remove_vm host1 +} + +scenario_run_tests() { + exit_if_image_not_found "${start_image}" + + run_tests host1 \ + suites/router/router-routes.robot \ + suites/router/router-config.robot +} diff --git a/test/scenarios-bootc/el9/presubmits/el98-src@router.sh b/test/scenarios-bootc/el9/presubmits/el98-src@router.sh index 1f2bc33ce9..7241c7c82d 100644 --- a/test/scenarios-bootc/el9/presubmits/el98-src@router.sh +++ b/test/scenarios-bootc/el9/presubmits/el98-src@router.sh @@ -13,5 +13,5 @@ scenario_remove_vms() { scenario_run_tests() { run_tests host1 \ - suites/router + suites/router/router.robot } diff --git a/test/scenarios-bootc/el9/releases/el98-lrel@osconfig-router.sh b/test/scenarios-bootc/el9/releases/el98-lrel@osconfig-router.sh index 629ce5417e..c6c3c78265 100644 --- a/test/scenarios-bootc/el9/releases/el98-lrel@osconfig-router.sh +++ b/test/scenarios-bootc/el9/releases/el98-lrel@osconfig-router.sh @@ -23,7 +23,7 @@ scenario_run_tests() { run_tests host1 \ suites/osconfig/clusterid.robot \ suites/osconfig/systemd-resolved.robot \ - suites/router/ \ + suites/router/router.robot \ suites/otp-workloads/oc-cli.robot \ suites/otp-workloads/statefulset-pvc.robot } diff --git a/test/scenarios-bootc/el9/releases/el98-lrel@router-extended.sh b/test/scenarios-bootc/el9/releases/el98-lrel@router-extended.sh new file mode 100644 index 0000000000..75986c5142 --- /dev/null +++ b/test/scenarios-bootc/el9/releases/el98-lrel@router-extended.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Sourced from scenario.sh and uses functions defined there. + +start_image="rhel98-bootc-brew-lrel-optional" + +scenario_create_vms() { + exit_if_image_not_found "${start_image}" + + prepare_kickstart host1 kickstart-bootc.ks.template "${start_image}" + launch_vm rhel98-bootc --vm_vcpus 4 +} + +scenario_remove_vms() { + exit_if_image_not_found "${start_image}" + + remove_vm host1 +} + +scenario_run_tests() { + exit_if_image_not_found "${start_image}" + + run_tests host1 \ + suites/router/router-routes.robot \ + suites/router/router-config.robot +} diff --git a/test/scenarios/presubmits/el98-src@router.sh b/test/scenarios/presubmits/el98-src@router.sh index 820af8566d..c18c108c5b 100644 --- a/test/scenarios/presubmits/el98-src@router.sh +++ b/test/scenarios/presubmits/el98-src@router.sh @@ -12,5 +12,5 @@ scenario_remove_vms() { } scenario_run_tests() { - run_tests host1 suites/router/ + run_tests host1 suites/router/router.robot } diff --git a/test/scenarios/releases/el98-lrel@router-extended.sh b/test/scenarios/releases/el98-lrel@router-extended.sh new file mode 100644 index 0000000000..302f93c830 --- /dev/null +++ b/test/scenarios/releases/el98-lrel@router-extended.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Sourced from scenario.sh and uses functions defined there. + +start_image="rhel98-brew-lrel-optional" + +scenario_create_vms() { + exit_if_commit_not_found "${start_image}" + + prepare_kickstart host1 kickstart.ks.template "${start_image}" + launch_vm rhel-9.8 --vm_vcpus 4 +} + +scenario_remove_vms() { + exit_if_commit_not_found "${start_image}" + + remove_vm host1 +} + +scenario_run_tests() { + exit_if_commit_not_found "${start_image}" + + run_tests host1 \ + suites/router/router-routes.robot \ + suites/router/router-config.robot +} diff --git a/test/scenarios/releases/el98-lrel@router.sh b/test/scenarios/releases/el98-lrel@router.sh index 91501d6085..999c597083 100644 --- a/test/scenarios/releases/el98-lrel@router.sh +++ b/test/scenarios/releases/el98-lrel@router.sh @@ -20,5 +20,5 @@ scenario_remove_vms() { scenario_run_tests() { exit_if_commit_not_found "${start_image}" - run_tests host1 suites/router/ + run_tests host1 suites/router/router.robot } diff --git a/test/suites/router/router-config.robot b/test/suites/router/router-config.robot new file mode 100644 index 0000000000..8992ad9e06 --- /dev/null +++ b/test/suites/router/router-config.robot @@ -0,0 +1,983 @@ +*** Settings *** +Documentation Router ingress controller configuration tests (disruptive) +... Migrated from openshift-tests-private: +... OCP-73203, OCP-73209, OCP-77349, OCP-80508, OCP-80510, OCP-80514, +... OCP-80517, OCP-80518, OCP-80520, OCP-81996, OCP-81997, OCP-82000, +... OCP-82003, OCP-82004, OCP-82014, OCP-82015, OCP-84260 + +Resource ../../resources/common.resource +Resource ../../resources/oc.resource +Resource ../../resources/microshift-config.resource +Resource ../../resources/microshift-process.resource +Resource ../../resources/ostree-health.resource +Resource ../../resources/router.resource + +Suite Setup Setup Suite With Namespace +Suite Teardown Teardown Suite With Namespace + +Test Tags restart slow + + +*** Variables *** +${BASE_DOMAIN} apps.example.com +${ALT_HTTP_PORT} 10080 +${ALT_HTTPS_PORT} 10443 + +${CONFIG_OLD_TLS} SEPARATOR=\n +... --- +... ingress: +... \ \ tlsSecurityProfile: +... \ \ \ \ old: {} +... \ \ \ \ type: Old + +${CONFIG_INTERMEDIATE_TLS} SEPARATOR=\n +... --- +... ingress: +... \ \ tlsSecurityProfile: +... \ \ \ \ intermediate: {} +... \ \ \ \ type: Intermediate + +${CONFIG_MODERN_TLS} SEPARATOR=\n +... --- +... ingress: +... \ \ tlsSecurityProfile: +... \ \ \ \ modern: {} +... \ \ \ \ type: Modern + +${CONFIG_CUSTOM_TLS} SEPARATOR=\n +... --- +... ingress: +... \ \ tlsSecurityProfile: +... \ \ \ \ custom: +... \ \ \ \ \ \ ciphers: +... \ \ \ \ \ \ - DHE-RSA-AES256-GCM-SHA384 +... \ \ \ \ \ \ - ECDHE-ECDSA-AES256-GCM-SHA384 +... \ \ \ \ \ \ minTLSVersion: VersionTLS12 +... \ \ \ \ type: Custom + +${CONFIG_WILDCARD_ALLOWED} SEPARATOR=\n +... --- +... ingress: +... \ \ routeAdmissionPolicy: +... \ \ \ \ wildcardPolicy: "WildcardsAllowed" + +${CONFIG_WILDCARD_DISALLOWED} SEPARATOR=\n +... --- +... ingress: +... \ \ routeAdmissionPolicy: +... \ \ \ \ wildcardPolicy: "WildcardsDisallowed" + +${CONFIG_TUNING_CUSTOM} SEPARATOR=\n +... --- +... ingress: +... \ \ forwardedHeaderPolicy: "Replace" +... \ \ httpCompression: +... \ \ \ \ mimeTypes: +... \ \ \ \ - "image" +... \ \ logEmptyRequests: "Ignore" +... \ \ tuningOptions: +... \ \ \ \ clientFinTimeout: "2s" +... \ \ \ \ clientTimeout: "60s" +... \ \ \ \ headerBufferBytes: 65536 +... \ \ \ \ headerBufferMaxRewriteBytes: 16384 +... \ \ \ \ healthCheckInterval: "10s" +... \ \ \ \ maxConnections: 100000 +... \ \ \ \ serverFinTimeout: "2s" +... \ \ \ \ serverTimeout: "60s" +... \ \ \ \ threadCount: 8 +... \ \ \ \ tlsInspectDelay: "10s" +... \ \ \ \ tunnelTimeout: "2h" + +${CONFIG_MTLS17_REQUIRED} SEPARATOR=\n +... --- +... ingress: +... \ \ clientTLS: +... \ \ \ \ clientCA: +... \ \ \ \ \ \ name: "ocp80517" +... \ \ \ \ clientCertificatePolicy: "Required" + +${CONFIG_MTLS17_OPTIONAL} SEPARATOR=\n +... --- +... ingress: +... \ \ clientTLS: +... \ \ \ \ clientCA: +... \ \ \ \ \ \ name: "ocp80517" +... \ \ \ \ clientCertificatePolicy: "Optional" + +${CONFIG_MTLS18_SUBJECT_FILTER} SEPARATOR=\n +... --- +... ingress: +... \ \ clientTLS: +... \ \ \ \ allowedSubjectPatterns: ["/CN=example-test.com"] +... \ \ \ \ clientCA: +... \ \ \ \ \ \ name: "ocp80518" +... \ \ \ \ clientCertificatePolicy: "Required" + +${CONFIG_COOKIE_EXACT_100} SEPARATOR=\n +... --- +... ingress: +... \ \ accessLogging: +... \ \ \ \ httpCaptureCookies: +... \ \ \ \ - matchType: Exact +... \ \ \ \ \ \ maxLength: 100 +... \ \ \ \ \ \ name: foo +... \ \ \ \ status: Enabled + +${CONFIG_COOKIE_EXACT_10} SEPARATOR=\n +... --- +... ingress: +... \ \ accessLogging: +... \ \ \ \ httpCaptureCookies: +... \ \ \ \ - matchType: Exact +... \ \ \ \ \ \ maxLength: 10 +... \ \ \ \ \ \ name: foo +... \ \ \ \ status: Enabled + +${CONFIG_LOG_FORMAT_1} SEPARATOR=\n +... --- +... ingress: +... \ \ accessLogging: +... \ \ \ \ httpLogFormat: "%{+Q}r" +... \ \ \ \ status: Enabled + +${CONFIG_LOG_FORMAT_2} SEPARATOR=\n +... --- +... ingress: +... \ \ accessLogging: +... \ \ \ \ httpLogFormat: "%ci:%cp %si:%sp %HU %ST" +... \ \ \ \ status: Enabled + +${LOGGING_INVALID_MAXLENGTH_NEG1} SEPARATOR=\n +... --- +... ingress: +... \ \ accessLogging: +... \ \ \ \ httpCaptureCookies: +... \ \ \ \ - matchType: Exact +... \ \ \ \ \ \ maxLength: -1 +... \ \ \ \ \ \ name: foo +... \ \ \ \ status: Enabled + +${LOGGING_INVALID_MAXLENGTH_ZERO} SEPARATOR=\n +... --- +... ingress: +... \ \ accessLogging: +... \ \ \ \ httpCaptureCookies: +... \ \ \ \ - matchType: Exact +... \ \ \ \ \ \ maxLength: 0 +... \ \ \ \ \ \ name: foo +... \ \ \ \ status: Enabled + +${LOGGING_INVALID_COOKIE_NAME} SEPARATOR=\n +... --- +... ingress: +... \ \ accessLogging: +... \ \ \ \ httpCaptureCookies: +... \ \ \ \ - matchType: Exact +... \ \ \ \ \ \ maxLength: 100 +... \ \ \ \ \ \ name: "foo 33#?-" +... \ \ \ \ status: Enabled + +${LOGGING_INVALID_HEADER_MAXLENGTH} SEPARATOR=\n +... --- +... ingress: +... \ \ accessLogging: +... \ \ \ \ httpCaptureHeaders: +... \ \ \ \ \ \ request: +... \ \ \ \ \ \ - maxLength: -1 +... \ \ \ \ \ \ \ \ name: Host +... \ \ \ \ \ \ response: +... \ \ \ \ \ \ - maxLength: 10 +... \ \ \ \ \ \ \ \ name: "Server" +... \ \ \ \ status: Enabled + +${LOGGING_INVALID_STATUS} SEPARATOR=\n +... --- +... ingress: +... \ \ accessLogging: +... \ \ \ \ httpCaptureHeaders: +... \ \ \ \ \ \ request: +... \ \ \ \ \ \ - maxLength: 10 +... \ \ \ \ \ \ \ \ name: Host +... \ \ \ \ status: Enable + +${LOGGING_COOKIES_NO_STATUS} SEPARATOR=\n +... --- +... ingress: +... \ \ accessLogging: +... \ \ \ \ httpCaptureCookies: +... \ \ \ \ - matchType: Prefix +... \ \ \ \ \ \ maxLength: 100 +... \ \ \ \ \ \ namePrefix: foo + + +*** Test Cases *** +Custom Listening IPs And Ports + [Documentation] Verify configuring a specific listen address and custom HTTP/HTTPS ports + ... causes the router LB to expose only that IP on the custom ports, and routes + ... are reachable through those ports. + ... OCP-73203 + + ${iface} ${host_ip}= Get First Host Interface And IP Via SSH + ${config}= Catenate SEPARATOR=\n + ... --- + ... ingress: + ... \ \ listenAddress: + ... \ \ - ${iface} + ... \ \ ports: + ... \ \ \ \ http: ${ALT_HTTP_PORT} + ... \ \ \ \ https: ${ALT_HTTPS_PORT} + Setup Router Config And Restart ${config} + + Verify Custom LB Ports And IP ${host_ip} + + Deploy Web Server Signed + Deploy Test Client Pod + VAR ${HTTP_HOST}= service-unsecure-ocp73203.${BASE_DOMAIN} scope=TEST + VAR ${EDGE_HOST}= route-edge-ocp73203.${BASE_DOMAIN} scope=TEST + VAR ${PASS_HOST}= route-passth-ocp73203.${BASE_DOMAIN} scope=TEST + VAR ${REEN_HOST}= route-reen-ocp73203.${BASE_DOMAIN} scope=TEST + Create And Admit Four Route Types ${HTTP_HOST} ${EDGE_HOST} ${PASS_HOST} ${REEN_HOST} + Curl Four Routes Via Custom Ports ${host_ip} + [Teardown] Remove Router Config And Restart + +Enable Disable Router + [Documentation] Verify setting ingress status to Removed deletes the openshift-ingress namespace, + ... and setting it back to Managed restores the router LB with correct IPs and ports. + ... OCP-73209 + + ${config_removed}= Catenate SEPARATOR=\n + ... --- + ... ingress: + ... \ \ status: Removed + Setup Router Config And Restart ${config_removed} + Oc Wait namespace/openshift-ingress --for=delete --timeout=300s + + ${config_managed}= Catenate SEPARATOR=\n + ... --- + ... ingress: + ... \ \ status: Managed + Setup Router Config And Restart ${config_managed} + + ${svc_type}= Oc Get JsonPath service ${ROUTER_NS} router-default .spec.type + Should Be Equal As Strings ${svc_type} LoadBalancer + ${http_port}= Get LB Port http + Should Be Equal As Strings ${http_port} 80 + ${https_port}= Get LB Port https + Should Be Equal As Strings ${https_port} 443 + ${lb_ips}= Get LB IPs + Should Not Be Empty ${lb_ips} + [Teardown] Remove Router Config And Restart + +Tuning Options Customization + [Documentation] Verify default tuning env vars and haproxy config values, then apply + ... custom tuning and verify all values are updated. + ... OCP-77349 + + VAR ${http_host}= service-unsecure-ocp77349.${BASE_DOMAIN} + Deploy Web Server + Create OC Route ${NAMESPACE} http route-http service-unsecure --hostname=${http_host} + Verify Default Router Tuning Env Vars + Verify Default Router Tuning Haproxy + Setup Router Config And Restart ${CONFIG_TUNING_CUSTOM} + Verify Custom Router Tuning Env Vars + Verify Custom Router Tuning Haproxy + [Teardown] Remove Router Config And Restart + +Custom Default Certificate + [Documentation] Verify a custom TLS certificate secret can be configured as the default + ... ingress certificate, and that routes use the custom cert for TLS. + ... OCP-80508 + [Setup] Prepare Custom Cert For Test 80508 route-edge80508.${BASE_DOMAIN} + + ${config}= Catenate SEPARATOR=\n + ... --- + ... ingress: + ... \ \ certificateSecret: "router-test-cert" + Setup Router Config And Restart ${config} + + Create OC Route ${NAMESPACE} edge route-edge service-unsecure --hostname=${CERT_EDGE_HOST} + Route Should Be Admitted route-edge + Verify Custom Cert Is Active + Deploy Test Client Pod + Copy Files To Pod ${NAMESPACE} ${CLIENT_POD_NAME} ${CERT_TMPDIR} /data/certs + ${router_ip}= Get Router Pod IP + VAR ${resolve}= ${CERT_EDGE_HOST}:443:${router_ip} + Wait Until Curl With Client Cert Succeeds + ... ${CLIENT_POD_NAME} ${NAMESPACE} https://${CERT_EDGE_HOST} ${resolve} + Wait Until HTTPS Curl Succeeds From Pod + ... ${CLIENT_POD_NAME} ${NAMESPACE} https://${CERT_EDGE_HOST} ${resolve} + [Teardown] Run Keywords + ... Remove Router Config And Restart + ... AND Run With Kubeconfig oc delete secret router-test-cert -n ${ROUTER_NS} --ignore-not-found + +Old And Intermediate TLS Profiles + [Documentation] Verify the default Intermediate TLS profile cipher settings, then apply Old + ... profile and verify updated cipher settings, then restore Intermediate. + ... OCP-80510 + + Router Pod Env Should Have Value SSL_MIN_VERSION TLSv1.2 + ${ciphers}= Oc Get JsonPath + ... pod ${ROUTER_NS} ${EMPTY} .items[*].spec.containers[*].env[?(@.name=="ROUTER_CIPHERSUITES")].value + Should Contain ${ciphers} TLS_AES_128_GCM_SHA256 + + Setup Router Config And Restart ${CONFIG_OLD_TLS} + Router Pod Env Should Have Value SSL_MIN_VERSION TLSv1.1 + ${ciphers}= Oc Get JsonPath + ... pod ${ROUTER_NS} ${EMPTY} .items[*].spec.containers[*].env[?(@.name=="ROUTER_CIPHERS")].value + Should Contain ${ciphers} DES-CBC3-SHA + + Setup Router Config And Restart ${CONFIG_INTERMEDIATE_TLS} + Router Pod Env Should Have Value SSL_MIN_VERSION TLSv1.2 + ${ciphers}= Oc Get JsonPath + ... pod ${ROUTER_NS} ${EMPTY} .items[*].spec.containers[*].env[?(@.name=="ROUTER_CIPHERSUITES")].value + Should Contain ${ciphers} TLS_AES_128_GCM_SHA256 + [Teardown] Remove Router Config And Restart + +Modern And Custom TLS Profiles + [Documentation] Verify Modern TLS profile enforces TLSv1.3 in env vars and haproxy config, + ... then apply a Custom profile with specific ciphers. + ... OCP-80514 + + Setup Router Config And Restart ${CONFIG_MODERN_TLS} + Router Pod Env Should Have Value SSL_MIN_VERSION TLSv1.3 + ${ciphers}= Oc Get JsonPath + ... pod ${ROUTER_NS} ${EMPTY} .items[*].spec.containers[*].env[?(@.name=="ROUTER_CIPHERSUITES")].value + Should Contain ${ciphers} TLS_AES_128_GCM_SHA256 + ${haproxy}= Read Haproxy Config + Should Contain ${haproxy} ssl-default-bind-options ssl-min-ver TLSv1.3 + + Setup Router Config And Restart ${CONFIG_CUSTOM_TLS} + Router Pod Env Should Have Value SSL_MIN_VERSION TLSv1.2 + ${ciphers}= Oc Get JsonPath + ... pod ${ROUTER_NS} ${EMPTY} .items[*].spec.containers[*].env[?(@.name=="ROUTER_CIPHERS")].value + Should Contain ${ciphers} DHE-RSA-AES256-GCM-SHA384 + [Teardown] Remove Router Config And Restart + +MTLS Optional And Required Policy + [Documentation] Verify mTLS with clientCertificatePolicy Required rejects connections without + ... client cert, and Optional policy allows them. + ... OCP-80517 + [Setup] Prepare MTLS Cert For Test 80517 route-edge80517.${BASE_DOMAIN} + + Run With Kubeconfig oc create configmap ocp80517 --from-file=ca-bundle.pem=${MTLS_TMPDIR}/ca.crt -n ${ROUTER_NS} + + Setup Router Config And Restart ${CONFIG_MTLS17_REQUIRED} + Router Pod Env Should Have Value ROUTER_MUTUAL_TLS_AUTH required + Deploy MTLS Test Workloads + ${router_ip}= Get Router Pod IP + VAR ${resolve}= ${MTLS_EDGE_HOST}:443:${router_ip} + Wait Until Curl With Client Cert Succeeds + ... ${CLIENT_POD_NAME} + ... ${NAMESPACE} + ... https://${MTLS_EDGE_HOST} + ... ${resolve} + Curl Without Cert Should Return SSL Error + ... ${CLIENT_POD_NAME} + ... ${NAMESPACE} + ... https://${MTLS_EDGE_HOST} + ... ${resolve} + + Setup Router Config And Restart ${CONFIG_MTLS17_OPTIONAL} + Router Pod Env Should Have Value ROUTER_MUTUAL_TLS_AUTH optional + ${router_ip}= Get Router Pod IP + VAR ${resolve}= ${MTLS_EDGE_HOST}:443:${router_ip} + Wait Until Curl With Client Cert Succeeds + ... ${CLIENT_POD_NAME} + ... ${NAMESPACE} + ... https://${MTLS_EDGE_HOST} + ... ${resolve} + Wait Until HTTPS Curl Succeeds From Pod + ... ${CLIENT_POD_NAME} + ... ${NAMESPACE} + ... https://${MTLS_EDGE_HOST} + ... ${resolve} + [Teardown] Run Keywords + ... Remove Router Config And Restart + ... AND Run With Kubeconfig oc delete configmap ocp80517 -n ${ROUTER_NS} --ignore-not-found + +MTLS Subject Filter + [Documentation] Verify mTLS with allowedSubjectPatterns allows a cert matching the CN filter + ... and blocks a cert not matching it. + ... OCP-80518 + [Setup] Prepare Two MTLS Certs For Test 80518 route-edge80518.${BASE_DOMAIN} route2-edge80518.${BASE_DOMAIN} + + Run With Kubeconfig + ... oc create configmap ocp80518 --from-file=ca-bundle.pem=${MTLS2_TMPDIR}/ca.crt -n ${ROUTER_NS} + Setup Router Config And Restart ${CONFIG_MTLS18_SUBJECT_FILTER} + + ${env}= Oc Get JsonPath + ... pod + ... ${ROUTER_NS} + ... ${EMPTY} + ... .items[*].spec.containers[*].env[?(@.name=="ROUTER_MUTUAL_TLS_AUTH_FILTER")].value + Should Contain ${env} example-test.com + + Deploy Web Server + Deploy Test Client Pod + Copy Files To Pod ${NAMESPACE} ${CLIENT_POD_NAME} ${MTLS2_TMPDIR} /data/certs + Create OC Route ${NAMESPACE} edge route-edge service-unsecure + ... --hostname=${MTLS2_HOST1} --cert=${MTLS2_TMPDIR}/usr1.crt --key=${MTLS2_TMPDIR}/usr1.key + Create OC Route ${NAMESPACE} edge route-edge2 service-unsecure + ... --hostname=${MTLS2_HOST2} --cert=${MTLS2_TMPDIR}/usr2.crt --key=${MTLS2_TMPDIR}/usr2.key + Route Should Be Admitted route-edge + Route Should Be Admitted route-edge2 + + ${router_ip}= Get Router Pod IP + Wait Until Curl With Cert File Succeeds + ... ${CLIENT_POD_NAME} ${NAMESPACE} https://${MTLS2_HOST1} ${MTLS2_HOST1}:443:${router_ip} usr1 + Wait Until Curl With Cert File Returns 403 + ... ${CLIENT_POD_NAME} ${NAMESPACE} https://${MTLS2_HOST2} ${MTLS2_HOST2}:443:${router_ip} usr2 + [Teardown] Run Keywords + ... Remove Router Config And Restart + ... AND Run With Kubeconfig oc delete configmap ocp80518 -n ${ROUTER_NS} --ignore-not-found + +Wildcard Route Admission Policy + [Documentation] Verify WildcardsDisallowed rejects wildcard routes, WildcardsAllowed admits them, + ... and reverting to WildcardsDisallowed rejects them again. + ... OCP-80520 + [Setup] Run Keywords + ... Deploy Web Server + ... AND Deploy Test Client Pod + + VAR ${wildcard_host}= wildcard.${BASE_DOMAIN} + VAR ${any_host}= any.${BASE_DOMAIN} + + ${env}= Oc Get JsonPath + ... pod + ... ${ROUTER_NS} + ... ${EMPTY} + ... .items[*].spec.containers[*].env[?(@.name=="ROUTER_ALLOW_WILDCARD_ROUTES")].value + Should Be Equal As Strings ${env} false + Create OC Route ${NAMESPACE} http unsecure80520 service-unsecure + ... --hostname=${wildcard_host} --wildcard-policy=Subdomain + Route Should Not Be Admitted unsecure80520 + + Setup Router Config And Restart ${CONFIG_WILDCARD_ALLOWED} + Router Pod Env Should Have Value ROUTER_ALLOW_WILDCARD_ROUTES true + Route Should Be Admitted unsecure80520 + ${router_ip}= Get Router Pod IP + Wait Until Curl Succeeds From Pod + ... ${CLIENT_POD_NAME} ${NAMESPACE} http://${wildcard_host} ${wildcard_host}:80:${router_ip} + Wait Until Curl Succeeds From Pod + ... ${CLIENT_POD_NAME} ${NAMESPACE} http://${any_host} ${any_host}:80:${router_ip} + + Setup Router Config And Restart ${CONFIG_WILDCARD_DISALLOWED} + Router Pod Env Should Have Value ROUTER_ALLOW_WILDCARD_ROUTES false + Route Should Not Be Admitted unsecure80520 + [Teardown] Remove Router Config And Restart + +HTTP Capture Cookies Prefix Match + [Documentation] Verify httpCaptureCookies with Prefix match captures cookies matching the + ... prefix in router logs across HTTP, edge, and reencrypt routes. + ... OCP-81996 + + ${config}= Catenate SEPARATOR=\n + ... --- + ... ingress: + ... \ \ accessLogging: + ... \ \ \ \ httpCaptureCookies: + ... \ \ \ \ - matchType: Prefix + ... \ \ \ \ \ \ maxLength: 100 + ... \ \ \ \ \ \ namePrefix: foo + ... \ \ \ \ status: Enabled + Setup Router Config And Restart ${config} + + Deploy Web Server Signed + Deploy Test Client Pod + VAR ${routehost}= route-unsec81996.${BASE_DOMAIN} + VAR ${edge_host}= route-edge81996.${BASE_DOMAIN} + VAR ${reen_host}= route-reen81996.${BASE_DOMAIN} + Create OC Route ${NAMESPACE} http route-http service-unsecure --hostname=${routehost} + Route Should Be Admitted route-http + Create OC Route ${NAMESPACE} edge route-edge service-unsecure --hostname=${edge_host} + Route Should Be Admitted route-edge + Create OC Route ${NAMESPACE} reencrypt route-reen service-secure --hostname=${reen_host} + Route Should Be Admitted route-reen + ${router_ip}= Get Router Pod IP + Curl All Cookie Routes And Verify Logs ${routehost} ${edge_host} ${reen_host} ${router_ip} + [Teardown] Remove Router Config And Restart + +HTTP Capture Cookies Exact Match And MaxLength + [Documentation] Verify httpCaptureCookies with Exact match captures only exact-named cookies, + ... and maxLength truncates cookie values in logs. + ... OCP-81997 + + Setup Router Config And Restart ${CONFIG_COOKIE_EXACT_100} + Deploy Web Server + Deploy Test Client Pod + VAR ${routehost}= route-unsec81997.${BASE_DOMAIN} + Create OC Route ${NAMESPACE} http route-http service-unsecure --hostname=${routehost} + Route Should Be Admitted route-http + ${router_ip}= Get Router Pod IP + Wait Until Curl With Cookie Succeeds From Pod + ... ${CLIENT_POD_NAME} + ... ${NAMESPACE} + ... http://${routehost}/index.html + ... ${routehost}:80:${router_ip} + ... fooor=nobar + Wait Until Curl With Cookie Succeeds From Pod + ... ${CLIENT_POD_NAME} + ... ${NAMESPACE} + ... http://${routehost}/index.html + ... ${routehost}:80:${router_ip} + ... foo=bar + Wait For Router Logs To Contain foo=bar + Router Logs Should Not Contain fooor=nobar + + Setup Router Config And Restart ${CONFIG_COOKIE_EXACT_10} + ${router_ip}= Get Router Pod IP + Wait Until Curl With Cookie Succeeds From Pod + ... ${CLIENT_POD_NAME} + ... ${NAMESPACE} + ... http://${routehost}/index.html + ... ${routehost}:80:${router_ip} + ... foo=bar89abdef + Wait For Router Logs To Contain foo=bar89a + Router Logs Should Not Contain foo=bar89ab + [Teardown] Remove Router Config And Restart + +HTTP Capture Headers Request And Response + [Documentation] Verify httpCaptureHeaders captures request Host and response Server headers + ... in router logs, including for edge and reencrypt routes. + ... OCP-82000 + + ${config}= Catenate SEPARATOR=\n + ... --- + ... ingress: + ... \ \ accessLogging: + ... \ \ \ \ httpCaptureHeaders: + ... \ \ \ \ \ \ request: + ... \ \ \ \ \ \ - maxLength: 120 + ... \ \ \ \ \ \ \ \ name: Host + ... \ \ \ \ \ \ response: + ... \ \ \ \ \ \ - maxLength: 120 + ... \ \ \ \ \ \ \ \ name: "Server" + ... \ \ \ \ status: Enabled + Setup Router Config And Restart ${config} + + Deploy Web Server Signed + Deploy Test Client Pod + VAR ${routehost}= route-unsec82000.${BASE_DOMAIN} + VAR ${edge_host}= route-edge82000.${BASE_DOMAIN} + VAR ${reen_host}= route-reen82000.${BASE_DOMAIN} + Create OC Route ${NAMESPACE} http route-http service-unsecure --hostname=${routehost} + Route Should Be Admitted route-http + Create OC Route ${NAMESPACE} edge route-edge service-unsecure --hostname=${edge_host} + Route Should Be Admitted route-edge + Create OC Route ${NAMESPACE} reencrypt route-reen service-secure --hostname=${reen_host} + Route Should Be Admitted route-reen + Verify Header Capture Config And Logs ${routehost} ${edge_host} ${reen_host} + [Teardown] Remove Router Config And Restart + +HTTP Capture Headers MaxLength Adherence + [Documentation] Verify httpCaptureHeaders maxLength truncates captured header values in logs. + ... OCP-82003 + + VAR ${routehost}= route-unsec82003.${BASE_DOMAIN} + ${config}= Catenate SEPARATOR=\n + ... --- + ... ingress: + ... \ \ accessLogging: + ... \ \ \ \ httpCaptureHeaders: + ... \ \ \ \ \ \ request: + ... \ \ \ \ \ \ - maxLength: 16 + ... \ \ \ \ \ \ \ \ name: Host + ... \ \ \ \ \ \ response: + ... \ \ \ \ \ \ - maxLength: 5 + ... \ \ \ \ \ \ \ \ name: "Server" + ... \ \ \ \ status: Enabled + Setup Router Config And Restart ${config} + + Deploy Web Server + Deploy Test Client Pod + Create OC Route ${NAMESPACE} http route-http service-unsecure --hostname=${routehost} + Route Should Be Admitted route-http + ${router_ip}= Get Router Pod IP + Wait Until Curl Succeeds From Pod + ... ${CLIENT_POD_NAME} ${NAMESPACE} http://${routehost}/index.html ${routehost}:80:${router_ip} + # maxLength is 16, so hostname is truncated to exactly 16 chars: route-unsec82003 + Wait For Router Logs To Contain route-unsec82003 + ${logs}= Get Router Access Logs + Should Not Contain ${logs} ${routehost} + # nginx server version is 5+ chars, so only "nginx" should appear without the version + Should Contain ${logs} nginx + Should Not Contain ${logs} nginx/ + [Teardown] Remove Router Config And Restart + +Custom HTTP Error Pages + [Documentation] Verify custom 503 and 404 error pages are served when configured via + ... httpErrorCodePages configmap. + ... OCP-82004 + + Create Configmap From Files ${ROUTER_NS} custom-82004-error-code-pages + ... --from-file=./assets/router/error-page-503.http --from-file=./assets/router/error-page-404.http + ${config}= Catenate SEPARATOR=\n + ... --- + ... ingress: + ... \ \ httpErrorCodePages: + ... \ \ \ \ name: custom-82004-error-code-pages + Setup Router Config And Restart ${config} + + Deploy Web Server + Deploy Test Client Pod + VAR ${routehost}= route-unsec82004.${BASE_DOMAIN} + VAR ${noexist_host}= not-exist82004.${BASE_DOMAIN} + Create OC Route ${NAMESPACE} http route-http service-unsecure --hostname=${routehost} + Route Should Be Admitted route-http + Verify Custom Error Pages ${routehost} ${noexist_host} + [Teardown] Run Keywords + ... Remove Router Config And Restart + ... AND Run With Kubeconfig oc delete configmap custom-82004-error-code-pages -n ${ROUTER_NS} --ignore-not-found + +HTTP Log Format + [Documentation] Verify httpLogFormat with HAProxy format directives produces structured log + ... output with the correct format. + ... OCP-82014 + + Setup Router Config And Restart ${CONFIG_LOG_FORMAT_1} + Deploy Web Server + Deploy Test Client Pod + VAR ${routehost}= route-unsec82014.${BASE_DOMAIN} + Create OC Route ${NAMESPACE} http route-http service-unsecure --hostname=${routehost} + Route Should Be Admitted route-http + ${router_ip}= Get Router Pod IP + Wait Until Curl Succeeds From Pod + ... ${CLIENT_POD_NAME} + ... ${NAMESPACE} + ... http://${routehost}/path/second/index.html + ... ${routehost}:80:${router_ip} + Wait For Router Logs To Contain /path/second/index.html + ${logs}= Get Router Access Logs + Should Match Regexp ${logs} haproxy\\[[0-9]+\\]: "HEAD /path/second/index.html HTTP + + Setup Router Config And Restart ${CONFIG_LOG_FORMAT_2} + ${router_ip}= Get Router Pod IP + Wait Until Curl Succeeds From Pod + ... ${CLIENT_POD_NAME} + ... ${NAMESPACE} + ... http://${routehost}/path/second/index.html + ... ${routehost}:80:${router_ip} + Wait For Router Logs To Contain /path/second/index.html + ${logs}= Get Router Access Logs + Should Match Regexp ${logs} + ... haproxy\\[[0-9]+\\]: [0-9\\.a-fA-F:]+:[0-9]+ [0-9\\.a-fA-F:]+:8080 /path/second/index.html 200 + [Teardown] Remove Router Config And Restart + +Syslog Logging Destination + [Documentation] Verify logging to a syslog server delivers router access logs to the syslog pod, + ... and changing the facility is reflected in the haproxy global config. + ... OCP-82015 + + Privileged Namespace + Deploy Rsyslogd Pod + Deploy Web Server + Deploy Test Client Pod + ${syslog_ip}= Oc Get JsonPath pod ${NAMESPACE} rsyslogd-pod .status.podIP + Verify Syslog Logging ${syslog_ip} + [Teardown] Remove Router Config And Restart + +Negative Logging Config Validation + [Documentation] Verify invalid logging configurations are rejected by microshift show-config, + ... and that setting httpCaptureCookies without status: Enabled does not activate logging. + ... OCP-84260 + + Show Invalid Drop In Config Should Fail With ${LOGGING_INVALID_MAXLENGTH_NEG1} Must be between 1 and 1024 + Show Invalid Drop In Config Should Fail With ${LOGGING_INVALID_MAXLENGTH_ZERO} Must be between 1 and 1024 + Show Invalid Drop In Config Should Fail With ${LOGGING_INVALID_COOKIE_NAME} contains invalid characters + Show Invalid Drop In Config Should Fail With ${LOGGING_INVALID_HEADER_MAXLENGTH} maxLength must be at least 1 + Show Invalid Drop In Config Should Fail With ${LOGGING_INVALID_STATUS} invalid access logging status: Enable + + ${gen_before}= Get Router Deployment Generation + Drop In MicroShift Config ${LOGGING_COOKIES_NO_STATUS} 10-router + Restart MicroShift + Wait For Router Ready + ${gen_after}= Get Router Deployment Generation + Should Be Equal As Strings ${gen_before} ${gen_after} + ${haproxy}= Read Haproxy Config + Should Not Contain ${haproxy} capture cookie foo len 100 + [Teardown] Remove Router Config And Restart + + +*** Keywords *** +Verify Custom LB Ports And IP + [Documentation] Check the router-default LB has the expected IP and custom port numbers. + [Arguments] ${host_ip} + ${lb_ips}= Get LB IPs + Should Be Equal As Strings ${lb_ips} ${host_ip} + ${http_port}= Get LB Port http + Should Be Equal As Strings ${http_port} ${ALT_HTTP_PORT} + ${https_port}= Get LB Port https + Should Be Equal As Strings ${https_port} ${ALT_HTTPS_PORT} + +Curl Four Routes Via Custom Ports + [Documentation] Curl all four route types through the custom ports. + [Arguments] ${host_ip} + Wait Until Curl Succeeds From Pod + ... ${CLIENT_POD_NAME} ${NAMESPACE} + ... http://${HTTP_HOST}:${ALT_HTTP_PORT} ${HTTP_HOST}:${ALT_HTTP_PORT}:${host_ip} + Wait Until HTTPS Curl Succeeds From Pod + ... ${CLIENT_POD_NAME} ${NAMESPACE} + ... https://${EDGE_HOST}:${ALT_HTTPS_PORT} ${EDGE_HOST}:${ALT_HTTPS_PORT}:${host_ip} + Wait Until HTTPS Curl Succeeds From Pod + ... ${CLIENT_POD_NAME} ${NAMESPACE} + ... https://${PASS_HOST}:${ALT_HTTPS_PORT} ${PASS_HOST}:${ALT_HTTPS_PORT}:${host_ip} + Wait Until HTTPS Curl Succeeds From Pod + ... ${CLIENT_POD_NAME} ${NAMESPACE} + ... https://${REEN_HOST}:${ALT_HTTPS_PORT} ${REEN_HOST}:${ALT_HTTPS_PORT}:${host_ip} + +Verify Default Router Tuning Env Vars + [Documentation] Verify the default router tuning environment variable values. + Router Pod Env Should Have Value ROUTER_BUF_SIZE 32768 + Router Pod Env Should Have Value ROUTER_MAX_REWRITE_SIZE 8192 + Router Pod Env Should Have Value ROUTER_DEFAULT_CLIENT_TIMEOUT 30s + Router Pod Env Should Have Value ROUTER_CLIENT_FIN_TIMEOUT 1s + Router Pod Env Should Have Value ROUTER_DEFAULT_SERVER_TIMEOUT 30s + Router Pod Env Should Have Value ROUTER_DEFAULT_SERVER_FIN_TIMEOUT 1s + Router Pod Env Should Have Value ROUTER_DEFAULT_TUNNEL_TIMEOUT 1h + Router Pod Env Should Have Value ROUTER_INSPECT_DELAY 5s + Router Pod Env Should Have Value ROUTER_THREADS 4 + Router Pod Env Should Have Value ROUTER_MAX_CONNECTIONS 50000 + +Verify Default Router Tuning Haproxy + [Documentation] Verify the default tuning values are present in haproxy.config. + ${haproxy}= Read Haproxy Config + Should Contain ${haproxy} tune.bufsize 32768 + Should Contain ${haproxy} tune.maxrewrite 8192 + Should Contain ${haproxy} timeout client 30s + Should Contain ${haproxy} timeout server 30s + Should Contain ${haproxy} timeout tunnel 1h + Should Contain ${haproxy} nbthread 4 + Should Contain ${haproxy} maxconn 50000 + +Verify Custom Router Tuning Env Vars + [Documentation] Verify the custom tuning environment variable values after config change. + Router Pod Env Should Have Value ROUTER_BUF_SIZE 65536 + Router Pod Env Should Have Value ROUTER_MAX_REWRITE_SIZE 16384 + Router Pod Env Should Have Value ROUTER_DEFAULT_CLIENT_TIMEOUT 1m + Router Pod Env Should Have Value ROUTER_CLIENT_FIN_TIMEOUT 2s + Router Pod Env Should Have Value ROUTER_DEFAULT_SERVER_TIMEOUT 1m + Router Pod Env Should Have Value ROUTER_DEFAULT_SERVER_FIN_TIMEOUT 2s + Router Pod Env Should Have Value ROUTER_DEFAULT_TUNNEL_TIMEOUT 2h + Router Pod Env Should Have Value ROUTER_INSPECT_DELAY 10s + Router Pod Env Should Have Value ROUTER_THREADS 8 + Router Pod Env Should Have Value ROUTER_MAX_CONNECTIONS 100000 + +Verify Custom Router Tuning Haproxy + [Documentation] Verify the custom tuning values are present in haproxy.config after config change. + ${haproxy}= Read Haproxy Config + Should Contain ${haproxy} tune.bufsize 65536 + Should Contain ${haproxy} tune.maxrewrite 16384 + Should Contain ${haproxy} timeout client 1m + Should Contain ${haproxy} timeout server 1m + Should Contain ${haproxy} timeout tunnel 2h + Should Contain ${haproxy} nbthread 8 + Should Contain ${haproxy} maxconn 100000 + +Prepare Custom Cert For Test + [Documentation] Generate CA and user cert for custom certificate test, create TLS secret. + [Arguments] ${case_id} ${edge_host} + VAR ${tmpdir}= /tmp/ocp-${case_id} + Create Directory ${tmpdir} + VAR ${CERT_TMPDIR}= ${tmpdir} scope=TEST + VAR ${CERT_EDGE_HOST}= ${edge_host} scope=TEST + Generate CA Certificate ${tmpdir}/ca.key ${tmpdir}/ca.crt /CN=MS-default-CA + ${san}= Catenate SEPARATOR=\n + ... [ v3_req ] + ... subjectAltName = @alt_names + ... [ alt_names ] + ... DNS.1 = *.${BASE_DOMAIN} + Generate CSR And Key ${tmpdir}/usr.key ${tmpdir}/usr.csr /CN=example-ne.com + Sign CSR With CA ${tmpdir}/usr.csr ${tmpdir}/ca.crt ${tmpdir}/ca.key ${tmpdir}/usr.crt ${san} + Run With Kubeconfig + ... oc create secret tls router-test-cert --cert=${tmpdir}/ca.crt --key=${tmpdir}/ca.key -n ${ROUTER_NS} + Deploy Web Server + +Verify Custom Cert Is Active + [Documentation] Verify the custom cert secret is mounted and the cert issuer is correct. + ${vol}= Oc Get JsonPath + ... deployment + ... ${ROUTER_NS} + ... router-default + ... ..volumes[?(@.name=="default-certificate")].secret.secretName + Should Contain ${vol} router-test-cert + ${router_pod}= Get Router Pod Name + ${cert_info}= Run With Kubeconfig + ... oc exec -n ${ROUTER_NS} ${router_pod} -- openssl x509 -noout -in /etc/pki/tls/private/tls.crt -text + Should Contain ${cert_info} CN = MS-default-CA + +Prepare MTLS Cert For Test + [Documentation] Generate CA and client cert for mTLS tests. + [Arguments] ${case_id} ${edge_host} + VAR ${tmpdir}= /tmp/ocp-${case_id}-ca + Create Directory ${tmpdir} + VAR ${MTLS_TMPDIR}= ${tmpdir} scope=TEST + VAR ${MTLS_EDGE_HOST}= ${edge_host} scope=TEST + Generate MTLS Client Cert ${tmpdir} ${edge_host} + +Prepare Two MTLS Certs For Test + [Documentation] Generate CA and two client certs with different subjects for mTLS subject filter test. + [Arguments] ${case_id} ${host1} ${host2} + VAR ${tmpdir}= /tmp/ocp-${case_id}-ca + Create Directory ${tmpdir} + VAR ${MTLS2_TMPDIR}= ${tmpdir} scope=TEST + VAR ${MTLS2_HOST1}= ${host1} scope=TEST + VAR ${MTLS2_HOST2}= ${host2} scope=TEST + Generate CA Certificate ${tmpdir}/ca.key ${tmpdir}/ca.crt /CN=MS-Test-Root-CA + Generate Client Cert File In Dir ${tmpdir} ${host1} example-test.com usr1 + Generate Client Cert File In Dir ${tmpdir} ${host2} example-test2.com usr2 + +Deploy MTLS Test Workloads + [Documentation] Deploy workloads and create the mTLS edge route. + Deploy Web Server + Deploy Test Client Pod + Copy Files To Pod ${NAMESPACE} ${CLIENT_POD_NAME} ${MTLS_TMPDIR} /data/certs + Create OC Route ${NAMESPACE} edge route-edge service-unsecure + ... --hostname=${MTLS_EDGE_HOST} --cert=${MTLS_TMPDIR}/usr.crt --key=${MTLS_TMPDIR}/usr.key + Route Should Be Admitted route-edge + +Curl All Cookie Routes And Verify Logs + [Documentation] Curl all routes with cookies and verify correct log entries. + [Arguments] ${routehost} ${edge_host} ${reen_host} ${router_ip} + Wait Until Curl With Cookie Succeeds From Pod + ... ${CLIENT_POD_NAME} + ... ${NAMESPACE} + ... http://${routehost}/index.html + ... ${routehost}:80:${router_ip} + ... fo=nobar + Wait Until Curl With Cookie Succeeds From Pod + ... ${CLIENT_POD_NAME} + ... ${NAMESPACE} + ... http://${routehost}/index.html + ... ${routehost}:80:${router_ip} + ... foo=bar + Wait Until Curl With Cookie Succeeds From Pod + ... ${CLIENT_POD_NAME} + ... ${NAMESPACE} + ... http://${routehost}/index.html + ... ${routehost}:80:${router_ip} + ... foo22=bar22 + Wait Until HTTPS Curl With Cookie Succeeds From Pod + ... ${CLIENT_POD_NAME} + ... ${NAMESPACE} + ... https://${edge_host}/index.html + ... ${edge_host}:443:${router_ip} + ... foo=barforedge + Wait Until HTTPS Curl With Cookie Succeeds From Pod + ... ${CLIENT_POD_NAME} + ... ${NAMESPACE} + ... https://${reen_host}/index.html + ... ${reen_host}:443:${router_ip} + ... foo=barforreen + Wait For Router Logs To Contain foo=bar + Wait For Router Logs To Contain foo22=bar22 + Wait For Router Logs To Contain foo=barforedge + Wait For Router Logs To Contain foo=barforreen + Router Logs Should Not Contain fo=nobar + +Verify Header Capture Config And Logs + [Documentation] Verify haproxy header capture config and check captured headers in logs. + [Arguments] ${routehost} ${edge_host} ${reen_host} + Verify Header Capture In Haproxy + ${router_ip}= Get Router Pod IP + Wait Until Curl Succeeds From Pod + ... ${CLIENT_POD_NAME} ${NAMESPACE} http://${routehost}/index.html ${routehost}:80:${router_ip} + Wait Until HTTPS Curl Succeeds From Pod + ... ${CLIENT_POD_NAME} ${NAMESPACE} https://${edge_host}/index.html ${edge_host}:443:${router_ip} + Wait Until HTTPS Curl Succeeds From Pod + ... ${CLIENT_POD_NAME} ${NAMESPACE} https://${reen_host}/index.html ${reen_host}:443:${router_ip} + Wait For Router Logs To Contain ${routehost} + Wait For Router Logs To Contain ${edge_host} + Wait For Router Logs To Contain ${reen_host} + +Verify Header Capture In Haproxy + [Documentation] Verify the haproxy frontend has the expected header capture configuration. + ${router_pod}= Get Router Pod Name + ${haproxy}= Run With Kubeconfig + ... oc exec -n ${ROUTER_NS} ${router_pod} -- grep -A 20 "frontend fe_sni" haproxy.config + Should Contain ${haproxy} capture request header Host len 120 + Should Contain ${haproxy} capture response header Server len 120 + +Verify Custom Error Pages + [Documentation] Verify custom 503 and 404 error pages are served by the router. + [Arguments] ${routehost} ${noexist_host} + ${router_pod}= Get Router Pod Name + ${error503}= Run With Kubeconfig + ... oc exec -n ${ROUTER_NS} ${router_pod} -- cat /var/lib/haproxy/errorfiles/error-page-503.http + Should Contain ${error503} Custom:Application Unavailable + ${error404}= Run With Kubeconfig + ... oc exec -n ${ROUTER_NS} ${router_pod} -- cat /var/lib/haproxy/errorfiles/error-page-404.http + Should Contain ${error404} Custom:Not Found + ${router_ip}= Get Router Pod IP + ${output}= Run With Kubeconfig + ... oc exec -n ${NAMESPACE} ${CLIENT_POD_NAME} -- curl http://${noexist_host} -s --resolve ${noexist_host}:80:${router_ip} --connect-timeout 10 + Should Contain ${output} Custom:Not Found + Scale Deployment ${NAMESPACE} web-server-deploy 0 + Wait Until Keyword Succeeds 60s 2s + ... Custom 503 Body Should Contain ${routehost} ${router_ip} + +Custom 503 Body Should Contain + [Documentation] Curl a route (GET, full response body) and assert the custom 503 error page body is returned. + [Arguments] ${routehost} ${router_ip} + ${output}= Run With Kubeconfig + ... oc exec -n ${NAMESPACE} ${CLIENT_POD_NAME} -- curl http://${routehost} -s --resolve ${routehost}:80:${router_ip} --connect-timeout 10 + Should Contain ${output} Custom:Application Unavailable + +Verify Syslog Logging + [Documentation] Apply syslog config and verify log delivery, then verify facility change. + [Arguments] ${syslog_ip} + VAR ${config1}= SEPARATOR=\n + ... --- + ... ingress: + ... \ \ accessLogging: + ... \ \ \ \ destination: + ... \ \ \ \ \ \ syslog: + ... \ \ \ \ \ \ \ \ address: ${syslog_ip} + ... \ \ \ \ \ \ \ \ port: 514 + ... \ \ \ \ \ \ type: Syslog + ... \ \ \ \ status: Enabled + Setup Router Config And Restart ${config1} + VAR ${routehost}= route-unsec82015.${BASE_DOMAIN} + Create OC Route ${NAMESPACE} http route-http service-unsecure --hostname=${routehost} + Route Should Be Admitted route-http + Verify Syslog Haproxy Config ${syslog_ip} local1 + Verify Syslog Log Delivery ${routehost} ${syslog_ip} + VAR ${config2}= SEPARATOR=\n + ... --- + ... ingress: + ... \ \ accessLogging: + ... \ \ \ \ destination: + ... \ \ \ \ \ \ syslog: + ... \ \ \ \ \ \ \ \ address: ${syslog_ip} + ... \ \ \ \ \ \ \ \ port: 514 + ... \ \ \ \ \ \ \ \ facility: local2 + ... \ \ \ \ \ \ type: Syslog + ... \ \ \ \ status: Enabled + Setup Router Config And Restart ${config2} + Verify Syslog Haproxy Config ${syslog_ip} local2 + +Verify Syslog Haproxy Config + [Documentation] Verify the syslog server address and facility in haproxy global config. + [Arguments] ${syslog_ip} ${facility} + ${haproxy}= Read Haproxy Config + Should Contain ${haproxy} log ${syslog_ip}:514 len 1024 ${facility} info + +Verify Syslog Log Delivery + [Documentation] Curl a route and verify the log entry appears in the syslog pod. + [Arguments] ${routehost} ${syslog_ip} + ${router_ip}= Get Router Pod IP + Wait Until Curl Succeeds From Pod + ... ${CLIENT_POD_NAME} ${NAMESPACE} + ... http://${routehost}/path/second/index.html ${routehost}:80:${router_ip} + Wait Until Keyword Succeeds 60s 3s Syslog Pod Should Contain /path/second/index.html + +Syslog Pod Should Contain + [Documentation] Check that the rsyslogd pod logs contain a pattern. + [Arguments] ${pattern} + ${logs}= Oc Logs rsyslogd-pod --tail=20 ${NAMESPACE} + Should Contain ${logs} ${pattern} + +Router Logs Should Not Contain + [Documentation] Verify the router access logs do NOT contain a given pattern. + [Arguments] ${pattern} + ${logs}= Get Router Access Logs + Should Not Contain ${logs} ${pattern} diff --git a/test/suites/router/router-routes.robot b/test/suites/router/router-routes.robot new file mode 100644 index 0000000000..ff79b66535 --- /dev/null +++ b/test/suites/router/router-routes.robot @@ -0,0 +1,249 @@ +*** Settings *** +Documentation Router end-to-end route tests +... Migrated from openshift-tests-private: +... OCP-60136, OCP-60266, OCP-60283, OCP-72802, OCP-73152, OCP-73202 + +Resource ../../resources/common.resource +Resource ../../resources/oc.resource +Resource ../../resources/microshift-network.resource +Resource ../../resources/router.resource + +Suite Setup Setup Suite With Namespace +Suite Teardown Teardown Suite With Namespace + +Test Tags slow + + +*** Variables *** +${BASE_DOMAIN} apps.example.com + + +*** Test Cases *** +Reencrypt Route Via Ingress With Destination CA + [Documentation] Verify a reencrypt route created via a Kubernetes Ingress resource with + ... destination CA certificate is admitted and reachable. + ... OCP-60136 + [Setup] Setup Reencrypt Ingress Test + + ${router_ip}= Get Router Pod IP + ${srv_pod}= Get Web Server Pod Name + Wait Until HTTPS Curl Succeeds From Pod + ... ${srv_pod} ${NAMESPACE} + ... https://service-secure-test.example.com:443 service-secure-test.example.com:443:${router_ip} + ${haproxy}= Read Haproxy Config + Should Contain ${haproxy} backend be_secure:${NAMESPACE}:ingress-ms-reen + [Teardown] Teardown Reencrypt Ingress Test + +Edge And Passthrough Routes + [Documentation] Verify edge-terminated and passthrough route creation and connectivity. + ... OCP-60266 + [Setup] Deploy Web Server + + ${router_ip}= Get Router Pod IP + ${srv_pod}= Get Web Server Pod Name + VAR ${pass_host}= route-pass-60266.${BASE_DOMAIN} + VAR ${edge_host}= route-edge-60266.${BASE_DOMAIN} + + Create OC Route ${NAMESPACE} passthrough ms-pass service-secure --hostname=${pass_host} + Route Should Be Admitted ms-pass + Wait Until HTTPS Curl Succeeds From Pod + ... ${srv_pod} ${NAMESPACE} https://${pass_host}:443 ${pass_host}:443:${router_ip} + ${haproxy}= Read Haproxy Config + Should Contain ${haproxy} backend be_tcp:${NAMESPACE}:ms-pass + + Create OC Route ${NAMESPACE} edge ms-edge service-unsecure --hostname=${edge_host} + Route Should Be Admitted ms-edge + Wait Until HTTPS Curl Succeeds From Pod + ... ${srv_pod} ${NAMESPACE} https://${edge_host}:443 ${edge_host}:443:${router_ip} + ${haproxy}= Read Haproxy Config + Should Contain ${haproxy} backend be_edge_http:${NAMESPACE}:ms-edge + [Teardown] Run Keywords + ... Oc Delete route/ms-pass route/ms-edge -n ${NAMESPACE} + ... AND Oc Delete -f ${WEB_SERVER_DEPLOY} -n ${NAMESPACE} + +HTTP And Reencrypt Routes + [Documentation] Verify HTTP route via oc expose and reencrypt route creation and connectivity. + ... OCP-60283 + [Setup] Deploy Web Server Signed + + ${router_ip}= Get Router Pod IP + ${srv_pod}= Get Web Server Pod Name + VAR ${http_host}= route-http-60283.${BASE_DOMAIN} + VAR ${reen_host}= route-reen-60283.${BASE_DOMAIN} + + Create OC Route ${NAMESPACE} http ms-http service-unsecure --hostname=${http_host} + Route Should Be Admitted ms-http + Wait Until Curl Succeeds From Pod + ... ${srv_pod} ${NAMESPACE} http://${http_host}:80 ${http_host}:80:${router_ip} + ${haproxy}= Read Haproxy Config + Should Contain ${haproxy} backend be_http:${NAMESPACE}:ms-http + + Create OC Route ${NAMESPACE} reencrypt ms-reen service-secure --hostname=${reen_host} + Route Should Be Admitted ms-reen + Wait Until HTTPS Curl Succeeds From Pod + ... ${srv_pod} ${NAMESPACE} https://${reen_host}:443 ${reen_host}:443:${router_ip} + ${haproxy}= Read Haproxy Config + Should Contain ${haproxy} backend be_secure:${NAMESPACE}:ms-reen + [Teardown] Run Keywords + ... Oc Delete route/ms-http route/ms-reen -n ${NAMESPACE} + ... AND Oc Delete -f ${WEB_SERVER_SIGNED_DEPLOY} -n ${NAMESPACE} + +Namespace Ownership Default Config + [Documentation] Verify the default InterNamespaceAllowed config allows routes from different + ... namespaces to share the same hostname with different paths. + ... OCP-72802 + [Setup] Setup Two Namespace Test + + ${router_ip}= Get Router Pod IP + VAR ${HTTP_HOST}= service-unsecure-ocp72802.${BASE_DOMAIN} scope=TEST + VAR ${EDGE_HOST}= route-edge-ocp72802.${BASE_DOMAIN} scope=TEST + VAR ${REEN_HOST}= route-reen-ocp72802.${BASE_DOMAIN} scope=TEST + + ${env}= Oc Get JsonPath + ... pod ${ROUTER_NS} ${EMPTY} + ... .items[*].spec.containers[*].env[?(@.name=="ROUTER_DISABLE_NAMESPACE_OWNERSHIP_CHECK")].value + Should Be Equal As Strings ${env} true + + Create NS Ownership Routes + All NS Ownership Routes Should Be Admitted + + Wait Until Curl Succeeds From Pod + ... ${CLIENT_POD_NAME} ${NS1} http://${HTTP_HOST}/path/index.html ${HTTP_HOST}:80:${router_ip} + Wait Until Curl Succeeds From Pod + ... ${CLIENT_POD_NAME} ${NS1} http://${HTTP_HOST}/test/index.html ${HTTP_HOST}:80:${router_ip} + [Teardown] Teardown Two Namespace Test + +Router Load Balancer Service Type + [Documentation] Verify the router-default service is of type LoadBalancer with IPs assigned, + ... and that all route types are reachable through the LB IP. + ... OCP-73152 + [Setup] Run Keywords + ... Deploy Web Server Signed + ... AND Deploy Test Client Pod + + ${svc_type}= Oc Get JsonPath service ${ROUTER_NS} router-default .spec.type + Should Be Equal As Strings ${svc_type} LoadBalancer + ${lb_ips}= Get LB IPs + Should Not Be Empty ${lb_ips} + + VAR ${HTTP_HOST}= service-unsecure-ocp73152.${BASE_DOMAIN} scope=TEST + VAR ${EDGE_HOST}= route-edge-ocp73152.${BASE_DOMAIN} scope=TEST + VAR ${PASS_HOST}= route-passth-ocp73152.${BASE_DOMAIN} scope=TEST + VAR ${REEN_HOST}= route-reen-ocp73152.${BASE_DOMAIN} scope=TEST + + Create And Admit Four Route Types ${HTTP_HOST} ${EDGE_HOST} ${PASS_HOST} ${REEN_HOST} + + ${lb_ip}= Fetch From Left ${lb_ips} ${SPACE} + Curl All LB Route Types Via IP ${lb_ip} + [Teardown] Run Keywords + ... Oc Delete route/route-http route/route-edge route/route-passth route/route-reen -n ${NAMESPACE} + ... AND Oc Delete -f ${WEB_SERVER_SIGNED_DEPLOY} -n ${NAMESPACE} + ... AND Oc Delete -f ${TEST_CLIENT_POD} -n ${NAMESPACE} + +Default Listening IPs And Ports + [Documentation] Verify the router-default service LB IPs match all host IPs, default ports + ... are 80 and 443, and all route types are reachable through each LB IP. + ... OCP-73202 + [Setup] Run Keywords + ... Deploy Web Server Signed + ... AND Deploy Test Client Pod + + Verify Default LB IPs And Ports + + VAR ${HTTP_HOST}= service-unsecure-ocp73202.${BASE_DOMAIN} scope=TEST + VAR ${EDGE_HOST}= route-edge-ocp73202.${BASE_DOMAIN} scope=TEST + VAR ${PASS_HOST}= route-passth-ocp73202.${BASE_DOMAIN} scope=TEST + VAR ${REEN_HOST}= route-reen-ocp73202.${BASE_DOMAIN} scope=TEST + + Create And Admit Four Route Types ${HTTP_HOST} ${EDGE_HOST} ${PASS_HOST} ${REEN_HOST} + + ${lb_ips}= Get LB IPs + @{ips}= Split String ${lb_ips} + FOR ${lb_ip} IN @{ips} + Curl All LB Route Types Via IP ${lb_ip} + END + [Teardown] Run Keywords + ... Oc Delete route/route-http route/route-edge route/route-passth route/route-reen -n ${NAMESPACE} + ... AND Oc Delete -f ${WEB_SERVER_SIGNED_DEPLOY} -n ${NAMESPACE} + ... AND Oc Delete -f ${TEST_CLIENT_POD} -n ${NAMESPACE} + + +*** Keywords *** +Setup Reencrypt Ingress Test + [Documentation] Deploy web-server-signed and apply the destCA ingress. + Deploy Web Server Signed + Oc Create -f ${INGRESS_DESTCA} -n ${NAMESPACE} + Route Should Be Admitted ingress-ms-reen + +Teardown Reencrypt Ingress Test + [Documentation] Delete the destCA ingress and web-server deployment. + Oc Delete -f ${INGRESS_DESTCA} -n ${NAMESPACE} + Oc Delete -f ${WEB_SERVER_SIGNED_DEPLOY} -n ${NAMESPACE} + +Setup Two Namespace Test + [Documentation] Create two extra namespaces and deploy workloads in each. + VAR ${NS1}= ${NAMESPACE}-ocp72802-1 scope=TEST + VAR ${NS2}= ${NAMESPACE}-ocp72802-2 scope=TEST + Create Namespace ${NS1} + Create Namespace ${NS2} + Deploy Test Client Pod ${NS1} + Deploy Web Server ${NS1} + Deploy Test Client Pod ${NS2} + Deploy Web Server ${NS2} + +Teardown Two Namespace Test + [Documentation] Delete the extra namespaces. + Remove Namespace ${NS1} + Remove Namespace ${NS2} + +Create NS Ownership Routes + [Documentation] Create HTTP, edge, and reencrypt routes in both test namespaces. + Create OC Route ${NS1} http service-unsecure service-unsecure + ... --hostname=${HTTP_HOST} --path=/path + Create OC Route ${NS1} edge route-edge service-unsecure + ... --hostname=${EDGE_HOST} --path=/path + Create OC Route ${NS1} reencrypt route-reen service-secure + ... --hostname=${REEN_HOST} --path=/path + Create OC Route ${NS2} http service-unsecure service-unsecure + ... --hostname=${HTTP_HOST} --path=/test + Create OC Route ${NS2} edge route-edge service-unsecure + ... --hostname=${EDGE_HOST} --path=/test + Create OC Route ${NS2} reencrypt route-reen service-secure + ... --hostname=${REEN_HOST} --path=/test + +All NS Ownership Routes Should Be Admitted + [Documentation] Verify all six routes across both namespaces are admitted. + Route Should Be Admitted service-unsecure ${NS1} + Route Should Be Admitted route-edge ${NS1} + Route Should Be Admitted route-reen ${NS1} + Route Should Be Admitted service-unsecure ${NS2} + Route Should Be Admitted route-edge ${NS2} + Route Should Be Admitted route-reen ${NS2} + +Curl All LB Route Types Via IP + [Documentation] Curl all four route types through a single LB IP. + [Arguments] ${ip} + Wait Until Curl Succeeds From Pod + ... ${CLIENT_POD_NAME} + ... ${NAMESPACE} + ... http://${HTTP_HOST} + ... ${HTTP_HOST}:80:${ip} + Wait Until HTTPS Curl Succeeds From Pod + ... ${CLIENT_POD_NAME} ${NAMESPACE} https://${EDGE_HOST} ${EDGE_HOST}:443:${ip} + Wait Until HTTPS Curl Succeeds From Pod + ... ${CLIENT_POD_NAME} ${NAMESPACE} https://${PASS_HOST} ${PASS_HOST}:443:${ip} + Wait Until HTTPS Curl Succeeds From Pod + ... ${CLIENT_POD_NAME} ${NAMESPACE} https://${REEN_HOST} ${REEN_HOST}:443:${ip} + +Verify Default LB IPs And Ports + [Documentation] Verify the router-default service LB IPs match all host IPs and ports are 80/443. + ${http_port}= Get LB Port http + Should Be Equal As Strings ${http_port} 80 + ${https_port}= Get LB Port https + Should Be Equal As Strings ${https_port} 443 + ${lb_ips}= Get LB IPs + Should Not Be Empty ${lb_ips} + @{host_ips}= Get Host IPs Via SSH + ${sorted_host_ips}= Evaluate " ".join(sorted(${host_ips})) + ${sorted_lb_ips}= Evaluate " ".join(sorted("${lb_ips}".split())) + Should Be Equal As Strings ${sorted_lb_ips} ${sorted_host_ips}