Set Ubuntu Noble as the default stemcell for HAProxy acceptance tests

.gitignore

@@ -4,7 +4,7 @@ config/settings.yml
 releases/*.tgz
 releases/**/*.tgz
 ci/scripts/stemcell/*.tgz
-ci/scripts/stemcell-bionic/*.tgz
+ci/scripts/stemcell-jammy/*.tgz
 dev_releases
 blobs/*
 .blobs

acceptance-tests/acceptance_tests_suite_test.go

@@ -247,10 +247,3 @@ func checkNetOpErr(err error, expectString string) {
 	Expect(errors.As(tlsErr, &opErr)).To(BeTrue())
 	Expect(opErr.Err.Error()).To(ContainSubstring(expectString))
 }
-
-func writeLog(s string) {
-	ginkgoConfig, _ := GinkgoConfiguration()
-	for _, line := range strings.Split(s, "\n") {
-		fmt.Printf("node %d/%d: %s\n", ginkgoConfig.ParallelProcess, ginkgoConfig.ParallelTotal, line)
-	}
-}

acceptance-tests/bionic_test.go → acceptance-tests/jammy_test.go

@@ -7,22 +7,22 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 )
 
-var _ = Describe("Bionic", func() {
-	It("Correctly proxies HTTP requests when using the Bionic stemcell", func() {
+var _ = Describe("Jammy", func() {
+	It("Correctly proxies HTTP requests when using the Jammy stemcell", func() {
 
-		opsfileBionic := `---
-# Configure Bionic stemcell
+		opsfileJammy := `---
+# Configure Jammy stemcell
 - type: replace
   path: /stemcells/alias=default/os
-  value: ubuntu-bionic
+  value: ubuntu-jammy
 `
 
 		haproxyBackendPort := 12000
 		haproxyInfo, _ := deployHAProxy(baseManifestVars{
 			haproxyBackendPort:    haproxyBackendPort,
 			haproxyBackendServers: []string{"127.0.0.1"},
 			deploymentName:        deploymentNameForTestNode(),
-		}, []string{opsfileBionic}, map[string]interface{}{}, true)
+		}, []string{opsfileJammy}, map[string]interface{}{}, true)
 
 		closeLocalServer, localPort := startDefaultTestServer()
 		defer closeLocalServer()

acceptance-tests/log_helpers.go

@@ -0,0 +1,15 @@
+package acceptance_tests
+
+import (
+	"fmt"
+	"strings"
+
+	. "github.com/onsi/ginkgo/v2"
+)
+
+func writeLog(s string) {
+	ginkgoConfig, _ := GinkgoConfiguration()
+	for _, line := range strings.Split(s, "\n") {
+		fmt.Printf("node %d/%d: %s\n", ginkgoConfig.ParallelProcess, ginkgoConfig.ParallelTotal, line)
+	}
+}

acceptance-tests/run-local.sh

@@ -3,21 +3,27 @@
 set -eu
 REPO_DIR="$(cd "$(dirname "$0")/.." && pwd)"
 source "${REPO_DIR}/ci/scripts/functions-ci.sh"
+FOCUS=""
+PARALLELISM=""
 KEEP_RUNNING=""
 
 usage() {
-    echo -e "Usage: $0 [-F <ginkgo focus target>] [-k]
+    echo -e "Usage: $0 [-F <ginkgo focus target>] [-P <ginkgo nodes>] [-k]
 
     -F      Focus on a particular test. Expects a Ginkgo test name. Keep bosh running afterwards.
+    -P      Set Ginkgo parallel node count. Default is '-p' (smart parallelism).
     -k      Keep bosh container running. Useful for debug." 1>&2; exit 1;
 }
 
-while getopts ":F:k" o; do
+while getopts ":F:P:k" o; do
     case "${o}" in
         F)
             FOCUS=${OPTARG}
             KEEP_RUNNING=true
             ;;
+        P)
+            PARALLELISM=${OPTARG}
+            ;;
         k)
             KEEP_RUNNING=true
             ;;
@@ -35,7 +41,7 @@ docker_mac_check_cgroupsv1() {
     SETTINGS=~/Library/Group\ Containers/group.com.docker/settings.json
 
     cgroupsV1Enabled=$(jq '.deprecatedCgroupv1' "$SETTINGS")
-    if [ "$cgroupsV1Enabled" != "true" ]; then 
+    if [ "$cgroupsV1Enabled" != "true" ]; then
         echo "deprecatedCgroupv1 should be enabled in $SETTINGS. Otherwise the acceptance tests will not run on Docker for Mac."
         echo "Check in the README.md for a convenient script to set deprecatedCgroupv1 and restart Docker."
         exit 1
@@ -45,8 +51,8 @@ docker_mac_check_cgroupsv1() {
 check_required_files() {
   PIDS=""
   REQUIRED_FILE_PATTERNS=(
-    ci/scripts/stemcell/bosh-stemcell-*-ubuntu-jammy-*.tgz!https://bosh.io/d/stemcells/bosh-warden-boshlite-ubuntu-jammy-go_agent
-    ci/scripts/stemcell-bionic/bosh-stemcell-*-ubuntu-bionic-*.tgz!https://bosh.io/d/stemcells/bosh-warden-boshlite-ubuntu-bionic-go_agent
+    ci/scripts/stemcell/bosh-stemcell-*-ubuntu-noble.tgz!https://bosh.io/d/stemcells/bosh-warden-boshlite-ubuntu-noble
+    ci/scripts/stemcell-jammy/bosh-stemcell-*-ubuntu-jammy-*.tgz!https://bosh.io/d/stemcells/bosh-warden-boshlite-ubuntu-jammy-go_agent
   )
 
   for entry in "${REQUIRED_FILE_PATTERNS[@]}"; do
@@ -63,9 +69,10 @@ check_required_files() {
     fi
 
     (
-      echo "$filepattern not found, downloading latest."
+      echo "$filepattern not found, downloading."
       cd "$folder" && \
-      resolved=$(curl -s --write-out '\n%{redirect_url}' "$url" | tail -n1) && \
+      resolved=$(curl -s --write-out '\n%{redirect_url}' "$url" | tail -n1 | tr -d '\n') && \
+      echo "Resolved URL: $resolved" && \
       curl -s --remote-name --remote-header-name --location "$resolved" && \
       echo "Downloaded '$url' successfully." && \
       ls -1lh "$folder/"$filepattern
@@ -93,9 +100,9 @@ if [ -n "$KEEP_RUNNING" ] ; then
   echo
   echo "*** KEEP_RUNNING enabled. Please clean up docker scratch after removing containers: ${DOCKER_SCRATCH}"
   echo
-  docker run --privileged -v "$REPO_DIR":/repo -v "${DOCKER_SCRATCH}":/scratch/docker -e REPO_ROOT=/repo -e FOCUS="$FOCUS" -e KEEP_RUNNING="${KEEP_RUNNING}" haproxy-boshrelease-testflight bash -c "cd /repo/ci/scripts && ./acceptance-tests ; sleep infinity"
+  docker run --privileged -v "$REPO_DIR":/repo -v "${DOCKER_SCRATCH}":/scratch/docker -e REPO_ROOT=/repo -e FOCUS="${FOCUS}" -e PARALLELISM="${PARALLELISM}" -e KEEP_RUNNING="${KEEP_RUNNING}" haproxy-boshrelease-testflight bash -c "cd /repo/ci/scripts && ./acceptance-tests ; sleep infinity"
 else
-  docker run --rm --privileged -v "$REPO_DIR":/repo -v "${DOCKER_SCRATCH}":/scratch/docker -e REPO_ROOT=/repo -e KEEP_RUNNING="" haproxy-boshrelease-testflight bash -c "cd /repo/ci/scripts && ./acceptance-tests"
+  docker run --rm --privileged -v "$REPO_DIR":/repo -v "${DOCKER_SCRATCH}":/scratch/docker -e REPO_ROOT=/repo -e KEEP_RUNNING="" -e PARALLELISM="${PARALLELISM}" haproxy-boshrelease-testflight bash -c "cd /repo/ci/scripts && ./acceptance-tests"
   echo "Cleaning up docker scratch: ${DOCKER_SCRATCH}"
   sudo rm -rf "${DOCKER_SCRATCH}"
 fi

ci/Dockerfile

@@ -1,18 +1,44 @@
-FROM bosh/docker-cpi:main
+FROM ghcr.io/cloudfoundry/bosh/docker-cpi:latest
 
 # Install all necessary tools for haproxy testflight and dependency autobump
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && \
-    apt-get install -y wget jq git vim nano python3-pip && \
+    apt-get install -y tar wget jq git vim nano python3-pip python3-venv && \
+    apt-get install -y build-essential zlib1g-dev ruby ruby-dev openssl libxslt1-dev libxml2-dev libssl-dev libreadline-dev libyaml-dev libsqlite3-dev sqlite3 && \
     apt-get clean
 
+# Install bosh cli v2
+RUN wget -O ./bosh https://github.com/cloudfoundry/bosh-cli/releases/download/v7.9.18/bosh-cli-7.9.18-linux-amd64 && \
+    chmod +x ./bosh && \
+    mv ./bosh /usr/local/bin/bosh && \
+    bosh -v
+
 # Set bosh env at login
+RUN echo "source /tmp/local-bosh/director/docker-env" >> /root/.bashrc
 RUN echo "source /tmp/local-bosh/director/env" >> /root/.bashrc
 
+RUN mkdir -p /usr/local/bosh-deployment/haproxy-boshrelease
+
+# Make local BPM release
+COPY local-releases/bpm-del-runc-state.patch ./bpm-del-runc-state.patch
+RUN git clone https://github.com/cloudfoundry/bpm-release.git && cd bpm-release && git apply ../bpm-del-runc-state.patch && \
+    bosh create-release --force --version=1.4.26+patch.runc.state --tarball=bpm-patched-dev-release.tgz && \
+    mv bpm-patched-dev-release.tgz /usr/local/bosh-deployment/haproxy-boshrelease/bpm-patched-dev-release.tgz && \
+    chmod 644 /usr/local/bosh-deployment/haproxy-boshrelease/bpm-patched-dev-release.tgz && \
+    cd .. && rm -rf bpm-release
+
+# Copy ops files
+COPY ops/bosh-scaled-out.yml /usr/local/bosh-deployment/haproxy-boshrelease/bosh-scaled-out.yml
+COPY ops/bosh-timeouts.yml /usr/local/bosh-deployment/haproxy-boshrelease/bosh-timeouts.yml
+COPY ops/compilation.yml /usr/local/bosh-deployment/haproxy-boshrelease/compilation.yml
+COPY ops/local-releases.yml /usr/local/bosh-deployment/haproxy-boshrelease/local-releases.yml
+
 # Install Python libraries needed for scripts
+RUN python3 -m venv /opt/venv
+ENV PATH="/opt/venv/bin:${PATH}"
 COPY scripts/requirements.txt /requirements.txt
-RUN /usr/bin/python3 -m pip install -r /requirements.txt
+RUN pip install -r /requirements.txt
 
 # Install go dependencies
 ENV GOBIN=/usr/local/bin
-RUN go install github.com/geofffranks/spruce/cmd/spruce@latest
+RUN go install github.com/geofffranks/spruce/cmd/spruce@latest

ci/local-releases/bpm-del-runc-state.patch

@@ -0,0 +1,64 @@
+diff --git a/src/bpm/runc/client/client.go b/src/bpm/runc/client/client.go
+index f9a2591c..06c15119 100644
+--- a/src/bpm/runc/client/client.go
++++ b/src/bpm/runc/client/client.go
+@@ -228,6 +228,35 @@ func (c *RuncClient) DeleteContainer(containerID string) error {
+ 	return runcCmd.Run()
+ }
+
++func (c *RuncClient) CleanupState(containerID string) error {
++	// kill all lingering processes in the job's cgroup scope dirs.
++	scopeName := fmt.Sprintf("runc-%s.scope", containerID)
++	cgroupDirs, _ := filepath.Glob(fmt.Sprintf("/sys/fs/cgroup/*/system.slice/%s", scopeName))
++	cgroupV2Dirs, _ := filepath.Glob(fmt.Sprintf("/sys/fs/cgroup/system.slice/%s", scopeName))
++	cgroupDirs = append(cgroupDirs, cgroupV2Dirs...)
++	killPidsFromFile := func(path string) {
++		if data, err := os.ReadFile(path); err == nil && len(data) > 0 {
++			for _, line := range strings.Split(string(data), "\n") {
++				var pid int
++				if _, err := fmt.Sscanf(strings.TrimSpace(line), "%d", &pid); err == nil && pid > 0 {
++					_ = syscall.Kill(pid, syscall.SIGKILL)
++				}
++			}
++		}
++	}
++	for _, cgDir := range cgroupDirs {
++		if info, err := os.Stat(cgDir); err == nil && info.IsDir() {
++			killPidsFromFile(filepath.Join(cgDir, "cgroup.procs"))
++			killPidsFromFile(filepath.Join(cgDir, "tasks"))
++		}
++	}
++
++	// remove the runc state dir so BPM can re-create the container.
++	args := []string{"-rf", fmt.Sprintf("%s/%s", c.runcRoot, containerID)}
++	cmd := exec.Command("rm", args...)
++	return cmd.Run()
++}
++
+ func (*RuncClient) DestroyBundle(bundlePath string) error {
+ 	return os.RemoveAll(bundlePath)
+ }
+diff --git a/src/bpm/runc/lifecycle/lifecycle.go b/src/bpm/runc/lifecycle/lifecycle.go
+index 0621e2d0..bd00d40d 100644
+--- a/src/bpm/runc/lifecycle/lifecycle.go
++++ b/src/bpm/runc/lifecycle/lifecycle.go
+@@ -74,6 +74,7 @@ type RuncClient interface {
+ 	ContainerState(containerID string) (*specs.State, error)
+ 	ListContainers() ([]client.ContainerState, error)
+ 	SignalContainer(containerID string, signal client.Signal) error
++	CleanupState(containerID string) error
+ 	DeleteContainer(containerID string) error
+ 	DestroyBundle(bundlePath string) error
+ }
+@@ -274,6 +275,11 @@ func (j *RuncLifecycle) StopProcess(logger lager.Logger, cfg *config.BPMConfig,
+ }
+
+ func (j *RuncLifecycle) RemoveProcess(logger lager.Logger, cfg *config.BPMConfig) error {
++	logger.Info("forcefully-cleanup-runc-state")
++	if err := j.runcClient.CleanupState(cfg.ContainerID()); err != nil {
++		return err
++	}
++
+ 	logger.Info("forcefully-deleting-container")
+ 	if err := j.runcClient.DeleteContainer(cfg.ContainerID()); err != nil {
+ 		return err

ci/ops/bosh-scaled-out.yml

@@ -0,0 +1,3 @@
+- type: replace
+  path: /instance_groups/name=bosh/properties/director/workers?
+  value: 12

ci/ops/bosh-timeouts.yml

@@ -0,0 +1,3 @@
+- type: replace
+  path: /instance_groups/name=bosh/properties/director/db/connection_wait_timeout?
+  value: 60

ci/ops/local-releases.yml

@@ -0,0 +1,6 @@
+- type: replace
+  path: /releases/name=bpm?
+  value:
+    name: bpm
+    version: 1.4.26+patch.runc.state
+    url: file:///usr/local/bosh-deployment/haproxy-boshrelease/bpm-patched-dev-release.tgz

ci/pipeline.yml

@@ -121,23 +121,25 @@ jobs:
       - in_parallel:
         - { get: git, trigger: true, passed: [unit-tests] }
         - { get: stemcell }
-        - { get: stemcell-bionic }
+        - { get: stemcell-jammy }
         - get: haproxy-boshrelease-testflight
       - task: acceptance-tests
         privileged: true
+        timeout: 4h
         image: haproxy-boshrelease-testflight
         config:
           platform: linux
           inputs:
             - { name: git }
             - { name: stemcell }
-            - { name: stemcell-bionic }
+            - { name: stemcell-jammy }
           run:
             path: ./git/ci/scripts/acceptance-tests
             args: []
           params:
             REPO_ROOT:            git
-      on_failure:
+            SUITE_TIMEOUT:        3h
+            GRACE_PERIOD:         10m
         put: notify
         params:
           channel:  "#haproxy-boshrelease"
@@ -152,7 +154,7 @@ jobs:
     - do:
       - { get: git-pull-requests, trigger: true, version: every }
       - { get: stemcell }
-      - { get: stemcell-bionic }
+      - { get: stemcell-jammy }
       - get: haproxy-boshrelease-testflight
       - put: git-pull-requests
         params:
@@ -163,18 +165,21 @@ jobs:
           list_changed_files: true
       - task: acceptance-tests
         privileged: true
+        timeout: 4h
         image: haproxy-boshrelease-testflight
         config:
           platform: linux
           inputs:
             - { name: git-pull-requests }
             - { name: stemcell }
-            - { name: stemcell-bionic }
+            - { name: stemcell-jammy }
           run:
             path: ./git-pull-requests/ci/scripts/acceptance-tests
             args: []
           params:
             REPO_ROOT:            git-pull-requests
+            SUITE_TIMEOUT:        3h
+            GRACE_PERIOD:         10m
     on_success:
       put: git-pull-requests
       params:
@@ -403,15 +408,15 @@ resources:
         - "dependabot"
         - "CFN-CI"
 
-  - name: stemcell-bionic
+  - name: stemcell-jammy
     type: bosh-io-stemcell
     source:
-      name: bosh-warden-boshlite-ubuntu-bionic-go_agent
+      name: bosh-warden-boshlite-ubuntu-jammy-go_agent
 
   - name: stemcell
     type: bosh-io-stemcell
     source:
-      name: bosh-warden-boshlite-ubuntu-jammy-go_agent
+      name: bosh-warden-boshlite-ubuntu-noble
 
   - name: version
     type: semver
@@ -453,7 +458,7 @@ resources:
   - name: docker-cpi-image
     type: docker-image
     source:
-      repository: bosh/docker-cpi
+      repository: ghcr.io/cloudfoundry/bosh/docker-cpi
 
   - name: git-ci
     type: git