From 7d293396977d2e9ad7762d2e62bc08136d4374e5 Mon Sep 17 00:00:00 2001 From: Basundhara Chakrabarty Date: Thu, 25 Sep 2025 23:28:15 +0000 Subject: [PATCH 1/5] reverse tunnel sandbox Signed-off-by: Basundhara Chakrabarty --- reverse_tunnel/Dockerfile.xds | 150 ++++++++++++++++++++++++++ reverse_tunnel/README.md | 2 + reverse_tunnel/docker-compose.yaml | 45 ++++++++ reverse_tunnel/example.rst | 158 ++++++++++++++++++++++++++++ reverse_tunnel/initiator-envoy.yaml | 91 ++++++++++++++++ reverse_tunnel/requirements.txt | 5 + reverse_tunnel/responder-envoy.yaml | 83 +++++++++++++++ 7 files changed, 534 insertions(+) create mode 100644 reverse_tunnel/Dockerfile.xds create mode 100644 reverse_tunnel/README.md create mode 100644 reverse_tunnel/docker-compose.yaml create mode 100644 reverse_tunnel/example.rst create mode 100644 reverse_tunnel/initiator-envoy.yaml create mode 100644 reverse_tunnel/requirements.txt create mode 100644 reverse_tunnel/responder-envoy.yaml diff --git a/reverse_tunnel/Dockerfile.xds b/reverse_tunnel/Dockerfile.xds new file mode 100644 index 00000000..4631d2a2 --- /dev/null +++ b/reverse_tunnel/Dockerfile.xds @@ -0,0 +1,150 @@ +FROM ubuntu:20.04 + +# Prevent interactive prompts during package installation +ENV DEBIAN_FRONTEND=noninteractive + +WORKDIR /app + +# Install Python and pip +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + && rm -rf /var/lib/apt/lists/* + +# Install dependencies +RUN pip3 install requests pyyaml + +# Create a simple xDS server script +RUN echo '#!/usr/bin/env python3\n\ +import json\n\ +import time\n\ +import threading\n\ +import http.server\n\ +import socketserver\n\ +import logging\n\ +\n\ +logging.basicConfig(level=logging.INFO)\n\ +logger = logging.getLogger(__name__)\n\ +\n\ +class XDSServer:\n\ + def __init__(self):\n\ + self.listeners = {}\n\ + self.version = 1\n\ + self._lock = threading.Lock()\n\ + self.server = None\n\ + \n\ + def start(self, port):\n\ + class XDSHandler(http.server.BaseHTTPRequestHandler):\n\ + def do_POST(self):\n\ + if self.path == "/v3/discovery:listeners":\n\ + content_length = int(self.headers["Content-Length"])\n\ + post_data = self.rfile.read(content_length)\n\ + response_data = self.server.xds_server.handle_lds_request(post_data)\n\ + self.send_response(200)\n\ + self.send_header("Content-type", "application/json")\n\ + self.end_headers()\n\ + self.wfile.write(response_data.encode())\n\ + elif self.path == "/add_listener":\n\ + content_length = int(self.headers["Content-Length"])\n\ + post_data = self.rfile.read(content_length)\n\ + data = json.loads(post_data.decode())\n\ + self.server.xds_server.add_listener(data["name"], data["config"])\n\ + self.send_response(200)\n\ + self.send_header("Content-type", "application/json")\n\ + self.end_headers()\n\ + self.wfile.write(json.dumps({"status": "success"}).encode())\n\ + elif self.path == "/remove_listener":\n\ + content_length = int(self.headers["Content-Length"])\n\ + post_data = self.rfile.read(content_length)\n\ + data = json.loads(post_data.decode())\n\ + success = self.server.xds_server.remove_listener(data["name"])\n\ + if success:\n\ + self.send_response(200)\n\ + self.send_header("Content-type", "application/json")\n\ + self.end_headers()\n\ + self.wfile.write(json.dumps({"status": "success"}).encode())\n\ + else:\n\ + self.send_response(404)\n\ + self.send_header("Content-type", "application/json")\n\ + self.end_headers()\n\ + self.wfile.write(json.dumps({"status": "not_found"}).encode())\n\ + elif self.path == "/state":\n\ + state = self.server.xds_server.get_state()\n\ + self.send_response(200)\n\ + self.send_header("Content-type", "application/json")\n\ + self.end_headers()\n\ + self.wfile.write(json.dumps(state).encode())\n\ + else:\n\ + self.send_response(404)\n\ + self.end_headers()\n\ + \n\ + def log_message(self, format, *args):\n\ + pass\n\ + \n\ + class XDSServer(socketserver.TCPServer):\n\ + def __init__(self, server_address, RequestHandlerClass, xds_server):\n\ + self.xds_server = xds_server\n\ + super().__init__(server_address, RequestHandlerClass)\n\ + \n\ + self.server = XDSServer(("0.0.0.0", port), XDSHandler, self)\n\ + self.server_thread = threading.Thread(target=self.server.serve_forever)\n\ + self.server_thread.daemon = True\n\ + self.server_thread.start()\n\ + logger.info(f"xDS server started on port {port}")\n\ + \n\ + def handle_lds_request(self, request_data):\n\ + with self._lock:\n\ + response = {\n\ + "version_info": str(self.version),\n\ + "resources": [],\n\ + "type_url": "type.googleapis.com/envoy.config.listener.v3.Listener"\n\ + }\n\ + for listener_name, listener_config in self.listeners.items():\n\ + # Wrap the listener config in a proper Any message\n\ + wrapped_config = {\n\ + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener",\n\ + **listener_config\n\ + }\n\ + response["resources"].append(wrapped_config)\n\ + return json.dumps(response)\n\ + \n\ + def add_listener(self, listener_name, listener_config):\n\ + with self._lock:\n\ + self.listeners[listener_name] = listener_config\n\ + self.version += 1\n\ + logger.info(f"Added listener {listener_name}, version {self.version}")\n\ + \n\ + def remove_listener(self, listener_name):\n\ + with self._lock:\n\ + if listener_name in self.listeners:\n\ + del self.listeners[listener_name]\n\ + self.version += 1\n\ + logger.info(f"Removed listener {listener_name}, version {self.version}")\n\ + return True\n\ + return False\n\ +\n\ + def get_state(self):\n\ + with self._lock:\n\ + return {\n\ + "version": self.version,\n\ + "listeners": list(self.listeners.keys())\n\ + }\n\ +\n\ +if __name__ == "__main__":\n\ + xds_server = XDSServer()\n\ + xds_server.start(18000)\n\ + try:\n\ + while True:\n\ + time.sleep(1)\n\ + except KeyboardInterrupt:\n\ + print("Shutting down xDS server...")\n\ +' > /app/xds_server.py + +# Make the script executable +RUN chmod +x /app/xds_server.py + +# Expose the xDS server port +EXPOSE 18000 + +# Run the xDS server +CMD ["python3", "/app/xds_server.py"] \ No newline at end of file diff --git a/reverse_tunnel/README.md b/reverse_tunnel/README.md new file mode 100644 index 00000000..cbac9410 --- /dev/null +++ b/reverse_tunnel/README.md @@ -0,0 +1,2 @@ +To learn about this sandbox and for instructions on how to run it please head over +to the [Envoy docs](https://www.envoyproxy.io/docs/envoy/latest/start/sandboxes/reverse_tunnel.html). \ No newline at end of file diff --git a/reverse_tunnel/docker-compose.yaml b/reverse_tunnel/docker-compose.yaml new file mode 100644 index 00000000..021ae249 --- /dev/null +++ b/reverse_tunnel/docker-compose.yaml @@ -0,0 +1,45 @@ +version: '2' +services: + + downstream-envoy: + image: debug/envoy:latest + volumes: + - ./initiator-envoy.yaml:/etc/downstream-envoy.yaml + command: envoy -c /etc/downstream-envoy.yaml --concurrency 1 -l trace --drain-time-s 3 + ports: + # Admin interface + - "8888:8888" + # Reverse connection API listener + - "9000:9000" + # Ingress HTTP listener + - "6060:6060" + extra_hosts: + - "host.docker.internal:host-gateway" + networks: + - envoy-network + depends_on: + - downstream-service + + downstream-service: + image: nginxdemos/hello:plain-text + networks: + - envoy-network + + upstream-envoy: + image: debug/envoy:latest + volumes: + - ./responder-envoy.yaml:/etc/upstream-envoy.yaml + command: envoy -c /etc/upstream-envoy.yaml --concurrency 1 -l trace --drain-time-s 3 + ports: + # Admin interface + - "8889:8888" + # Reverse connection API listener + - "9001:9000" + # Egress listener + - "8085:8085" + networks: + - envoy-network + +networks: + envoy-network: + driver: bridge \ No newline at end of file diff --git a/reverse_tunnel/example.rst b/reverse_tunnel/example.rst new file mode 100644 index 00000000..07f828f8 --- /dev/null +++ b/reverse_tunnel/example.rst @@ -0,0 +1,158 @@ +.. _install_sandboxes_reverse_tunnel: + +Reverse Tunnels +=============== + +.. sidebar:: Requirements + + .. include:: _include/docker-env-setup-link.rst + + :ref:`curl ` + Used to make HTTP requests. + +This sandbox demonstrates Envoy's :ref:`reverse tunnels ` feature, which allows establishing long-lived connections from downstream to upstream in scenarios where direct connectivity from upstream to downstream is not possible, and using these cached connection sockets to send data traffic from upstream to downstream Envoy. + +In this example, a downstream Envoy proxy initiates reverse tunnels to an upstream Envoy using a custom address resolver. The configuration includes bootstrap extensions and reverse connection listeners with specialized address formats. + +Step 1: Build Envoy with reverse tunnels feature +************************************************ + +Build Envoy with the reverse tunnels feature enabled: + +.. code-block:: console + + $ ./ci/run_envoy_docker.sh './ci/do_ci.sh bazel.release.server_only' + +Step 2: Build Envoy Docker image +******************************** + +Build the Envoy Docker image: + +.. code-block:: console + + $ docker build -f ci/Dockerfile-envoy-image -t envoy:latest . + +Step 3: Understanding the configuration +************************************** + +The reverse tunnel configuration is explained in the :ref:`Reverse Tunnels ` section. + +Step 4: Launch test containers +****************************** + +Change to the ``reverse_tunnel`` directory and bring up the docker composition. + +.. code-block:: console + + $ pwd + examples/reverse_tunnel + $ docker compose up + +.. note:: + The docker-compose maps the following ports: + + - **downstream-envoy**: Host port 9000 → Container port 9000 (reverse connection API) + - **upstream-envoy**: Host port 9001 → Container port 9000 (reverse connection API) + +Verify the containers are running: + +.. code-block:: console + + $ docker ps + +Expected output showing all containers are up: + +.. code-block:: console + + CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES + ae15eab504f8 debug/envoy:latest "/docker-entrypoint.…" 27 seconds ago Up 3 seconds 0.0.0.0:6060->6060/tcp, :::6060->6060/tcp, 0.0.0.0:8888->8888/tcp, :::8888->8888/tcp, 0.0.0.0:9000->9000/tcp, :::9000->9000/tcp, 10000/tcp reverse_tunnel-downstream-envoy-1 + 58eba3678f20 nginxdemos/hello:plain-text "/docker-entrypoint.…" 27 seconds ago Up 3 seconds 80/tcp reverse_tunnel-downstream-service-1 + 49145cc8a9d1 debug/envoy:latest "/docker-entrypoint.…" 27 seconds ago Up 3 seconds 0.0.0.0:8085->8085/tcp, :::8085->8085/tcp, 10000/tcp, 0.0.0.0:8889->8888/tcp, :::8889->8888/tcp, 0.0.0.0:9001->9000/tcp, :::9001->9000/tcp reverse_tunnel-upstream-envoy-1 + +Step 5: Validate reverse tunnel establishment +********************************************* + +Verify that reverse tunnels have been successfully established by checking the stats counters on both downstream and upstream Envoy proxies. + +Check downstream Envoy stats (port 8888): + +.. code-block:: console + + $ curl http://localhost:8888/stats | grep reverse_connection + +Expected downstream stats showing connected reverse tunnels: + +.. code-block:: console + + downstream_reverse_connection.cluster.upstream-cluster.connected: 1 + downstream_reverse_connection.cluster.upstream-cluster.connecting: 0 + downstream_reverse_connection.host.172.27.0.2:9000.connected: 1 + downstream_reverse_connection.host.172.27.0.2:9000.connecting: 0 + downstream_reverse_connection.worker_0.cluster.upstream-cluster.connected: 1 + downstream_reverse_connection.worker_0.cluster.upstream-cluster.connecting: 0 + downstream_reverse_connection.worker_0.host.172.27.0.2:9000.connected: 1 + downstream_reverse_connection.worker_0.host.172.27.0.2:9000.connecting: 0 + +Check upstream Envoy stats (port 8889): + +.. code-block:: console + + $ curl http://localhost:8889/stats | grep reverse_connections + +Expected upstream stats showing received reverse connections: + +.. code-block:: console + + reverse_connections.clusters.downstream-cluster: 1 + reverse_connections.nodes.downstream-node: 1 + reverse_connections.worker_0.cluster.downstream-cluster: 1 + reverse_connections.worker_0.node.downstream-node: 1 + +The stats confirm that: + +- **Downstream Envoy**: Has successfully connected (``connected: 1``) to the upstream cluster with no pending connections (``connecting: 0``) +- **Upstream Envoy**: Has received reverse connections from the downstream node and cluster, as indicated by the reverse connection counters + +Step 6: Test reverse tunnel +*************************** + +Perform an HTTP request for the service behind downstream Envoy, to upstream Envoy. This request will be sent over a reverse tunnel. + +.. code-block:: console + + $ curl -H "x-remote-node-id: downstream-node" -H "x-dst-cluster-uuid: downstream-cluster" http://localhost:8085/downstream_service -v + +Expected response: + +.. code-block:: console + + * Trying ::1... + * TCP_NODELAY set + * Connected to localhost (::1) port 8085 (#0) + > GET /downstream_service HTTP/1.1 + > Host: localhost:8085 + > User-Agent: curl/7.61.1 + > Accept: */* + > x-remote-node-id: downstream-node + > x-dst-cluster-uuid: downstream-cluster + > + < HTTP/1.1 200 OK + < server: envoy + < date: Thu, 25 Sep 2025 21:25:38 GMT + < content-type: text/plain + < content-length: 159 + < expires: Thu, 25 Sep 2025 21:25:37 GMT + < cache-control: no-cache + < x-envoy-upstream-service-time: 13 + < + Server address: 172.27.0.3:80 + Server name: b490f264caf9 + Date: 25/Sep/2025:21:25:38 +0000 + URI: /downstream_service + Request ID: 41807e3cd1f6a0b601597b80f7e51513 + * Connection #0 to host localhost left intact + +.. seealso:: + + :ref:`Reverse Tunnels architecture overview ` + Learn more about Envoy's reverse tunnel functionality. diff --git a/reverse_tunnel/initiator-envoy.yaml b/reverse_tunnel/initiator-envoy.yaml new file mode 100644 index 00000000..03c05037 --- /dev/null +++ b/reverse_tunnel/initiator-envoy.yaml @@ -0,0 +1,91 @@ +--- +node: + id: downstream-node + cluster: downstream-cluster + +# Enable reverse connection bootstrap extension which registers the custom resolver +bootstrap_extensions: +- name: envoy.bootstrap.reverse_tunnel.downstream_socket_interface + typed_config: + "@type": type.googleapis.com/envoy.extensions.bootstrap.reverse_tunnel.downstream_socket_interface.v3.DownstreamReverseConnectionSocketInterface + stat_prefix: "downstream_reverse_connection" + +static_resources: + listeners: + # Initiates reverse connections to upstream using custom resolver + - name: reverse_conn_listener + listener_filters_timeout: 0s + listener_filters: + # Use custom address with reverse connection metadata encoded in URL format + address: + socket_address: + # This encodes: src_node_id=downstream-node, src_cluster_id=downstream, src_tenant_id=downstream + # and remote clusters: upstream with 1 connection + address: "rc://downstream-node:downstream-cluster:downstream-tenant@upstream-cluster:1" + port_value: 0 + # Use custom resolver that can parse reverse connection metadata + resolver_name: "envoy.resolvers.reverse_connection" + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: reverse_conn_listener + route_config: + virtual_hosts: + - name: backend + domains: + - "*" + routes: + - match: + prefix: '/downstream_service' + route: + cluster: downstream-service + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + + # Cluster designating upstream-envoy + clusters: + - name: upstream-cluster + type: STRICT_DNS + connect_timeout: 30s + load_assignment: + cluster_name: upstream-cluster + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: upstream-envoy # Container name of upstream-envoy in docker-compose + port_value: 9000 # Port where upstream-envoy's rev_conn_api_listener listens + + # Backend HTTP service behind downstream which + # we will access via reverse connections + - name: downstream-service + type: STRICT_DNS + connect_timeout: 30s + load_assignment: + cluster_name: downstream-service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: downstream-service + port_value: 80 + +admin: + access_log_path: "/dev/stdout" + address: + socket_address: + protocol: TCP + address: 0.0.0.0 + port_value: 8888 + +layered_runtime: + layers: + - name: layer + static_layer: + re2.max_program_size.error_level: 1000 \ No newline at end of file diff --git a/reverse_tunnel/requirements.txt b/reverse_tunnel/requirements.txt new file mode 100644 index 00000000..faee1f60 --- /dev/null +++ b/reverse_tunnel/requirements.txt @@ -0,0 +1,5 @@ +requests>=2.25.0 +PyYAML>=5.4.0 +grpcio>=1.43.0 +grpcio-tools>=1.43.0 +protobuf>=3.19.0 \ No newline at end of file diff --git a/reverse_tunnel/responder-envoy.yaml b/reverse_tunnel/responder-envoy.yaml new file mode 100644 index 00000000..8b732342 --- /dev/null +++ b/reverse_tunnel/responder-envoy.yaml @@ -0,0 +1,83 @@ +--- +node: + id: upstream-node + cluster: upstream-cluster +static_resources: + listeners: + # Accepts reverse tunnel requests + - name: rev_conn_api_listener + address: + socket_address: + address: 0.0.0.0 + port_value: 9000 + filter_chains: + - filters: + - name: envoy.filters.network.reverse_tunnel + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.reverse_tunnel.v3.ReverseTunnel + ping_interval: 2s + + # Listener that will route the downstream request to the reverse connection cluster + - name: egress_listener + address: + socket_address: + address: 0.0.0.0 + port_value: 8085 + filter_chains: + - filters: + - name: envoy.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: egress_http + route_config: + virtual_hosts: + - name: backend + domains: + - "*" + routes: + - match: + prefix: "/downstream_service" + route: + cluster: reverse_connection_cluster + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + # Cluster used to write requests to cached sockets + clusters: + - name: reverse_connection_cluster + connect_timeout: 200s + lb_policy: CLUSTER_PROVIDED + cluster_type: + name: envoy.clusters.reverse_connection + typed_config: + "@type": type.googleapis.com/envoy.extensions.clusters.reverse_connection.v3.RevConClusterConfig + # The following headers are expected in downstream requests + # to be sent over reverse connections + http_header_names: + - x-remote-node-id # Should be set to the node ID of the downstream envoy node, ie., downstream-node + - x-dst-cluster-uuid # Should be set to the cluster ID of the downstream envoy node, ie., downstream + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicit_http_config: + # Right the moment, reverse connections are supported over HTTP/2 only + http2_protocol_options: {} +admin: + access_log_path: "/dev/stdout" + address: + socket_address: + address: 0.0.0.0 + port_value: 8888 +layered_runtime: + layers: + - name: layer + static_layer: + re2.max_program_size.error_level: 1000 + envoy.reloadable_features.reverse_conn_force_local_reply: true +# Enable reverse connection bootstrap extension +bootstrap_extensions: +- name: envoy.bootstrap.reverse_tunnel.upstream_socket_interface + typed_config: + "@type": type.googleapis.com/envoy.extensions.bootstrap.reverse_tunnel.upstream_socket_interface.v3.UpstreamReverseConnectionSocketInterface + stat_prefix: "upstream_reverse_connection" \ No newline at end of file From a4acedac93e85a937b8352f9b5aa8efe6c1c3262 Mon Sep 17 00:00:00 2001 From: Basundhara Chakrabarty Date: Thu, 25 Sep 2025 23:37:11 +0000 Subject: [PATCH 2/5] remove extraneous files Signed-off-by: Basundhara Chakrabarty --- reverse_tunnel/Dockerfile.xds | 150 -------------------------------- reverse_tunnel/requirements.txt | 5 -- 2 files changed, 155 deletions(-) delete mode 100644 reverse_tunnel/Dockerfile.xds delete mode 100644 reverse_tunnel/requirements.txt diff --git a/reverse_tunnel/Dockerfile.xds b/reverse_tunnel/Dockerfile.xds deleted file mode 100644 index 4631d2a2..00000000 --- a/reverse_tunnel/Dockerfile.xds +++ /dev/null @@ -1,150 +0,0 @@ -FROM ubuntu:20.04 - -# Prevent interactive prompts during package installation -ENV DEBIAN_FRONTEND=noninteractive - -WORKDIR /app - -# Install Python and pip -RUN apt-get update && apt-get install -y \ - python3 \ - python3-pip \ - && rm -rf /var/lib/apt/lists/* - -# Install dependencies -RUN pip3 install requests pyyaml - -# Create a simple xDS server script -RUN echo '#!/usr/bin/env python3\n\ -import json\n\ -import time\n\ -import threading\n\ -import http.server\n\ -import socketserver\n\ -import logging\n\ -\n\ -logging.basicConfig(level=logging.INFO)\n\ -logger = logging.getLogger(__name__)\n\ -\n\ -class XDSServer:\n\ - def __init__(self):\n\ - self.listeners = {}\n\ - self.version = 1\n\ - self._lock = threading.Lock()\n\ - self.server = None\n\ - \n\ - def start(self, port):\n\ - class XDSHandler(http.server.BaseHTTPRequestHandler):\n\ - def do_POST(self):\n\ - if self.path == "/v3/discovery:listeners":\n\ - content_length = int(self.headers["Content-Length"])\n\ - post_data = self.rfile.read(content_length)\n\ - response_data = self.server.xds_server.handle_lds_request(post_data)\n\ - self.send_response(200)\n\ - self.send_header("Content-type", "application/json")\n\ - self.end_headers()\n\ - self.wfile.write(response_data.encode())\n\ - elif self.path == "/add_listener":\n\ - content_length = int(self.headers["Content-Length"])\n\ - post_data = self.rfile.read(content_length)\n\ - data = json.loads(post_data.decode())\n\ - self.server.xds_server.add_listener(data["name"], data["config"])\n\ - self.send_response(200)\n\ - self.send_header("Content-type", "application/json")\n\ - self.end_headers()\n\ - self.wfile.write(json.dumps({"status": "success"}).encode())\n\ - elif self.path == "/remove_listener":\n\ - content_length = int(self.headers["Content-Length"])\n\ - post_data = self.rfile.read(content_length)\n\ - data = json.loads(post_data.decode())\n\ - success = self.server.xds_server.remove_listener(data["name"])\n\ - if success:\n\ - self.send_response(200)\n\ - self.send_header("Content-type", "application/json")\n\ - self.end_headers()\n\ - self.wfile.write(json.dumps({"status": "success"}).encode())\n\ - else:\n\ - self.send_response(404)\n\ - self.send_header("Content-type", "application/json")\n\ - self.end_headers()\n\ - self.wfile.write(json.dumps({"status": "not_found"}).encode())\n\ - elif self.path == "/state":\n\ - state = self.server.xds_server.get_state()\n\ - self.send_response(200)\n\ - self.send_header("Content-type", "application/json")\n\ - self.end_headers()\n\ - self.wfile.write(json.dumps(state).encode())\n\ - else:\n\ - self.send_response(404)\n\ - self.end_headers()\n\ - \n\ - def log_message(self, format, *args):\n\ - pass\n\ - \n\ - class XDSServer(socketserver.TCPServer):\n\ - def __init__(self, server_address, RequestHandlerClass, xds_server):\n\ - self.xds_server = xds_server\n\ - super().__init__(server_address, RequestHandlerClass)\n\ - \n\ - self.server = XDSServer(("0.0.0.0", port), XDSHandler, self)\n\ - self.server_thread = threading.Thread(target=self.server.serve_forever)\n\ - self.server_thread.daemon = True\n\ - self.server_thread.start()\n\ - logger.info(f"xDS server started on port {port}")\n\ - \n\ - def handle_lds_request(self, request_data):\n\ - with self._lock:\n\ - response = {\n\ - "version_info": str(self.version),\n\ - "resources": [],\n\ - "type_url": "type.googleapis.com/envoy.config.listener.v3.Listener"\n\ - }\n\ - for listener_name, listener_config in self.listeners.items():\n\ - # Wrap the listener config in a proper Any message\n\ - wrapped_config = {\n\ - "@type": "type.googleapis.com/envoy.config.listener.v3.Listener",\n\ - **listener_config\n\ - }\n\ - response["resources"].append(wrapped_config)\n\ - return json.dumps(response)\n\ - \n\ - def add_listener(self, listener_name, listener_config):\n\ - with self._lock:\n\ - self.listeners[listener_name] = listener_config\n\ - self.version += 1\n\ - logger.info(f"Added listener {listener_name}, version {self.version}")\n\ - \n\ - def remove_listener(self, listener_name):\n\ - with self._lock:\n\ - if listener_name in self.listeners:\n\ - del self.listeners[listener_name]\n\ - self.version += 1\n\ - logger.info(f"Removed listener {listener_name}, version {self.version}")\n\ - return True\n\ - return False\n\ -\n\ - def get_state(self):\n\ - with self._lock:\n\ - return {\n\ - "version": self.version,\n\ - "listeners": list(self.listeners.keys())\n\ - }\n\ -\n\ -if __name__ == "__main__":\n\ - xds_server = XDSServer()\n\ - xds_server.start(18000)\n\ - try:\n\ - while True:\n\ - time.sleep(1)\n\ - except KeyboardInterrupt:\n\ - print("Shutting down xDS server...")\n\ -' > /app/xds_server.py - -# Make the script executable -RUN chmod +x /app/xds_server.py - -# Expose the xDS server port -EXPOSE 18000 - -# Run the xDS server -CMD ["python3", "/app/xds_server.py"] \ No newline at end of file diff --git a/reverse_tunnel/requirements.txt b/reverse_tunnel/requirements.txt deleted file mode 100644 index faee1f60..00000000 --- a/reverse_tunnel/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -requests>=2.25.0 -PyYAML>=5.4.0 -grpcio>=1.43.0 -grpcio-tools>=1.43.0 -protobuf>=3.19.0 \ No newline at end of file From c755d4f53598d0833da770cd1ceecc0fc4a477c3 Mon Sep 17 00:00:00 2001 From: Basundhara Chakrabarty Date: Fri, 26 Sep 2025 03:55:00 +0000 Subject: [PATCH 3/5] yamllint fixes Signed-off-by: Basundhara Chakrabarty --- reverse_tunnel/initiator-envoy.yaml | 125 +++++++++++++------------- reverse_tunnel/responder-envoy.yaml | 135 +++++++++++++++------------- 2 files changed, 137 insertions(+), 123 deletions(-) diff --git a/reverse_tunnel/initiator-envoy.yaml b/reverse_tunnel/initiator-envoy.yaml index 03c05037..317ea56d 100644 --- a/reverse_tunnel/initiator-envoy.yaml +++ b/reverse_tunnel/initiator-envoy.yaml @@ -7,74 +7,77 @@ node: bootstrap_extensions: - name: envoy.bootstrap.reverse_tunnel.downstream_socket_interface typed_config: - "@type": type.googleapis.com/envoy.extensions.bootstrap.reverse_tunnel.downstream_socket_interface.v3.DownstreamReverseConnectionSocketInterface + "@type": >- + type.googleapis.com/envoy.extensions.bootstrap.reverse_tunnel.downstream_socket_interface.v3.DownstreamReverseConnectionSocketInterface stat_prefix: "downstream_reverse_connection" static_resources: listeners: - # Initiates reverse connections to upstream using custom resolver - - name: reverse_conn_listener - listener_filters_timeout: 0s - listener_filters: - # Use custom address with reverse connection metadata encoded in URL format - address: - socket_address: - # This encodes: src_node_id=downstream-node, src_cluster_id=downstream, src_tenant_id=downstream - # and remote clusters: upstream with 1 connection - address: "rc://downstream-node:downstream-cluster:downstream-tenant@upstream-cluster:1" - port_value: 0 - # Use custom resolver that can parse reverse connection metadata - resolver_name: "envoy.resolvers.reverse_connection" - filter_chains: - - filters: - - name: envoy.filters.network.http_connection_manager - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager - stat_prefix: reverse_conn_listener - route_config: - virtual_hosts: - - name: backend - domains: - - "*" - routes: - - match: - prefix: '/downstream_service' - route: - cluster: downstream-service - http_filters: - - name: envoy.filters.http.router - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + # Initiates reverse connections to upstream using custom resolver + - name: reverse_conn_listener + listener_filters_timeout: 0s + listener_filters: [] + # Use custom address with reverse connection metadata encoded in URL format + address: + socket_address: + # This encodes: src_node_id, src_cluster_id, src_tenant_id + # and remote clusters: upstream-cluster with 1 connection + address: "rc://downstream-node:downstream-cluster:downstream-tenant@upstream-cluster:1" + port_value: 0 + # Use custom resolver that can parse reverse connection metadata + resolver_name: "envoy.resolvers.reverse_connection" + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": >- + type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: reverse_conn_listener + route_config: + virtual_hosts: + - name: backend + domains: + - "*" + routes: + - match: + prefix: '/downstream_service' + route: + cluster: downstream-service + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": >- + type.googleapis.com/envoy.extensions.filters.http.router.v3.Router # Cluster designating upstream-envoy clusters: - - name: upstream-cluster - type: STRICT_DNS - connect_timeout: 30s - load_assignment: - cluster_name: upstream-cluster - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: upstream-envoy # Container name of upstream-envoy in docker-compose - port_value: 9000 # Port where upstream-envoy's rev_conn_api_listener listens + - name: upstream-cluster + type: STRICT_DNS + connect_timeout: 30s + load_assignment: + cluster_name: upstream-cluster + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: upstream-envoy # Address of upstream-envoy + port_value: 9000 # Port for rev_conn_api_listener - # Backend HTTP service behind downstream which - # we will access via reverse connections - - name: downstream-service - type: STRICT_DNS - connect_timeout: 30s - load_assignment: - cluster_name: downstream-service - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: downstream-service - port_value: 80 + # Backend HTTP service behind downstream which + # we will access via reverse connections + - name: downstream-service + type: STRICT_DNS + connect_timeout: 30s + load_assignment: + cluster_name: downstream-service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: downstream-service + port_value: 80 admin: access_log_path: "/dev/stdout" @@ -88,4 +91,4 @@ layered_runtime: layers: - name: layer static_layer: - re2.max_program_size.error_level: 1000 \ No newline at end of file + re2.max_program_size.error_level: 1000 diff --git a/reverse_tunnel/responder-envoy.yaml b/reverse_tunnel/responder-envoy.yaml index 8b732342..5416f6c5 100644 --- a/reverse_tunnel/responder-envoy.yaml +++ b/reverse_tunnel/responder-envoy.yaml @@ -2,82 +2,93 @@ node: id: upstream-node cluster: upstream-cluster + +# Enable reverse connection bootstrap extension +bootstrap_extensions: +- name: envoy.bootstrap.reverse_tunnel.upstream_socket_interface + typed_config: + "@type": >- + type.googleapis.com/envoy.extensions.bootstrap.reverse_tunnel.upstream_socket_interface.v3.UpstreamReverseConnectionSocketInterface + stat_prefix: "upstream_reverse_connection" + static_resources: listeners: - # Accepts reverse tunnel requests - - name: rev_conn_api_listener - address: - socket_address: - address: 0.0.0.0 - port_value: 9000 - filter_chains: - - filters: - - name: envoy.filters.network.reverse_tunnel - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.reverse_tunnel.v3.ReverseTunnel - ping_interval: 2s + # Accepts reverse tunnel requests + - name: rev_conn_api_listener + address: + socket_address: + address: 0.0.0.0 + port_value: 9000 + filter_chains: + - filters: + - name: envoy.filters.network.reverse_tunnel + typed_config: + "@type": >- + type.googleapis.com/envoy.extensions.filters.network.reverse_tunnel.v3.ReverseTunnel + ping_interval: 2s + + # Listener that will route the downstream request to the reverse connection cluster + - name: egress_listener + address: + socket_address: + address: 0.0.0.0 + port_value: 8085 + filter_chains: + - filters: + - name: envoy.http_connection_manager + typed_config: + "@type": >- + type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: egress_http + route_config: + virtual_hosts: + - name: backend + domains: + - "*" + routes: + - match: + prefix: "/downstream_service" + route: + cluster: reverse_connection_cluster + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": >- + type.googleapis.com/envoy.extensions.filters.http.router.v3.Router - # Listener that will route the downstream request to the reverse connection cluster - - name: egress_listener - address: - socket_address: - address: 0.0.0.0 - port_value: 8085 - filter_chains: - - filters: - - name: envoy.http_connection_manager - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager - stat_prefix: egress_http - route_config: - virtual_hosts: - - name: backend - domains: - - "*" - routes: - - match: - prefix: "/downstream_service" - route: - cluster: reverse_connection_cluster - http_filters: - - name: envoy.filters.http.router - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router # Cluster used to write requests to cached sockets clusters: - - name: reverse_connection_cluster - connect_timeout: 200s - lb_policy: CLUSTER_PROVIDED - cluster_type: - name: envoy.clusters.reverse_connection - typed_config: - "@type": type.googleapis.com/envoy.extensions.clusters.reverse_connection.v3.RevConClusterConfig - # The following headers are expected in downstream requests - # to be sent over reverse connections - http_header_names: - - x-remote-node-id # Should be set to the node ID of the downstream envoy node, ie., downstream-node - - x-dst-cluster-uuid # Should be set to the cluster ID of the downstream envoy node, ie., downstream - typed_extension_protocol_options: - envoy.extensions.upstreams.http.v3.HttpProtocolOptions: - "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions - explicit_http_config: - # Right the moment, reverse connections are supported over HTTP/2 only - http2_protocol_options: {} + - name: reverse_connection_cluster + connect_timeout: 200s + lb_policy: CLUSTER_PROVIDED + cluster_type: + name: envoy.clusters.reverse_connection + typed_config: + "@type": >- + type.googleapis.com/envoy.extensions.clusters.reverse_connection.v3.RevConClusterConfig + # The following headers are expected in downstream requests + # to be sent over reverse connections + http_header_names: + - x-remote-node-id # Should be set to downstream-node + - x-dst-cluster-uuid # Should be set to downstream + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": >- + type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicit_http_config: + # Right the moment, reverse connections are supported over HTTP/2 only + http2_protocol_options: {} + admin: access_log_path: "/dev/stdout" address: socket_address: address: 0.0.0.0 port_value: 8888 + layered_runtime: layers: - name: layer static_layer: re2.max_program_size.error_level: 1000 envoy.reloadable_features.reverse_conn_force_local_reply: true -# Enable reverse connection bootstrap extension -bootstrap_extensions: -- name: envoy.bootstrap.reverse_tunnel.upstream_socket_interface - typed_config: - "@type": type.googleapis.com/envoy.extensions.bootstrap.reverse_tunnel.upstream_socket_interface.v3.UpstreamReverseConnectionSocketInterface - stat_prefix: "upstream_reverse_connection" \ No newline at end of file From 4f3db532239f4e9907d991d806c4bd44de700032 Mon Sep 17 00:00:00 2001 From: Basundhara Chakrabarty Date: Fri, 3 Oct 2025 04:34:03 +0000 Subject: [PATCH 4/5] fixes and resolving linter errors Signed-off-by: Basundhara Chakrabarty --- reverse_tunnel/docker-compose.yaml | 40 ++++----- reverse_tunnel/example.rst | 129 +++++++++++++++++++++------- reverse_tunnel/initiator-envoy.yaml | 8 +- reverse_tunnel/responder-envoy.yaml | 54 ++++++++++-- 4 files changed, 168 insertions(+), 63 deletions(-) diff --git a/reverse_tunnel/docker-compose.yaml b/reverse_tunnel/docker-compose.yaml index 021ae249..35734cc1 100644 --- a/reverse_tunnel/docker-compose.yaml +++ b/reverse_tunnel/docker-compose.yaml @@ -4,42 +4,42 @@ services: downstream-envoy: image: debug/envoy:latest volumes: - - ./initiator-envoy.yaml:/etc/downstream-envoy.yaml + - ./initiator-envoy.yaml:/etc/downstream-envoy.yaml command: envoy -c /etc/downstream-envoy.yaml --concurrency 1 -l trace --drain-time-s 3 ports: - # Admin interface - - "8888:8888" - # Reverse connection API listener - - "9000:9000" - # Ingress HTTP listener - - "6060:6060" + # Admin interface + - "8888:8888" + # Reverse connection API listener + - "9000:9000" + # Ingress HTTP listener + - "6060:6060" extra_hosts: - - "host.docker.internal:host-gateway" + - "host.docker.internal:host-gateway" networks: - - envoy-network + - envoy-network depends_on: - - downstream-service + - downstream-service downstream-service: image: nginxdemos/hello:plain-text networks: - - envoy-network + - envoy-network upstream-envoy: image: debug/envoy:latest volumes: - - ./responder-envoy.yaml:/etc/upstream-envoy.yaml + - ./responder-envoy.yaml:/etc/upstream-envoy.yaml command: envoy -c /etc/upstream-envoy.yaml --concurrency 1 -l trace --drain-time-s 3 ports: - # Admin interface - - "8889:8888" - # Reverse connection API listener - - "9001:9000" - # Egress listener - - "8085:8085" + # Admin interface + - "8889:8888" + # Reverse connection API listener + - "9001:9000" + # Egress listener + - "8085:8085" networks: - - envoy-network + - envoy-network networks: envoy-network: - driver: bridge \ No newline at end of file + driver: bridge diff --git a/reverse_tunnel/example.rst b/reverse_tunnel/example.rst index 07f828f8..e8cf219b 100644 --- a/reverse_tunnel/example.rst +++ b/reverse_tunnel/example.rst @@ -10,7 +10,9 @@ Reverse Tunnels :ref:`curl ` Used to make HTTP requests. -This sandbox demonstrates Envoy's :ref:`reverse tunnels ` feature, which allows establishing long-lived connections from downstream to upstream in scenarios where direct connectivity from upstream to downstream is not possible, and using these cached connection sockets to send data traffic from upstream to downstream Envoy. +This sandbox demonstrates Envoy's :ref:`reverse tunnels ` feature, which allows establishing +long-lived connections from downstream to upstream in scenarios where direct connectivity from upstream to downstream +is not possible, and using these cached connection sockets to send data traffic from upstream to downstream Envoy. In this example, a downstream Envoy proxy initiates reverse tunnels to an upstream Envoy using a custom address resolver. The configuration includes bootstrap extensions and reverse connection listeners with specialized address formats. @@ -26,11 +28,30 @@ Build Envoy with the reverse tunnels feature enabled: Step 2: Build Envoy Docker image ******************************** -Build the Envoy Docker image: +Build the Envoy Docker image and tag it as ``debug/envoy:latest``. .. code-block:: console - $ docker build -f ci/Dockerfile-envoy-image -t envoy:latest . + $ ENVOY_DOCKER_IN_DOCKER=1 ./ci/run_envoy_docker.sh './ci/do_ci.sh docker' + +Verify the built images: + +.. code-block:: console + + $ docker image ls + +Expected output showing the built Envoy images: + +.. code-block:: console + + REPOSITORY TAG IMAGE ID CREATED SIZE + envoyproxy/envoy dev-57894704c2fc60ead4818d8d77e7ec7fa abc8392a56c4 2 minutes ago 193MB + +Tag the development image as ``debug/envoy:latest`` for use in the sandbox: + +.. code-block:: console + + $ docker tag envoyproxy/envoy:dev-57894704c2fc60ead4818d8d77e7ec7fa debug/envoy:latest Step 3: Understanding the configuration ************************************** @@ -50,7 +71,7 @@ Change to the ``reverse_tunnel`` directory and bring up the docker composition. .. note:: The docker-compose maps the following ports: - + - **downstream-envoy**: Host port 9000 → Container port 9000 (reverse connection API) - **upstream-envoy**: Host port 9001 → Container port 9000 (reverse connection API) @@ -64,10 +85,10 @@ Expected output showing all containers are up: .. code-block:: console - CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES - ae15eab504f8 debug/envoy:latest "/docker-entrypoint.…" 27 seconds ago Up 3 seconds 0.0.0.0:6060->6060/tcp, :::6060->6060/tcp, 0.0.0.0:8888->8888/tcp, :::8888->8888/tcp, 0.0.0.0:9000->9000/tcp, :::9000->9000/tcp, 10000/tcp reverse_tunnel-downstream-envoy-1 - 58eba3678f20 nginxdemos/hello:plain-text "/docker-entrypoint.…" 27 seconds ago Up 3 seconds 80/tcp reverse_tunnel-downstream-service-1 - 49145cc8a9d1 debug/envoy:latest "/docker-entrypoint.…" 27 seconds ago Up 3 seconds 0.0.0.0:8085->8085/tcp, :::8085->8085/tcp, 10000/tcp, 0.0.0.0:8889->8888/tcp, :::8889->8888/tcp, 0.0.0.0:9001->9000/tcp, :::9001->9000/tcp reverse_tunnel-upstream-envoy-1 + CONTAINER ID IMAGE COMMAND CREATED STATUS NAMES + ae15eab504f8 debug/envoy:latest "/docker-entrypoint.…" 27 seconds ago Up 3 seconds reverse_tunnel-downstream-envoy-1 + 58eba3678f20 nginxdemos/hello:plain-text "/docker-entrypoint.…" 27 seconds ago Up 3 seconds reverse_tunnel-downstream-service-1 + 49145cc8a9d1 debug/envoy:latest "/docker-entrypoint.…" 27 seconds ago Up 3 seconds reverse_tunnel-upstream-envoy-1 Step 5: Validate reverse tunnel establishment ********************************************* @@ -110,17 +131,23 @@ Expected upstream stats showing received reverse connections: The stats confirm that: -- **Downstream Envoy**: Has successfully connected (``connected: 1``) to the upstream cluster with no pending connections (``connecting: 0``) -- **Upstream Envoy**: Has received reverse connections from the downstream node and cluster, as indicated by the reverse connection counters +- **Downstream Envoy**: Has successfully connected (``connected: 1``) to the upstream cluster + with no pending connections (``connecting: 0``) +- **Upstream Envoy**: Has received reverse connections from the downstream node and cluster, + as indicated by the reverse connection counters Step 6: Test reverse tunnel *************************** -Perform an HTTP request for the service behind downstream Envoy, to upstream Envoy. This request will be sent over a reverse tunnel. +Perform HTTP requests for the service behind downstream Envoy, to upstream Envoy. +These requests will be sent over a reverse tunnel. You can route requests using either +cluster ID or node ID headers. + +**Option 1: Route by cluster ID** .. code-block:: console - $ curl -H "x-remote-node-id: downstream-node" -H "x-dst-cluster-uuid: downstream-cluster" http://localhost:8085/downstream_service -v + $ curl -H "x-cluster-id: downstream-cluster" http://localhost:8085/downstream_service -v Expected response: @@ -129,29 +156,67 @@ Expected response: * Trying ::1... * TCP_NODELAY set * Connected to localhost (::1) port 8085 (#0) - > GET /downstream_service HTTP/1.1 - > Host: localhost:8085 - > User-Agent: curl/7.61.1 - > Accept: */* - > x-remote-node-id: downstream-node - > x-dst-cluster-uuid: downstream-cluster - > - < HTTP/1.1 200 OK - < server: envoy - < date: Thu, 25 Sep 2025 21:25:38 GMT - < content-type: text/plain - < content-length: 159 - < expires: Thu, 25 Sep 2025 21:25:37 GMT - < cache-control: no-cache - < x-envoy-upstream-service-time: 13 - < - Server address: 172.27.0.3:80 - Server name: b490f264caf9 - Date: 25/Sep/2025:21:25:38 +0000 + >GET /downstream_service HTTP/1.1 + >Host: localhost:8085 + >User-Agent: curl/7.61.1 + >Accept: */* + >x-cluster-id: downstream-cluster + > + GET /downstream_service HTTP/1.1 + >Host: localhost:8085 + >User-Agent: curl/7.61.1 + >Accept: */* + >x-node-id: downstream-node + > + ` diff --git a/reverse_tunnel/initiator-envoy.yaml b/reverse_tunnel/initiator-envoy.yaml index 317ea56d..61675e60 100644 --- a/reverse_tunnel/initiator-envoy.yaml +++ b/reverse_tunnel/initiator-envoy.yaml @@ -80,7 +80,11 @@ static_resources: port_value: 80 admin: - access_log_path: "/dev/stdout" + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: "/dev/stdout" address: socket_address: protocol: TCP @@ -91,4 +95,4 @@ layered_runtime: layers: - name: layer static_layer: - re2.max_program_size.error_level: 1000 + re2.max_program_size.error_level: 1000 \ No newline at end of file diff --git a/reverse_tunnel/responder-envoy.yaml b/reverse_tunnel/responder-envoy.yaml index 5416f6c5..64b05b9a 100644 --- a/reverse_tunnel/responder-envoy.yaml +++ b/reverse_tunnel/responder-envoy.yaml @@ -51,6 +51,37 @@ static_resources: route: cluster: reverse_connection_cluster http_filters: + # The Lua filter is used to extract the host ID from the headers and set it in the x-computed-host-id header. + # This header is then used by the reverse connection cluster to look up a socket. + # The reverse connection cluster checks if there are cached sockets for this host. + - name: envoy.filters.http.lua + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua + inline_code: | + function envoy_on_request(request_handle) + local headers = request_handle:headers() + local node_id = headers:get("x-node-id") + local cluster_id = headers:get("x-cluster-id") + + local host_id = "" + + -- Priority 1: x-node-id header + if node_id then + host_id = node_id + request_handle:logInfo("Using x-node-id as host_id: " .. host_id) + -- Priority 2: x-cluster-id header + elseif cluster_id then + host_id = cluster_id + request_handle:logInfo("Using x-cluster-id as host_id: " .. host_id) + else + request_handle:logError("No valid headers found: x-node-id or x-cluster-id") + -- Don't set x-computed-host-id, which will cause cluster matching to fail + return + end + + -- Set the computed host ID for the reverse connection cluster + headers:add("x-computed-host-id", host_id) + end - name: envoy.filters.http.router typed_config: "@type": >- @@ -65,22 +96,27 @@ static_resources: name: envoy.clusters.reverse_connection typed_config: "@type": >- - type.googleapis.com/envoy.extensions.clusters.reverse_connection.v3.RevConClusterConfig - # The following headers are expected in downstream requests - # to be sent over reverse connections - http_header_names: - - x-remote-node-id # Should be set to downstream-node - - x-dst-cluster-uuid # Should be set to downstream + type.googleapis.com/envoy.extensions.clusters.reverse_connection.v3.ReverseConnectionClusterConfig + cleanup_interval: 60s + # This is the actual host ID that will be used by the reverse connection cluster to look up a socket. + # The reverse connection cluster checks if there are cached sockets for this cluster, if so, it will + # use the socket. Otherwise, it assumes this is a downstream node and looks for cached sockets with + # this as the node instead. + host_id_format: "%REQ(x-computed-host-id)%" typed_extension_protocol_options: envoy.extensions.upstreams.http.v3.HttpProtocolOptions: "@type": >- type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions explicit_http_config: - # Right the moment, reverse connections are supported over HTTP/2 only + # Right the moment, reverse connections are supported over HTTP/2 only. http2_protocol_options: {} admin: - access_log_path: "/dev/stdout" + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: "/dev/stdout" address: socket_address: address: 0.0.0.0 @@ -91,4 +127,4 @@ layered_runtime: - name: layer static_layer: re2.max_program_size.error_level: 1000 - envoy.reloadable_features.reverse_conn_force_local_reply: true + envoy.reloadable_features.reverse_conn_force_local_reply: true \ No newline at end of file From 0ea057a8c37335f0a7fe6b7c8e9b5c16cb827de1 Mon Sep 17 00:00:00 2001 From: Basundhara Chakrabarty Date: Mon, 13 Oct 2025 09:01:20 +0000 Subject: [PATCH 5/5] add verify.sh and some more fixes Signed-off-by: Basundhara Chakrabarty --- reverse_tunnel/docker-compose.yaml | 37 +++++------ reverse_tunnel/example.rst | 96 ++++++++--------------------- reverse_tunnel/initiator-envoy.yaml | 5 +- reverse_tunnel/responder-envoy.yaml | 1 + reverse_tunnel/verify.sh | 57 +++++++++++++++++ 5 files changed, 105 insertions(+), 91 deletions(-) create mode 100755 reverse_tunnel/verify.sh diff --git a/reverse_tunnel/docker-compose.yaml b/reverse_tunnel/docker-compose.yaml index 35734cc1..32f15343 100644 --- a/reverse_tunnel/docker-compose.yaml +++ b/reverse_tunnel/docker-compose.yaml @@ -1,18 +1,17 @@ -version: '2' services: - downstream-envoy: - image: debug/envoy:latest - volumes: - - ./initiator-envoy.yaml:/etc/downstream-envoy.yaml - command: envoy -c /etc/downstream-envoy.yaml --concurrency 1 -l trace --drain-time-s 3 + initiator-envoy: + build: + context: . + dockerfile: ../shared/envoy/Dockerfile + args: + ENVOY_CONFIG: initiator-envoy.yaml + command: ["envoy", "-c", "/etc/envoy.yaml", "--concurrency", "1", "-l", "trace", "--drain-time-s", "3"] ports: # Admin interface - - "8888:8888" + - "${PORT_ADMIN_DOWNSTREAM:-8888}:8888" # Reverse connection API listener - - "9000:9000" - # Ingress HTTP listener - - "6060:6060" + - "${PORT_REVERSE_API_DOWNSTREAM:-9000}:9000" extra_hosts: - "host.docker.internal:host-gateway" networks: @@ -25,18 +24,20 @@ services: networks: - envoy-network - upstream-envoy: - image: debug/envoy:latest - volumes: - - ./responder-envoy.yaml:/etc/upstream-envoy.yaml - command: envoy -c /etc/upstream-envoy.yaml --concurrency 1 -l trace --drain-time-s 3 + responder-envoy: + build: + context: . + dockerfile: ../shared/envoy/Dockerfile + args: + ENVOY_CONFIG: responder-envoy.yaml + command: ["envoy", "-c", "/etc/envoy.yaml", "--concurrency", "1", "-l", "trace", "--drain-time-s", "3"] ports: # Admin interface - - "8889:8888" + - "${PORT_ADMIN_UPSTREAM:-8889}:8888" # Reverse connection API listener - - "9001:9000" + - "${PORT_REVERSE_API_UPSTREAM:-9001}:9000" # Egress listener - - "8085:8085" + - "${PORT_PROXY:-8085}:8085" networks: - envoy-network diff --git a/reverse_tunnel/example.rst b/reverse_tunnel/example.rst index e8cf219b..bf41462c 100644 --- a/reverse_tunnel/example.rst +++ b/reverse_tunnel/example.rst @@ -16,81 +16,35 @@ is not possible, and using these cached connection sockets to send data traffic In this example, a downstream Envoy proxy initiates reverse tunnels to an upstream Envoy using a custom address resolver. The configuration includes bootstrap extensions and reverse connection listeners with specialized address formats. -Step 1: Build Envoy with reverse tunnels feature -************************************************ +Step 1: Build the sandbox +************************* -Build Envoy with the reverse tunnels feature enabled: +Change to the ``reverse_tunnel`` directory. -.. code-block:: console - - $ ./ci/run_envoy_docker.sh './ci/do_ci.sh bazel.release.server_only' - -Step 2: Build Envoy Docker image -******************************** - -Build the Envoy Docker image and tag it as ``debug/envoy:latest``. - -.. code-block:: console - - $ ENVOY_DOCKER_IN_DOCKER=1 ./ci/run_envoy_docker.sh './ci/do_ci.sh docker' - -Verify the built images: +To build this sandbox example, and start the example services run the following commands: .. code-block:: console - $ docker image ls - -Expected output showing the built Envoy images: - -.. code-block:: console - - REPOSITORY TAG IMAGE ID CREATED SIZE - envoyproxy/envoy dev-57894704c2fc60ead4818d8d77e7ec7fa abc8392a56c4 2 minutes ago 193MB - -Tag the development image as ``debug/envoy:latest`` for use in the sandbox: - -.. code-block:: console + $ pwd + examples/reverse_tunnel + $ docker compose pull + $ docker compose up --build -d + $ docker compose ps - $ docker tag envoyproxy/envoy:dev-57894704c2fc60ead4818d8d77e7ec7fa debug/envoy:latest + Name Command State Ports + ------------------------------------------------------------------------------------------------------------------------------- + reverse_tunnel_initiator-envoy_1 /docker-entrypoint.sh /usr ... Up 0.0.0.0:8888->8888/tcp, 0.0.0.0:9000->9000/tcp + reverse_tunnel_responder-envoy_1 /docker-entrypoint.sh /usr ... Up 0.0.0.0:8085->8085/tcp, + 0.0.0.0:8889->8888/tcp, + 0.0.0.0:9001->9000/tcp + reverse_tunnel_downstream-service_1 /docker-entrypoint.sh /usr ... Up 80/tcp -Step 3: Understanding the configuration +Step 2: Understanding the configuration ************************************** The reverse tunnel configuration is explained in the :ref:`Reverse Tunnels ` section. -Step 4: Launch test containers -****************************** - -Change to the ``reverse_tunnel`` directory and bring up the docker composition. - -.. code-block:: console - - $ pwd - examples/reverse_tunnel - $ docker compose up - -.. note:: - The docker-compose maps the following ports: - - - **downstream-envoy**: Host port 9000 → Container port 9000 (reverse connection API) - - **upstream-envoy**: Host port 9001 → Container port 9000 (reverse connection API) - -Verify the containers are running: - -.. code-block:: console - - $ docker ps - -Expected output showing all containers are up: - -.. code-block:: console - - CONTAINER ID IMAGE COMMAND CREATED STATUS NAMES - ae15eab504f8 debug/envoy:latest "/docker-entrypoint.…" 27 seconds ago Up 3 seconds reverse_tunnel-downstream-envoy-1 - 58eba3678f20 nginxdemos/hello:plain-text "/docker-entrypoint.…" 27 seconds ago Up 3 seconds reverse_tunnel-downstream-service-1 - 49145cc8a9d1 debug/envoy:latest "/docker-entrypoint.…" 27 seconds ago Up 3 seconds reverse_tunnel-upstream-envoy-1 - -Step 5: Validate reverse tunnel establishment +Step 3: Validate reverse tunnel establishment ********************************************* Verify that reverse tunnels have been successfully established by checking the stats counters on both downstream and upstream Envoy proxies. @@ -99,7 +53,7 @@ Check downstream Envoy stats (port 8888): .. code-block:: console - $ curl http://localhost:8888/stats | grep reverse_connection + $ curl "http://localhost:8888/stats?hidden=include" | grep downstream_reverse_connection Expected downstream stats showing connected reverse tunnels: @@ -118,16 +72,16 @@ Check upstream Envoy stats (port 8889): .. code-block:: console - $ curl http://localhost:8889/stats | grep reverse_connections + $ curl "http://localhost:8889/stats?hidden=include" | grep upstream_reverse_connection Expected upstream stats showing received reverse connections: .. code-block:: console - reverse_connections.clusters.downstream-cluster: 1 - reverse_connections.nodes.downstream-node: 1 - reverse_connections.worker_0.cluster.downstream-cluster: 1 - reverse_connections.worker_0.node.downstream-node: 1 + upstream_reverse_connection.clusters.downstream-cluster: 1 + upstream_reverse_connection.nodes.downstream-node: 1 + upstream_reverse_connection.worker_0.cluster.downstream-cluster: 1 + upstream_reverse_connection.worker_0.node.downstream-node: 1 The stats confirm that: @@ -136,7 +90,7 @@ The stats confirm that: - **Upstream Envoy**: Has received reverse connections from the downstream node and cluster, as indicated by the reverse connection counters -Step 6: Test reverse tunnel +Step 4: Test reverse tunnel *************************** Perform HTTP requests for the service behind downstream Envoy, to upstream Envoy. diff --git a/reverse_tunnel/initiator-envoy.yaml b/reverse_tunnel/initiator-envoy.yaml index 61675e60..28229f07 100644 --- a/reverse_tunnel/initiator-envoy.yaml +++ b/reverse_tunnel/initiator-envoy.yaml @@ -10,10 +10,11 @@ bootstrap_extensions: "@type": >- type.googleapis.com/envoy.extensions.bootstrap.reverse_tunnel.downstream_socket_interface.v3.DownstreamReverseConnectionSocketInterface stat_prefix: "downstream_reverse_connection" + enable_detailed_stats: true static_resources: listeners: - # Initiates reverse connections to upstream using custom resolver + # Initiates rev connere hierse connections to upstream using custom resolver - name: reverse_conn_listener listener_filters_timeout: 0s listener_filters: [] @@ -61,7 +62,7 @@ static_resources: - endpoint: address: socket_address: - address: upstream-envoy # Address of upstream-envoy + address: responder-envoy # Address of responder-envoy service port_value: 9000 # Port for rev_conn_api_listener # Backend HTTP service behind downstream which diff --git a/reverse_tunnel/responder-envoy.yaml b/reverse_tunnel/responder-envoy.yaml index 64b05b9a..2d694bce 100644 --- a/reverse_tunnel/responder-envoy.yaml +++ b/reverse_tunnel/responder-envoy.yaml @@ -10,6 +10,7 @@ bootstrap_extensions: "@type": >- type.googleapis.com/envoy.extensions.bootstrap.reverse_tunnel.upstream_socket_interface.v3.UpstreamReverseConnectionSocketInterface stat_prefix: "upstream_reverse_connection" + enable_detailed_stats: true static_resources: listeners: diff --git a/reverse_tunnel/verify.sh b/reverse_tunnel/verify.sh new file mode 100755 index 00000000..4427e356 --- /dev/null +++ b/reverse_tunnel/verify.sh @@ -0,0 +1,57 @@ +#!/bin/bash -e + +export NAME=reverse_tunnel +# Port on upstream envoy which accepts requests to be sent to downstream +# services through reverse tunnels. +export PORT_PROXY="${REVERSE_TUNNEL_PORT_PROXY:-8085}" +# Admin port on downstream envoy. +export PORT_ADMIN_DOWNSTREAM="${REVERSE_TUNNEL_ADMIN_DOWNSTREAM:-8888}" +# Admin port on upstream envoy. +export PORT_ADMIN_UPSTREAM="${REVERSE_TUNNEL_ADMIN_UPSTREAM:-8889}" +# Reverse connection API port on downstream envoy. +export PORT_REVERSE_API_DOWNSTREAM="${REVERSE_TUNNEL_REVERSE_API_DOWNSTREAM:-9000}" +# Reverse connection API port on upstream envoy. +export PORT_REVERSE_API_UPSTREAM="${REVERSE_TUNNEL_REVERSE_API_UPSTREAM:-9001}" + + +# shellcheck source=verify-common.sh +. "$(dirname "${BASH_SOURCE[0]}")/../verify-common.sh" + +# Wait for reverse tunnel establishment - check downstream stats first +run_log "Wait for reverse tunnel establishment" +wait_for 60 bash -c "responds_with 'downstream_reverse_connection.cluster.upstream-cluster.connected: 1' 'http://localhost:${PORT_ADMIN_DOWNSTREAM}/stats?hidden=include'" + +# Verify downstream stats. +run_log "Verify reverse tunnel establishment - downstream stats" +responds_with \ + "downstream_reverse_connection.cluster.upstream-cluster.connected: 1" \ + "http://localhost:${PORT_ADMIN_DOWNSTREAM}/stats?hidden=include" + +run_log "Verify no pending downstream connections" +responds_with \ + "downstream_reverse_connection.cluster.upstream-cluster.connecting: 0" \ + "http://localhost:${PORT_ADMIN_DOWNSTREAM}/stats?hidden=include" + +# Verify upstream stats. +run_log "Verify reverse tunnel establishment - upstream stats" +responds_with \ + "upstream_reverse_connection.clusters.downstream-cluster: 1" \ + "http://localhost:${PORT_ADMIN_UPSTREAM}/stats?hidden=include" + +run_log "Verify upstream received connections from downstream node" +responds_with \ + "upstream_reverse_connection.nodes.downstream-node: 1" \ + "http://localhost:${PORT_ADMIN_UPSTREAM}/stats?hidden=include" + +# Verify data requests through reverse tunnel. +run_log "Test reverse tunnel with cluster ID routing" +responds_with \ + "URI: /downstream_service" \ + "http://localhost:${PORT_PROXY}/downstream_service" \ + -H "x-cluster-id: downstream-cluster" + +run_log "Test reverse tunnel with node ID routing" +responds_with \ + "URI: /downstream_service" \ + "http://localhost:${PORT_PROXY}/downstream_service" \ + -H "x-node-id: downstream-node"