From b2a44eb4431c85f008b8b6735e5ab1c92539adfe Mon Sep 17 00:00:00 2001 From: Ivan Medina Date: Tue, 2 Jun 2026 10:57:40 -0300 Subject: [PATCH 1/5] fix(devtools-proxy-support): reconnect SSH tunnel after unrecoverable client error COMPASS-8355 - Add sshClient.on('error') handler in SSHAgent constructor to prevent unhandled 'error' events from crashing the process when the SSH session dies unexpectedly (e.g. after hibernate) - Add reinitializeClient flag to initialize() to recreate the ssh2 Client instance when it enters an unrecoverable "Instance unusable" state - Expand retryable error patterns to include "Instance unusable after fatal error", "read ECONNRESET", and "Socket closed" - Extract client setup into createSshClient() to ensure close/error handlers and forwardOut binding are consistent on both initial setup and recreation --- .../devtools-proxy-support/src/ssh.spec.ts | 42 +++++++++++++++++ packages/devtools-proxy-support/src/ssh.ts | 47 +++++++++++++++---- 2 files changed, 79 insertions(+), 10 deletions(-) diff --git a/packages/devtools-proxy-support/src/ssh.spec.ts b/packages/devtools-proxy-support/src/ssh.spec.ts index 277b13e6..6155e119 100644 --- a/packages/devtools-proxy-support/src/ssh.spec.ts +++ b/packages/devtools-proxy-support/src/ssh.spec.ts @@ -125,4 +125,46 @@ describe('SSHAgent', function () { expect(setup.authHandler).to.have.been.calledTwice; expect(setup.canTunnel).to.have.been.calledTwice; }); + + it('does not crash on unexpected sshClient error events', async function () { + agent = new SSHAgent({ + proxy: `ssh://foo:bar@127.0.0.1:${setup.sshProxyPort}/`, + }); + await agent.initialize(); + expect(() => { + (agent as any).sshClient.emit( + 'error', + new Error('some unexpected ssh error'), + ); + }).not.to.throw(); + expect((agent as any).connected).to.be.false; + }); + + it('reconnects with a fresh SSH client after "Instance unusable after fatal error"', async function () { + setup.authHandler = sinon.stub().returns(true); + agent = new SSHAgent({ + proxy: `ssh://foo:bar@127.0.0.1:${setup.sshProxyPort}/`, + }); + await agent.initialize(); + const fetch = createFetch(agent); + await fetch('http://example.com/hello'); + + // Simulate the ssh2 client entering an unrecoverable "unusable" state + // (e.g. parser received truncated data during hibernate). When connect() + // is called on such a client, ssh2 emits 'error' instead of 'ready'. + const brokenClient = (agent as any).sshClient; + brokenClient.connect = function () { + process.nextTick(() => + brokenClient.emit( + 'error', + new Error('Instance unusable after fatal error'), + ), + ); + }; + (agent as any).connected = false; + + await fetch('http://example.com/hello'); + // A fresh client was created and a new SSH handshake performed + expect(setup.authHandler).to.have.been.calledTwice; + }); }); diff --git a/packages/devtools-proxy-support/src/ssh.ts b/packages/devtools-proxy-support/src/ssh.ts index c0e9d832..4ca271c1 100644 --- a/packages/devtools-proxy-support/src/ssh.ts +++ b/packages/devtools-proxy-support/src/ssh.ts @@ -37,7 +37,7 @@ export class SSHAgent extends AgentBase implements AgentWithInitialize { private connected = false; private connectingPromise?: Promise; private closed = false; - private forwardOut: ( + private forwardOut!: ( srcIP: string, srcPort: number, dstIP: string, @@ -52,21 +52,33 @@ export class SSHAgent extends AgentBase implements AgentWithInitialize { this.logger = logger ?? new EventEmitter().setMaxListeners(Infinity); this.proxyOptions = options; this.url = new URL(options.proxy ?? ''); - this.sshClient = new (ssh2().Client)(); - this.sshClient.on('close', () => { + this.sshClient = this.createSshClient(); + } + + private createSshClient(): SshClient { + const client = new (ssh2().Client)(); + client.on('close', () => { this.logger.emit('ssh:client-closed'); this.connected = false; }); - - this.forwardOut = promisify(this.sshClient.forwardOut.bind(this.sshClient)); + client.on('error', () => { + // Errors during connection setup are handled through initialize()'s + // connectingPromise race, and post-connection errors through _connect()'s + // catch block. This listener prevents unhandled 'error' events from + // crashing the process when the SSH session dies unexpectedly (e.g. after + // the host machine resumes from hibernate). + this.connected = false; + }); + this.forwardOut = promisify(client.forwardOut.bind(client)); + return client; } - async initialize(): Promise { - if (this.connected) { + async initialize(reinitializeClient = false): Promise { + if (this.connected && !reinitializeClient) { return; } - if (this.connectingPromise) { + if (this.connectingPromise && !reinitializeClient) { return this.connectingPromise; } @@ -75,6 +87,16 @@ export class SSHAgent extends AgentBase implements AgentWithInitialize { throw new Error('Disconnected.'); } + if (reinitializeClient) { + // The previous ssh2 Client instance is in an unrecoverable state (e.g. + // "Instance unusable after fatal error" after the TCP connection was killed + // mid-stream during hibernate). Create a fresh client before reconnecting. + delete this.connectingPromise; + this.connected = false; + this.sshClient.end(); + this.sshClient = this.createSshClient(); + } + const sshConnectConfig: ConnectConfig = { readyTimeout: 20000, keepaliveInterval: 20000, @@ -161,9 +183,14 @@ export class SSHAgent extends AgentBase implements AgentWithInitialize { } return sock; } catch (err: unknown) { - const retryableError = /Not connected|Channel open failure/.test( + const requiresNewClient = /Instance unusable after fatal error/.test( (err as Error).message, ); + const retryableError = + requiresNewClient || + /Not connected|Channel open failure|read ECONNRESET|Socket closed/.test( + (err as Error).message, + ); this.logger.emit('ssh:failed-forward', { host, error: String((err as Error).stack), @@ -173,7 +200,7 @@ export class SSHAgent extends AgentBase implements AgentWithInitialize { if (retryableError) { this.connected = false; if (retriesLeft > 0) { - await this.initialize(); + await this.initialize(requiresNewClient); return await this._connect(req, connectOpts, retriesLeft - 1); } } From 2dde42a9dddce031e1e8bfaf45497d6186b3f3fc Mon Sep 17 00:00:00 2001 From: Ivan Medina Date: Tue, 2 Jun 2026 12:53:49 -0300 Subject: [PATCH 2/5] address review feedback: use = undefined, improve test coverage - Replace delete this.connectingPromise with = undefined assignment - Improve "Instance unusable" test: patch connect() to simulate broken client state, verify fresh client is created and auth happens twice - Track initializedSuccessfully in _connect() to detect when initialize() itself failed, requiring a new client on retry --- .../devtools-proxy-support/src/ssh.spec.ts | 7 ++++-- packages/devtools-proxy-support/src/ssh.ts | 23 ++++++++++++------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/packages/devtools-proxy-support/src/ssh.spec.ts b/packages/devtools-proxy-support/src/ssh.spec.ts index 6155e119..c3f01446 100644 --- a/packages/devtools-proxy-support/src/ssh.spec.ts +++ b/packages/devtools-proxy-support/src/ssh.spec.ts @@ -2,6 +2,7 @@ import { HTTPServerProxyTestSetup } from '../test/helpers'; import { SSHAgent } from './ssh'; import { createFetch } from './fetch'; import { expect } from 'chai'; +import { once } from 'events'; import sinon from 'sinon'; describe('SSHAgent', function () { @@ -150,8 +151,10 @@ describe('SSHAgent', function () { await fetch('http://example.com/hello'); // Simulate the ssh2 client entering an unrecoverable "unusable" state - // (e.g. parser received truncated data during hibernate). When connect() - // is called on such a client, ssh2 emits 'error' instead of 'ready'. + // (e.g. the TCP connection was killed mid-stream during hibernate). When + // connect() is called on such a client, ssh2 emits 'error' instead of + // 'ready', which our new createSshClient() path must handle by discarding + // the broken instance and creating a fresh one. const brokenClient = (agent as any).sshClient; brokenClient.connect = function () { process.nextTick(() => diff --git a/packages/devtools-proxy-support/src/ssh.ts b/packages/devtools-proxy-support/src/ssh.ts index 4ca271c1..83f61919 100644 --- a/packages/devtools-proxy-support/src/ssh.ts +++ b/packages/devtools-proxy-support/src/ssh.ts @@ -90,10 +90,12 @@ export class SSHAgent extends AgentBase implements AgentWithInitialize { if (reinitializeClient) { // The previous ssh2 Client instance is in an unrecoverable state (e.g. // "Instance unusable after fatal error" after the TCP connection was killed - // mid-stream during hibernate). Create a fresh client before reconnecting. - delete this.connectingPromise; + // mid-stream during hibernate). Discard it and create a fresh one. + // We do not call end() here: the underlying socket is already dead + // (forceful RST or broken parser), and calling end() on it may emit + // unhandled socket-level errors. + this.connectingPromise = undefined; this.connected = false; - this.sshClient.end(); this.sshClient = this.createSshClient(); } @@ -139,11 +141,11 @@ export class SSHAgent extends AgentBase implements AgentWithInitialize { this.logger.emit('ssh:failed-connection', { error: (err as any)?.stack ?? String(err), }); - delete this.connectingPromise; + this.connectingPromise = undefined; throw err; } - delete this.connectingPromise; + this.connectingPromise = undefined; this.connected = true; this.logger.emit('ssh:established-connection'); } @@ -161,12 +163,14 @@ export class SSHAgent extends AgentBase implements AgentWithInitialize { retriesLeft = 1, ): Promise { let host = ''; + let initializedSuccessfully = false; try { // Using the `host` header matches what proxy-agent does host = connectOpts.host || (req.getHeader('host') as string); const url = new URL(req.path, `tcp://${host}:${connectOpts.port}`); await this.initialize(); + initializedSuccessfully = true; let sock: Duplex & Partial> = await this.forwardOut('127.0.0.1', 0, url.hostname, +url.port); @@ -183,9 +187,12 @@ export class SSHAgent extends AgentBase implements AgentWithInitialize { } return sock; } catch (err: unknown) { - const requiresNewClient = /Instance unusable after fatal error/.test( - (err as Error).message, - ); + // If initialize() itself failed, the ssh2 Client instance may be in a + // broken state (e.g. after a forceful TCP reset during hibernate) and + // needs to be recreated before the next attempt. + const requiresNewClient = + !initializedSuccessfully || + /Instance unusable after fatal error/.test((err as Error).message); const retryableError = requiresNewClient || /Not connected|Channel open failure|read ECONNRESET|Socket closed/.test( From 8c3e38e495080bc49de429f0c707733c076525e5 Mon Sep 17 00:00:00 2001 From: Ivan Medina Date: Tue, 2 Jun 2026 14:23:58 -0300 Subject: [PATCH 3/5] fix(devtools-proxy-support): remove unused once import COMPASS-8355 --- packages/devtools-proxy-support/src/ssh.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/devtools-proxy-support/src/ssh.spec.ts b/packages/devtools-proxy-support/src/ssh.spec.ts index c3f01446..6c04e087 100644 --- a/packages/devtools-proxy-support/src/ssh.spec.ts +++ b/packages/devtools-proxy-support/src/ssh.spec.ts @@ -2,7 +2,6 @@ import { HTTPServerProxyTestSetup } from '../test/helpers'; import { SSHAgent } from './ssh'; import { createFetch } from './fetch'; import { expect } from 'chai'; -import { once } from 'events'; import sinon from 'sinon'; describe('SSHAgent', function () { From 9d5a356b90054aea5f2ba070a85a448d6fd15f50 Mon Sep 17 00:00:00 2001 From: Ivan Medina Date: Tue, 2 Jun 2026 16:31:43 -0300 Subject: [PATCH 4/5] fix(devtools-proxy-support): restore end() call in reinitializeClient to clear keepalive timer COMPASS-8355 Without this, the old ssh2 Client's keepalive setInterval keeps firing after initialize(true) replaces it, preventing process exit. ssh2's end() guards with isWritable() so it is a no-op when the socket is already dead. --- packages/devtools-proxy-support/src/ssh.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/devtools-proxy-support/src/ssh.ts b/packages/devtools-proxy-support/src/ssh.ts index 83f61919..93e3f4bc 100644 --- a/packages/devtools-proxy-support/src/ssh.ts +++ b/packages/devtools-proxy-support/src/ssh.ts @@ -90,12 +90,13 @@ export class SSHAgent extends AgentBase implements AgentWithInitialize { if (reinitializeClient) { // The previous ssh2 Client instance is in an unrecoverable state (e.g. // "Instance unusable after fatal error" after the TCP connection was killed - // mid-stream during hibernate). Discard it and create a fresh one. - // We do not call end() here: the underlying socket is already dead - // (forceful RST or broken parser), and calling end() on it may emit - // unhandled socket-level errors. + // mid-stream during hibernate). End it to release the keepalive timer + // and any other internal handles, then create a fresh instance. + // ssh2's end() is a no-op when the underlying socket is already dead + // (it guards with isWritable()), so this is safe in all cases. this.connectingPromise = undefined; this.connected = false; + this.sshClient.end(); this.sshClient = this.createSshClient(); } From e34875633b530c3d22c16e2a950e9f20a7f624c4 Mon Sep 17 00:00:00 2001 From: Ivan Medina Date: Wed, 3 Jun 2026 09:57:03 -0300 Subject: [PATCH 5/5] test(devtools-proxy-support): verify SSH reconnect after server-side connection reset COMPASS-8355 --- .../devtools-proxy-support/src/ssh.spec.ts | 24 ++++++++++++++++ .../devtools-proxy-support/test/helpers.ts | 28 +++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/packages/devtools-proxy-support/src/ssh.spec.ts b/packages/devtools-proxy-support/src/ssh.spec.ts index 6c04e087..a1684b91 100644 --- a/packages/devtools-proxy-support/src/ssh.spec.ts +++ b/packages/devtools-proxy-support/src/ssh.spec.ts @@ -1,3 +1,4 @@ +import { once } from 'events'; import { HTTPServerProxyTestSetup } from '../test/helpers'; import { SSHAgent } from './ssh'; import { createFetch } from './fetch'; @@ -169,4 +170,27 @@ describe('SSHAgent', function () { // A fresh client was created and a new SSH handshake performed expect(setup.authHandler).to.have.been.calledTwice; }); + + it('reconnects after the underlying connection is destroyed server-side', async function () { + setup.authHandler = sinon.stub().returns(true); + agent = new SSHAgent({ + proxy: `ssh://foo:bar@127.0.0.1:${setup.sshProxyPort}/`, + }); + await agent.initialize(); + const fetch = createFetch(agent); + await fetch('http://example.com/hello'); + expect(setup.authHandler).to.have.been.calledOnce; + + // Simulate the OS killing network connections during hibernate by + // forcibly destroying the SSH server's TCP sockets (with a TCP reset) + // from the server side, then wait for the agent to notice the loss. + const clientClosed = once(agent.logger, 'ssh:client-closed'); + setup.destroySSHConnections(); + await clientClosed; + + // The next request must transparently re-establish the SSH connection. + const response = await fetch('http://example.com/hello'); + expect(await response.text()).to.equal('OK /hello'); + expect(setup.authHandler).to.have.been.calledTwice; + }); }); diff --git a/packages/devtools-proxy-support/test/helpers.ts b/packages/devtools-proxy-support/test/helpers.ts index a00e2f56..b46288a1 100644 --- a/packages/devtools-proxy-support/test/helpers.ts +++ b/packages/devtools-proxy-support/test/helpers.ts @@ -39,6 +39,11 @@ export class HTTPServerProxyTestSetup { readonly sshServer: SSHServer; readonly sshTunnelInfos: TcpipRequestInfo[] = []; readonly connections: Duplex[] = []; + // Raw TCP sockets accepted by the SSH server, kept so that tests can + // forcibly destroy them (simulating the OS dropping connections during + // hibernate). These are the underlying net.Sockets, not the high-level + // ssh2 Client objects emitted by the SSH server's 'connection' event. + readonly sshServerSockets: Socket[] = []; canTunnel: () => boolean = () => true; authHandler: undefined | ((username: string, password: string) => boolean); @@ -169,6 +174,29 @@ export class HTTPServerProxyTestSetup { }); }, ); + // The ssh2 Server's public 'connection' event hands out high-level Client + // objects, but to simulate an abrupt network interruption we need the raw + // TCP sockets. ssh2 exposes the underlying net.Server as `_srv`, which + // emits 'connection' with the raw socket for every accepted connection. + (this.sshServer as unknown as { _srv: Server })._srv.on( + 'connection', + (socket: Socket) => { + this.sshServerSockets.push(socket); + socket.once('close', () => { + const index = this.sshServerSockets.indexOf(socket); + if (index !== -1) this.sshServerSockets.splice(index, 1); + }); + }, + ); + } + + // Forcibly destroys all currently open SSH server-side TCP sockets using a + // TCP reset, simulating an abrupt network interruption such as the OS + // killing connections when a laptop hibernates. + destroySSHConnections(): void { + for (const socket of [...this.sshServerSockets]) { + if (!socket.destroyed) socket.resetAndDestroy(); + } } async listen(): Promise {