Application crashing when No Response from server

[nick22985]

Hey,
I am trying to handle all errors to stop this from crashing my application if the server goes down. Instead it will gracefully handle them and reconnect.

Below is what I currently have but I am facing a issue with if the server goes down on the server it will crash my application. I have purposly stopped mongo on the server that it is connecting to simulate this.

  bot:info ----------------------Starting bot---------------------- +0ms
  bot:sshTunnel SSH Tunnel found in .env file, creating tunnel... +0ms
try connection
  bot:sshTunnel SSH Tunnel created! +0ms
  bot:mongo Connecting to the database... +0ms
  bot:mongoosy:connecting Connecting to MongoDB.... +0ms
  bot:sshTunnel SSH Tunnel Server Connected +0ms
  bot:sshTunnel:warn SSH Tunnel Conn Error (handled) Error: (SSH) Channel open failure: Connection refused
    at onChannelOpenFailure (C:\bot\node_modules\ssh2\lib\utils.js:16:11)
    at CHANNEL_OPEN_FAILURE (C:\bot\node_modules\ssh2\lib\client.js:572:11)
    at 92 (C:\bot\node_modules\ssh2\lib\protocol\handlers.misc.js:881:16)
    at Protocol.onPayload (C:\bot\node_modules\ssh2\lib\protocol\Protocol.js:2052:10)
    at AESGCMDecipherNative.decrypt (C:\bot\node_modules\ssh2\lib\protocol\crypto.js:987:26)
    at Protocol.parsePacket [as _parse] (C:\bot\node_modules\ssh2\lib\protocol\Protocol.js:2021:25)
    at Protocol.parse (C:\bot\node_modules\ssh2\lib\protocol\Protocol.js:306:16)
    at Socket.<anonymous> (C:\bot\node_modules\ssh2\lib\client.js:775:21)
    at Socket.emit (node:events:511:28)
    at Socket.emit (node:domain:489:12) {
  reason: 2
} +0ms
  bot:sshTunnel SSH Tunnel found in .env file, creating tunnel... +0ms
try connection
  bot:sshTunnel:warn SSH Tunnel Conn Closed +0ms
  bot:sshTunnel SSH Tunnel created! +0ms
  bot:sshTunnel SSH Tunnel Server Connected +0ms
  bot:sshTunnel SSH Tunnel Server Connected +0ms
  bot:sshTunnel:warn SSH Tunnel Conn Error (handled) Error: (SSH) Channel open failure: Connection refused
    at onChannelOpenFailure (C:\bot\node_modules\ssh2\lib\utils.js:16:11)
    at CHANNEL_OPEN_FAILURE (C:\bot\node_modules\ssh2\lib\client.js:572:11)
    at 92 (C:\bot\node_modules\ssh2\lib\protocol\handlers.misc.js:881:16)
    at Protocol.onPayload (C:\bot\node_modules\ssh2\lib\protocol\Protocol.js:2052:10)
    at AESGCMDecipherNative.decrypt (C:\bot\node_modules\ssh2\lib\protocol\crypto.js:987:26)
    at Protocol.parsePacket [as _parse] (C:\bot\node_modules\ssh2\lib\protocol\Protocol.js:2021:25)
    at Protocol.parse (C:\bot\node_modules\ssh2\lib\protocol\Protocol.js:306:16)
    at Socket.<anonymous> (C:\bot\node_modules\ssh2\lib\client.js:775:21)
    at Socket.emit (node:events:511:28)
    at Socket.emit (node:domain:489:12) {
  reason: 2
} +0ms
  bot:sshTunnel SSH Tunnel found in .env file, creating tunnel... +0ms
try connection
  bot:sshTunnel:warn SSH Tunnel Conn Closed +0ms
C:\bot\node_modules\ssh2\lib\client.js:826
      const err = new Error('No response from server');
                  ^
Error: No response from server
    at Socket.<anonymous> (C:\bot\node_modules\ssh2\lib\client.js:826:19)
    at Socket.emit (node:events:511:28)
    at Socket.emit (node:domain:489:12)
    at TCP.<anonymous> (node:net:335:12)
[nodemon] app crashed - waiting for file changes before starting...

async setupTunnel() {
		if (process.env.SSH_TUNNEL_HOST) {
			this.$debug.extend('sshTunnel')('SSH Tunnel found in .env file, creating tunnel...');
			if (!process.env.SSH_TUNNEL_HOST) throw new Error('SSH_TUNNEL_HOST not found in .env file');
			if (!process.env.SSH_TUNNEL_PORT) throw new Error('SSH_TUNNEL_PORT not found in .env file');
			if (!process.env.SSH_TUNNEL_SSH_PORT) throw new Error('SSH_TUNNEL_SSH_PORT not found in .env file');
			if (!process.env.SSH_TUNNEL_USERNAME) throw new Error('SSH_TUNNEL_USERNAME not found in .env file');
			if (!process.env.SSH_TUNNEL_PRIVATEKEY) throw new Error('SSH_TUNNEL_PRIVATEKEY not found in .env file');
			if (!process.env.SSH_TUNNEL_SRC_ADDR) throw new Error('SSH_TUNNEL_SRC_ADDR not found in .env file');
			if (!process.env.SSH_TUNNEL_DST_ADDR) throw new Error('SSH_TUNNEL_DST_ADDR not found in .env file');
			const tunnelOptions = {
				autoClose: false,
			};
			const serverOptions = {
				port: parseInt(process.env.SSH_TUNNEL_PORT),
			};
			const sshOptions = {
				host: process.env.SSH_TUNNEL_HOST,
				port: process.env.SSH_TUNNEL_SSH_PORT,
				username: process.env.SSH_TUNNEL_USERNAME,
				privateKey: require('fs').readFileSync(process.env.SSH_TUNNEL_PRIVATEKEY),
				passphrase: process.env.SSH_TUNNEL_PASSPHRASE,
				keepaliveInterval: 10000,
				readyTimeout: 30000,
				keepaliveCountMax: 10,
				debug: (msg) => {
					this.$debug.extend('sshTunnel:debug')(msg);
				},
			};
			const forwardOptions = {
				srcAddr: process.env.SSH_TUNNEL_SRC_ADDR,
				srcPort: parseInt(process.env.SSH_TUNNEL_PORT),
				dstAddr: process.env.SSH_TUNNEL_DST_ADDR,
				dstPort: parseInt(process.env.SSH_TUNNEL_PORT),
			};
			try {
				console.log('try connection');
				this.sshTunnel = await createTunnel(tunnelOptions, serverOptions, sshOptions, forwardOptions)
					.then((sshTunnel) => {
						let [server, client] = sshTunnel;
						if (!server || !client) return this.$debug.extend('sshTunnel:error')('SSH Tunnel', 'server or client not found');
						client.on('error', (err) => {
							Promise.resolve()
								.then(() => (server ? server.close() : null))
								.then(() => (client ? client.end() : null))
								.then(() => this.setupTunnel());
							this.$debug.extend('sshTunnel:warn')('SSH Tunnel Conn Error (handled)', err);
						});

						client.on('close', () => {
							if (server) server.close();
							this.$debug.extend('sshTunnel:warn')('SSH Tunnel Conn Closed');
						});

						server.on('close', () => {
							this.$debug.extend('sshTunnel:warn')('SSH Tunnel Server', 'closed');
							client.end();
						});

						server.on('error', (err) => {
							Promise.resolve()
								.then(() => (server ? server.close() : null))
								.then(() => (server ? client.end() : null))
								.then(() => this.setupTunnel());
							this.$debug.extend('sshTunnel:warn')('SSH Tunnel Server Error (Handled)', err);
						});
						server.on('connection', (connection) => {
							connection.on('error', (err) => {
								this.$debug.extend('sshTunnel:warn')('SSH server connection error:', err);
							});
							connection.on('close', () => {
								this.$debug.extend('sshTunnel:warn')('SSH server connection closed');
							});
							connection.on('end', () => {
								this.$debug.extend('sshTunnel:warn')('SSH server connection ended');
							});
							this.$debug.extend('sshTunnel')('SSH Tunnel Server Connected');
						});
						return sshTunnel;
					})
					.catch((err) => {
						this.$debug.extend('sshTunnel:error')('SSH Tunnel', err);
					});
			} catch (e) {
				this.$debug.extend('sshTunnel:error')('SSH Tunnel', e);
			}
			this.$debug.extend('sshTunnel')('SSH Tunnel created!');
		} else this.$debug.extend('sshTunnel')('No SSH Tunnel found in .env file');

[nick22985]

Also with the new update it is possible for the ssConnection.isBroken to be undefined and causes the package to fall over

            if (sshConnection.isBroken) {
                              ^
TypeError: Cannot read properties of undefined (reading 'isBroken')

[agebrock]

Hi there, you did some impressive job here :-)

I will do some additional testing on that topic, and I am planning to release a pre-release version , that could make your life much easier.

Thanks for the effort I will come back to you !

[negativems]

Also with the new update it is possible for the ssConnection.isBroken to be undefined and causes the package to fall over

            if (sshConnection.isBroken) {
                              ^
TypeError: Cannot read properties of undefined (reading 'isBroken')

I had this error multiple times

[nick22985]

I ended up rewriting this to suit my needs. I did not do all the auto connection close things but this may help with solving the issues that I was facing here.

import { Client, type ConnectConfig } from 'ssh2';
import HandlerClient from './handlers/client.js';
import net, { type ListenOptions, type Server } from 'net';
import { type Debugger } from 'debug';
export interface ITunnelOptions {
	autoClose: boolean;
	reconnectOnError: boolean;
}

export interface IForwardOptions {
	srcAddr: string;
	srcPort: number;
	dstAddr: string;
	dstPort: number;
}

export default class sshTunnel {
	discordClient: HandlerClient;
	debug: Debugger;
	tunnelOptions: ITunnelOptions;
	listenOptions: ListenOptions;
	connectConfig: ConnectConfig;
	forwardOptions: IForwardOptions;

	// Client, Server
	server: Server | undefined;
	client: Client | undefined;

	clientReconnectInterval: NodeJS.Timeout | undefined;

	constructor(options: { discordClient: HandlerClient; tunnelOptions: ITunnelOptions; listenOptions: ListenOptions; connectConfig: ConnectConfig; forwardOptions: IForwardOptions }) {
		this.discordClient = options.discordClient;
		this.tunnelOptions = Object.assign({ autoClose: false, reconnectOnError: false }, options.tunnelOptions || {});
		this.listenOptions = options.listenOptions;
		this.connectConfig = Object.assign({ port: 22, username: 'root' }, options.connectConfig);
		this.forwardOptions = Object.assign({ dstAddr: '0.0.0.0' }, options.forwardOptions);

		this.debug = this.discordClient.$debug.extend('sshTunnel');
		this.debug('Starting SSH Tunnel');
	}

	createServer() {
		const $debug = this.debug.extend('createServer');
		$debug('Creating Server');
		return Promise.resolve()
			.then(() => net.createServer())
			.then((server) => {
				return new Promise((resolve, reject) => {
					let errorHandler = (err) => {
						$debug('Error', err);
						reject(err);
					};
					server.on('error', errorHandler);
					process.on('uncaughtException', errorHandler);

					server.on('close', () => {
						$debug('Server Close');
						if (this.tunnelOptions.reconnectOnError) {
							this.createServer().then(() => this.serverEventListeners());
						}
					});

					server.on('drop', () => {
						$debug('Server Drop');
					});

					server.listen(this.listenOptions);

					server.on('listening', () => {
						process.removeListener('uncaughtException', errorHandler);
						$debug('Server Listening');
						this.server = server;
						resolve(server);
					});
				});
			});
	}

	createSSHClient() {
		const $debug = this.debug.extend('createSSHClient');
		$debug('Creating SSH Client');
		// make sure client is closed
		if (this.client) {
			$debug('Client has old connection and is trying to reconnect killing old client');
			this.client.end();
			this.client = undefined;
		}
		return new Promise((resolve, reject) => {
			let conn: Client = new Client();
			conn.on('ready', () => {
				this.client = conn;
				resolve(conn);
			});
			conn.on('error', reject);

			conn.on('close', () => {
				$debug('Client Close');
				if (this.tunnelOptions.reconnectOnError) {
					setTimeout(() => {
						return this.createSSHClient()
							.then(() => this.clientEventListeners())
							.then(() => $debug('reconnected to client'))
							.catch(() => {
								$debug.extend('error')('failed to reconnect to client');
							});
					}, 10000);
				}
			});

			conn.on('end', () => {
				$debug('Client End');
			});
			Promise.resolve()
				.then(() => conn.connect(this.connectConfig))
				.catch((err: Error) => {
					$debug.extend('error')('failed to connect', err);
					throw err;
				});
		}).catch((err: Error) => {
			$debug.extend('error')('failed to createSSHClient', err);
			throw err;
		});
	}

	serverEventListeners() {
		const $debug = this.debug.extend('serverEventListeners');
		$debug('Creating Server Event Listeners');
		if (!this.server) throw new Error('No Server found');
		// reco logic
		if (this.tunnelOptions.reconnectOnError)
			this.server.on('error', (err) => {
				$debug.extend('error')('Server Error', err);
				return Promise.resolve().then(() => this.createServer().then(() => this.serverEventListeners()));
			});

		this.server.on('connection', (connection) => {
			$debug('Server Connection', connection.address());
			if (!this.client) {
				$debug('No Client Connection');
				return connection.end(); // Kick the connection trying to connect to the server
			}
			connection.on('error', (err) => {
				$debug('Connection Closed error', err);
			});
			// This is the mongo Connection itself
			return this.client.forwardOut(this.forwardOptions.srcAddr, this.forwardOptions.srcPort, this.forwardOptions.dstAddr, this.forwardOptions.dstPort, (err, stream) => {
				if (err) {
					$debug.extend('error')('Server Connection Error', connection.address());
					$debug('server con err', err);

					setTimeout(() => {
						return connection.end(); // end user connection to server
					}, 10000);
				} else
					connection
						.pipe(stream)
						.pipe(connection)
						.on('close', (test) => {
							$debug('Server Connection Close', test);
						})
						.on('error', (err: Error) => {
							$debug('Connection closed from server (Usually this is from the client not closing the connection) (usually ignore this)', err);
						});
			});
		});
		this.server.on('close', () => {
			$debug('Server Close');
			if (this.client) this.client.end();
		});
	}

	clientEventListeners() {
		const $debug = this.debug.extend('clientEventListeners');
		$debug('Creating Client Event Listeners');
		if (!this.client) throw new Error('No Client found');

		this.client.on('ready', () => {
			$debug('Client Ready');
		});
		this.client.on('close', () => {
			$debug('Client Close');
			// if (this.server) this.server.close();
		});
	}

	createTunnel() {
		this.debug('Creating SSH Tunnel');
		return Promise.resolve()
			.then(() => this.createSSHClient()) // Create SSH Client
			.then(() => this.createServer()) // Create Server
			.then(() => this.serverEventListeners()) // server event listners
			.then(() => this.debug('SSH Tunnel Created'));
	}
}

[ghusse]

I experience the same issue in my tests (and possibly in the app I'm working on):

        it('should throw an error when the destination url is incorrect', async () => {
          const privateKey = await readFile(path.resolve(__dirname, 'ssh-config', 'id_rsa'));
          await expect(
            buildMongooseInstance({
              uri: 'mongodb://username:secret@invalid_host:27017/movies?authSource=admin',
              connection: {
                ssh: {
                  host: '127.0.0.1',
                  port: 2224,
                  username: 'forest',
                  privateKey,
                },
                socketTimeoutMS: 10,
                connectTimeoutMS: 10,
                serverSelectionTimeoutMS: 10,
              },
            }),
          ).rejects.toThrow(
            new ConnectionError(
              'mongodb://forest:***@invalid_host:27017/movies?authSource=admin',
              'Server selection timed out after 10 ms',
            ),
          );
        });

        it('should pass', async () => {
          await new Promise(resolve => {
            setTimeout(resolve, 1000);
          });
          expect(true).toBe(true);
        });

The first test checks that the SSH error is correctly handled by our code when the URL is incorrect.

BUT when running this test suite, the second test actually fails!

I think this is caused by this line: https://github.com/agebrock/tunnel-ssh/blob/master/index.js#L123

sshConnection.forwardOut(
                forwardOptionsLocal.srcAddr,
                forwardOptionsLocal.srcPort,
                forwardOptionsLocal.dstAddr,
                forwardOptionsLocal.dstPort, (err, stream) => {
                    if (err) {
                        if (server) {
                            server.close()
                        }
                        throw err;
                    } else {
                        clientConnection.pipe(stream).pipe(clientConnection);
                    }
                });

We should not throw the error in this callback, but instead send it back to sshConnection