From 49eb26694a7308e5aa790a671a51e4d60fbe1e44 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 15 Aug 2025 16:19:05 -0400 Subject: [PATCH 1/7] add error handler for ws library --- .../src/server/api/StreamingApiServerService.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/packages/backend/src/server/api/StreamingApiServerService.ts b/packages/backend/src/server/api/StreamingApiServerService.ts index 783f6af34e..90f9f2e68a 100644 --- a/packages/backend/src/server/api/StreamingApiServerService.ts +++ b/packages/backend/src/server/api/StreamingApiServerService.ts @@ -105,6 +105,10 @@ export class StreamingApiServerService implements OnApplicationShutdown { perMessageDeflate: this.config.websocketCompression, }); + // ws library will kill the process if we don't catch unhandled exceptions. + // https://github.com/websockets/ws/issues/1354#issuecomment-1343117738 + this.#wss.on('error', this.onWsError); + server.on('upgrade', async (request, socket, head) => { if (request.url == null) { socket.write('HTTP/1.1 400 Bad Request\r\n\r\n'); @@ -307,6 +311,7 @@ export class StreamingApiServerService implements OnApplicationShutdown { this.#connections.clear(); this.#connectionsByClient.clear(); + this.#wss.off('error', this.onWsError); await new Promise((resolve, reject) => { this.#wss.close(err => { @@ -315,4 +320,10 @@ export class StreamingApiServerService implements OnApplicationShutdown { }); }); } + + @bindThis + private async onWsError(error: unknown) { + this.#logger.error(`Unhandled error in streaming api: ${renderInlineError(error)}`); + this.#logger.debug('Error details:', { error }); + } } From d18b910791cde354b408129299ad9f26162c14e1 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 15 Aug 2025 22:58:18 -0400 Subject: [PATCH 2/7] don't detach error handler until *after* socket close --- packages/backend/src/server/api/StreamingApiServerService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/server/api/StreamingApiServerService.ts b/packages/backend/src/server/api/StreamingApiServerService.ts index 90f9f2e68a..7ee72ed30d 100644 --- a/packages/backend/src/server/api/StreamingApiServerService.ts +++ b/packages/backend/src/server/api/StreamingApiServerService.ts @@ -311,10 +311,10 @@ export class StreamingApiServerService implements OnApplicationShutdown { this.#connections.clear(); this.#connectionsByClient.clear(); - this.#wss.off('error', this.onWsError); await new Promise((resolve, reject) => { this.#wss.close(err => { + this.#wss.off('error', this.onWsError); if (err) reject(err); else resolve(); }); From 840f589651d4f34419cd710ebbe2b49b3ebda493 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 15 Aug 2025 23:32:09 -0400 Subject: [PATCH 3/7] disconnect message callback when disposing Connection instance --- packages/backend/src/server/api/stream/Connection.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/backend/src/server/api/stream/Connection.ts b/packages/backend/src/server/api/stream/Connection.ts index 8d8f211af3..7c73014012 100644 --- a/packages/backend/src/server/api/stream/Connection.ts +++ b/packages/backend/src/server/api/stream/Connection.ts @@ -382,6 +382,7 @@ export default class Connection { for (const k of this.subscribingNotes.keys()) { this.subscriber.off(`noteStream:${k}`, this.onNoteStreamMessage); } + this.wsConnection.off('message', this.onWsConnectionMessage); this.fetchIntervalId = null; this.channels.clear(); From 3770f4a86ad826dc23b3dfc54086a0700f0ef01d Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 15 Aug 2025 23:32:50 -0400 Subject: [PATCH 4/7] disconnect pong() listener when closing connection --- .../backend/src/server/api/StreamingApiServerService.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/server/api/StreamingApiServerService.ts b/packages/backend/src/server/api/StreamingApiServerService.ts index 7ee72ed30d..fdb16107c7 100644 --- a/packages/backend/src/server/api/StreamingApiServerService.ts +++ b/packages/backend/src/server/api/StreamingApiServerService.ts @@ -270,8 +270,12 @@ export class StreamingApiServerService implements OnApplicationShutdown { if (user) { this.usersService.updateLastActiveDate(user); } + const pong = () => { + this.#connections.set(connection, Date.now()); + }; connection.once('close', () => { + connection.off('pong', pong); ev.removeAllListeners(); stream.dispose(); this.#globalEv.off('message', onRedisMessage); @@ -279,9 +283,7 @@ export class StreamingApiServerService implements OnApplicationShutdown { if (userUpdateIntervalId) clearInterval(userUpdateIntervalId); }); - connection.on('pong', () => { - this.#connections.set(connection, Date.now()); - }); + connection.on('pong', pong); }); // 一定期間通信が無いコネクションは実際には切断されている可能性があるため定期的にterminateする From 272f5cc5abff4f95a16e4a75b55804074fea5f65 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 15 Aug 2025 23:33:09 -0400 Subject: [PATCH 5/7] add error handler for connection --- packages/backend/src/server/api/StreamingApiServerService.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/backend/src/server/api/StreamingApiServerService.ts b/packages/backend/src/server/api/StreamingApiServerService.ts index fdb16107c7..c061f8914b 100644 --- a/packages/backend/src/server/api/StreamingApiServerService.ts +++ b/packages/backend/src/server/api/StreamingApiServerService.ts @@ -275,6 +275,7 @@ export class StreamingApiServerService implements OnApplicationShutdown { }; connection.once('close', () => { + connection.off('error', this.onWsError); connection.off('pong', pong); ev.removeAllListeners(); stream.dispose(); @@ -283,6 +284,7 @@ export class StreamingApiServerService implements OnApplicationShutdown { if (userUpdateIntervalId) clearInterval(userUpdateIntervalId); }); + connection.on('error', this.onWsError); connection.on('pong', pong); }); From 4f68bbfd524225d3580ee0ad452647cb486377ce Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 15 Aug 2025 23:33:28 -0400 Subject: [PATCH 6/7] disconnect startup error handler after establishing connection --- .../src/server/api/StreamingApiServerService.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/backend/src/server/api/StreamingApiServerService.ts b/packages/backend/src/server/api/StreamingApiServerService.ts index c061f8914b..780f7a7e9e 100644 --- a/packages/backend/src/server/api/StreamingApiServerService.ts +++ b/packages/backend/src/server/api/StreamingApiServerService.ts @@ -228,19 +228,23 @@ export class StreamingApiServerService implements OnApplicationShutdown { stream.dispose(); }); - ws.once('error', (e) => { - this.#logger.error(`Unhandled error in Streaming Api: ${renderInlineError(e)}`); - ws.terminate(); - }); - if (dieInstantly !== null) { ws.close(...dieInstantly); return; } + // Special handler to hard-terminate the connection if it fails during initialization. + // Disconnect immediately after because the connection() handler below defines its own error handler. + const onWsInitError = (error: unknown) => { + this.onWsError(error); + ws.terminate(); + }; + + ws.on('error', onWsInitError); this.#wss.emit('connection', ws, request, { stream, user, app, }); + ws.off('error', onWsInitError); }); }); From f93693535ea7e41b6f3a2b39598da8de0af0aabc Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 15 Aug 2025 23:33:44 -0400 Subject: [PATCH 7/7] disconnect ws error event after promise resolves --- packages/backend/src/server/api/StreamingApiServerService.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/server/api/StreamingApiServerService.ts b/packages/backend/src/server/api/StreamingApiServerService.ts index 780f7a7e9e..42c0ef58f2 100644 --- a/packages/backend/src/server/api/StreamingApiServerService.ts +++ b/packages/backend/src/server/api/StreamingApiServerService.ts @@ -322,11 +322,13 @@ export class StreamingApiServerService implements OnApplicationShutdown { await new Promise((resolve, reject) => { this.#wss.close(err => { - this.#wss.off('error', this.onWsError); if (err) reject(err); else resolve(); }); }); + + // Don't disconnect this until *after* close returns + this.#wss.off('error', this.onWsError); } @bindThis