From 0dcf069c3eb7a9df8cd005e4358020c454381ca0 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Wed, 1 Oct 2025 14:49:43 -0400 Subject: [PATCH 01/17] fix misskey-js tests (unit and type) --- packages/misskey-js/api-extractor.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/misskey-js/api-extractor.json b/packages/misskey-js/api-extractor.json index 0961dd24b9..70f9a08490 100644 --- a/packages/misskey-js/api-extractor.json +++ b/packages/misskey-js/api-extractor.json @@ -45,7 +45,7 @@ * * SUPPORTED TOKENS: , , */ - "mainEntryPointFilePath": "/built/index.d.ts", + "mainEntryPointFilePath": "/built/src/index.d.ts", /** * A list of NPM package names whose exports should be treated as part of this package. From c65f5b5fb259aa659fc396de94dc4a70d3f07492 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Wed, 1 Oct 2025 14:53:07 -0400 Subject: [PATCH 02/17] test misskey-js in pipelines --- .gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 31be935c47..9697a08965 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -184,6 +184,12 @@ megalodon_tests: - pnpm run --filter megalodon build - pnpm run --filter megalodon test +misskey-js_tests: + <<: *test_common + script: + - pnpm run --filter misskey-js build + - pnpm run --filter misskey-js test + get_image_tag: <<: *deploy_common image: From 7810e1aad074b0424bd5e62a975da44a6c902f0a Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Wed, 1 Oct 2025 14:53:07 -0400 Subject: [PATCH 03/17] test misskey-js in pipelines --- .gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9697a08965..66eff82fc9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -190,6 +190,12 @@ misskey-js_tests: - pnpm run --filter misskey-js build - pnpm run --filter misskey-js test +misskey-js_tests: + <<: *test_common + script: + - pnpm run --filter misskey-js build + - pnpm run --filter misskey-js test + get_image_tag: <<: *deploy_common image: From 529325aac2bc9cc8fa5a4dc5592328aa957ae5cf Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Wed, 1 Oct 2025 15:55:45 -0400 Subject: [PATCH 04/17] rework misskey-js build to preserve original package structure --- packages/misskey-js/api-extractor.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/misskey-js/api-extractor.json b/packages/misskey-js/api-extractor.json index 70f9a08490..0961dd24b9 100644 --- a/packages/misskey-js/api-extractor.json +++ b/packages/misskey-js/api-extractor.json @@ -45,7 +45,7 @@ * * SUPPORTED TOKENS: , , */ - "mainEntryPointFilePath": "/built/src/index.d.ts", + "mainEntryPointFilePath": "/built/index.d.ts", /** * A list of NPM package names whose exports should be treated as part of this package. From c62af1c5281b761fca8577585851ba594f7a6e21 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 6 Oct 2025 00:01:25 -0400 Subject: [PATCH 05/17] add megalodon tests to CI --- .gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 66eff82fc9..e994b6d8f9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -196,6 +196,12 @@ misskey-js_tests: - pnpm run --filter misskey-js build - pnpm run --filter misskey-js test +megalodon_tests: + <<: *test_common + script: + - pnpm run --filter megalodon build + - pnpm run --filter megalodon test + get_image_tag: <<: *deploy_common image: From c5bbd667a7d096976ea8453881f1cb83fa6fae58 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Thu, 26 Jun 2025 10:44:14 -0400 Subject: [PATCH 06/17] replace deprecated repeatable jobs with job schedulers --- packages/backend/src/core/QueueService.ts | 128 ++++++++++++++-------- 1 file changed, 80 insertions(+), 48 deletions(-) diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 5020614676..67d1901408 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -78,62 +78,94 @@ export class QueueService implements OnModuleInit { @bindThis public async onModuleInit() { - await this.systemQueue.add('tickCharts', { - }, { - repeat: { pattern: '55 * * * *' }, - removeOnComplete: 10, - removeOnFail: 30, - }); + await this.systemQueue.upsertJobScheduler( + 'tickCharts-scheduler', + { pattern: '55 * * * *' }, + { + name: 'tickCharts', + opts: { + removeOnComplete: 10, + removeOnFail: 30, + }, + }); - await this.systemQueue.add('resyncCharts', { - }, { - repeat: { pattern: '0 0 * * *' }, - removeOnComplete: 10, - removeOnFail: 30, - }); + await this.systemQueue.upsertJobScheduler( + 'resyncCharts-scheduler', + { pattern: '0 0 * * *' }, + { + name: 'resyncCharts', + opts: { + removeOnComplete: 10, + removeOnFail: 30, + }, + }); - await this.systemQueue.add('cleanCharts', { - }, { - repeat: { pattern: '0 0 * * *' }, - removeOnComplete: 10, - removeOnFail: 30, - }); + await this.systemQueue.upsertJobScheduler( + 'cleanCharts-scheduler', + { pattern: '0 0 * * *' }, + { + name: 'cleanCharts', + opts: { + removeOnComplete: 10, + removeOnFail: 30, + }, + }); - await this.systemQueue.add('aggregateRetention', { - }, { - repeat: { pattern: '0 0 * * *' }, - removeOnComplete: 10, - removeOnFail: 30, - }); + await this.systemQueue.upsertJobScheduler( + 'aggregateRetention-scheduler', + { pattern: '0 0 * * *' }, + { + name: 'aggregateRetention', + opts: { + removeOnComplete: 10, + removeOnFail: 30, + }, + }); - await this.systemQueue.add('clean', { - }, { - repeat: { pattern: '0 0 * * *' }, - removeOnComplete: 10, - removeOnFail: 30, - }); + await this.systemQueue.upsertJobScheduler( + 'clean-scheduler', + { pattern: '0 0 * * *' }, + { + name: 'clean', + opts: { + removeOnComplete: 10, + removeOnFail: 30, + }, + }); - await this.systemQueue.add('checkExpiredMutings', { - }, { - repeat: { pattern: '*/5 * * * *' }, - removeOnComplete: 10, - removeOnFail: 30, - }); + await this.systemQueue.upsertJobScheduler( + 'checkExpiredMutings-scheduler', + { pattern: '*/5 * * * *' }, + { + name: 'checkExpiredMutings', + opts: { + removeOnComplete: 10, + removeOnFail: 30, + }, + }); - await this.systemQueue.add('bakeBufferedReactions', { - }, { - repeat: { pattern: '0 0 * * *' }, - removeOnComplete: 10, - removeOnFail: 30, - }); + await this.systemQueue.upsertJobScheduler( + 'backBufferedReactions-scheduler', + { pattern: '0 0 * * *' }, + { + name: 'backBufferedReactions', + opts: { + removeOnComplete: 10, + removeOnFail: 30, + }, + }); - await this.systemQueue.add('checkModeratorsActivity', { - }, { + await this.systemQueue.upsertJobScheduler( + 'checkModeratorsActivity-scheduler', // 毎時30分に起動 - repeat: { pattern: '30 * * * *' }, - removeOnComplete: 10, - removeOnFail: 30, - }); + { pattern: '30 * * * *' }, + { + name: 'checkModeratorsActivity', + opts: { + removeOnComplete: 10, + removeOnFail: 30, + }, + }); } @bindThis From edcaef072751c531dcb3d5fc3129fb7aa42610bf Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Thu, 26 Jun 2025 10:55:30 -0400 Subject: [PATCH 07/17] space out system queue tasks to not crush the instance all at once --- packages/backend/src/core/QueueService.ts | 14 +++++++------- .../AggregateRetentionProcessorService.ts | 2 +- .../src/queue/processors/CleanProcessorService.ts | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 67d1901408..b7f8fb6567 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -80,7 +80,7 @@ export class QueueService implements OnModuleInit { public async onModuleInit() { await this.systemQueue.upsertJobScheduler( 'tickCharts-scheduler', - { pattern: '55 * * * *' }, + { pattern: '0 * * * *' }, { name: 'tickCharts', opts: { @@ -91,7 +91,7 @@ export class QueueService implements OnModuleInit { await this.systemQueue.upsertJobScheduler( 'resyncCharts-scheduler', - { pattern: '0 0 * * *' }, + { pattern: '10 0 * * *' }, { name: 'resyncCharts', opts: { @@ -102,7 +102,7 @@ export class QueueService implements OnModuleInit { await this.systemQueue.upsertJobScheduler( 'cleanCharts-scheduler', - { pattern: '0 0 * * *' }, + { pattern: '30 0 * * *' }, { name: 'cleanCharts', opts: { @@ -113,7 +113,7 @@ export class QueueService implements OnModuleInit { await this.systemQueue.upsertJobScheduler( 'aggregateRetention-scheduler', - { pattern: '0 0 * * *' }, + { pattern: '0 1 * * *' }, { name: 'aggregateRetention', opts: { @@ -124,7 +124,7 @@ export class QueueService implements OnModuleInit { await this.systemQueue.upsertJobScheduler( 'clean-scheduler', - { pattern: '0 0 * * *' }, + { pattern: '15 1 * * *' }, { name: 'clean', opts: { @@ -146,7 +146,7 @@ export class QueueService implements OnModuleInit { await this.systemQueue.upsertJobScheduler( 'backBufferedReactions-scheduler', - { pattern: '0 0 * * *' }, + { pattern: '30 1 * * *' }, { name: 'backBufferedReactions', opts: { @@ -158,7 +158,7 @@ export class QueueService implements OnModuleInit { await this.systemQueue.upsertJobScheduler( 'checkModeratorsActivity-scheduler', // 毎時30分に起動 - { pattern: '30 * * * *' }, + { pattern: '45 1 * * *' }, { name: 'checkModeratorsActivity', opts: { diff --git a/packages/backend/src/queue/processors/AggregateRetentionProcessorService.ts b/packages/backend/src/queue/processors/AggregateRetentionProcessorService.ts index 6f71b42fa9..7d346a2f5d 100644 --- a/packages/backend/src/queue/processors/AggregateRetentionProcessorService.ts +++ b/packages/backend/src/queue/processors/AggregateRetentionProcessorService.ts @@ -83,7 +83,7 @@ export class AggregateRetentionProcessorService { const data = deepClone(record.data); data[dateKey] = retention; - this.retentionAggregationsRepository.update(record.id, { + await this.retentionAggregationsRepository.update(record.id, { updatedAt: now, data, }); diff --git a/packages/backend/src/queue/processors/CleanProcessorService.ts b/packages/backend/src/queue/processors/CleanProcessorService.ts index 149d72de6a..f678801e08 100644 --- a/packages/backend/src/queue/processors/CleanProcessorService.ts +++ b/packages/backend/src/queue/processors/CleanProcessorService.ts @@ -45,13 +45,13 @@ export class CleanProcessorService { public async process(): Promise { this.logger.info('Cleaning...'); - this.userIpsRepository.delete({ + await this.userIpsRepository.delete({ createdAt: LessThan(new Date(this.timeService.now - (1000 * 60 * 60 * 24 * 90))), }); // 使われてないアンテナを停止 if (this.config.deactivateAntennaThreshold > 0) { - this.antennasRepository.update({ + await this.antennasRepository.update({ lastUsedAt: LessThan(new Date(this.timeService.now - this.config.deactivateAntennaThreshold)), }, { isActive: false, @@ -69,7 +69,7 @@ export class CleanProcessorService { }); } - this.reversiService.cleanOutdatedGames(); + await this.reversiService.cleanOutdatedGames(); this.logger.info('Cleaned.'); } From 7050583b86140451352bfe5ee77587675bc61b68 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Thu, 26 Jun 2025 11:02:18 -0400 Subject: [PATCH 08/17] convert ApLogsCleanupService into a queue task --- packages/backend/src/core/QueueService.ts | 17 ++++- .../src/daemons/ApLogCleanupService.ts | 65 ------------------- packages/backend/src/daemons/DaemonModule.ts | 3 - .../backend/src/queue/QueueProcessorModule.ts | 2 + .../src/queue/QueueProcessorService.ts | 3 + .../CleanupApLogsProcessorService.ts | 32 +++++++++ 6 files changed, 51 insertions(+), 71 deletions(-) delete mode 100644 packages/backend/src/daemons/ApLogCleanupService.ts create mode 100644 packages/backend/src/queue/processors/CleanupApLogsProcessorService.ts diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index b7f8fb6567..7e17114641 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -124,7 +124,7 @@ export class QueueService implements OnModuleInit { await this.systemQueue.upsertJobScheduler( 'clean-scheduler', - { pattern: '15 1 * * *' }, + { pattern: '10 1 * * *' }, { name: 'clean', opts: { @@ -144,9 +144,20 @@ export class QueueService implements OnModuleInit { }, }); + await this.systemQueue.upsertJobScheduler( + 'cleanupApLogs-scheduler', + { pattern: '*/10 * * *' }, + { + name: 'cleanupApLogs', + opts: { + removeOnComplete: 10, + removeOnFail: 30, + }, + }); + await this.systemQueue.upsertJobScheduler( 'backBufferedReactions-scheduler', - { pattern: '30 1 * * *' }, + { pattern: '20 1 * * *' }, { name: 'backBufferedReactions', opts: { @@ -158,7 +169,7 @@ export class QueueService implements OnModuleInit { await this.systemQueue.upsertJobScheduler( 'checkModeratorsActivity-scheduler', // 毎時30分に起動 - { pattern: '45 1 * * *' }, + { pattern: '30 1 * * *' }, { name: 'checkModeratorsActivity', opts: { diff --git a/packages/backend/src/daemons/ApLogCleanupService.ts b/packages/backend/src/daemons/ApLogCleanupService.ts deleted file mode 100644 index d3f09bf660..0000000000 --- a/packages/backend/src/daemons/ApLogCleanupService.ts +++ /dev/null @@ -1,65 +0,0 @@ -/* - * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors - * SPDX-License-Identifier: AGPL-3.0-only - */ - -import { Injectable, type OnApplicationShutdown } from '@nestjs/common'; -import { bindThis } from '@/decorators.js'; -import { LoggerService } from '@/core/LoggerService.js'; -import Logger from '@/logger.js'; -import { ApLogService } from '@/core/ApLogService.js'; -import { TimeService, type TimerHandle } from '@/global/TimeService.js'; - -// 10 minutes -export const scanInterval = 1000 * 60 * 10; - -@Injectable() -export class ApLogCleanupService implements OnApplicationShutdown { - private readonly logger: Logger; - private scanTimer: TimerHandle | null = null; - - constructor( - private readonly apLogService: ApLogService, - private readonly timeService: TimeService, - - loggerService: LoggerService, - ) { - this.logger = loggerService.getLogger('activity-log-cleanup'); - } - - @bindThis - public async start(): Promise { - // Just in case start() gets called multiple times. - this.dispose(); - - // Prune at startup, in case the server was rebooted during the interval. - // noinspection ES6MissingAwait - this.tick(); - - // Prune on a regular interval for the lifetime of the server. - this.scanTimer = this.timeService.startTimer(this.tick, scanInterval, { repeated: true }); - } - - @bindThis - private async tick(): Promise { - try { - const affected = await this.apLogService.deleteExpiredLogs(); - this.logger.info(`Activity Log cleanup complete; removed ${affected} expired logs.`); - } catch (err) { - this.logger.error('Activity Log cleanup failed:', err as Error); - } - } - - @bindThis - public onApplicationShutdown(): void { - this.dispose(); - } - - @bindThis - public dispose(): void { - if (this.scanTimer) { - this.timeService.stopTimer(this.scanTimer); - this.scanTimer = null; - } - } -} diff --git a/packages/backend/src/daemons/DaemonModule.ts b/packages/backend/src/daemons/DaemonModule.ts index 286fba56f3..00c5e2c847 100644 --- a/packages/backend/src/daemons/DaemonModule.ts +++ b/packages/backend/src/daemons/DaemonModule.ts @@ -7,7 +7,6 @@ import { Module } from '@nestjs/common'; import { CoreModule } from '@/core/CoreModule.js'; import { QueueStatsService } from './QueueStatsService.js'; import { ServerStatsService } from './ServerStatsService.js'; -import { ApLogCleanupService } from './ApLogCleanupService.js'; @Module({ imports: [ @@ -16,12 +15,10 @@ import { ApLogCleanupService } from './ApLogCleanupService.js'; providers: [ QueueStatsService, ServerStatsService, - ApLogCleanupService, ], exports: [ QueueStatsService, ServerStatsService, - ApLogCleanupService, ], }) export class DaemonModule {} diff --git a/packages/backend/src/queue/QueueProcessorModule.ts b/packages/backend/src/queue/QueueProcessorModule.ts index cb26a06529..792d128560 100644 --- a/packages/backend/src/queue/QueueProcessorModule.ts +++ b/packages/backend/src/queue/QueueProcessorModule.ts @@ -44,6 +44,7 @@ import { AggregateRetentionProcessorService } from './processors/AggregateRetent import { ExportFavoritesProcessorService } from './processors/ExportFavoritesProcessorService.js'; import { RelationshipProcessorService } from './processors/RelationshipProcessorService.js'; import { ScheduleNotePostProcessorService } from './processors/ScheduleNotePostProcessorService.js'; +import { CleanupApLogsProcessorService } from './processors/CleanupApLogsProcessorService.js'; @Module({ imports: [ @@ -89,6 +90,7 @@ import { ScheduleNotePostProcessorService } from './processors/ScheduleNotePostP CheckModeratorsActivityProcessorService, QueueProcessorService, ScheduleNotePostProcessorService, + CleanupApLogsProcessorService, ], exports: [ QueueProcessorService, diff --git a/packages/backend/src/queue/QueueProcessorService.ts b/packages/backend/src/queue/QueueProcessorService.ts index 35dc812652..b71544061f 100644 --- a/packages/backend/src/queue/QueueProcessorService.ts +++ b/packages/backend/src/queue/QueueProcessorService.ts @@ -51,6 +51,7 @@ import { ScheduleNotePostProcessorService } from './processors/ScheduleNotePostP import { QueueLoggerService } from './QueueLoggerService.js'; import { QUEUE, baseWorkerOptions } from './const.js'; import { ImportNotesProcessorService } from './processors/ImportNotesProcessorService.js'; +import { CleanupApLogsProcessorService } from './processors/CleanupApLogsProcessorService.js'; // ref. https://github.com/misskey-dev/misskey/pull/7635#issue-971097019 function httpRelatedBackoff(attemptsMade: number) { @@ -136,6 +137,7 @@ export class QueueProcessorService implements OnApplicationShutdown { private cleanProcessorService: CleanProcessorService, private scheduleNotePostProcessorService: ScheduleNotePostProcessorService, private readonly timeService: TimeService, + private readonly cleanupApLogsProcessorService: CleanupApLogsProcessorService, ) { this.logger = this.queueLoggerService.logger; @@ -156,6 +158,7 @@ export class QueueProcessorService implements OnApplicationShutdown { case 'bakeBufferedReactions': return this.bakeBufferedReactionsProcessorService.process(); case 'checkModeratorsActivity': return this.checkModeratorsActivityProcessorService.process(); case 'clean': return this.cleanProcessorService.process(); + case 'cleanupApLogs': return this.cleanupApLogsProcessorService.process(); default: throw new Error(`unrecognized job type ${job.name} for system`); } }; diff --git a/packages/backend/src/queue/processors/CleanupApLogsProcessorService.ts b/packages/backend/src/queue/processors/CleanupApLogsProcessorService.ts new file mode 100644 index 0000000000..1869d8bd79 --- /dev/null +++ b/packages/backend/src/queue/processors/CleanupApLogsProcessorService.ts @@ -0,0 +1,32 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Injectable } from '@nestjs/common'; +import { bindThis } from '@/decorators.js'; +import { QueueLoggerService } from '@/queue/QueueLoggerService.js'; +import Logger from '@/logger.js'; +import { ApLogService } from '@/core/ApLogService.js'; + +@Injectable() +export class CleanupApLogsProcessorService { + private readonly logger: Logger; + + constructor( + private readonly apLogService: ApLogService, + queueLoggerService: QueueLoggerService, + ) { + this.logger = queueLoggerService.logger.createSubLogger('activity-log-cleanup'); + } + + @bindThis + public async process(): Promise { + try { + const affected = await this.apLogService.deleteExpiredLogs(); + this.logger.info(`Activity Log cleanup complete; removed ${affected} expired logs.`); + } catch (err) { + this.logger.error('Activity Log cleanup failed:', err as Error); + } + } +} From 4a341a53d0f942c813641a624b2f182b58b1fe12 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Thu, 26 Jun 2025 11:36:54 -0400 Subject: [PATCH 09/17] move user hibernation into HibernateUsersProcessorService.ts --- .../backend/src/core/NoteCreateService.ts | 46 +----------- packages/backend/src/core/NoteEditService.ts | 46 +----------- packages/backend/src/core/QueueService.ts | 13 ++++ .../backend/src/queue/QueueProcessorModule.ts | 2 + .../src/queue/QueueProcessorService.ts | 3 + .../HibernateUsersProcessorService.ts | 71 +++++++++++++++++++ 6 files changed, 93 insertions(+), 88 deletions(-) create mode 100644 packages/backend/src/queue/processors/HibernateUsersProcessorService.ts diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index e7cfbd136b..fd55c33bfb 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -1037,55 +1037,13 @@ export class NoteCreateService implements OnApplicationShutdown { } } - if (Math.random() < 0.1) { - process.nextTick(() => { - this.checkHibernation(followings); - }); - } + // checkHibernation moved to HibernateUsersProcessorService } r.exec(); } - @bindThis - public async checkHibernation(followings: MiFollowing[]) { - if (followings.length === 0) return; - - const shuffle = (array: MiFollowing[]) => { - for (let i = array.length - 1; i > 0; i--) { - const j = Math.floor(Math.random() * (i + 1)); - [array[i], array[j]] = [array[j], array[i]]; - } - return array; - }; - - // ランダムに最大1000件サンプリング - const samples = shuffle(followings).slice(0, Math.min(followings.length, 1000)); - - const hibernatedUsers = await this.usersRepository.find({ - where: { - id: In(samples.map(x => x.followerId)), - lastActiveDate: LessThan(new Date(this.timeService.now - (1000 * 60 * 60 * 24 * 50))), - }, - select: ['id'], - }); - - if (hibernatedUsers.length > 0) { - await Promise.all([ - this.usersRepository.update({ - id: In(hibernatedUsers.map(x => x.id)), - }, { - isHibernated: true, - }), - this.followingsRepository.update({ - followerId: In(hibernatedUsers.map(x => x.id)), - }, { - isFollowerHibernated: true, - }), - this.cacheService.hibernatedUserCache.setMany(hibernatedUsers.map(x => [x.id, true])), - ]); - } - } + // checkHibernation moved to HibernateUsersProcessorService public checkProhibitedWordsContain(content: Parameters[0], prohibitedWords?: string[]) { if (prohibitedWords == null) { diff --git a/packages/backend/src/core/NoteEditService.ts b/packages/backend/src/core/NoteEditService.ts index 17daf386d6..20afc4e63c 100644 --- a/packages/backend/src/core/NoteEditService.ts +++ b/packages/backend/src/core/NoteEditService.ts @@ -909,55 +909,13 @@ export class NoteEditService implements OnApplicationShutdown { } } - if (Math.random() < 0.1) { - process.nextTick(() => { - this.checkHibernation(followings); - }); - } + // checkHibernation moved to HibernateUsersProcessorService } r.exec(); } - @bindThis - public async checkHibernation(followings: MiFollowing[]) { - if (followings.length === 0) return; - - const shuffle = (array: MiFollowing[]) => { - for (let i = array.length - 1; i > 0; i--) { - const j = Math.floor(Math.random() * (i + 1)); - [array[i], array[j]] = [array[j], array[i]]; - } - return array; - }; - - // ランダムに最大1000件サンプリング - const samples = shuffle(followings).slice(0, Math.min(followings.length, 1000)); - - const hibernatedUsers = await this.usersRepository.find({ - where: { - id: In(samples.map(x => x.followerId)), - lastActiveDate: LessThan(new Date(this.timeService.now - (1000 * 60 * 60 * 24 * 50))), - }, - select: ['id'], - }); - - if (hibernatedUsers.length > 0) { - await Promise.all([ - this.usersRepository.update({ - id: In(hibernatedUsers.map(x => x.id)), - }, { - isHibernated: true, - }), - this.followingsRepository.update({ - followerId: In(hibernatedUsers.map(x => x.id)), - }, { - isFollowerHibernated: true, - }), - this.cacheService.hibernatedUserCache.setMany(hibernatedUsers.map(x => [x.id, true])), - ]); - } - } + // checkHibernation moved to HibernateUsersProcessorService @bindThis private collapseNotesCount(oldValue: number, newValue: number) { diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 7e17114641..d0b2ae9f2f 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -177,6 +177,19 @@ export class QueueService implements OnModuleInit { removeOnFail: 30, }, }); + + await this.systemQueue.upsertJobScheduler( + 'hibernateUsers-scheduler', + { pattern: '40 1 * * *' }, + { + name: 'hibernateUsers', + opts: { + removeOnComplete: 10, + removeOnFail: 30, + }, + }); + + // Slot '50 1 * * *' is available for future work } @bindThis diff --git a/packages/backend/src/queue/QueueProcessorModule.ts b/packages/backend/src/queue/QueueProcessorModule.ts index 792d128560..b6469229d2 100644 --- a/packages/backend/src/queue/QueueProcessorModule.ts +++ b/packages/backend/src/queue/QueueProcessorModule.ts @@ -45,6 +45,7 @@ import { ExportFavoritesProcessorService } from './processors/ExportFavoritesPro import { RelationshipProcessorService } from './processors/RelationshipProcessorService.js'; import { ScheduleNotePostProcessorService } from './processors/ScheduleNotePostProcessorService.js'; import { CleanupApLogsProcessorService } from './processors/CleanupApLogsProcessorService.js'; +import { HibernateUsersProcessorService } from './processors/HibernateUsersProcessorService.js'; @Module({ imports: [ @@ -91,6 +92,7 @@ import { CleanupApLogsProcessorService } from './processors/CleanupApLogsProcess QueueProcessorService, ScheduleNotePostProcessorService, CleanupApLogsProcessorService, + HibernateUsersProcessorService, ], exports: [ QueueProcessorService, diff --git a/packages/backend/src/queue/QueueProcessorService.ts b/packages/backend/src/queue/QueueProcessorService.ts index b71544061f..dd94fffb36 100644 --- a/packages/backend/src/queue/QueueProcessorService.ts +++ b/packages/backend/src/queue/QueueProcessorService.ts @@ -52,6 +52,7 @@ import { QueueLoggerService } from './QueueLoggerService.js'; import { QUEUE, baseWorkerOptions } from './const.js'; import { ImportNotesProcessorService } from './processors/ImportNotesProcessorService.js'; import { CleanupApLogsProcessorService } from './processors/CleanupApLogsProcessorService.js'; +import { HibernateUsersProcessorService } from './processors/HibernateUsersProcessorService.js'; // ref. https://github.com/misskey-dev/misskey/pull/7635#issue-971097019 function httpRelatedBackoff(attemptsMade: number) { @@ -138,6 +139,7 @@ export class QueueProcessorService implements OnApplicationShutdown { private scheduleNotePostProcessorService: ScheduleNotePostProcessorService, private readonly timeService: TimeService, private readonly cleanupApLogsProcessorService: CleanupApLogsProcessorService, + private readonly hibernateUsersProcessorService: HibernateUsersProcessorService, ) { this.logger = this.queueLoggerService.logger; @@ -159,6 +161,7 @@ export class QueueProcessorService implements OnApplicationShutdown { case 'checkModeratorsActivity': return this.checkModeratorsActivityProcessorService.process(); case 'clean': return this.cleanProcessorService.process(); case 'cleanupApLogs': return this.cleanupApLogsProcessorService.process(); + case 'hibernateUsers': return this.hibernateUsersProcessorService.process(); default: throw new Error(`unrecognized job type ${job.name} for system`); } }; diff --git a/packages/backend/src/queue/processors/HibernateUsersProcessorService.ts b/packages/backend/src/queue/processors/HibernateUsersProcessorService.ts new file mode 100644 index 0000000000..e4c0ee11ae --- /dev/null +++ b/packages/backend/src/queue/processors/HibernateUsersProcessorService.ts @@ -0,0 +1,71 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Inject, Injectable } from '@nestjs/common'; +import { In, LessThan } from 'typeorm'; +import { QueueLoggerService } from '@/queue/QueueLoggerService.js'; +import type Logger from '@/logger.js'; +import { bindThis } from '@/decorators.js'; +import { renderInlineError } from '@/misc/render-inline-error.js'; +import { CacheService } from '@/core/CacheService.js'; +import type { FollowingsRepository, UsersRepository } from '@/models/_.js'; +import { DI } from '@/di-symbols.js'; + +@Injectable() +export class HibernateUsersProcessorService { + private readonly logger: Logger; + + constructor( + @Inject(DI.usersRepository) + private readonly usersRepository: UsersRepository, + + @Inject(DI.followingsRepository) + private readonly followingsRepository: FollowingsRepository, + + private readonly cacheService: CacheService, + + queueLoggerService: QueueLoggerService, + ) { + this.logger = queueLoggerService.logger.createSubLogger('hibernate-users'); + } + + @bindThis + public async process() { + try { + let totalHibernated = 0; + + // Any users last active *before* this date should be hibernated + const hibernationThreshold = new Date(Date.now() - (1000 * 60 * 60 * 24 * 50)); + + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + while (true) { + // Work in batches of 100 + const page = await this.usersRepository.find({ + where: { isHibernated: false, lastActiveDate: LessThan(hibernationThreshold) }, + select: { id: true }, + take: 100, + }) as { id: string }[]; + const ids = page.map(u => u.id); + + // Stop when we get them all + if (ids.length < 1) break; + + await this.usersRepository.update({ id: In(ids) }, { isHibernated: true }); + await this.followingsRepository.update({ followerId: In(ids) }, { isFollowerHibernated: true }); + await this.cacheService.hibernatedUserCache.refreshMany(ids); + + totalHibernated += ids.length; + } + + if (totalHibernated > 0) { + this.logger.info(`Hibernated ${totalHibernated} inactive users`); + } else { + this.logger.debug('Skipping hibernation: nothing to do'); + } + } catch (err) { + this.logger.error(`Error hibernating users: ${renderInlineError(err)}`); + } + } +} From 9fcdd8e513e74d96f121c9862d0572e46e207e58 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Thu, 26 Jun 2025 11:42:16 -0400 Subject: [PATCH 10/17] remove dangling reference to ApLogCleanupService --- packages/backend/src/boot/common.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/backend/src/boot/common.ts b/packages/backend/src/boot/common.ts index 8ba25c7c0b..34c8307dfc 100644 --- a/packages/backend/src/boot/common.ts +++ b/packages/backend/src/boot/common.ts @@ -13,7 +13,6 @@ import { ServerStatsService } from '@/daemons/ServerStatsService.js'; import { ServerService } from '@/server/ServerService.js'; import { MainModule } from '@/MainModule.js'; import { EnvService } from '@/global/EnvService.js'; -import { ApLogCleanupService } from '@/daemons/ApLogCleanupService.js'; export async function server() { const app = await NestFactory.createApplicationContext(MainModule, { @@ -32,7 +31,6 @@ export async function server() { if (!envService.options.noDaemons) { app.get(QueueStatsService).start(); app.get(ServerStatsService).start(); - app.get(ApLogCleanupService).start(); } return app; From 8eeaa12ca0710707a60739c78ea7a73cbeb015d1 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 21 Jul 2025 16:00:42 -0400 Subject: [PATCH 11/17] fix typo: backBufferedReactions -> bakeBufferedReactions --- packages/backend/src/core/QueueService.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index d0b2ae9f2f..c0387ded44 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -156,10 +156,10 @@ export class QueueService implements OnModuleInit { }); await this.systemQueue.upsertJobScheduler( - 'backBufferedReactions-scheduler', + 'bakeBufferedReactions-scheduler', { pattern: '20 1 * * *' }, { - name: 'backBufferedReactions', + name: 'bakeBufferedReactions', opts: { removeOnComplete: 10, removeOnFail: 30, From a91be6581c37cd69d31d3c860628f6d70d6800e2 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 6 Oct 2025 15:34:23 -0400 Subject: [PATCH 12/17] fix merge errors --- .../src/queue/processors/HibernateUsersProcessorService.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/queue/processors/HibernateUsersProcessorService.ts b/packages/backend/src/queue/processors/HibernateUsersProcessorService.ts index e4c0ee11ae..bed8e8cff5 100644 --- a/packages/backend/src/queue/processors/HibernateUsersProcessorService.ts +++ b/packages/backend/src/queue/processors/HibernateUsersProcessorService.ts @@ -10,6 +10,7 @@ import type Logger from '@/logger.js'; import { bindThis } from '@/decorators.js'; import { renderInlineError } from '@/misc/render-inline-error.js'; import { CacheService } from '@/core/CacheService.js'; +import { TimeService } from '@/core/TimeService.js'; import type { FollowingsRepository, UsersRepository } from '@/models/_.js'; import { DI } from '@/di-symbols.js'; @@ -25,6 +26,7 @@ export class HibernateUsersProcessorService { private readonly followingsRepository: FollowingsRepository, private readonly cacheService: CacheService, + private readonly timeService: TimeService, queueLoggerService: QueueLoggerService, ) { @@ -37,7 +39,7 @@ export class HibernateUsersProcessorService { let totalHibernated = 0; // Any users last active *before* this date should be hibernated - const hibernationThreshold = new Date(Date.now() - (1000 * 60 * 60 * 24 * 50)); + const hibernationThreshold = new Date(this.timeService.now - (1000 * 60 * 60 * 24 * 50)); // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition while (true) { From 397f7fdf132d03d9086e5a7f00ed589790952921 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 7 Nov 2025 02:38:11 -0500 Subject: [PATCH 13/17] fix rebase error --- .gitlab-ci.yml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e994b6d8f9..31be935c47 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -184,24 +184,6 @@ megalodon_tests: - pnpm run --filter megalodon build - pnpm run --filter megalodon test -misskey-js_tests: - <<: *test_common - script: - - pnpm run --filter misskey-js build - - pnpm run --filter misskey-js test - -misskey-js_tests: - <<: *test_common - script: - - pnpm run --filter misskey-js build - - pnpm run --filter misskey-js test - -megalodon_tests: - <<: *test_common - script: - - pnpm run --filter megalodon build - - pnpm run --filter megalodon test - get_image_tag: <<: *deploy_common image: From 4e03a128cc310f41ea624f119b1bb556d863f2fb Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 7 Nov 2025 02:44:57 -0500 Subject: [PATCH 14/17] add comments to job scheduler times --- packages/backend/src/core/QueueService.ts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index c0387ded44..278f95efd1 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -80,7 +80,7 @@ export class QueueService implements OnModuleInit { public async onModuleInit() { await this.systemQueue.upsertJobScheduler( 'tickCharts-scheduler', - { pattern: '0 * * * *' }, + { pattern: '0 * * * *' }, // every hour at :00 { name: 'tickCharts', opts: { @@ -91,7 +91,7 @@ export class QueueService implements OnModuleInit { await this.systemQueue.upsertJobScheduler( 'resyncCharts-scheduler', - { pattern: '10 0 * * *' }, + { pattern: '10 0 * * *' }, // every day at 00:10 (avoid tickCharts) { name: 'resyncCharts', opts: { @@ -102,7 +102,7 @@ export class QueueService implements OnModuleInit { await this.systemQueue.upsertJobScheduler( 'cleanCharts-scheduler', - { pattern: '30 0 * * *' }, + { pattern: '30 0 * * *' }, // every day at 00:30 (wait for resyncCharts) { name: 'cleanCharts', opts: { @@ -113,7 +113,7 @@ export class QueueService implements OnModuleInit { await this.systemQueue.upsertJobScheduler( 'aggregateRetention-scheduler', - { pattern: '0 1 * * *' }, + { pattern: '5 1 * * *' }, // every day at 01:05 (avoid chart jobs) { name: 'aggregateRetention', opts: { @@ -124,7 +124,7 @@ export class QueueService implements OnModuleInit { await this.systemQueue.upsertJobScheduler( 'clean-scheduler', - { pattern: '10 1 * * *' }, + { pattern: '10 1 * * *' }, // every day at 01:10 (avoid aggregateRetention) { name: 'clean', opts: { @@ -135,7 +135,7 @@ export class QueueService implements OnModuleInit { await this.systemQueue.upsertJobScheduler( 'checkExpiredMutings-scheduler', - { pattern: '*/5 * * * *' }, + { pattern: '*/5 * * * *' }, // every 5 minutes { name: 'checkExpiredMutings', opts: { @@ -146,7 +146,7 @@ export class QueueService implements OnModuleInit { await this.systemQueue.upsertJobScheduler( 'cleanupApLogs-scheduler', - { pattern: '*/10 * * *' }, + { pattern: '*/10 * * *' }, // every 10 minutes { name: 'cleanupApLogs', opts: { @@ -157,7 +157,7 @@ export class QueueService implements OnModuleInit { await this.systemQueue.upsertJobScheduler( 'bakeBufferedReactions-scheduler', - { pattern: '20 1 * * *' }, + { pattern: '20 1 * * *' }, // every day at 01:20 (avoid clean) { name: 'bakeBufferedReactions', opts: { @@ -169,7 +169,7 @@ export class QueueService implements OnModuleInit { await this.systemQueue.upsertJobScheduler( 'checkModeratorsActivity-scheduler', // 毎時30分に起動 - { pattern: '30 1 * * *' }, + { pattern: '30 1 * * *' }, // every day at 01:30 (avoid cleanupApLogs) { name: 'checkModeratorsActivity', opts: { @@ -180,7 +180,7 @@ export class QueueService implements OnModuleInit { await this.systemQueue.upsertJobScheduler( 'hibernateUsers-scheduler', - { pattern: '40 1 * * *' }, + { pattern: '40 1 * * *' }, // every day at 01:40 (avoid checkModeratorsActivity) { name: 'hibernateUsers', opts: { From 26db8cb96eabc4532a5b2abdeb6269941ec1441f Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 7 Nov 2025 02:51:47 -0500 Subject: [PATCH 15/17] fix rebase error in HibernateUsersProcessorService --- .../src/queue/processors/HibernateUsersProcessorService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/queue/processors/HibernateUsersProcessorService.ts b/packages/backend/src/queue/processors/HibernateUsersProcessorService.ts index bed8e8cff5..892c3128aa 100644 --- a/packages/backend/src/queue/processors/HibernateUsersProcessorService.ts +++ b/packages/backend/src/queue/processors/HibernateUsersProcessorService.ts @@ -10,7 +10,7 @@ import type Logger from '@/logger.js'; import { bindThis } from '@/decorators.js'; import { renderInlineError } from '@/misc/render-inline-error.js'; import { CacheService } from '@/core/CacheService.js'; -import { TimeService } from '@/core/TimeService.js'; +import { TimeService } from '@/global/TimeService.js'; import type { FollowingsRepository, UsersRepository } from '@/models/_.js'; import { DI } from '@/di-symbols.js'; From 10a9e423d97d8601799ab8cfdd6c214cd5f0ee05 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 7 Nov 2025 02:51:58 -0500 Subject: [PATCH 16/17] adjust job scheduler timings --- packages/backend/src/core/QueueService.ts | 37 ++++++++++++----------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 278f95efd1..f9ac58fb66 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -91,7 +91,7 @@ export class QueueService implements OnModuleInit { await this.systemQueue.upsertJobScheduler( 'resyncCharts-scheduler', - { pattern: '10 0 * * *' }, // every day at 00:10 (avoid tickCharts) + { pattern: '20 0 * * *' }, // every day at 00:20 (wait for tickCharts) { name: 'resyncCharts', opts: { @@ -102,7 +102,7 @@ export class QueueService implements OnModuleInit { await this.systemQueue.upsertJobScheduler( 'cleanCharts-scheduler', - { pattern: '30 0 * * *' }, // every day at 00:30 (wait for resyncCharts) + { pattern: '40 0 * * *' }, // every day at 00:40 (wait for resyncCharts) { name: 'cleanCharts', opts: { @@ -113,7 +113,7 @@ export class QueueService implements OnModuleInit { await this.systemQueue.upsertJobScheduler( 'aggregateRetention-scheduler', - { pattern: '5 1 * * *' }, // every day at 01:05 (avoid chart jobs) + { pattern: '0 1 * * *' }, // every day at 01:00 { name: 'aggregateRetention', opts: { @@ -124,7 +124,7 @@ export class QueueService implements OnModuleInit { await this.systemQueue.upsertJobScheduler( 'clean-scheduler', - { pattern: '10 1 * * *' }, // every day at 01:10 (avoid aggregateRetention) + { pattern: '10 1 * * *' }, // every day at 01:10 (wait for aggregateRetention) { name: 'clean', opts: { @@ -144,20 +144,9 @@ export class QueueService implements OnModuleInit { }, }); - await this.systemQueue.upsertJobScheduler( - 'cleanupApLogs-scheduler', - { pattern: '*/10 * * *' }, // every 10 minutes - { - name: 'cleanupApLogs', - opts: { - removeOnComplete: 10, - removeOnFail: 30, - }, - }); - await this.systemQueue.upsertJobScheduler( 'bakeBufferedReactions-scheduler', - { pattern: '20 1 * * *' }, // every day at 01:20 (avoid clean) + { pattern: '20 1 * * *' }, // every day at 01:40 (wait for clean) { name: 'bakeBufferedReactions', opts: { @@ -169,7 +158,7 @@ export class QueueService implements OnModuleInit { await this.systemQueue.upsertJobScheduler( 'checkModeratorsActivity-scheduler', // 毎時30分に起動 - { pattern: '30 1 * * *' }, // every day at 01:30 (avoid cleanupApLogs) + { pattern: '30 * * * *' }, // every hour at :30 { name: 'checkModeratorsActivity', opts: { @@ -178,9 +167,20 @@ export class QueueService implements OnModuleInit { }, }); + await this.systemQueue.upsertJobScheduler( + 'cleanupApLogs-scheduler', + { pattern: '*/10 * * *' }, // every 10 minutes + { + name: 'cleanupApLogs', + opts: { + removeOnComplete: 10, + removeOnFail: 30, + }, + }); + await this.systemQueue.upsertJobScheduler( 'hibernateUsers-scheduler', - { pattern: '40 1 * * *' }, // every day at 01:40 (avoid checkModeratorsActivity) + { pattern: '30 1 * * *' }, // every day at 01:30 (avoid bakeBufferedReactions) { name: 'hibernateUsers', opts: { @@ -189,6 +189,7 @@ export class QueueService implements OnModuleInit { }, }); + // Slot '40 1 * * *' is available for future work // Slot '50 1 * * *' is available for future work } From d71a4fa41d8927a43cd6b95cc4068effb7209eb7 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 7 Nov 2025 02:53:03 -0500 Subject: [PATCH 17/17] fix indentation --- packages/backend/src/core/QueueService.ts | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index f9ac58fb66..da5788f143 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -179,18 +179,18 @@ export class QueueService implements OnModuleInit { }); await this.systemQueue.upsertJobScheduler( - 'hibernateUsers-scheduler', - { pattern: '30 1 * * *' }, // every day at 01:30 (avoid bakeBufferedReactions) - { - name: 'hibernateUsers', - opts: { - removeOnComplete: 10, - removeOnFail: 30, - }, - }); + 'hibernateUsers-scheduler', + { pattern: '30 1 * * *' }, // every day at 01:30 (avoid bakeBufferedReactions) + { + name: 'hibernateUsers', + opts: { + removeOnComplete: 10, + removeOnFail: 30, + }, + }); - // Slot '40 1 * * *' is available for future work - // Slot '50 1 * * *' is available for future work + // Slot '40 1 * * *' is available for future work + // Slot '50 1 * * *' is available for future work } @bindThis