From 4b08e978ce7a17f644f2b391d94605619448e400 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Tue, 17 Jun 2025 14:06:15 -0400 Subject: [PATCH 001/107] implement background task queue --- .config/ci.yml | 41 ++++++-- .config/cypress-devcontainer.yml | 37 ++++++-- .config/docker_example.yml | 25 ++++- .config/example.yml | 25 ++++- packages/backend/src/config.ts | 9 ++ .../src/core/FetchInstanceMetadataService.ts | 15 +++ packages/backend/src/core/QueueModule.ts | 12 +++ packages/backend/src/core/QueueService.ts | 76 +++++++++++++++ .../src/core/activitypub/ApInboxService.ts | 29 ++---- .../activitypub/models/ApPersonService.ts | 50 +++++++--- .../backend/src/queue/QueueProcessorModule.ts | 3 +- .../src/queue/QueueProcessorService.ts | 47 +++++++++- packages/backend/src/queue/const.ts | 1 + .../BackgroundTaskProcessorService.ts | 93 +++++++++++++++++++ .../processors/DeliverProcessorService.ts | 8 +- .../queue/processors/InboxProcessorService.ts | 4 +- packages/backend/src/queue/types.ts | 20 ++++ .../test/misc/immediateBackgroundTasks.ts | 50 ++++++++++ 18 files changed, 482 insertions(+), 63 deletions(-) create mode 100644 packages/backend/src/queue/processors/BackgroundTaskProcessorService.ts create mode 100644 packages/backend/test/misc/immediateBackgroundTasks.ts diff --git a/.config/ci.yml b/.config/ci.yml index 8543205b17..d5f288fc05 100644 --- a/.config/ci.yml +++ b/.config/ci.yml @@ -253,21 +253,42 @@ id: 'aidx' # Number of worker processes #clusterLimit: 1 +# +-------------------------+ +# | Job concurrency options | +# +-------------------------+ +# +### Available options: +# [type]JobConcurrency - limits the number jobs that can run at the same time. +# Sharkey will allow this many jobs of this type *per worker process*. +# [type]JobPerSec - limits the total number of jobs that may complete within a single second. +# If this limit is exceeded, then Sharkey will pause this type of job until the next second. +# [type]JobMaxAttempts - limits the number of times that a job is allowed to fail and re-try before it's permanently stopped. +# If this limit is exceeded, then the job is considered "failed" and recorded for debugging. +# +### Job types: +# inbox - processes ActivityPub messages (AKA "Activities") received from remote instances. +# All inbound activities are queued and processed in chronological order by this job. +# deliver - processes ActivityPub messages (AKA "Activities") being set to remote instances. +# All outbound activities are queued and processed in chronological order by this job. +# relationship - processes user-to-user tasks including follow/unfollow, block/unblock, account migrations, and all follow import jobs. +# background - processes background synchronization tasks that need to happen soon (but not immediately), such as remote user updates and instance metadata updates. + # Job concurrency per worker -# deliverJobConcurrency: 128 -# inboxJobConcurrency: 16 -# relashionshipJobConcurrency: 16 -# What's relashionshipJob?: -# Follow, unfollow, block and unblock(ings) while following-imports, etc. or account migrations. +#deliverJobConcurrency: 128 +#inboxJobConcurrency: 16 +#relationshipJobConcurrency: 16 +#backgroundJobConcurrency: 32 # Job rate limiter -# deliverJobPerSec: 128 -# inboxJobPerSec: 32 -# relashionshipJobPerSec: 64 +#deliverJobPerSec: 128 +#inboxJobPerSec: 32 +#relationshipJobPerSec: 64 +#backgroundJobPerSec: 256 # Job attempts -# deliverJobMaxAttempts: 12 -# inboxJobMaxAttempts: 8 +#deliverJobMaxAttempts: 12 +#inboxJobMaxAttempts: 8 +#backgroundJobMaxAttempts: 8 # Local address used for outgoing requests #outgoingAddress: 127.0.0.1 diff --git a/.config/cypress-devcontainer.yml b/.config/cypress-devcontainer.yml index f705d06d45..ba6d51959c 100644 --- a/.config/cypress-devcontainer.yml +++ b/.config/cypress-devcontainer.yml @@ -223,17 +223,42 @@ id: 'aidx' # Number of worker processes #clusterLimit: 1 +# +-------------------------+ +# | Job concurrency options | +# +-------------------------+ +# +### Available options: +# [type]JobConcurrency - limits the number jobs that can run at the same time. +# Sharkey will allow this many jobs of this type *per worker process*. +# [type]JobPerSec - limits the total number of jobs that may complete within a single second. +# If this limit is exceeded, then Sharkey will pause this type of job until the next second. +# [type]JobMaxAttempts - limits the number of times that a job is allowed to fail and re-try before it's permanently stopped. +# If this limit is exceeded, then the job is considered "failed" and recorded for debugging. +# +### Job types: +# inbox - processes ActivityPub messages (AKA "Activities") received from remote instances. +# All inbound activities are queued and processed in chronological order by this job. +# deliver - processes ActivityPub messages (AKA "Activities") being set to remote instances. +# All outbound activities are queued and processed in chronological order by this job. +# relationship - processes user-to-user tasks including follow/unfollow, block/unblock, account migrations, and all follow import jobs. +# background - processes background synchronization tasks that need to happen soon (but not immediately), such as remote user updates and instance metadata updates. + # Job concurrency per worker -# deliverJobConcurrency: 128 -# inboxJobConcurrency: 16 +#deliverJobConcurrency: 128 +#inboxJobConcurrency: 16 +#relationshipJobConcurrency: 16 +#backgroundJobConcurrency: 32 # Job rate limiter -# deliverJobPerSec: 128 -# inboxJobPerSec: 32 +#deliverJobPerSec: 128 +#inboxJobPerSec: 32 +#relationshipJobPerSec: 64 +#backgroundJobPerSec: 256 # Job attempts -# deliverJobMaxAttempts: 12 -# inboxJobMaxAttempts: 8 +#deliverJobMaxAttempts: 12 +#inboxJobMaxAttempts: 8 +#backgroundJobMaxAttempts: 8 # IP address family used for outgoing request (ipv4, ipv6 or dual) #outgoingAddressFamily: ipv4 diff --git a/.config/docker_example.yml b/.config/docker_example.yml index 5905e3deed..8ca8d7ff50 100644 --- a/.config/docker_example.yml +++ b/.config/docker_example.yml @@ -307,21 +307,42 @@ id: 'aidx' # Number of worker processes #clusterLimit: 1 +# +-------------------------+ +# | Job concurrency options | +# +-------------------------+ +# +### Available options: +# [type]JobConcurrency - limits the number jobs that can run at the same time. +# Sharkey will allow this many jobs of this type *per worker process*. +# [type]JobPerSec - limits the total number of jobs that may complete within a single second. +# If this limit is exceeded, then Sharkey will pause this type of job until the next second. +# [type]JobMaxAttempts - limits the number of times that a job is allowed to fail and re-try before it's permanently stopped. +# If this limit is exceeded, then the job is considered "failed" and recorded for debugging. +# +### Job types: +# inbox - processes ActivityPub messages (AKA "Activities") received from remote instances. +# All inbound activities are queued and processed in chronological order by this job. +# deliver - processes ActivityPub messages (AKA "Activities") being set to remote instances. +# All outbound activities are queued and processed in chronological order by this job. +# relationship - processes user-to-user tasks including follow/unfollow, block/unblock, account migrations, and all follow import jobs. +# background - processes background synchronization tasks that need to happen soon (but not immediately), such as remote user updates and instance metadata updates. + # Job concurrency per worker #deliverJobConcurrency: 128 #inboxJobConcurrency: 16 #relationshipJobConcurrency: 16 -# What's relationshipJob?: -# Follow, unfollow, block and unblock(ings) while following-imports, etc. or account migrations. +#backgroundJobConcurrency: 32 # Job rate limiter #deliverJobPerSec: 128 #inboxJobPerSec: 32 #relationshipJobPerSec: 64 +#backgroundJobPerSec: 256 # Job attempts #deliverJobMaxAttempts: 12 #inboxJobMaxAttempts: 8 +#backgroundJobMaxAttempts: 8 # Local address used for outgoing requests #outgoingAddress: 127.0.0.1 diff --git a/.config/example.yml b/.config/example.yml index cffc333d14..6fa3f02026 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -310,21 +310,42 @@ id: 'aidx' # Number of worker processes #clusterLimit: 1 +# +-------------------------+ +# | Job concurrency options | +# +-------------------------+ +# +### Available options: +# [type]JobConcurrency - limits the number jobs that can run at the same time. +# Sharkey will allow this many jobs of this type *per worker process*. +# [type]JobPerSec - limits the total number of jobs that may complete within a single second. +# If this limit is exceeded, then Sharkey will pause this type of job until the next second. +# [type]JobMaxAttempts - limits the number of times that a job is allowed to fail and re-try before it's permanently stopped. +# If this limit is exceeded, then the job is considered "failed" and recorded for debugging. +# +### Job types: +# inbox - processes ActivityPub messages (AKA "Activities") received from remote instances. +# All inbound activities are queued and processed in chronological order by this job. +# deliver - processes ActivityPub messages (AKA "Activities") being set to remote instances. +# All outbound activities are queued and processed in chronological order by this job. +# relationship - processes user-to-user tasks including follow/unfollow, block/unblock, account migrations, and all follow import jobs. +# background - processes background synchronization tasks that need to happen soon (but not immediately), such as remote user updates and instance metadata updates. + # Job concurrency per worker #deliverJobConcurrency: 128 #inboxJobConcurrency: 16 #relationshipJobConcurrency: 16 -# What's relationshipJob?: -# Follow, unfollow, block and unblock(ings) while following-imports, etc. or account migrations. +#backgroundJobConcurrency: 32 # Job rate limiter #deliverJobPerSec: 128 #inboxJobPerSec: 32 #relationshipJobPerSec: 64 +#backgroundJobPerSec: 256 # Job attempts #deliverJobMaxAttempts: 12 #inboxJobMaxAttempts: 8 +#backgroundJobMaxAttempts: 8 # Local address used for outgoing requests #outgoingAddress: 127.0.0.1 diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 5607f50eb7..be3892e942 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -111,11 +111,14 @@ type Source = { deliverJobConcurrency?: number; inboxJobConcurrency?: number; relationshipJobConcurrency?: number; + backgroundJobConcurrency?: number; deliverJobPerSec?: number; inboxJobPerSec?: number; relationshipJobPerSec?: number; + backgroundJobPerSec?: number; deliverJobMaxAttempts?: number; inboxJobMaxAttempts?: number; + backgroundJobMaxAttempts?: number; mediaDirectory?: string; mediaProxy?: string; @@ -272,11 +275,14 @@ export type Config = { deliverJobConcurrency: number | undefined; inboxJobConcurrency: number | undefined; relationshipJobConcurrency: number | undefined; + backgroundJobConcurrency: number | undefined; deliverJobPerSec: number | undefined; inboxJobPerSec: number | undefined; relationshipJobPerSec: number | undefined; + backgroundJobPerSec: number | undefined; deliverJobMaxAttempts: number | undefined; inboxJobMaxAttempts: number | undefined; + backgroundJobMaxAttempts: number | undefined; proxyRemoteFiles: boolean | undefined; customMOTD: string[] | undefined; signToActivityPubGet: boolean; @@ -475,11 +481,14 @@ export function loadConfig(loggerService: LoggerService): Config { deliverJobConcurrency: config.deliverJobConcurrency, inboxJobConcurrency: config.inboxJobConcurrency, relationshipJobConcurrency: config.relationshipJobConcurrency, + backgroundJobConcurrency: config.backgroundJobConcurrency, deliverJobPerSec: config.deliverJobPerSec, inboxJobPerSec: config.inboxJobPerSec, relationshipJobPerSec: config.relationshipJobPerSec, + backgroundJobPerSec: config.backgroundJobPerSec, deliverJobMaxAttempts: config.deliverJobMaxAttempts, inboxJobMaxAttempts: config.inboxJobMaxAttempts, + backgroundJobMaxAttempts: config.backgroundJobMaxAttempts, proxyRemoteFiles: config.proxyRemoteFiles, customMOTD: config.customMOTD, signToActivityPubGet: config.signToActivityPubGet ?? true, diff --git a/packages/backend/src/core/FetchInstanceMetadataService.ts b/packages/backend/src/core/FetchInstanceMetadataService.ts index d288c5d231..0ba61ce4f9 100644 --- a/packages/backend/src/core/FetchInstanceMetadataService.ts +++ b/packages/backend/src/core/FetchInstanceMetadataService.ts @@ -17,6 +17,7 @@ import { bindThis } from '@/decorators.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; import { TimeService } from '@/global/TimeService.js'; import { renderInlineError } from '@/misc/render-inline-error.js'; +import { QueueService } from '@/core/QueueService.js'; import type { CheerioAPI } from 'cheerio/slim'; type NodeInfo = { @@ -50,6 +51,7 @@ export class FetchInstanceMetadataService { private redisClient: Redis.Redis, private readonly timeService: TimeService, + private readonly queueService: QueueService, ) { this.logger = this.loggerService.getLogger('metadata', 'cyan'); } @@ -73,8 +75,21 @@ export class FetchInstanceMetadataService { return this.redisClient.del(`fetchInstanceMetadata:mutex:v2:${host}`); } + /** + * Schedules a deferred update on the background task worker. + * Duplicate updates are automatically skipped. + */ + @bindThis + public async fetchInstanceMetadataLazy(instance: MiInstance): Promise { + if (!instance.isBlocked) { + await this.queueService.createUpdateInstanceJob(instance.host); + } + } + @bindThis public async fetchInstanceMetadata(instance: MiInstance, force = false): Promise { + if (instance.isBlocked) return; + const host = instance.host; // finallyでunlockされてしまうのでtry内でロックチェックをしない diff --git a/packages/backend/src/core/QueueModule.ts b/packages/backend/src/core/QueueModule.ts index 2f594394a6..078c6002e8 100644 --- a/packages/backend/src/core/QueueModule.ts +++ b/packages/backend/src/core/QueueModule.ts @@ -20,6 +20,7 @@ import { UserWebhookDeliverJobData, SystemWebhookDeliverJobData, ScheduleNotePostJobData, + BackgroundTaskJobData, } from '../queue/types.js'; import type { Provider } from '@nestjs/common'; @@ -33,6 +34,7 @@ export type ObjectStorageQueue = Bull.Queue; export type UserWebhookDeliverQueue = Bull.Queue; export type SystemWebhookDeliverQueue = Bull.Queue; export type ScheduleNotePostQueue = Bull.Queue; +export type BackgroundTaskQueue = Bull.Queue; const $system: Provider = { provide: 'queue:system', @@ -94,6 +96,12 @@ const $scheduleNotePost: Provider = { inject: [DI.config], }; +const $backgroundTask: Provider = { + provide: 'queue:backgroundTask', + useFactory: (config: Config) => new Bull.Queue(QUEUE.BACKGROUND_TASK, baseQueueOptions(config, QUEUE.BACKGROUND_TASK)), + inject: [DI.config], +}; + @Module({ imports: [ ], @@ -108,6 +116,7 @@ const $scheduleNotePost: Provider = { $userWebhookDeliver, $systemWebhookDeliver, $scheduleNotePost, + $backgroundTask, ], exports: [ $system, @@ -120,6 +129,7 @@ const $scheduleNotePost: Provider = { $userWebhookDeliver, $systemWebhookDeliver, $scheduleNotePost, + $backgroundTask, ], }) export class QueueModule implements OnApplicationShutdown { @@ -136,6 +146,7 @@ export class QueueModule implements OnApplicationShutdown { @Inject('queue:userWebhookDeliver') public userWebhookDeliverQueue: UserWebhookDeliverQueue, @Inject('queue:systemWebhookDeliver') public systemWebhookDeliverQueue: SystemWebhookDeliverQueue, @Inject('queue:scheduleNotePost') public scheduleNotePostQueue: ScheduleNotePostQueue, + @Inject('queue:backgroundTask') public readonly backgroundTaskQueue: BackgroundTaskQueue, ) {} public async dispose(): Promise { @@ -155,6 +166,7 @@ export class QueueModule implements OnApplicationShutdown { this.userWebhookDeliverQueue.close(), this.systemWebhookDeliverQueue.close(), this.scheduleNotePostQueue.close(), + this.backgroundTaskQueue.close(), ]).then(res => { for (const result of res) { if (result.status === 'rejected') { diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 9fd646e655..71716f0f6b 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -21,6 +21,7 @@ import type { SystemWebhookPayload } from '@/core/SystemWebhookService.js'; import type { MiNote } from '@/models/Note.js'; import { type UserWebhookPayload } from './UserWebhookService.js'; import type { + BackgroundTaskJobData, DbJobData, DeliverJobData, RelationshipJobData, @@ -39,6 +40,7 @@ import type { SystemWebhookDeliverQueue, UserWebhookDeliverQueue, ScheduleNotePostQueue, + BackgroundTaskQueue, } from './QueueModule.js'; import type httpSignature from '@peertube/http-signature'; import type * as Bull from 'bullmq'; @@ -54,6 +56,7 @@ export const QUEUE_TYPES = [ 'userWebhookDeliver', 'systemWebhookDeliver', 'scheduleNotePost', + 'backgroundTask', ] as const; @Injectable() @@ -72,6 +75,7 @@ export class QueueService implements OnModuleInit { @Inject('queue:userWebhookDeliver') public userWebhookDeliverQueue: UserWebhookDeliverQueue, @Inject('queue:systemWebhookDeliver') public systemWebhookDeliverQueue: SystemWebhookDeliverQueue, @Inject('queue:scheduleNotePost') public ScheduleNotePostQueue: ScheduleNotePostQueue, + @Inject('queue:backgroundTask') public readonly backgroundTaskQueue: BackgroundTaskQueue, private readonly timeService: TimeService, ) {} @@ -839,6 +843,78 @@ export class QueueService implements OnModuleInit { }); } + @bindThis + public async createUpdateUserJob(userId: string) { + return await this.createBackgroundTask( + 'update-user', + { + type: 'update-user', + userId, + }, + { + id: `update-user:${userId}`, + // ttl: 1000 * 60 * 60 * 24, + }, + ); + } + + @bindThis + public async createUpdateFeaturedJob(userId: string) { + return await this.createBackgroundTask( + 'update-featured', + { + type: 'update-featured', + userId, + }, + { + id: `update-featured:${userId}`, + // ttl: 1000 * 60 * 60 * 24, + }, + ); + } + + @bindThis + public async createUpdateInstanceJob(host: string) { + return await this.createBackgroundTask( + 'update-instance', + { + type: 'update-instance', + host, + }, + { + id: `update-instance:${host}`, + // ttl: 1000 * 60 * 60 * 24, + }, + ); + } + + private async createBackgroundTask(name: string, data: BackgroundTaskJobData, duplication: { id: string, ttl?: number }) { + return await this.backgroundTaskQueue.add( + name, + data, + { + removeOnComplete: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 30, + }, + removeOnFail: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 100, + }, + + // https://docs.bullmq.io/guide/retrying-failing-jobs#custom-back-off-strategies + attempts: this.config.backgroundJobMaxAttempts ?? 8, + backoff: { + // Resolves to QueueProcessorService::HttpRelatedBackoff() + type: 'custom', + }, + + // https://docs.bullmq.io/guide/jobs/deduplication + deduplication: duplication, + }, + ); + }; + /** * @see UserWebhookDeliverJobData * @see UserWebhookDeliverProcessorService diff --git a/packages/backend/src/core/activitypub/ApInboxService.ts b/packages/backend/src/core/activitypub/ApInboxService.ts index 91eba793d6..309259120d 100644 --- a/packages/backend/src/core/activitypub/ApInboxService.ts +++ b/packages/backend/src/core/activitypub/ApInboxService.ts @@ -153,11 +153,10 @@ export class ApInboxService { // ついでにリモートユーザーの情報が古かったら更新しておく if (actor.uri) { if (actor.lastFetchedAt == null || this.timeService.now - actor.lastFetchedAt.getTime() > 1000 * 60 * 60 * 24) { - setImmediate(() => { + { // 同一ユーザーの情報を再度処理するので、使用済みのresolverを再利用してはいけない - this.apPersonService.updatePerson(actor.uri) - .catch(err => this.logger.error(`Failed to update person: ${renderInlineError(err)}`)); - }); + await this.apPersonService.updatePersonLazy(actor); + } } } return result; @@ -441,25 +440,15 @@ export class ApInboxService { this.instanceChart.requestReceived(i.host).then(); } - this.fetchInstanceMetadataService.fetchInstanceMetadata(i).then(); + await this.fetchInstanceMetadataService.fetchInstanceMetadataLazy(i); }); // Process it! - return await this.performOneActivity(actor, activity, resolver) - .finally(() => { - // Update user (adapted from performActivity) - if (actor.lastFetchedAt == null || this.timeService.now - actor.lastFetchedAt.getTime() > 1000 * 60 * 60 * 24) { - setImmediate(() => { - // Don't re-use the resolver, or it may throw recursion errors. - // Instead, create a new resolver with an appropriately-reduced recursion limit. - const subResolver = this.apResolverService.createResolver({ - recursionLimit: resolver.getRecursionLimit() - resolver.getHistory().length, - }); - this.apPersonService.updatePerson(actor.uri, subResolver) - .catch(err => this.logger.error(`Failed to update person: ${renderInlineError(err)}`)); - }); - } - }); + try { + return await this.performOneActivity(actor, activity, resolver); + } finally { + await this.apPersonService.updatePersonLazy(actor); + } } @bindThis diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index 4ce0e8db3f..096480047f 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -579,7 +579,7 @@ export class ApPersonService implements OnModuleInit { if (this.meta.enableChartsForFederatedInstances) { this.instanceChart.newUser(i.host); } - this.fetchInstanceMetadataService.fetchInstanceMetadata(i); + this.fetchInstanceMetadataService.fetchInstanceMetadataLazy(i); }); } @@ -604,16 +604,26 @@ export class ApPersonService implements OnModuleInit { } //#endregion - await this.updateFeatured(user.id, resolver).catch(err => { - // Permanent error implies hidden or inaccessible, which is a normal thing. - if (isRetryableError(err)) { - this.logger.error(`Error updating featured notes: ${renderInlineError(err)}`); - } - }); + await this.updateFeaturedLazy(user); return user; } + /** + * Schedules a deferred update on the background task worker. + * Duplicate updates are automatically skipped. + */ + @bindThis + public async updatePersonLazy(uriOrUser: string | MiUser): Promise { + const user = typeof(uriOrUser) === 'string' + ? await this.fetchPerson(uriOrUser) + : uriOrUser; + + if (user && user.host != null) { + await this.queueService.createUpdateUserJob(user.id); + } + } + /** * Personの情報を更新します。 * Misskeyに対象のPersonが登録されていなければ無視します。 @@ -817,12 +827,7 @@ export class ApPersonService implements OnModuleInit { await this.cacheService.refreshFollowRelationsFor(exist.id); } - await this.updateFeatured(exist.id, resolver).catch(err => { - // Permanent error implies hidden or inaccessible, which is a normal thing. - if (isRetryableError(err)) { - this.logger.error(`Error updating featured notes: ${renderInlineError(err)}`); - } - }); + await this.updateFeaturedLazy(exist); const updated = { ...exist, ...updates }; @@ -902,9 +907,24 @@ export class ApPersonService implements OnModuleInit { return fields; } + /** + * Schedules a deferred update on the background task worker. + * Duplicate updates are automatically skipped. + */ @bindThis - public async updateFeatured(userId: MiUser['id'], resolver?: Resolver): Promise { - const user = await this.usersRepository.findOneByOrFail({ id: userId, isDeleted: false }); + public async updateFeaturedLazy(userOrId: MiUser | MiUser['id']): Promise { + const userId = typeof(userOrId) === 'object' ? userOrId.id : userOrId; + const user = typeof(userOrId) === 'object' ? userOrId : await this.usersRepository.findOneByOrFail({ id: userId, isDeleted: false }); + + if (isRemoteUser(user) && user.featured) { + await this.queueService.createUpdateFeaturedJob(userId); + } + } + + @bindThis + public async updateFeatured(userOrId: MiUser | MiUser['id'], resolver?: Resolver): Promise { + const userId = typeof(userOrId) === 'object' ? userOrId.id : userOrId; + const user = typeof(userOrId) === 'object' ? userOrId : await this.usersRepository.findOneByOrFail({ id: userId, isDeleted: false }); if (!isRemoteUser(user)) return; if (!user.featured) return; diff --git a/packages/backend/src/queue/QueueProcessorModule.ts b/packages/backend/src/queue/QueueProcessorModule.ts index b6469229d2..d76b29340d 100644 --- a/packages/backend/src/queue/QueueProcessorModule.ts +++ b/packages/backend/src/queue/QueueProcessorModule.ts @@ -46,7 +46,7 @@ import { RelationshipProcessorService } from './processors/RelationshipProcessor import { ScheduleNotePostProcessorService } from './processors/ScheduleNotePostProcessorService.js'; import { CleanupApLogsProcessorService } from './processors/CleanupApLogsProcessorService.js'; import { HibernateUsersProcessorService } from './processors/HibernateUsersProcessorService.js'; - +import { BackgroundTaskProcessorService } from './processors/BackgroundTaskProcessorService.js'; @Module({ imports: [ CoreModule, @@ -93,6 +93,7 @@ import { HibernateUsersProcessorService } from './processors/HibernateUsersProce ScheduleNotePostProcessorService, CleanupApLogsProcessorService, HibernateUsersProcessorService, + BackgroundTaskProcessorService, ], exports: [ QueueProcessorService, diff --git a/packages/backend/src/queue/QueueProcessorService.ts b/packages/backend/src/queue/QueueProcessorService.ts index cd85db3122..b04d19618f 100644 --- a/packages/backend/src/queue/QueueProcessorService.ts +++ b/packages/backend/src/queue/QueueProcessorService.ts @@ -14,6 +14,7 @@ import { CheckModeratorsActivityProcessorService } from '@/queue/processors/Chec import { TimeService } from '@/global/TimeService.js'; import { renderFullError } from '@/misc/render-full-error.js'; import { renderInlineError } from '@/misc/render-inline-error.js'; +import { isRetryableError } from '@/misc/is-retryable-error.js'; import { UserWebhookDeliverProcessorService } from './processors/UserWebhookDeliverProcessorService.js'; import { SystemWebhookDeliverProcessorService } from './processors/SystemWebhookDeliverProcessorService.js'; import { EndedPollNotificationProcessorService } from './processors/EndedPollNotificationProcessorService.js'; @@ -53,9 +54,16 @@ import { QUEUE, baseWorkerOptions } from './const.js'; import { ImportNotesProcessorService } from './processors/ImportNotesProcessorService.js'; import { CleanupApLogsProcessorService } from './processors/CleanupApLogsProcessorService.js'; import { HibernateUsersProcessorService } from './processors/HibernateUsersProcessorService.js'; +import { BackgroundTaskProcessorService } from './processors/BackgroundTaskProcessorService.js'; // ref. https://github.com/misskey-dev/misskey/pull/7635#issue-971097019 -function httpRelatedBackoff(attemptsMade: number) { +function httpRelatedBackoff(attemptsMade: number, type?: string, error?: Error) { + // Don't retry permanent errors + // https://docs.bullmq.io/guide/retrying-failing-jobs#custom-back-off-strategies + if (error && !isRetryableError(error)) { + return -1; + } + const baseDelay = 60 * 1000; // 1min const maxBackoff = 8 * 60 * 60 * 1000; // 8hours let backoff = (Math.pow(2, attemptsMade) - 1) * baseDelay; @@ -95,6 +103,7 @@ export class QueueProcessorService implements OnApplicationShutdown { private objectStorageQueueWorker: Bull.Worker; private endedPollNotificationQueueWorker: Bull.Worker; private schedulerNotePostQueueWorker: Bull.Worker; + private readonly backgroundTaskWorker: Bull.Worker; constructor( @Inject(DI.config) @@ -140,6 +149,7 @@ export class QueueProcessorService implements OnApplicationShutdown { private readonly timeService: TimeService, private readonly cleanupApLogsProcessorService: CleanupApLogsProcessorService, private readonly hibernateUsersProcessorService: HibernateUsersProcessorService, + private readonly backgroundTaskProcessorService: BackgroundTaskProcessorService, ) { this.logger = this.queueLoggerService.logger; @@ -565,6 +575,39 @@ export class QueueProcessorService implements OnApplicationShutdown { .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); } //#endregion + + //#region background tasks + { + const logger = this.logger.createSubLogger('backgroundTask'); + + this.backgroundTaskWorker = new Bull.Worker(QUEUE.BACKGROUND_TASK, (job) => this.backgroundTaskProcessorService.process(job), { + ...baseWorkerOptions(this.config, QUEUE.BACKGROUND_TASK), + autorun: false, + concurrency: this.config.backgroundJobConcurrency ?? 32, + limiter: { + max: this.config.backgroundJobPerSec ?? 256, + duration: 1000, + }, + settings: { + backoffStrategy: httpRelatedBackoff, + }, + }); + this.backgroundTaskWorker + .on('active', (job) => logger.debug(`active id=${job.id}`)) + .on('completed', (job, result) => logger.debug(`completed(${result}) id=${job.id}`)) + .on('failed', (job, err) => { + this.logError(logger, err, job); + if (config.sentryForBackend) { + Sentry.captureMessage(`Queue: ${QUEUE.BACKGROUND_TASK}: ${job?.name ?? '?'}: ${err.name}: ${err.message}`, { + level: 'error', + extra: { job, err }, + }); + } + }) + .on('error', (err: Error) => this.logError(logger, err)) + .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); + } + //#endregion } private logError(logger: Logger, err: unknown, job?: Bull.Job | null): void { @@ -606,6 +649,7 @@ export class QueueProcessorService implements OnApplicationShutdown { this.objectStorageQueueWorker.run(), this.endedPollNotificationQueueWorker.run(), this.schedulerNotePostQueueWorker.run(), + this.backgroundTaskWorker.run(), ]); } @@ -622,6 +666,7 @@ export class QueueProcessorService implements OnApplicationShutdown { this.objectStorageQueueWorker.close(), this.endedPollNotificationQueueWorker.close(), this.schedulerNotePostQueueWorker.close(), + this.backgroundTaskWorker.close(), ]).then(res => { for (const result of res) { if (result.status === 'rejected') { diff --git a/packages/backend/src/queue/const.ts b/packages/backend/src/queue/const.ts index 17c6b81736..44192c280e 100644 --- a/packages/backend/src/queue/const.ts +++ b/packages/backend/src/queue/const.ts @@ -18,6 +18,7 @@ export const QUEUE = { USER_WEBHOOK_DELIVER: 'userWebhookDeliver', SYSTEM_WEBHOOK_DELIVER: 'systemWebhookDeliver', SCHEDULE_NOTE_POST: 'scheduleNotePost', + BACKGROUND_TASK: 'backgroundTask', }; export function baseQueueOptions(config: Config, queueName: typeof QUEUE[keyof typeof QUEUE]): Bull.QueueOptions { diff --git a/packages/backend/src/queue/processors/BackgroundTaskProcessorService.ts b/packages/backend/src/queue/processors/BackgroundTaskProcessorService.ts new file mode 100644 index 0000000000..b237990a4c --- /dev/null +++ b/packages/backend/src/queue/processors/BackgroundTaskProcessorService.ts @@ -0,0 +1,93 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Inject, Injectable } from '@nestjs/common'; +import * as Bull from 'bullmq'; +import { BackgroundTaskJobData, UpdateFeaturedBackgroundTask, UpdateInstanceBackgroundTask, UpdateUserBackgroundTask } from '@/queue/types.js'; +import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js'; +import { QueueLoggerService } from '@/queue/QueueLoggerService.js'; +import Logger from '@/logger.js'; +import { isRetryableError } from '@/misc/is-retryable-error.js'; +import { DI } from '@/di-symbols.js'; +import type { Config } from '@/config.js'; +import { CacheService } from '@/core/CacheService.js'; +import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; +import { FetchInstanceMetadataService } from '@/core/FetchInstanceMetadataService.js'; +import { renderInlineError } from '@/misc/render-inline-error.js'; + +@Injectable() +export class BackgroundTaskProcessorService { + private readonly logger: Logger; + + constructor( + @Inject(DI.config) + private readonly config: Config, + + private readonly apPersonService: ApPersonService, + private readonly cacheService: CacheService, + private readonly federatedInstanceService: FederatedInstanceService, + private readonly fetchInstanceMetadataService: FetchInstanceMetadataService, + + queueLoggerService: QueueLoggerService, + ) { + this.logger = queueLoggerService.logger.createSubLogger('background-task'); + } + + public async process(job: Bull.Job): Promise { + if (job.data.type === 'update-user') { + return await this.processUpdateUser(job.data); + } else if (job.data.type === 'update-featured') { + return await this.processUpdateFeatured(job.data); + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + } else if (job.data.type === 'update-instance') { + return await this.processUpdateInstance(job.data); + } else { + this.logger.warn(`Can't process unknown job type "${job.data}"; this is likely a bug. Full job data:`, job.data); + throw new Error(`Unknown job type ${job.data}, see system logs for details`); + } + } + + private async processUpdateUser(task: UpdateUserBackgroundTask): Promise { + const user = await this.cacheService.findOptionalUserById(task.userId); + if (!user || user.isDeleted) return `Skipping update-user task: user ${task.userId} has been deleted`; + if (user.isSuspended) return `Skipping update-user task: user ${task.userId} is suspended`; + if (!user.uri) return `Skipping update-user task: user ${task.userId} is local`; + + if (user.lastFetchedAt && Date.now() - user.lastFetchedAt.getTime() < 1000 * 60 * 60 * 24) { + return `Skipping update-user task: user ${task.userId} was recently updated`; + } + + await this.apPersonService.updatePerson(user.uri); + return 'ok'; + } + + private async processUpdateFeatured(task: UpdateFeaturedBackgroundTask): Promise { + const user = await this.cacheService.findOptionalUserById(task.userId); + if (!user || user.isDeleted) return `Skipping update-featured task: user ${task.userId} has been deleted`; + if (user.isSuspended) return `Skipping update-featured task: user ${task.userId} is suspended`; + if (!user.uri) return `Skipping update-featured task: user ${task.userId} is local`; + if (!user.featured) return `Skipping update-featured task: user ${task.userId} has no featured collection`; + + if (user.lastFetchedAt && Date.now() - user.lastFetchedAt.getTime() < 1000 * 60 * 60 * 24) { + return `Skipping update-featured task: user ${task.userId} was recently updated`; + } + + await this.apPersonService.updateFeatured(user); + return 'ok'; + } + + private async processUpdateInstance(task: UpdateInstanceBackgroundTask): Promise { + const instance = await this.federatedInstanceService.fetch(task.host); + if (!instance) return `Skipping update-instance task: instance ${task.host} has been deleted`; + if (instance.isBlocked) return `Skipping update-instance task: instance ${task.host} is blocked`; + + if (instance.infoUpdatedAt && Date.now() - instance.infoUpdatedAt.getTime() < 1000 * 60 * 60 * 24) { + return `Skipping update-instance task: instance ${task.host} was recently updated`; + } + + await this.fetchInstanceMetadataService.fetchInstanceMetadata(instance); + return 'ok'; + } +} diff --git a/packages/backend/src/queue/processors/DeliverProcessorService.ts b/packages/backend/src/queue/processors/DeliverProcessorService.ts index 0b1ef03a7a..792ec4b015 100644 --- a/packages/backend/src/queue/processors/DeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/DeliverProcessorService.ts @@ -80,21 +80,21 @@ export class DeliverProcessorService { if (i == null) return; if (i.isNotResponding) { - this.federatedInstanceService.update(i.id, { + await this.federatedInstanceService.update(i.id, { isNotResponding: false, notRespondingSince: null, }); } if (this.meta.enableChartsForFederatedInstances) { - this.instanceChart.requestSent(i.host, true); + await this.instanceChart.requestSent(i.host, true); } }); return 'Success'; } catch (res) { - this.apRequestChart.deliverFail(); - this.federationChart.deliverd(host, false); + await this.apRequestChart.deliverFail(); + await this.federationChart.deliverd(host, false); // Update instance stats this.federatedInstanceService.fetchOrRegister(host).then(i => { diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index 13b2885263..3b73e98e5c 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -275,10 +275,10 @@ export class InboxProcessorService implements OnApplicationShutdown { }); if (this.meta.enableChartsForFederatedInstances) { - this.instanceChart.requestReceived(i.host); + await this.instanceChart.requestReceived(i.host); } - this.fetchInstanceMetadataService.fetchInstanceMetadata(i); + await this.fetchInstanceMetadataService.fetchInstanceMetadataLazy(i); }); // アクティビティを処理 diff --git a/packages/backend/src/queue/types.ts b/packages/backend/src/queue/types.ts index 6dc9f88034..0c3d790a97 100644 --- a/packages/backend/src/queue/types.ts +++ b/packages/backend/src/queue/types.ts @@ -168,3 +168,23 @@ export type ThinUser = { export type ScheduleNotePostJobData = { scheduleNoteId: MiNote['id']; }; + +export type BackgroundTaskJobData = + UpdateUserBackgroundTask | + UpdateFeaturedBackgroundTask | + UpdateInstanceBackgroundTask; + +export type UpdateUserBackgroundTask = { + type: 'update-user'; + userId: string; +}; + +export type UpdateFeaturedBackgroundTask = { + type: 'update-featured'; + userId: string; +}; + +export type UpdateInstanceBackgroundTask = { + type: 'update-instance'; + host: string; +}; diff --git a/packages/backend/test/misc/immediateBackgroundTasks.ts b/packages/backend/test/misc/immediateBackgroundTasks.ts new file mode 100644 index 0000000000..fd059e3f10 --- /dev/null +++ b/packages/backend/test/misc/immediateBackgroundTasks.ts @@ -0,0 +1,50 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import type { IObject } from '@/core/activitypub/type.js'; +import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js'; +import { MiRemoteUser, MiUser } from '@/models/User.js'; +import { FetchInstanceMetadataService } from '@/core/FetchInstanceMetadataService.js'; +import { MiInstance } from '@/models/Instance.js'; +import { Resolver } from '@/core/activitypub/ApResolverService.js'; +import { bindThis } from '@/decorators.js'; + +export class ImmediateApPersonService extends ApPersonService { + @bindThis + async createPerson(uri: string, resolver?: Resolver): Promise { + const user = await super.createPerson(uri, resolver); + await this.updateFeatured(user, resolver); + return user; + } + + @bindThis + async updatePerson(uri: string, resolver?: Resolver | null, hint?: IObject, movePreventUris: string[] = []): Promise { + const result = await super.updatePerson(uri, resolver, hint, movePreventUris); + + const user = await this.fetchPerson(uri); + if (user == null) throw new Error('updated user is null, did you forget to mock out caches?'); + await this.updateFeatured(user, resolver ?? undefined); + + return result; + } + + @bindThis + async updatePersonLazy(uriOrUser: string | MiUser): Promise { + const userId = typeof(uriOrUser) === 'object' ? uriOrUser.id : uriOrUser; + await this.updatePerson(userId); + } + + @bindThis + async updateFeaturedLazy(userOrId: string | MiUser): Promise { + await this.updateFeatured(userOrId); + } +} + +export class ImmediateFetchInstanceMetadataService extends FetchInstanceMetadataService { + @bindThis + async fetchInstanceMetadataLazy(instance: MiInstance): Promise { + return await this.fetchInstanceMetadata(instance); + } +} From 8f436ef8cac6e68ed322e9c740dfbc1ac43eab43 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Tue, 17 Jun 2025 20:30:32 -0400 Subject: [PATCH 002/107] implement background queue charts --- packages/backend/src/core/QueueService.ts | 1 + packages/backend/src/core/SponsorsService.ts | 2 ++ .../backend/src/daemons/QueueStatsService.ts | 28 ++++++++++++++++- .../frontend/src/pages/admin/job-queue.vue | 1 + .../frontend/src/widgets/WidgetJobQueue.vue | 31 +++++++++++++++++-- 5 files changed, 60 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 71716f0f6b..15c50e4d83 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -1003,6 +1003,7 @@ export class QueueService implements OnModuleInit { case 'userWebhookDeliver': return this.userWebhookDeliverQueue; case 'systemWebhookDeliver': return this.systemWebhookDeliverQueue; case 'scheduleNotePost': return this.ScheduleNotePostQueue; + case 'backgroundTask': return this.backgroundTaskQueue; default: throw new Error(`Unrecognized queue type: ${type}`); } } diff --git a/packages/backend/src/core/SponsorsService.ts b/packages/backend/src/core/SponsorsService.ts index 551768bdc9..23994b5761 100644 --- a/packages/backend/src/core/SponsorsService.ts +++ b/packages/backend/src/core/SponsorsService.ts @@ -61,6 +61,7 @@ export class SponsorsService { } try { + // TODO use HTTP service const backers = await fetch(`${this.meta.donationUrl}/members/users.json`).then((response) => response.json() as Promise); // Merge both together into one array and make sure it only has Active subscriptions @@ -76,6 +77,7 @@ export class SponsorsService { @bindThis private async fetchSharkeySponsors(): Promise { try { + // TODO use HTTP service const backers = await fetch('https://opencollective.com/sharkey/tiers/backer/all.json').then((response) => response.json() as Promise); const sponsorsOC = await fetch('https://opencollective.com/sharkey/tiers/sponsor/all.json').then((response) => response.json() as Promise); diff --git a/packages/backend/src/daemons/QueueStatsService.ts b/packages/backend/src/daemons/QueueStatsService.ts index 3779172517..9d2266e7e7 100644 --- a/packages/backend/src/daemons/QueueStatsService.ts +++ b/packages/backend/src/daemons/QueueStatsService.ts @@ -24,6 +24,7 @@ export interface StatsEntry { export interface Stats { deliver: StatsEntry, inbox: StatsEntry, + background: StatsEntry, } const ev = new Xev(); @@ -35,9 +36,11 @@ export class QueueStatsService implements OnApplicationShutdown { private intervalId?: TimerHandle; private activeDeliverJobs = 0; private activeInboxJobs = 0; + private activeBackgroundJobs = 0; private deliverQueueEvents?: Bull.QueueEvents; private inboxQueueEvents?: Bull.QueueEvents; + private backgroundQueueEvents?: Bull.QueueEvents; private log?: Stats[]; @@ -60,6 +63,11 @@ export class QueueStatsService implements OnApplicationShutdown { this.activeInboxJobs++; } + @bindThis + private onBackgroundActive() { + this.activeBackgroundJobs++; + } + @bindThis private onRequestQueueStatsLog(x: { id: string, length?: number }) { if (this.log) { @@ -80,13 +88,16 @@ export class QueueStatsService implements OnApplicationShutdown { this.deliverQueueEvents = new Bull.QueueEvents(QUEUE.DELIVER, baseQueueOptions(this.config, QUEUE.DELIVER)); this.inboxQueueEvents = new Bull.QueueEvents(QUEUE.INBOX, baseQueueOptions(this.config, QUEUE.INBOX)); + this.backgroundQueueEvents = new Bull.QueueEvents(QUEUE.BACKGROUND_TASK, baseQueueOptions(this.config, QUEUE.BACKGROUND_TASK)); this.deliverQueueEvents.on('active', this.onDeliverActive); this.inboxQueueEvents.on('active', this.onInboxActive); + this.backgroundQueueEvents.on('active', this.onBackgroundActive); const tick = async () => { const deliverJobCounts = await this.queueService.deliverQueue.getJobCounts(); const inboxJobCounts = await this.queueService.inboxQueue.getJobCounts(); + const backgroundJobCounts = await this.queueService.backgroundTaskQueue.getJobCounts(); const stats = { deliver: { @@ -101,6 +112,12 @@ export class QueueStatsService implements OnApplicationShutdown { waiting: inboxJobCounts.waiting, delayed: inboxJobCounts.delayed, }, + background: { + activeSincePrevTick: this.activeBackgroundJobs, + active: backgroundJobCounts.active, + waiting: backgroundJobCounts.waiting, + delayed: backgroundJobCounts.delayed, + }, }; ev.emit('queueStats', stats); @@ -112,6 +129,7 @@ export class QueueStatsService implements OnApplicationShutdown { this.activeDeliverJobs = 0; this.activeInboxJobs = 0; + this.activeBackgroundJobs = 0; }; tick(); @@ -120,7 +138,7 @@ export class QueueStatsService implements OnApplicationShutdown { } @bindThis - public async stop() { + public async stop(): void { if (this.intervalId) { this.timeService.stopTimer(this.intervalId); } @@ -130,12 +148,20 @@ export class QueueStatsService implements OnApplicationShutdown { this.deliverQueueEvents?.off('active', this.onDeliverActive); this.inboxQueueEvents?.off('active', this.onInboxActive); + this.backgroundQueueEvents?.off('active', this.onBackgroundActive); await this.deliverQueueEvents?.close(); await this.inboxQueueEvents?.close(); + await this.backgroundQueueEvents?.close(); this.activeDeliverJobs = 0; this.activeInboxJobs = 0; + this.activeBackgroundJobs = 0; + } + + @bindThis + public async dispose(): void { + await this.stop(); } @bindThis diff --git a/packages/frontend/src/pages/admin/job-queue.vue b/packages/frontend/src/pages/admin/job-queue.vue index 155277c976..a37c8f872b 100644 --- a/packages/frontend/src/pages/admin/job-queue.vue +++ b/packages/frontend/src/pages/admin/job-queue.vue @@ -204,6 +204,7 @@ const QUEUE_TYPES = [ 'userWebhookDeliver', 'systemWebhookDeliver', 'scheduleNotePost', + 'backgroundTask', ] as const; const tab: Ref = ref('-'); diff --git a/packages/frontend/src/widgets/WidgetJobQueue.vue b/packages/frontend/src/widgets/WidgetJobQueue.vue index 485e532d51..7ecc663f45 100644 --- a/packages/frontend/src/widgets/WidgetJobQueue.vue +++ b/packages/frontend/src/widgets/WidgetJobQueue.vue @@ -47,6 +47,27 @@ SPDX-License-Identifier: AGPL-3.0-only +
+
Background queue
+
+
+
Process
+
{{ kmg(current.background.activeSincePrevTick, 2) }}
+
+
+
Active
+
{{ kmg(current.background.active, 2) }}
+
+
+
Delayed
+
{{ kmg(current.background.delayed, 2) }}
+
+
+
Waiting
+
{{ kmg(current.background.waiting, 2) }}
+
+
+
@@ -99,6 +120,12 @@ const current = reactive({ waiting: 0, delayed: 0, }, + background: { + activeSincePrevTick: 0, + active: 0, + waiting: 0, + delayed: 0, + }, }); const prev = reactive({} as typeof current); const jammedAudioBuffer = ref(null); @@ -111,12 +138,12 @@ if (prefer.s['sound.masterVolume']) { }); } -for (const domain of ['inbox', 'deliver']) { +for (const domain of ['inbox', 'deliver', 'background']) { prev[domain] = deepClone(current[domain]); } const onStats = (stats) => { - for (const domain of ['inbox', 'deliver']) { + for (const domain of ['inbox', 'deliver', 'background']) { prev[domain] = deepClone(current[domain]); current[domain].activeSincePrevTick = stats[domain].activeSincePrevTick; current[domain].active = stats[domain].active; From 86b8a8a734d64589b61678f92fa28a398fbd8f4f Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Wed, 18 Jun 2025 01:34:38 -0400 Subject: [PATCH 003/107] use QuantumKVCache in FederatedInstanceService --- packages/backend/src/core/MetaService.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/backend/src/core/MetaService.ts b/packages/backend/src/core/MetaService.ts index 68902ef539..519994e57e 100644 --- a/packages/backend/src/core/MetaService.ts +++ b/packages/backend/src/core/MetaService.ts @@ -16,6 +16,7 @@ import { TimeService, type TimerHandle } from '@/global/TimeService.js'; import { MiInstance } from '@/models/Instance.js'; import { diffArrays } from '@/misc/diff-arrays.js'; import type { MetasRepository } from '@/models/_.js'; +import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; import type { OnApplicationShutdown } from '@nestjs/common'; @Injectable() @@ -36,6 +37,7 @@ export class MetaService implements OnApplicationShutdown { private featuredService: FeaturedService, private globalEventService: GlobalEventService, private readonly timeService: TimeService, + private readonly federatedInstanceService: FederatedInstanceService, ) { //this.onMessage = this.onMessage.bind(this); @@ -156,6 +158,7 @@ export class MetaService implements OnApplicationShutdown { }); } + await this.federatedInstanceService.syncCache(before, updated); this.globalEventService.publishInternalEvent('metaUpdated', { before, after: updated }); return updated; From ce8c8e9851de5df8153a822a146df4399ee76d20 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Wed, 18 Jun 2025 01:37:28 -0400 Subject: [PATCH 004/107] slightly reduce stalls when updating MiInstance --- ...50217001651-enable-instance-HOT-updates.js | 29 +++++++++++++++++++ .../src/core/FetchInstanceMetadataService.ts | 23 ++++++++------- 2 files changed, 42 insertions(+), 10 deletions(-) create mode 100644 packages/backend/migration/1750217001651-enable-instance-HOT-updates.js diff --git a/packages/backend/migration/1750217001651-enable-instance-HOT-updates.js b/packages/backend/migration/1750217001651-enable-instance-HOT-updates.js new file mode 100644 index 0000000000..995c3ed445 --- /dev/null +++ b/packages/backend/migration/1750217001651-enable-instance-HOT-updates.js @@ -0,0 +1,29 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +// https://www.cybertec-postgresql.com/en/hot-updates-in-postgresql-for-better-performance/ + +/** + * @typedef {import('typeorm').MigrationInterface} MigrationInterface + */ + +/** + * @class + * @implements {MigrationInterface} + */ +export class EnableInstanceHOTUpdates1750217001651 { + name = 'EnableInstanceHOTUpdates1750217001651'; + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "instance" SET (fillfactor = 50)`); + + // Vacuum can't run inside a transaction block, so query directly from the connection. + await queryRunner.connection.query(`VACUUM (FULL, VERBOSE) "instance"`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "instance" SET (fillfactor = 100)`); + } +} diff --git a/packages/backend/src/core/FetchInstanceMetadataService.ts b/packages/backend/src/core/FetchInstanceMetadataService.ts index 0ba61ce4f9..855ecd7553 100644 --- a/packages/backend/src/core/FetchInstanceMetadataService.ts +++ b/packages/backend/src/core/FetchInstanceMetadataService.ts @@ -132,18 +132,21 @@ export class FetchInstanceMetadataService { } as Record; if (info) { - updates.softwareName = typeof info.software?.name === 'string' ? info.software.name.toLowerCase() : '?'; - updates.softwareVersion = info.software?.version; - updates.openRegistrations = info.openRegistrations; - updates.maintainerName = info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.name ?? null) : null : null; - updates.maintainerEmail = info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.email ?? null) : null : null; + const softwareName = typeof info.software?.name === 'string' ? info.software.name.toLowerCase() : '?'; + if (softwareName !== instance.softwareName) updates.softwareName = updates.softwareVersion = info.software?.version; + if (info.openRegistrations !== instance.openRegistrations) updates.openRegistrations = info.openRegistrations; + const maintainerName = info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.name ?? null) : null : null; + if (maintainerName !== instance.maintainerName) updates.maintainerName = maintainerName; + const maintainerEmail = info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.email ?? null) : null : null; + if (maintainerEmail !== instance.maintainerEmail) updates.maintainerEmail = maintainerEmail; } - if (name) updates.name = name; - if (description) updates.description = description; - if (icon ?? favicon) updates.iconUrl = (icon && !icon.includes('data:image/png;base64')) ? icon : favicon; - if (favicon) updates.faviconUrl = favicon; - if (themeColor) updates.themeColor = themeColor; + if (name !== instance.name) updates.name = name; + if (description !== instance.description) updates.description = description; + const iconUrl = (icon && !icon.includes('data:image/png;base64')) ? icon : favicon; + if (iconUrl !== instance.iconUrl) updates.iconUrl = iconUrl; + if (favicon !== instance.faviconUrl) updates.faviconUrl = favicon; + if (themeColor !== instance.themeColor) updates.themeColor = themeColor; await this.federatedInstanceService.update(instance.id, updates); From 41e50eeb0ee7a3a149db6544b1d1a35e52da1a2e Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Wed, 18 Jun 2025 01:53:59 -0400 Subject: [PATCH 005/107] add update-user-tags, update-note-tags, post-deliver, post-inbox, post-note, and check-hibernation background tasks --- ...1748990662839-fix-IDX_instance_host_key.js | 2 + ...991828473-create-IDX_note_for_timelines.js | 2 + ...017688-create-IDX_instance_host_filters.js | 2 + .../1748992128683-create-statistics.js | 2 + ...1749097536193-fix-IDX_note_for_timeline.js | 2 + ...016885-remove-IDX_instance_host_filters.js | 2 + packages/backend/src/core/HashtagService.ts | 12 +- .../backend/src/core/NoteCreateService.ts | 19 +- packages/backend/src/core/NoteEditService.ts | 13 +- packages/backend/src/core/QueueService.ts | 87 +++++++++- packages/backend/src/core/ReactionService.ts | 1 + .../src/core/activitypub/ApInboxService.ts | 28 +-- .../activitypub/models/ApPersonService.ts | 14 +- .../BackgroundTaskProcessorService.ts | 162 +++++++++++++++++- .../processors/DeliverProcessorService.ts | 57 +----- .../queue/processors/InboxProcessorService.ts | 26 +-- packages/backend/src/queue/types.ts | 41 ++++- .../src/server/api/endpoints/i/update.ts | 8 +- 18 files changed, 331 insertions(+), 149 deletions(-) diff --git a/packages/backend/migration/1748990662839-fix-IDX_instance_host_key.js b/packages/backend/migration/1748990662839-fix-IDX_instance_host_key.js index fc6d303743..e423ecd1b6 100644 --- a/packages/backend/migration/1748990662839-fix-IDX_instance_host_key.js +++ b/packages/backend/migration/1748990662839-fix-IDX_instance_host_key.js @@ -4,6 +4,8 @@ */ export class FixIDXInstanceHostKey1748990662839 { + name = 'FixIDXInstanceHostKey1748990662839'; + async up(queryRunner) { // must include host for index-only scans: https://www.postgresql.org/docs/current/indexes-index-only-scans.html await queryRunner.query(`DROP INDEX "public"."IDX_instance_host_key"`); diff --git a/packages/backend/migration/1748991828473-create-IDX_note_for_timelines.js b/packages/backend/migration/1748991828473-create-IDX_note_for_timelines.js index 2ea7fe95d2..54debcee27 100644 --- a/packages/backend/migration/1748991828473-create-IDX_note_for_timelines.js +++ b/packages/backend/migration/1748991828473-create-IDX_note_for_timelines.js @@ -4,6 +4,8 @@ */ export class CreateIDXNoteForTimelines1748991828473 { + name = 'CreateIDXNoteForTimelines1748991828473'; + async up(queryRunner) { await queryRunner.query(` create index "IDX_note_for_timelines" diff --git a/packages/backend/migration/1748992017688-create-IDX_instance_host_filters.js b/packages/backend/migration/1748992017688-create-IDX_instance_host_filters.js index 76cf16a6de..24b2b1894f 100644 --- a/packages/backend/migration/1748992017688-create-IDX_instance_host_filters.js +++ b/packages/backend/migration/1748992017688-create-IDX_instance_host_filters.js @@ -4,6 +4,8 @@ */ export class CreateIDXInstanceHostFilters1748992017688 { + name = 'CreateIDXInstanceHostFilters1748992017688'; + async up(queryRunner) { await queryRunner.query(` create index "IDX_instance_host_filters" diff --git a/packages/backend/migration/1748992128683-create-statistics.js b/packages/backend/migration/1748992128683-create-statistics.js index 5d08868536..daa50332ff 100644 --- a/packages/backend/migration/1748992128683-create-statistics.js +++ b/packages/backend/migration/1748992128683-create-statistics.js @@ -4,6 +4,8 @@ */ export class CreateStatistics1748992128683 { + name = 'CreateStatistics1748992128683'; + async up(queryRunner) { await queryRunner.query(`CREATE STATISTICS "STTS_instance_isBlocked_isBubbled" (mcv) ON "isBlocked", "isBubbled" FROM "instance"`); await queryRunner.query(`CREATE STATISTICS "STTS_instance_isBlocked_isSilenced" (mcv) ON "isBlocked", "isSilenced" FROM "instance"`); diff --git a/packages/backend/migration/1749097536193-fix-IDX_note_for_timeline.js b/packages/backend/migration/1749097536193-fix-IDX_note_for_timeline.js index 9a651e5871..57c5579110 100644 --- a/packages/backend/migration/1749097536193-fix-IDX_note_for_timeline.js +++ b/packages/backend/migration/1749097536193-fix-IDX_note_for_timeline.js @@ -4,6 +4,8 @@ */ export class FixIDXNoteForTimeline1749097536193 { + name = 'FixIDXNoteForTimeline1749097536193'; + async up(queryRunner) { await queryRunner.query('drop index "IDX_note_for_timelines"'); await queryRunner.query(` diff --git a/packages/backend/migration/1749267016885-remove-IDX_instance_host_filters.js b/packages/backend/migration/1749267016885-remove-IDX_instance_host_filters.js index d0a4e4f91e..4236399a6e 100644 --- a/packages/backend/migration/1749267016885-remove-IDX_instance_host_filters.js +++ b/packages/backend/migration/1749267016885-remove-IDX_instance_host_filters.js @@ -4,6 +4,8 @@ */ export class RemoveIDXInstanceHostFilters1749267016885 { + name = 'RemoveIDXInstanceHostFilters1749267016885'; + async up(queryRunner) { await queryRunner.query(`DROP INDEX IF EXISTS "IDX_instance_host_filters"`); } diff --git a/packages/backend/src/core/HashtagService.ts b/packages/backend/src/core/HashtagService.ts index 0035c4b0d5..b9945d34b3 100644 --- a/packages/backend/src/core/HashtagService.ts +++ b/packages/backend/src/core/HashtagService.ts @@ -59,7 +59,7 @@ export class HashtagService { tag = normalizeForSearch(tag); // TODO: サンプリング - this.updateHashtagsRanking(tag, user.id); + await this.updateHashtagsRanking(tag, user.id); const index = await this.hashtagsRepository.findOneBy({ name: tag }); @@ -119,11 +119,11 @@ export class HashtagService { if (Object.keys(set).length > 0) { q.set(set); - q.execute(); + await q.execute(); } } else { if (isUserAttached) { - this.hashtagsRepository.insert({ + await this.hashtagsRepository.insert({ id: this.idService.gen(), name: tag, mentionedUserIds: [], @@ -140,7 +140,7 @@ export class HashtagService { attachedRemoteUsersCount: isRemoteUser(user) ? 1 : 0, } as MiHashtag); } else { - this.hashtagsRepository.insert({ + await this.hashtagsRepository.insert({ id: this.idService.gen(), name: tag, mentionedUserIds: [user.id], @@ -174,7 +174,7 @@ export class HashtagService { const exist = await this.redisClient.sismember(`hashtagUsers:${hashtag}`, userId); if (exist === 1) return; - this.featuredService.updateHashtagsRanking(hashtag, 1); + await this.featuredService.updateHashtagsRanking(hashtag, 1); const redisPipeline = this.redisClient.pipeline(); @@ -193,7 +193,7 @@ export class HashtagService { 'NX', // "NX -- Set expiry only when the key has no expiry" = 有効期限がないときだけ設定 ); - redisPipeline.exec(); + await redisPipeline.exec(); } @bindThis diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index fd55c33bfb..484f7e01d0 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -458,10 +458,10 @@ export class NoteCreateService implements OnApplicationShutdown { const note = await this.insertNote(user, data, tags, emojis, mentionedUsers); - setImmediate('post created', { signal: this.#shutdownController.signal }).then( - () => this.postNoteCreated(note, user, data, silent, tags!, mentionedUsers!), - () => { /* aborted, ignore this */ }, - ); + // Update the Latest Note index / following feed + this.latestNoteService.handleCreatedNoteBG(note); + + await this.queueService.createPostNoteJob(note.id, silent, 'create'); return note; } @@ -577,7 +577,7 @@ export class NoteCreateService implements OnApplicationShutdown { } @bindThis - private async postNoteCreated(note: MiNote, user: MiUser & { + public async postNoteCreated(note: MiNote, user: MiUser & { id: MiUser['id']; username: MiUser['username']; host: MiUser['host']; @@ -606,7 +606,7 @@ export class NoteCreateService implements OnApplicationShutdown { // ハッシュタグ更新 if (data.visibility === 'public' || data.visibility === 'home') { if (!user.isBot || this.meta.enableBotTrending) { - this.hashtagService.updateHashtags(user, tags); + await this.queueService.createUpdateNoteTagsJob(note.id); } } @@ -807,9 +807,6 @@ export class NoteCreateService implements OnApplicationShutdown { }); } - // Update the Latest Note index / following feed - this.latestNoteService.handleCreatedNoteBG(note); - // Register to search database if (!user.noindex) this.index(note); } @@ -1100,8 +1097,8 @@ export class NoteCreateService implements OnApplicationShutdown { // Instance cannot quote if (user.host) { - const instance = await this.federatedInstanceService.fetch(user.host); - if (instance?.rejectQuotes) { + const instance = await this.federatedInstanceService.fetchOrRegister(user.host); + if (instance.rejectQuotes) { (data as Option).renote = null; (data.processErrors ??= []).push('quoteUnavailable'); } diff --git a/packages/backend/src/core/NoteEditService.ts b/packages/backend/src/core/NoteEditService.ts index 20afc4e63c..21991e2966 100644 --- a/packages/backend/src/core/NoteEditService.ts +++ b/packages/backend/src/core/NoteEditService.ts @@ -588,10 +588,10 @@ export class NoteEditService implements OnApplicationShutdown { // Re-fetch note to get the default values of null / unset fields. const edited = await this.notesRepository.findOneByOrFail({ id: note.id }); - setImmediate('post edited', { signal: this.#shutdownController.signal }).then( - () => this.postNoteEdited(edited, oldnote, user, data, silent, tags!, mentionedUsers!), - () => { /* aborted, ignore this */ }, - ); + // Update the Latest Note index / following feed + this.latestNoteService.handleUpdatedNoteBG(edited, oldnote); + + await this.queueService.createPostNoteJob(note.id, silent, 'edit'); return edited; } else { @@ -600,7 +600,7 @@ export class NoteEditService implements OnApplicationShutdown { } @bindThis - private async postNoteEdited(note: MiNote, oldNote: MiNote, user: MiUser & { + public async postNoteEdited(note: MiNote, user: MiUser & { id: MiUser['id']; username: MiUser['username']; host: MiUser['host']; @@ -754,9 +754,6 @@ export class NoteEditService implements OnApplicationShutdown { }); } - // Update the Latest Note index / following feed - this.latestNoteService.handleUpdatedNoteBG(oldNote, note); - // Register to search database if (!user.noindex) this.index(note); } diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 15c50e4d83..98b4d6cd8c 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -853,7 +853,6 @@ export class QueueService implements OnModuleInit { }, { id: `update-user:${userId}`, - // ttl: 1000 * 60 * 60 * 24, }, ); } @@ -868,7 +867,6 @@ export class QueueService implements OnModuleInit { }, { id: `update-featured:${userId}`, - // ttl: 1000 * 60 * 60 * 24, }, ); } @@ -883,12 +881,93 @@ export class QueueService implements OnModuleInit { }, { id: `update-instance:${host}`, - // ttl: 1000 * 60 * 60 * 24, }, ); } - private async createBackgroundTask(name: string, data: BackgroundTaskJobData, duplication: { id: string, ttl?: number }) { + @bindThis + public async createPostDeliverJob(host: string, result: 'success' | 'temp-fail' | 'perm-fail') { + return await this.createBackgroundTask( + 'post-deliver', + { + type: 'post-deliver', + host, + result, + }, + ); + } + + @bindThis + public async createPostInboxJob(host: string) { + return await this.createBackgroundTask( + 'post-inbox', + { + type: 'post-inbox', + host, + }, + ); + } + + @bindThis + public async createPostNoteJob(noteId: string, silent: boolean, type: 'create' | 'edit') { + return await this.createBackgroundTask( + 'post-note', + { + type: 'post-note', + noteId, + silent, + edit: type === 'edit', + }, + { + id: `post-note:${noteId}:${type}`, + }, + ); + } + + @bindThis + public async createCheckHibernationJob(userId: string) { + return await this.createBackgroundTask( + 'check-hibernation', + { + type: 'check-hibernation', + userId, + }, + { + id: `check-hibernation:${userId}`, + ttl: 1000 * 60 * 60 * 24, // This is a very heavy task, so only run once per day per user + }, + ); + } + + @bindThis + public async createUpdateUserTagsJob(userId: string) { + return await this.createBackgroundTask( + 'update-user-tags', + { + type: 'update-user-tags', + userId, + }, + { + id: `update-user-tags:${userId}`, + }, + ); + } + + @bindThis + public async createUpdateNoteTagsJob(noteId: string) { + return await this.createBackgroundTask( + 'update-note-tags', + { + type: 'update-note-tags', + noteId, + }, + { + id: `update-note-tags:${noteId}`, + }, + ); + } + + private async createBackgroundTask(name: string, data: BackgroundTaskJobData, duplication?: { id: string, ttl?: number }) { return await this.backgroundTaskQueue.add( name, data, diff --git a/packages/backend/src/core/ReactionService.ts b/packages/backend/src/core/ReactionService.ts index d3ab48e3ff..27a6d7b514 100644 --- a/packages/backend/src/core/ReactionService.ts +++ b/packages/backend/src/core/ReactionService.ts @@ -340,6 +340,7 @@ export class ReactionService implements OnModuleInit { .execute(); } + // TODO update caches this.usersRepository.update({ id: user.id }, { updatedAt: this.timeService.date }); this.globalEventService.publishNoteStream(note.id, 'unreacted', { diff --git a/packages/backend/src/core/activitypub/ApInboxService.ts b/packages/backend/src/core/activitypub/ApInboxService.ts index 309259120d..de37944344 100644 --- a/packages/backend/src/core/activitypub/ApInboxService.ts +++ b/packages/backend/src/core/activitypub/ApInboxService.ts @@ -33,10 +33,6 @@ import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; import { fromTuple } from '@/misc/from-tuple.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; import { renderInlineError } from '@/misc/render-inline-error.js'; -import InstanceChart from '@/core/chart/charts/instance.js'; -import FederationChart from '@/core/chart/charts/federation.js'; -import { FetchInstanceMetadataService } from '@/core/FetchInstanceMetadataService.js'; -import { UpdateInstanceQueue } from '@/core/UpdateInstanceQueue.js'; import { CacheService } from '@/core/CacheService.js'; import { NoteVisibilityService } from '@/core/NoteVisibilityService.js'; import { TimeService } from '@/global/TimeService.js'; @@ -97,10 +93,6 @@ export class ApInboxService { private queueService: QueueService, private globalEventService: GlobalEventService, private readonly federatedInstanceService: FederatedInstanceService, - private readonly fetchInstanceMetadataService: FetchInstanceMetadataService, - private readonly instanceChart: InstanceChart, - private readonly federationChart: FederationChart, - private readonly updateInstanceQueue: UpdateInstanceQueue, private readonly cacheService: CacheService, private readonly noteVisibilityService: NoteVisibilityService, private readonly timeService: TimeService, @@ -423,25 +415,7 @@ export class ApInboxService { } // Update stats (adapted from InboxProcessorService) - this.federationChart.inbox(actor.host).then(); - process.nextTick(async () => { - const i = await (this.meta.enableStatsForFederatedInstances - ? this.federatedInstanceService.fetchOrRegister(actor.host) - : this.federatedInstanceService.fetch(actor.host)); - - if (i == null) return; - - this.updateInstanceQueue.enqueue(i.id, { - latestRequestReceivedAt: this.timeService.date, - shouldUnsuspend: i.suspensionState === 'autoSuspendedForNotResponding', - }); - - if (this.meta.enableChartsForFederatedInstances) { - this.instanceChart.requestReceived(i.host).then(); - } - - await this.fetchInstanceMetadataService.fetchInstanceMetadataLazy(i); - }); + await this.queueService.createPostInboxJob(actor.host); // Process it! try { diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index 096480047f..f153d51c9d 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -585,9 +585,6 @@ export class ApPersonService implements OnModuleInit { this.usersChart.update(user, true); - // ハッシュタグ更新 - this.hashtagService.updateUsertags(user, tags); - //#region アバターとヘッダー画像をフェッチ try { const updates = await this.resolveAvatarAndBanner(user, person.icon, person.image, person.backgroundUrl); @@ -604,6 +601,9 @@ export class ApPersonService implements OnModuleInit { } //#endregion + // ハッシュタグ更新 + await this.queueService.createUpdateUserTagsJob(user.id); + await this.updateFeaturedLazy(user); return user; @@ -811,9 +811,6 @@ export class ApPersonService implements OnModuleInit { this.globalEventService.publishInternalEvent('remoteUserUpdated', { id: exist.id }); - // ハッシュタグ更新 - this.hashtagService.updateUsertags(exist, tags); - // 該当ユーザーが既にフォロワーになっていた場合はFollowingもアップデートする if (exist.inbox !== person.inbox || exist.sharedInbox !== (person.sharedInbox ?? person.endpoints?.sharedInbox)) { await this.followingsRepository.update( @@ -827,6 +824,9 @@ export class ApPersonService implements OnModuleInit { await this.cacheService.refreshFollowRelationsFor(exist.id); } + // ハッシュタグ更新 + await this.queueService.createUpdateUserTagsJob(exist.id); + await this.updateFeaturedLazy(exist); const updated = { ...exist, ...updates }; @@ -967,7 +967,7 @@ export class ApPersonService implements OnModuleInit { let td = 0; for (const note of featuredNotes.filter(x => x != null)) { td -= 1000; - transactionalEntityManager.insert(MiUserNotePining, { + await transactionalEntityManager.insert(MiUserNotePining, { id: this.idService.gen(this.timeService.now + td), userId: user.id, noteId: note.id, diff --git a/packages/backend/src/queue/processors/BackgroundTaskProcessorService.ts b/packages/backend/src/queue/processors/BackgroundTaskProcessorService.ts index b237990a4c..341c54883e 100644 --- a/packages/backend/src/queue/processors/BackgroundTaskProcessorService.ts +++ b/packages/backend/src/queue/processors/BackgroundTaskProcessorService.ts @@ -5,30 +5,46 @@ import { Inject, Injectable } from '@nestjs/common'; import * as Bull from 'bullmq'; -import { BackgroundTaskJobData, UpdateFeaturedBackgroundTask, UpdateInstanceBackgroundTask, UpdateUserBackgroundTask } from '@/queue/types.js'; +import { BackgroundTaskJobData, CheckHibernationBackgroundTask, PostDeliverBackgroundTask, PostInboxBackgroundTask, PostNoteBackgroundTask, UpdateFeaturedBackgroundTask, UpdateInstanceBackgroundTask, UpdateUserTagsBackgroundTask, UpdateUserBackgroundTask, UpdateNoteTagsBackgroundTask } from '@/queue/types.js'; import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js'; import { QueueLoggerService } from '@/queue/QueueLoggerService.js'; import Logger from '@/logger.js'; -import { isRetryableError } from '@/misc/is-retryable-error.js'; import { DI } from '@/di-symbols.js'; -import type { Config } from '@/config.js'; import { CacheService } from '@/core/CacheService.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; import { FetchInstanceMetadataService } from '@/core/FetchInstanceMetadataService.js'; -import { renderInlineError } from '@/misc/render-inline-error.js'; +import { MiMeta } from '@/models/Meta.js'; +import InstanceChart from '@/core/chart/charts/instance.js'; +import ApRequestChart from '@/core/chart/charts/ap-request.js'; +import FederationChart from '@/core/chart/charts/federation.js'; +import { UpdateInstanceQueue } from '@/core/UpdateInstanceQueue.js'; +import { NoteCreateService } from '@/core/NoteCreateService.js'; +import type { NotesRepository } from '@/models/_.js'; +import { NoteEditService } from '@/core/NoteEditService.js'; +import { HashtagService } from '@/core/HashtagService.js'; @Injectable() export class BackgroundTaskProcessorService { private readonly logger: Logger; constructor( - @Inject(DI.config) - private readonly config: Config, + @Inject(DI.meta) + private readonly meta: MiMeta, + + @Inject(DI.notesRepository) + private readonly notesRepository: NotesRepository, private readonly apPersonService: ApPersonService, private readonly cacheService: CacheService, private readonly federatedInstanceService: FederatedInstanceService, private readonly fetchInstanceMetadataService: FetchInstanceMetadataService, + private readonly instanceChart: InstanceChart, + private readonly apRequestChart: ApRequestChart, + private readonly federationChart: FederationChart, + private readonly updateInstanceQueue: UpdateInstanceQueue, + private readonly noteCreateService: NoteCreateService, + private readonly noteEditService: NoteEditService, + private readonly hashtagService: HashtagService, queueLoggerService: QueueLoggerService, ) { @@ -40,9 +56,21 @@ export class BackgroundTaskProcessorService { return await this.processUpdateUser(job.data); } else if (job.data.type === 'update-featured') { return await this.processUpdateFeatured(job.data); - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + } else if (job.data.type === 'update-user-tags') { + return await this.processUpdateUserTags(job.data); + } else if (job.data.type === 'update-note-tags') { + return await this.processUpdateNoteTags(job.data); } else if (job.data.type === 'update-instance') { return await this.processUpdateInstance(job.data); + } else if (job.data.type === 'post-deliver') { + return await this.processPostDeliver(job.data); + } else if (job.data.type === 'post-inbox') { + return await this.processPostInbox(job.data); + } else if (job.data.type === 'post-note') { + return await this.processPostNote(job.data); + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + } else if (job.data.type === 'check-hibernation') { + return await this.processCheckHibernation(job.data); } else { this.logger.warn(`Can't process unknown job type "${job.data}"; this is likely a bug. Full job data:`, job.data); throw new Error(`Unknown job type ${job.data}, see system logs for details`); @@ -78,10 +106,30 @@ export class BackgroundTaskProcessorService { return 'ok'; } + private async processUpdateUserTags(task: UpdateUserTagsBackgroundTask): Promise { + const user = await this.cacheService.findOptionalUserById(task.userId); + if (!user || user.isDeleted) return `Skipping update-user-tags task: user ${task.userId} has been deleted`; + if (user.isSuspended) return `Skipping update-user-tags task: user ${task.userId} is suspended`; + if (!user.uri) return `Skipping update-user-tags task: user ${task.userId} is local`; + + await this.hashtagService.updateUsertags(user, user.tags); + return 'ok'; + } + + private async processUpdateNoteTags(task: UpdateNoteTagsBackgroundTask): Promise { + const note = await this.notesRepository.findOneBy({ id: task.noteId }); + if (!note) return `Skipping update-note-tags task: note ${task.noteId} has been deleted`; + const user = await this.cacheService.findUserById(note.userId); + if (user.isSuspended) return `Skipping update-note-tags task: note ${task.noteId}'s user ${note.userId} is suspended`; + + await this.hashtagService.updateHashtags(user, note.tags); + return 'ok'; + } + private async processUpdateInstance(task: UpdateInstanceBackgroundTask): Promise { const instance = await this.federatedInstanceService.fetch(task.host); - if (!instance) return `Skipping update-instance task: instance ${task.host} has been deleted`; if (instance.isBlocked) return `Skipping update-instance task: instance ${task.host} is blocked`; + if (instance.suspensionState === 'goneSuspended') return `Skipping update-instance task: instance ${task.host} is gone`; if (instance.infoUpdatedAt && Date.now() - instance.infoUpdatedAt.getTime() < 1000 * 60 * 60 * 24) { return `Skipping update-instance task: instance ${task.host} was recently updated`; @@ -90,4 +138,102 @@ export class BackgroundTaskProcessorService { await this.fetchInstanceMetadataService.fetchInstanceMetadata(instance); return 'ok'; } + + private async processPostDeliver(task: PostDeliverBackgroundTask): Promise { + let instance = await this.federatedInstanceService.fetchOrRegister(task.host); + if (instance.isBlocked) return `Skipping post-deliver task: instance ${task.host} is blocked`; + + const success = task.result === 'success'; + + // isNotResponding should be the inverse of success, because: + // 1. We expect success (success=true) from a responding instance (isNotResponding=false). + // 2. We expect failure (success=false) from a non-responding instance (isNotResponding=true). + // If they are equal, then we need to update the cached state. + const updateNotResponding = success === instance.isNotResponding; + + // If we get a permanent failure, then we need to immediately suspend the instance + const updateGoneSuspended = task.result === 'perm-fail' && instance.suspensionState !== 'goneSuspended'; + + // Check if we need to auto-suspend the instance + const updateAutoSuspended = instance.isNotResponding && instance.notRespondingSince && instance.suspensionState === 'none' && instance.notRespondingSince.getTime() <= Date.now() - 1000 * 60 * 60 * 24 * 7; + + // This is messy, but we need to minimize updates to space in Postgres blocks. + if (updateNotResponding || updateGoneSuspended || updateAutoSuspended) { + instance = await this.federatedInstanceService.update(instance.id, { + isNotResponding: updateNotResponding ? !success : undefined, + notRespondingSince: updateNotResponding ? (success ? null : new Date()) : undefined, + suspensionState: updateGoneSuspended + ? 'goneSuspended' + : updateAutoSuspended + ? 'autoSuspendedForNotResponding' + : undefined, + }); + } + + // Update instance metadata (deferred) + if (success && this.meta.enableStatsForFederatedInstances) { + await this.fetchInstanceMetadataService.fetchInstanceMetadataLazy(instance); + } + + // Update charts + if (this.meta.enableChartsForFederatedInstances) { + await this.instanceChart.requestSent(task.host, success); + } + if (success) { + await this.apRequestChart.deliverSucc(); + } else { + await this.apRequestChart.deliverFail(); + } + await this.federationChart.deliverd(task.host, success); + + return 'ok'; + } + + private async processPostInbox(task: PostInboxBackgroundTask): Promise { + const instance = await this.federatedInstanceService.fetchOrRegister(task.host); + if (instance.isBlocked) return `Skipping post-inbox task: instance ${task.host} is blocked`; + + // Update charts + if (this.meta.enableChartsForFederatedInstances) { + await this.instanceChart.requestReceived(task.host); + } + await this.apRequestChart.inbox(); + await this.federationChart.inbox(task.host); + + // Update instance metadata (deferred) + await this.fetchInstanceMetadataService.fetchInstanceMetadataLazy(instance); + + // Unsuspend instance (deferred) + this.updateInstanceQueue.enqueue(instance.id, { + latestRequestReceivedAt: new Date(), + shouldUnsuspend: instance.suspensionState === 'autoSuspendedForNotResponding', + }); + + return 'ok'; + } + + private async processPostNote(task: PostNoteBackgroundTask): Promise { + const note = await this.notesRepository.findOneBy({ id: task.noteId }); + if (!note) return `Skipping post-note task: note ${task.noteId} has been deleted`; + const user = await this.cacheService.findUserById(note.userId); + if (user.isSuspended) return `Skipping post-note task: note ${task.noteId}'s user ${note.userId} is suspended`; + + const mentionedUsers = await this.cacheService.getUsers(note.mentions); + + if (task.edit) { + await this.noteEditService.postNoteEdited(note, user, note, task.silent, note.tags, Array.from(mentionedUsers.values())); + } else { + await this.noteCreateService.postNoteCreated(note, user, note, task.silent, note.tags, Array.from(mentionedUsers.values())); + } + + return 'ok'; + } + + private async processCheckHibernation(task: CheckHibernationBackgroundTask): Promise { + const followers = await this.cacheService.getNonHibernatedFollowers(task.userId); + if (followers.length < 1) return `Skipping check-hibernation task: user ${task.userId} has no non-hibernated followers`; + + await this.noteCreateService.checkHibernation(followers); + return 'ok'; + } } diff --git a/packages/backend/src/queue/processors/DeliverProcessorService.ts b/packages/backend/src/queue/processors/DeliverProcessorService.ts index 792ec4b015..7e75a2105c 100644 --- a/packages/backend/src/queue/processors/DeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/DeliverProcessorService.ts @@ -21,6 +21,7 @@ import { StatusError } from '@/misc/status-error.js'; import { UtilityService } from '@/core/UtilityService.js'; import { TimeService } from '@/global/TimeService.js'; import { bindThis } from '@/decorators.js'; +import { QueueService } from '@/core/QueueService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type { DeliverJobData } from '../types.js'; @@ -44,13 +45,14 @@ export class DeliverProcessorService { private federationChart: FederationChart, private queueLoggerService: QueueLoggerService, private readonly timeService: TimeService, + private readonly queueService: QueueService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('deliver'); } @bindThis public async process(job: Bull.Job): Promise { - const { host } = new URL(job.data.to); + const host = this.utilityService.extractDbHost(job.data.to); if (!this.utilityService.isFederationAllowedUri(job.data.to)) { return 'skip (blocked)'; @@ -72,66 +74,19 @@ export class DeliverProcessorService { try { await this.apRequestService.signedPost(job.data.user, job.data.to, job.data.content, job.data.digest); - this.apRequestChart.deliverSucc(); - this.federationChart.deliverd(host, true); - // Update instance stats - process.nextTick(async () => { - if (i == null) return; - - if (i.isNotResponding) { - await this.federatedInstanceService.update(i.id, { - isNotResponding: false, - notRespondingSince: null, - }); - } - - if (this.meta.enableChartsForFederatedInstances) { - await this.instanceChart.requestSent(i.host, true); - } - }); + await this.queueService.createPostDeliverJob(host, 'success'); return 'Success'; } catch (res) { - await this.apRequestChart.deliverFail(); - await this.federationChart.deliverd(host, false); - // Update instance stats - this.federatedInstanceService.fetchOrRegister(host).then(i => { - if (!i.isNotResponding) { - this.federatedInstanceService.update(i.id, { - isNotResponding: true, - notRespondingSince: this.timeService.date, - }); - } else if (i.notRespondingSince) { - // 1週間以上不通ならサスペンド - if (i.suspensionState === 'none' && i.notRespondingSince.getTime() <= this.timeService.now - 1000 * 60 * 60 * 24 * 7) { - this.federatedInstanceService.update(i.id, { - suspensionState: 'autoSuspendedForNotResponding', - }); - } - } else { - // isNotRespondingがtrueでnotRespondingSinceがnullの場合はnotRespondingSinceをセット - // notRespondingSinceは新たな機能なので、それ以前のデータにはnotRespondingSinceがない場合がある - this.federatedInstanceService.update(i.id, { - notRespondingSince: this.timeService.date, - }); - } - - if (this.meta.enableChartsForFederatedInstances) { - this.instanceChart.requestSent(i.host, false); - } - }); + const isPerm = job.data.isSharedInbox && res instanceof StatusError && res.statusCode === 410; + await this.queueService.createPostDeliverJob(host, isPerm ? 'perm-fail' : 'temp-fail'); if (res instanceof StatusError && !res.isRetryable) { // 4xx // 相手が閉鎖していることを明示しているため、配送停止する if (job.data.isSharedInbox && res.statusCode === 410) { - this.federatedInstanceService.fetchOrRegister(host).then(i => { - this.federatedInstanceService.update(i.id, { - suspensionState: 'goneSuspended', - }); - }); throw new Bull.UnrecoverableError(`${host} is gone`); } throw new Bull.UnrecoverableError(`${res.statusCode} ${res.statusMessage}`); diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index 3b73e98e5c..1c7765fddf 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -30,10 +30,10 @@ import { DI } from '@/di-symbols.js'; import { SkApInboxLog } from '@/models/_.js'; import type { Config } from '@/config.js'; import { ApLogService, calculateDurationSince } from '@/core/ApLogService.js'; -import { UpdateInstanceQueue } from '@/core/UpdateInstanceQueue.js'; import { TimeService } from '@/global/TimeService.js'; import { isRetryableError } from '@/misc/is-retryable-error.js'; import { renderInlineError } from '@/misc/render-inline-error.js'; +import { QueueService } from '@/core/QueueService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type { InboxJobData } from '../types.js'; @@ -66,8 +66,8 @@ export class InboxProcessorService implements OnApplicationShutdown { private federationChart: FederationChart, private queueLoggerService: QueueLoggerService, private readonly apLogService: ApLogService, - private readonly updateInstanceQueue: UpdateInstanceQueue, private readonly timeService: TimeService, + private readonly queueService: QueueService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('inbox'); } @@ -258,28 +258,8 @@ export class InboxProcessorService implements OnApplicationShutdown { log.authUserId = authUser.user.id; } - this.apRequestChart.inbox(); - this.federationChart.inbox(authUser.user.host); - // Update instance stats - process.nextTick(async () => { - const i = await (this.meta.enableStatsForFederatedInstances - ? this.federatedInstanceService.fetchOrRegister(authUser.user.host) - : this.federatedInstanceService.fetch(authUser.user.host)); - - if (i == null) return; - - this.updateInstanceQueue.enqueue(i.id, { - latestRequestReceivedAt: this.timeService.date, - shouldUnsuspend: i.suspensionState === 'autoSuspendedForNotResponding', - }); - - if (this.meta.enableChartsForFederatedInstances) { - await this.instanceChart.requestReceived(i.host); - } - - await this.fetchInstanceMetadataService.fetchInstanceMetadataLazy(i); - }); + await this.queueService.createPostInboxJob(authUser.user.host); // アクティビティを処理 try { diff --git a/packages/backend/src/queue/types.ts b/packages/backend/src/queue/types.ts index 0c3d790a97..64ae679033 100644 --- a/packages/backend/src/queue/types.ts +++ b/packages/backend/src/queue/types.ts @@ -172,7 +172,13 @@ export type ScheduleNotePostJobData = { export type BackgroundTaskJobData = UpdateUserBackgroundTask | UpdateFeaturedBackgroundTask | - UpdateInstanceBackgroundTask; + UpdateUserTagsBackgroundTask | + UpdateNoteTagsBackgroundTask | + UpdateInstanceBackgroundTask | + PostDeliverBackgroundTask | + PostInboxBackgroundTask | + PostNoteBackgroundTask | + CheckHibernationBackgroundTask; export type UpdateUserBackgroundTask = { type: 'update-user'; @@ -184,7 +190,40 @@ export type UpdateFeaturedBackgroundTask = { userId: string; }; +export type UpdateUserTagsBackgroundTask = { + type: 'update-user-tags'; + userId: string; +}; + +export type UpdateNoteTagsBackgroundTask = { + type: 'update-note-tags'; + noteId: string; +}; + export type UpdateInstanceBackgroundTask = { type: 'update-instance'; host: string; }; + +export type PostDeliverBackgroundTask = { + type: 'post-deliver'; + host: string; + result: 'success' | 'temp-fail' | 'perm-fail'; +}; + +export type PostInboxBackgroundTask = { + type: 'post-inbox'; + host: string; +}; + +export type PostNoteBackgroundTask = { + type: 'post-note'; + noteId: string; + silent: boolean; + edit: boolean; +}; + +export type CheckHibernationBackgroundTask = { + type: 'check-hibernation'; + userId: string; +}; diff --git a/packages/backend/src/server/api/endpoints/i/update.ts b/packages/backend/src/server/api/endpoints/i/update.ts index b2396d93eb..fe3f180aca 100644 --- a/packages/backend/src/server/api/endpoints/i/update.ts +++ b/packages/backend/src/server/api/endpoints/i/update.ts @@ -36,6 +36,7 @@ import { notificationRecieveConfig } from '@/models/json-schema/user.js'; import { userUnsignedFetchOptions } from '@/const.js'; import { renderInlineError } from '@/misc/render-inline-error.js'; import { trackPromise } from '@/misc/promise-tracker.js'; +import { QueueService } from '@/core/QueueService.js'; import { ApiLoggerService } from '../../ApiLoggerService.js'; import { ApiError } from '../../error.js'; @@ -318,6 +319,7 @@ export default class extends Endpoint { // eslint- private httpRequestService: HttpRequestService, private avatarDecorationService: AvatarDecorationService, private utilityService: UtilityService, + private readonly queueService: QueueService, ) { super(meta, paramDef, async (ps, _user, token) => { const user = await this.usersRepository.findOneByOrFail({ id: _user.id }) as MiLocalUser; @@ -606,9 +608,6 @@ export default class extends Endpoint { // eslint- updates.emojis = emojis; updates.tags = tags; - - // ハッシュタグ更新 - this.hashtagService.updateUsertags(user, tags); //#endregion if (Object.keys(updates).length > 0) { @@ -639,6 +638,9 @@ export default class extends Endpoint { // eslint- // Publish meUpdated event this.globalEventService.publishMainStream(user.id, 'meUpdated', iObj); + // ハッシュタグ更新 + await this.queueService.createUpdateUserTagsJob(user.id); + // 鍵垢を解除したとき、溜まっていたフォローリクエストがあるならすべて承認 if (user.isLocked && ps.isLocked === false) { await this.userFollowingService.acceptAllFollowRequests(user); From fe25e6f3677738b3f9493ec9053881acf494f4a9 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Wed, 18 Jun 2025 22:53:04 -0400 Subject: [PATCH 006/107] remove useless async/await from charts --- .../src/core/chart/charts/active-users.ts | 8 +++--- .../src/core/chart/charts/ap-request.ts | 12 ++++---- .../backend/src/core/chart/charts/drive.ts | 4 +-- .../src/core/chart/charts/federation.ts | 8 +++--- .../backend/src/core/chart/charts/instance.ts | 28 +++++++++---------- .../backend/src/core/chart/charts/notes.ts | 4 +-- .../src/core/chart/charts/per-user-drive.ts | 4 +-- .../core/chart/charts/per-user-following.ts | 2 +- .../src/core/chart/charts/per-user-pv.ts | 8 +++--- .../core/chart/charts/per-user-reactions.ts | 2 +- .../src/core/chart/charts/test-grouped.ts | 4 +-- .../core/chart/charts/test-intersection.ts | 8 +++--- .../src/core/chart/charts/test-unique.ts | 4 +-- .../backend/src/core/chart/charts/test.ts | 8 +++--- .../backend/src/core/chart/charts/users.ts | 4 +-- 15 files changed, 54 insertions(+), 54 deletions(-) diff --git a/packages/backend/src/core/chart/charts/active-users.ts b/packages/backend/src/core/chart/charts/active-users.ts index 20432fb293..4858633973 100644 --- a/packages/backend/src/core/chart/charts/active-users.ts +++ b/packages/backend/src/core/chart/charts/active-users.ts @@ -50,9 +50,9 @@ export default class ActiveUsersChart extends Chart { // eslint-d } @bindThis - public async read(user: { id: MiUser['id'], host: null }): Promise { + public read(user: { id: MiUser['id'], host: null }): void { const createdAt = this.idService.parse(user.id).date; - await this.commit({ + this.commit({ 'read': [user.id], 'registeredWithinWeek': (this.timeService.now - createdAt.getTime() < week) ? [user.id] : [], 'registeredWithinMonth': (this.timeService.now - createdAt.getTime() < month) ? [user.id] : [], @@ -64,8 +64,8 @@ export default class ActiveUsersChart extends Chart { // eslint-d } @bindThis - public async write(user: { id: MiUser['id'], host: null }): Promise { - await this.commit({ + public write(user: { id: MiUser['id'], host: null }): void { + this.commit({ 'write': [user.id], }); } diff --git a/packages/backend/src/core/chart/charts/ap-request.ts b/packages/backend/src/core/chart/charts/ap-request.ts index 8cae5753c7..f4177955b7 100644 --- a/packages/backend/src/core/chart/charts/ap-request.ts +++ b/packages/backend/src/core/chart/charts/ap-request.ts @@ -43,22 +43,22 @@ export default class ApRequestChart extends Chart { // eslint-dis } @bindThis - public async deliverSucc(): Promise { - await this.commit({ + public deliverSucc(): void { + this.commit({ 'deliverSucceeded': 1, }); } @bindThis - public async deliverFail(): Promise { - await this.commit({ + public deliverFail(): void { + this.commit({ 'deliverFailed': 1, }); } @bindThis - public async inbox(): Promise { - await this.commit({ + public inbox(): void { + this.commit({ 'inboxReceived': 1, }); } diff --git a/packages/backend/src/core/chart/charts/drive.ts b/packages/backend/src/core/chart/charts/drive.ts index cce07f3b5b..2818faf1cb 100644 --- a/packages/backend/src/core/chart/charts/drive.ts +++ b/packages/backend/src/core/chart/charts/drive.ts @@ -44,9 +44,9 @@ export default class DriveChart extends Chart { // eslint-disable } @bindThis - public async update(file: MiDriveFile, isAdditional: boolean): Promise { + public update(file: MiDriveFile, isAdditional: boolean): void { const fileSizeKb = file.size / 1000; - await this.commit(file.userHost === null ? { + this.commit(file.userHost === null ? { 'local.incCount': isAdditional ? 1 : 0, 'local.incSize': isAdditional ? fileSizeKb : 0, 'local.decCount': isAdditional ? 0 : 1, diff --git a/packages/backend/src/core/chart/charts/federation.ts b/packages/backend/src/core/chart/charts/federation.ts index 199c263cce..ab160128f5 100644 --- a/packages/backend/src/core/chart/charts/federation.ts +++ b/packages/backend/src/core/chart/charts/federation.ts @@ -118,8 +118,8 @@ export default class FederationChart extends Chart { // eslint-di } @bindThis - public async deliverd(host: string, succeeded: boolean): Promise { - await this.commit(succeeded ? { + public deliverd(host: string, succeeded: boolean): void { + this.commit(succeeded ? { 'deliveredInstances': [host], } : { 'stalled': [host], @@ -127,8 +127,8 @@ export default class FederationChart extends Chart { // eslint-di } @bindThis - public async inbox(host: string): Promise { - await this.commit({ + public inbox(host: string): void { + this.commit({ 'inboxInstances': [host], }); } diff --git a/packages/backend/src/core/chart/charts/instance.ts b/packages/backend/src/core/chart/charts/instance.ts index ca6c1c5026..d3690820e8 100644 --- a/packages/backend/src/core/chart/charts/instance.ts +++ b/packages/backend/src/core/chart/charts/instance.ts @@ -80,31 +80,31 @@ export default class InstanceChart extends Chart { // eslint-disa } @bindThis - public async requestReceived(host: string): Promise { - await this.commit({ + public requestReceived(host: string): void { + this.commit({ 'requests.received': 1, }, this.utilityService.toPuny(host)); } @bindThis - public async requestSent(host: string, isSucceeded: boolean): Promise { - await this.commit({ + public requestSent(host: string, isSucceeded: boolean): void { + this.commit({ 'requests.succeeded': isSucceeded ? 1 : 0, 'requests.failed': isSucceeded ? 0 : 1, }, this.utilityService.toPuny(host)); } @bindThis - public async newUser(host: string): Promise { - await this.commit({ + public newUser(host: string): void { + this.commit({ 'users.total': 1, 'users.inc': 1, }, this.utilityService.toPuny(host)); } @bindThis - public async updateNote(host: string, note: MiNote, isAdditional: boolean): Promise { - await this.commit({ + public updateNote(host: string, note: MiNote, isAdditional: boolean): void { + this.commit({ 'notes.total': isAdditional ? 1 : -1, 'notes.inc': isAdditional ? 1 : 0, 'notes.dec': isAdditional ? 0 : 1, @@ -116,8 +116,8 @@ export default class InstanceChart extends Chart { // eslint-disa } @bindThis - public async updateFollowing(host: string, isAdditional: boolean): Promise { - await this.commit({ + public updateFollowing(host: string, isAdditional: boolean): void { + this.commit({ 'following.total': isAdditional ? 1 : -1, 'following.inc': isAdditional ? 1 : 0, 'following.dec': isAdditional ? 0 : 1, @@ -125,8 +125,8 @@ export default class InstanceChart extends Chart { // eslint-disa } @bindThis - public async updateFollowers(host: string, isAdditional: boolean): Promise { - await this.commit({ + public updateFollowers(host: string, isAdditional: boolean): void { + this.commit({ 'followers.total': isAdditional ? 1 : -1, 'followers.inc': isAdditional ? 1 : 0, 'followers.dec': isAdditional ? 0 : 1, @@ -134,9 +134,9 @@ export default class InstanceChart extends Chart { // eslint-disa } @bindThis - public async updateDrive(file: MiDriveFile, isAdditional: boolean): Promise { + public updateDrive(file: MiDriveFile, isAdditional: boolean): void { const fileSizeKb = file.size / 1000; - await this.commit({ + this.commit({ 'drive.totalFiles': isAdditional ? 1 : -1, 'drive.incFiles': isAdditional ? 1 : 0, 'drive.incUsage': isAdditional ? fileSizeKb : 0, diff --git a/packages/backend/src/core/chart/charts/notes.ts b/packages/backend/src/core/chart/charts/notes.ts index 43cabd0b98..f8c3676009 100644 --- a/packages/backend/src/core/chart/charts/notes.ts +++ b/packages/backend/src/core/chart/charts/notes.ts @@ -56,10 +56,10 @@ export default class NotesChart extends Chart { // eslint-disable } @bindThis - public async update(note: MiNote, isAdditional: boolean): Promise { + public update(note: MiNote, isAdditional: boolean): void { const prefix = note.userHost === null ? 'local' : 'remote'; - await this.commit({ + this.commit({ [`${prefix}.total`]: isAdditional ? 1 : -1, [`${prefix}.inc`]: isAdditional ? 1 : 0, [`${prefix}.dec`]: isAdditional ? 0 : 1, diff --git a/packages/backend/src/core/chart/charts/per-user-drive.ts b/packages/backend/src/core/chart/charts/per-user-drive.ts index 663abc5f00..b1bd7c6173 100644 --- a/packages/backend/src/core/chart/charts/per-user-drive.ts +++ b/packages/backend/src/core/chart/charts/per-user-drive.ts @@ -58,9 +58,9 @@ export default class PerUserDriveChart extends Chart { // eslint- } @bindThis - public async update(file: MiDriveFile, isAdditional: boolean): Promise { + public update(file: MiDriveFile, isAdditional: boolean): void { const fileSizeKb = file.size / 1000; - await this.commit({ + this.commit({ 'totalCount': isAdditional ? 1 : -1, 'totalSize': isAdditional ? fileSizeKb : -fileSizeKb, 'incCount': isAdditional ? 1 : 0, diff --git a/packages/backend/src/core/chart/charts/per-user-following.ts b/packages/backend/src/core/chart/charts/per-user-following.ts index 71678b0573..05e971616a 100644 --- a/packages/backend/src/core/chart/charts/per-user-following.ts +++ b/packages/backend/src/core/chart/charts/per-user-following.ts @@ -70,7 +70,7 @@ export default class PerUserFollowingChart extends Chart { // esl } @bindThis - public async update(follower: { id: MiUser['id']; host: MiUser['host']; }, followee: { id: MiUser['id']; host: MiUser['host']; }, isFollow: boolean): Promise { + public update(follower: { id: MiUser['id']; host: MiUser['host']; }, followee: { id: MiUser['id']; host: MiUser['host']; }, isFollow: boolean): void { const prefixFollower = this.userEntityService.isLocalUser(follower) ? 'local' : 'remote'; const prefixFollowee = this.userEntityService.isLocalUser(followee) ? 'local' : 'remote'; diff --git a/packages/backend/src/core/chart/charts/per-user-pv.ts b/packages/backend/src/core/chart/charts/per-user-pv.ts index 75a61aae07..272bc9180b 100644 --- a/packages/backend/src/core/chart/charts/per-user-pv.ts +++ b/packages/backend/src/core/chart/charts/per-user-pv.ts @@ -44,16 +44,16 @@ export default class PerUserPvChart extends Chart { // eslint-dis } @bindThis - public async commitByUser(user: { id: MiUser['id'] }, key: string): Promise { - await this.commit({ + public commitByUser(user: { id: MiUser['id'] }, key: string): void { + this.commit({ 'upv.user': [key], 'pv.user': 1, }, user.id); } @bindThis - public async commitByVisitor(user: { id: MiUser['id'] }, key: string): Promise { - await this.commit({ + public commitByVisitor(user: { id: MiUser['id'] }, key: string): void { + this.commit({ 'upv.visitor': [key], 'pv.visitor': 1, }, user.id); diff --git a/packages/backend/src/core/chart/charts/per-user-reactions.ts b/packages/backend/src/core/chart/charts/per-user-reactions.ts index 9fb78a28e9..89f6567f50 100644 --- a/packages/backend/src/core/chart/charts/per-user-reactions.ts +++ b/packages/backend/src/core/chart/charts/per-user-reactions.ts @@ -47,7 +47,7 @@ export default class PerUserReactionsChart extends Chart { // esl } @bindThis - public async update(user: { id: MiUser['id'], host: MiUser['host'] }, note: MiNote): Promise { + public update(user: { id: MiUser['id'], host: MiUser['host'] }, note: MiNote): void { const prefix = this.userEntityService.isLocalUser(user) ? 'local' : 'remote'; this.commit({ [`${prefix}.count`]: 1, diff --git a/packages/backend/src/core/chart/charts/test-grouped.ts b/packages/backend/src/core/chart/charts/test-grouped.ts index 6cc48d483d..186e056efc 100644 --- a/packages/backend/src/core/chart/charts/test-grouped.ts +++ b/packages/backend/src/core/chart/charts/test-grouped.ts @@ -48,12 +48,12 @@ export default class TestGroupedChart extends Chart { // eslint-d } @bindThis - public async increment(group: string): Promise { + public increment(group: string): void { if (this.total[group] == null) this.total[group] = 0; this.total[group]++; - await this.commit({ + this.commit({ 'foo.total': 1, 'foo.inc': 1, }, group); diff --git a/packages/backend/src/core/chart/charts/test-intersection.ts b/packages/backend/src/core/chart/charts/test-intersection.ts index d0ae1dab24..f84d498d52 100644 --- a/packages/backend/src/core/chart/charts/test-intersection.ts +++ b/packages/backend/src/core/chart/charts/test-intersection.ts @@ -44,15 +44,15 @@ export default class TestIntersectionChart extends Chart { // esl } @bindThis - public async addA(key: string): Promise { - await this.commit({ + public addA(key: string): void { + this.commit({ a: [key], }); } @bindThis - public async addB(key: string): Promise { - await this.commit({ + public addB(key: string): void { + this.commit({ b: [key], }); } diff --git a/packages/backend/src/core/chart/charts/test-unique.ts b/packages/backend/src/core/chart/charts/test-unique.ts index 54a081fe2a..10a7e0dad2 100644 --- a/packages/backend/src/core/chart/charts/test-unique.ts +++ b/packages/backend/src/core/chart/charts/test-unique.ts @@ -44,8 +44,8 @@ export default class TestUniqueChart extends Chart { // eslint-di } @bindThis - public async uniqueIncrement(key: string): Promise { - await this.commit({ + public uniqueIncrement(key: string): void { + this.commit({ foo: [key], }); } diff --git a/packages/backend/src/core/chart/charts/test.ts b/packages/backend/src/core/chart/charts/test.ts index e95259f3b2..b281d50e67 100644 --- a/packages/backend/src/core/chart/charts/test.ts +++ b/packages/backend/src/core/chart/charts/test.ts @@ -48,20 +48,20 @@ export default class TestChart extends Chart { // eslint-disable- } @bindThis - public async increment(): Promise { + public increment(): void { this.total++; - await this.commit({ + this.commit({ 'foo.total': 1, 'foo.inc': 1, }); } @bindThis - public async decrement(): Promise { + public decrement(): void { this.total--; - await this.commit({ + this.commit({ 'foo.total': -1, 'foo.dec': 1, }); diff --git a/packages/backend/src/core/chart/charts/users.ts b/packages/backend/src/core/chart/charts/users.ts index 91bf972371..ca5e58f07b 100644 --- a/packages/backend/src/core/chart/charts/users.ts +++ b/packages/backend/src/core/chart/charts/users.ts @@ -61,10 +61,10 @@ export default class UsersChart extends Chart { // eslint-disable } @bindThis - public async update(user: { id: MiUser['id'], host: MiUser['host'] }, isAdditional: boolean): Promise { + public update(user: { id: MiUser['id'], host: MiUser['host'] }, isAdditional: boolean): void { const prefix = this.userEntityService.isLocalUser(user) ? 'local' : 'remote'; - await this.commit({ + this.commit({ [`${prefix}.total`]: isAdditional ? 1 : -1, [`${prefix}.inc`]: isAdditional ? 1 : 0, [`${prefix}.dec`]: isAdditional ? 0 : 1, From 9c1208b6caee2cf746b86a2737d21b09fb425ccf Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Wed, 18 Jun 2025 22:55:56 -0400 Subject: [PATCH 007/107] implement delete-file background task --- packages/backend/src/core/QueueService.ts | 17 +++++++++++ .../BackgroundTaskProcessorService.ts | 30 +++++++++++++++++-- packages/backend/src/queue/types.ts | 10 ++++++- 3 files changed, 53 insertions(+), 4 deletions(-) diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 98b4d6cd8c..16b43ce4f7 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -44,6 +44,7 @@ import type { } from './QueueModule.js'; import type httpSignature from '@peertube/http-signature'; import type * as Bull from 'bullmq'; +import type { MiUser } from '@/models/User.js'; export const QUEUE_TYPES = [ 'system', @@ -967,6 +968,22 @@ export class QueueService implements OnModuleInit { ); } + @bindThis + public async createDeleteFileJob(fileId: string, isExpired?: boolean, deleterId?: string) { + return await this.createBackgroundTask( + 'delete-file', + { + type: 'delete-file', + fileId, + isExpired, + deleterId, + }, + { + id: `delete-file:${fileId}`, + }, + ); + } + private async createBackgroundTask(name: string, data: BackgroundTaskJobData, duplication?: { id: string, ttl?: number }) { return await this.backgroundTaskQueue.add( name, diff --git a/packages/backend/src/queue/processors/BackgroundTaskProcessorService.ts b/packages/backend/src/queue/processors/BackgroundTaskProcessorService.ts index 341c54883e..721e28a73d 100644 --- a/packages/backend/src/queue/processors/BackgroundTaskProcessorService.ts +++ b/packages/backend/src/queue/processors/BackgroundTaskProcessorService.ts @@ -5,7 +5,7 @@ import { Inject, Injectable } from '@nestjs/common'; import * as Bull from 'bullmq'; -import { BackgroundTaskJobData, CheckHibernationBackgroundTask, PostDeliverBackgroundTask, PostInboxBackgroundTask, PostNoteBackgroundTask, UpdateFeaturedBackgroundTask, UpdateInstanceBackgroundTask, UpdateUserTagsBackgroundTask, UpdateUserBackgroundTask, UpdateNoteTagsBackgroundTask } from '@/queue/types.js'; +import { BackgroundTaskJobData, CheckHibernationBackgroundTask, PostDeliverBackgroundTask, PostInboxBackgroundTask, PostNoteBackgroundTask, UpdateFeaturedBackgroundTask, UpdateInstanceBackgroundTask, UpdateUserTagsBackgroundTask, UpdateUserBackgroundTask, UpdateNoteTagsBackgroundTask, DeleteFileBackgroundTask } from '@/queue/types.js'; import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js'; import { QueueLoggerService } from '@/queue/QueueLoggerService.js'; import Logger from '@/logger.js'; @@ -19,9 +19,11 @@ import ApRequestChart from '@/core/chart/charts/ap-request.js'; import FederationChart from '@/core/chart/charts/federation.js'; import { UpdateInstanceQueue } from '@/core/UpdateInstanceQueue.js'; import { NoteCreateService } from '@/core/NoteCreateService.js'; -import type { NotesRepository } from '@/models/_.js'; +import type { DriveFilesRepository, NotesRepository } from '@/models/_.js'; +import { MiUser } from '@/models/_.js'; import { NoteEditService } from '@/core/NoteEditService.js'; import { HashtagService } from '@/core/HashtagService.js'; +import { DriveService } from '@/core/DriveService.js'; @Injectable() export class BackgroundTaskProcessorService { @@ -34,6 +36,9 @@ export class BackgroundTaskProcessorService { @Inject(DI.notesRepository) private readonly notesRepository: NotesRepository, + @Inject(DI.driveFilesRepository) + private readonly driveFilesRepository: DriveFilesRepository, + private readonly apPersonService: ApPersonService, private readonly cacheService: CacheService, private readonly federatedInstanceService: FederatedInstanceService, @@ -45,6 +50,7 @@ export class BackgroundTaskProcessorService { private readonly noteCreateService: NoteCreateService, private readonly noteEditService: NoteEditService, private readonly hashtagService: HashtagService, + private readonly driveService: DriveService, queueLoggerService: QueueLoggerService, ) { @@ -68,9 +74,11 @@ export class BackgroundTaskProcessorService { return await this.processPostInbox(job.data); } else if (job.data.type === 'post-note') { return await this.processPostNote(job.data); - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition } else if (job.data.type === 'check-hibernation') { return await this.processCheckHibernation(job.data); + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + } else if (job.data.type === 'delete-file') { + return await this.processDeleteFile(job.data); } else { this.logger.warn(`Can't process unknown job type "${job.data}"; this is likely a bug. Full job data:`, job.data); throw new Error(`Unknown job type ${job.data}, see system logs for details`); @@ -236,4 +244,20 @@ export class BackgroundTaskProcessorService { await this.noteCreateService.checkHibernation(followers); return 'ok'; } + + private async processDeleteFile(task: DeleteFileBackgroundTask): Promise { + const file = await this.driveFilesRepository.findOneBy({ id: task.fileId }); + if (!file) return `Skipping delete-file task: file ${task.fileId} has been deleted`; + + let deleter: MiUser | undefined = undefined; + if (task.deleterId) { + deleter = await this.cacheService.findOptionalUserById(task.deleterId); + if (!deleter) { + this.logger.warn(`[delete-file] Deleting user ${task.deleterId} has been deleted; proceeding with null deleter`); + } + } + + await this.driveService.deleteFileSync(file, task.isExpired, deleter); + return 'ok'; + } } diff --git a/packages/backend/src/queue/types.ts b/packages/backend/src/queue/types.ts index 64ae679033..7f7f1b8a74 100644 --- a/packages/backend/src/queue/types.ts +++ b/packages/backend/src/queue/types.ts @@ -178,7 +178,8 @@ export type BackgroundTaskJobData = PostDeliverBackgroundTask | PostInboxBackgroundTask | PostNoteBackgroundTask | - CheckHibernationBackgroundTask; + CheckHibernationBackgroundTask | + DeleteFileBackgroundTask; export type UpdateUserBackgroundTask = { type: 'update-user'; @@ -227,3 +228,10 @@ export type CheckHibernationBackgroundTask = { type: 'check-hibernation'; userId: string; }; + +export type DeleteFileBackgroundTask = { + type: 'delete-file'; + fileId: string; + isExpired?: boolean; + deleterId?: string; +}; From 7934643090dddc2973e7625e411561325af1a64e Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Wed, 18 Jun 2025 22:56:46 -0400 Subject: [PATCH 008/107] use delete-file task for async file deletions --- packages/backend/src/core/DriveService.ts | 40 +++++------------------ 1 file changed, 9 insertions(+), 31 deletions(-) diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts index 94289f237d..468c63a9cc 100644 --- a/packages/backend/src/core/DriveService.ts +++ b/packages/backend/src/core/DriveService.ts @@ -470,7 +470,7 @@ export class DriveService { for (const fileId of exceedFileIds) { const file = await this.driveFilesRepository.findOneBy({ id: fileId }); if (file == null) continue; - this.deleteFile(file, true); + await this.deleteFile(file, true); } } @@ -718,14 +718,14 @@ export class DriveService { if (values.isSensitive !== undefined && values.isSensitive !== file.isSensitive) { const user = file.userId ? await this.usersRepository.findOneByOrFail({ id: file.userId }) : null; if (values.isSensitive) { - this.moderationLogService.log(updater, 'markSensitiveDriveFile', { + await this.moderationLogService.log(updater, 'markSensitiveDriveFile', { fileId: file.id, fileUserId: file.userId, fileUserUsername: user?.username ?? null, fileUserHost: user?.host ?? null, }); } else { - this.moderationLogService.log(updater, 'unmarkSensitiveDriveFile', { + await this.moderationLogService.log(updater, 'unmarkSensitiveDriveFile', { fileId: file.id, fileUserId: file.userId, fileUserUsername: user?.username ?? null, @@ -739,30 +739,8 @@ export class DriveService { } @bindThis - public async deleteFile(file: MiDriveFile, isExpired = false, deleter?: { id: string }) { - if (file.storedInternal) { - this.deleteLocalFile(file.accessKey!); - - if (file.thumbnailUrl) { - this.deleteLocalFile(file.thumbnailAccessKey!); - } - - if (file.webpublicUrl) { - this.deleteLocalFile(file.webpublicAccessKey!); - } - } else if (!file.isLink) { - this.queueService.createDeleteObjectStorageFileJob(file.accessKey!); - - if (file.thumbnailUrl) { - this.queueService.createDeleteObjectStorageFileJob(file.thumbnailAccessKey!); - } - - if (file.webpublicUrl) { - this.queueService.createDeleteObjectStorageFileJob(file.webpublicAccessKey!); - } - } - - this.deletePostProcess(file, isExpired, deleter); + public async deleteFile(file: MiDriveFile, isExpired = false, deleter?: MiUser) { + await this.queueService.createDeleteFileJob(file.id, isExpired, deleter?.id); } @bindThis @@ -793,14 +771,14 @@ export class DriveService { await Promise.all(promises); - this.deletePostProcess(file, isExpired, deleter); + await this.deletePostProcess(file, isExpired, deleter); } @bindThis private async deletePostProcess(file: MiDriveFile, isExpired = false, deleter?: { id: string }) { // リモートファイル期限切れ削除後は直リンクにする if (isExpired && file.userHost !== null && file.uri != null) { - this.driveFilesRepository.update(file.id, { + await this.driveFilesRepository.update(file.id, { isLink: true, url: file.uri, thumbnailUrl: null, @@ -812,7 +790,7 @@ export class DriveService { webpublicAccessKey: 'webpublic-' + randomUUID(), }); } else { - this.driveFilesRepository.delete(file.id); + await this.driveFilesRepository.delete(file.id); } this.driveChart.update(file, false); @@ -831,7 +809,7 @@ export class DriveService { if (deleter && await this.roleService.isModerator(deleter) && (file.userId !== deleter.id)) { const user = file.userId ? await this.usersRepository.findOneByOrFail({ id: file.userId }) : null; - this.moderationLogService.log(deleter, 'deleteDriveFile', { + await this.moderationLogService.log(deleter, 'deleteDriveFile', { fileId: file.id, fileUserId: file.userId, fileUserUsername: user?.username ?? null, From 8e75d6149ec24e99dd6ba188143c6000c4582cc8 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Thu, 19 Jun 2025 13:16:19 -0400 Subject: [PATCH 009/107] fix stall in EnableInstanceHOTUpdates1750217001651 migration --- .../migration/1750217001651-enable-instance-HOT-updates.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/backend/migration/1750217001651-enable-instance-HOT-updates.js b/packages/backend/migration/1750217001651-enable-instance-HOT-updates.js index 995c3ed445..12b5f65ac2 100644 --- a/packages/backend/migration/1750217001651-enable-instance-HOT-updates.js +++ b/packages/backend/migration/1750217001651-enable-instance-HOT-updates.js @@ -18,9 +18,6 @@ export class EnableInstanceHOTUpdates1750217001651 { async up(queryRunner) { await queryRunner.query(`ALTER TABLE "instance" SET (fillfactor = 50)`); - - // Vacuum can't run inside a transaction block, so query directly from the connection. - await queryRunner.connection.query(`VACUUM (FULL, VERBOSE) "instance"`); } async down(queryRunner) { From a4440e43a6f5e4843420f58530d6b039a33e5050 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Thu, 19 Jun 2025 13:38:03 -0400 Subject: [PATCH 010/107] add UpdateLatestNote, PostSuspend, and PostUnsuspend background tasks --- .../1750353421706-more-note_edit-columns.js | 75 +++++++++ .../backend/src/core/LatestNoteService.ts | 50 +++--- .../backend/src/core/NoteCreateService.ts | 66 ++++---- .../backend/src/core/NoteDeleteService.ts | 14 +- packages/backend/src/core/NoteEditService.ts | 147 +++++++++-------- packages/backend/src/core/QueueService.ts | 152 ++++++------------ .../backend/src/core/UserSuspendService.ts | 30 ++-- packages/backend/src/misc/is-renote.ts | 101 ++++++++++-- packages/backend/src/models/LatestNote.ts | 6 +- packages/backend/src/models/NoteEdit.ts | 62 ++++++- .../BackgroundTaskProcessorService.ts | 77 ++++++++- packages/backend/src/queue/types.ts | 21 ++- 12 files changed, 512 insertions(+), 289 deletions(-) create mode 100644 packages/backend/migration/1750353421706-more-note_edit-columns.js diff --git a/packages/backend/migration/1750353421706-more-note_edit-columns.js b/packages/backend/migration/1750353421706-more-note_edit-columns.js new file mode 100644 index 0000000000..3bc24b1acc --- /dev/null +++ b/packages/backend/migration/1750353421706-more-note_edit-columns.js @@ -0,0 +1,75 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class MoreNoteEditColumns1750353421706 { + name = 'MoreNoteEditColumns1750353421706' + + async up(queryRunner) { + // Update column types + await queryRunner.query(`ALTER TABLE "note_edit" ALTER COLUMN "cw" TYPE text USING "cw"::text`); + + // Rename columns + await queryRunner.query(`ALTER TABLE "note_edit" RENAME COLUMN "oldText" TO "text"`); + await queryRunner.query(`ALTER TABLE "note_edit" RENAME COLUMN "cw" TO "newCw"`); + + // Add new fields + await queryRunner.query(`ALTER TABLE "note_edit" ADD "userId" character varying(32)`); + await queryRunner.query(`COMMENT ON COLUMN "note_edit"."userId" IS 'The ID of author.'`); + await queryRunner.query(`ALTER TABLE "note_edit" ADD CONSTRAINT "FK_7f1ded0f6e8a5bef701b7e698ab" FOREIGN KEY ("userId") REFERENCES "user"("id") ON DELETE CASCADE ON UPDATE NO ACTION`); + + await queryRunner.query(`ALTER TABLE "note_edit" ADD "renoteId" character varying(32)`); + await queryRunner.query(`COMMENT ON COLUMN "note_edit"."renoteId" IS 'The ID of renote target. Will always be null for older edits'`); + await queryRunner.query(`ALTER TABLE "note_edit" ADD CONSTRAINT "FK_d3003e5256bcbfad6c3588835c0" FOREIGN KEY ("renoteId") REFERENCES "note"("id") ON DELETE CASCADE ON UPDATE NO ACTION`); + + await queryRunner.query(`ALTER TABLE "note_edit" ADD "replyId" character varying(32)`); + await queryRunner.query(`COMMENT ON COLUMN "note_edit"."replyId" IS 'The ID of reply target. Will always be null for older edits'`); + await queryRunner.query(`ALTER TABLE "note_edit" ADD CONSTRAINT "FK_f34b53ab9b39774ca014972ad84" FOREIGN KEY ("replyId") REFERENCES "note"("id") ON DELETE CASCADE ON UPDATE NO ACTION`); + + await queryRunner.query(`ALTER TABLE "note_edit" ADD "visibility" "public"."note_visibility_enum"`); + + await queryRunner.query(`ALTER TABLE "note_edit" ADD "cw" text`); + await queryRunner.query(`COMMENT ON COLUMN "note_edit"."cw" IS 'Will always be null for older edits'`); + + await queryRunner.query(`ALTER TABLE "note_edit" ADD "hasPoll" boolean NOT NULL DEFAULT false`); + await queryRunner.query(`COMMENT ON COLUMN "note_edit"."hasPoll" IS 'Whether this revision had a poll. Will always be false for older edits'`); + + // Populate non-nullable fields + await queryRunner.query(` + UPDATE "note_edit" "e" + SET + "visibility" = "n"."visibility", + "userId" = "n"."userId" + FROM "note" "n" + WHERE "n"."id" = "e"."noteId" + `); + await queryRunner.query(`ALTER TABLE "note_edit" ALTER COLUMN "visibility" SET NOT NULL`); + await queryRunner.query(`ALTER TABLE "note_edit" ALTER COLUMN "userId" SET NOT NULL`); + } + + async down(queryRunner) { + // Drop new columns + await queryRunner.query(`ALTER TABLE "note_edit" DROP COLUMN "visibility"`); + await queryRunner.query(`ALTER TABLE "note_edit" DROP COLUMN "hasPoll"`); + await queryRunner.query(`ALTER TABLE "note_edit" DROP COLUMN "cw"`); + + await queryRunner.query(`ALTER TABLE "note_edit" DROP COLUMN "userId"`); + + await queryRunner.query(`ALTER TABLE "note_edit" DROP CONSTRAINT "FK_f34b53ab9b39774ca014972ad84"`); + await queryRunner.query(`ALTER TABLE "note_edit" DROP COLUMN "replyId"`); + + await queryRunner.query(`ALTER TABLE "note_edit" DROP CONSTRAINT "FK_d3003e5256bcbfad6c3588835c0"`); + await queryRunner.query(`ALTER TABLE "note_edit" DROP COLUMN "renoteId"`); + + await queryRunner.query(`ALTER TABLE "note_edit" DROP CONSTRAINT "FK_7f1ded0f6e8a5bef701b7e698ab"`); + await queryRunner.query(`ALTER TABLE "note_edit" DROP COLUMN "userId"`); + + // Rename new columns + await queryRunner.query(`ALTER TABLE "note_edit" RENAME COLUMN "text" TO "oldText"`); + await queryRunner.query(`ALTER TABLE "note_edit" RENAME COLUMN "newCw" TO "cw"`); + + // Revert column types + await queryRunner.query(`ALTER TABLE "note_edit" ALTER COLUMN "cw" TYPE varchar(512) USING "cw"::varchar(512)`); + } +} diff --git a/packages/backend/src/core/LatestNoteService.ts b/packages/backend/src/core/LatestNoteService.ts index 63f973c6c6..e714b50320 100644 --- a/packages/backend/src/core/LatestNoteService.ts +++ b/packages/backend/src/core/LatestNoteService.ts @@ -1,18 +1,14 @@ import { Inject, Injectable } from '@nestjs/common'; import { Not } from 'typeorm'; -import { MiNote } from '@/models/Note.js'; -import { isPureRenote } from '@/misc/is-renote.js'; +import { isPureRenote, MinimalNote } from '@/misc/is-renote.js'; import { SkLatestNote } from '@/models/LatestNote.js'; import { DI } from '@/di-symbols.js'; -import type { LatestNotesRepository, NotesRepository } from '@/models/_.js'; -import { LoggerService } from '@/core/LoggerService.js'; -import Logger from '@/logger.js'; -import { QueryService } from './QueryService.js'; +import type { LatestNotesRepository, MiNote, NotesRepository } from '@/models/_.js'; +import { QueryService } from '@/core/QueryService.js'; +import { QueueService } from '@/core/QueueService.js'; @Injectable() export class LatestNoteService { - private readonly logger: Logger; - constructor( @Inject(DI.notesRepository) private readonly notesRepository: NotesRepository, @@ -21,19 +17,23 @@ export class LatestNoteService { private readonly latestNotesRepository: LatestNotesRepository, private readonly queryService: QueryService, - loggerService: LoggerService, - ) { - this.logger = loggerService.getLogger('LatestNoteService'); + private readonly queueService: QueueService, + ) {} + + async handleUpdatedNoteDeferred(note: MiNote): Promise { + await this.queueService.createUpdateLatestNoteJob(note); } - handleUpdatedNoteBG(before: MiNote, after: MiNote): void { - this - .handleUpdatedNote(before, after) - .catch(err => this.logger.error('Unhandled exception while updating latest_note (after update):', err)); + async handleCreatedNoteDeferred(note: MiNote): Promise { + await this.queueService.createUpdateLatestNoteJob(note); } - async handleUpdatedNote(before: MiNote, after: MiNote): Promise { - // If the key didn't change, then there's nothing to update + async handleDeletedNoteDeferred(note: MiNote): Promise { + await this.queueService.createUpdateLatestNoteJob(note); + } + + async handleUpdatedNote(before: MinimalNote, after: MinimalNote): Promise { + // If the key didn't change, then there's nothing to update. if (SkLatestNote.areEquivalent(before, after)) return; // Simulate update as delete + create @@ -41,13 +41,7 @@ export class LatestNoteService { await this.handleCreatedNote(after); } - handleCreatedNoteBG(note: MiNote): void { - this - .handleCreatedNote(note) - .catch(err => this.logger.error('Unhandled exception while updating latest_note (after create):', err)); - } - - async handleCreatedNote(note: MiNote): Promise { + async handleCreatedNote(note: MinimalNote): Promise { // Ignore DMs. // Followers-only posts are *included*, as this table is used to back the "following" feed. if (note.visibility === 'specified') return; @@ -71,13 +65,7 @@ export class LatestNoteService { await this.latestNotesRepository.upsert(latestNote, ['userId', 'isPublic', 'isReply', 'isQuote']); } - handleDeletedNoteBG(note: MiNote): void { - this - .handleDeletedNote(note) - .catch(err => this.logger.error('Unhandled exception while updating latest_note (after delete):', err)); - } - - async handleDeletedNote(note: MiNote): Promise { + async handleDeletedNote(note: MinimalNote): Promise { // If it's a DM, then it can't possibly be the latest note so we can safely skip this. if (note.visibility === 'specified') return; diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 484f7e01d0..9b7ef57d9c 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -458,9 +458,6 @@ export class NoteCreateService implements OnApplicationShutdown { const note = await this.insertNote(user, data, tags, emojis, mentionedUsers); - // Update the Latest Note index / following feed - this.latestNoteService.handleCreatedNoteBG(note); - await this.queueService.createPostNoteJob(note.id, silent, 'create'); return note; @@ -474,7 +471,7 @@ export class NoteCreateService implements OnApplicationShutdown { isBot: MiUser['isBot']; noindex: MiUser['noindex']; }, data: Option): Promise { - return this.create(user, data, true); + return await this.create(user, data, true); } @bindThis @@ -583,7 +580,7 @@ export class NoteCreateService implements OnApplicationShutdown { host: MiUser['host']; isBot: MiUser['isBot']; noindex: MiUser['noindex']; - }, data: Option, silent: boolean, tags: string[], mentionedUsers: MinimumUser[]) { + }, data: Option, silent: boolean, mentionedUsers: MinimumUser[]) { this.notesChart.update(note, true); if (note.visibility !== 'specified' && (this.meta.enableChartsForRemoteUser || (user.host == null))) { this.perUserNotesChart.update(user, note, true); @@ -612,20 +609,20 @@ export class NoteCreateService implements OnApplicationShutdown { if (!this.isRenote(note) || this.isQuote(note)) { // Increment notes count (user) - this.incNotesCountOfUser(user); + await this.incNotesCountOfUser(user); } else { - this.usersRepository.update({ id: user.id }, { updatedAt: this.timeService.date }); + await this.usersRepository.update({ id: user.id }, { updatedAt: this.timeService.date }); } - this.pushToTl(note, user); + await this.pushToTl(note, user); - this.antennaService.addNoteToAntennas({ + await this.antennaService.addNoteToAntennas({ ...note, channel: data.channel ?? null, }, user); if (data.reply) { - this.saveReply(data.reply, note); + await this.saveReply(data.reply, note); } if (data.reply == null) { @@ -654,12 +651,12 @@ export class NoteCreateService implements OnApplicationShutdown { } if (this.isRenote(data) && !this.isQuote(data) && data.renote.userId !== user.id && !user.isBot) { - this.incRenoteCount(data.renote, user); + await this.incRenoteCount(data.renote, user); } if (data.poll && data.poll.expiresAt) { const delay = data.poll.expiresAt.getTime() - this.timeService.now; - this.queueService.endedPollNotificationQueue.add(note.id, { + await this.queueService.endedPollNotificationQueue.add(note.id, { noteId: note.id, }, { jobId: `pollEnd_${note.id}`, @@ -683,9 +680,9 @@ export class NoteCreateService implements OnApplicationShutdown { this.globalEventService.publishNotesStream(noteObj); - this.roleService.addNoteToRoleTimeline(noteObj); + await this.roleService.addNoteToRoleTimeline(noteObj); - this.webhookService.enqueueUserWebhook(user.id, 'note', { note: noteObj }); + await this.webhookService.enqueueUserWebhook(user.id, 'note', { note: noteObj }); const nm = new NotificationManager(this.mutingsRepository, this.notificationService, user, note); @@ -714,7 +711,7 @@ export class NoteCreateService implements OnApplicationShutdown { if (!isThreadMuted && !muted) { nm.push(data.reply.userId, 'reply'); this.globalEventService.publishMainStream(data.reply.userId, 'reply', noteObj); - this.webhookService.enqueueUserWebhook(data.reply.userId, 'reply', { note: noteObj }); + await this.webhookService.enqueueUserWebhook(data.reply.userId, 'reply', { note: noteObj }); } } } @@ -745,15 +742,15 @@ export class NoteCreateService implements OnApplicationShutdown { // Publish event if ((user.id !== data.renote.userId) && data.renote.userHost === null) { this.globalEventService.publishMainStream(data.renote.userId, 'renote', noteObj); - this.webhookService.enqueueUserWebhook(data.renote.userId, 'renote', { note: noteObj }); + await this.webhookService.enqueueUserWebhook(data.renote.userId, 'renote', { note: noteObj }); } } - nm.notify(); + await nm.notify(); //#region AP deliver if (!data.localOnly && isLocalUser(user)) { - trackTask(async () => { + await trackTask(async () => { const noteActivity = await this.apRendererService.renderNoteOrRenoteActivity(note, user, { renote: data.renote }); const dm = this.apDeliverManagerService.createDeliverManager(user, noteActivity); @@ -790,12 +787,12 @@ export class NoteCreateService implements OnApplicationShutdown { } if (data.channel) { - this.channelsRepository.increment({ id: data.channel.id }, 'notesCount', 1); - this.channelsRepository.update(data.channel.id, { + await this.channelsRepository.increment({ id: data.channel.id }, 'notesCount', 1); + await this.channelsRepository.update(data.channel.id, { lastNotedAt: this.timeService.date, }); - this.notesRepository.countBy({ + await this.notesRepository.countBy({ userId: user.id, channelId: data.channel.id, }).then(count => { @@ -807,8 +804,11 @@ export class NoteCreateService implements OnApplicationShutdown { }); } + // Update the Latest Note index / following feed + await this.latestNoteService.handleCreatedNoteDeferred(note); + // Register to search database - if (!user.noindex) this.index(note); + if (!user.noindex) await this.index(note); } /** @@ -841,12 +841,12 @@ export class NoteCreateService implements OnApplicationShutdown { if (policies.canTrend) { if (renote.channelId != null) { if (renote.replyId == null) { - this.featuredService.updateInChannelNotesRanking(renote.channelId, renote, 5); + await this.featuredService.updateInChannelNotesRanking(renote.channelId, renote, 5); } } else { if (renote.visibility === 'public' && renote.userHost == null && renote.replyId == null) { - this.featuredService.updateGlobalNotesRanking(renote, 5); - this.featuredService.updatePerUserNotesRanking(renote.userId, renote, 5); + await this.featuredService.updateGlobalNotesRanking(renote, 5); + await this.featuredService.updatePerUserNotesRanking(renote.userId, renote, 5); } } } @@ -880,7 +880,7 @@ export class NoteCreateService implements OnApplicationShutdown { }); this.globalEventService.publishMainStream(u.id, 'mention', detailPackedNote); - this.webhookService.enqueueUserWebhook(u.id, 'mention', { note: detailPackedNote }); + await this.webhookService.enqueueUserWebhook(u.id, 'mention', { note: detailPackedNote }); // Create notification nm.push(u.id, 'mention'); @@ -888,20 +888,20 @@ export class NoteCreateService implements OnApplicationShutdown { } @bindThis - private saveReply(reply: MiNote, note: MiNote) { - this.notesRepository.increment({ id: reply.id }, 'repliesCount', 1); + private async saveReply(reply: MiNote, note: MiNote) { + await this.notesRepository.increment({ id: reply.id }, 'repliesCount', 1); } @bindThis - private index(note: MiNote) { + private async index(note: MiNote) { if (note.text == null && note.cw == null) return; - this.searchService.indexNote(note); + await this.searchService.indexNote(note); } @bindThis - private incNotesCountOfUser(user: { id: MiUser['id']; }) { - this.usersRepository.createQueryBuilder().update() + private async incNotesCountOfUser(user: { id: MiUser['id']; }) { + await this.usersRepository.createQueryBuilder().update() .set({ updatedAt: this.timeService.date, notesCount: () => '"notesCount" + 1', @@ -1037,7 +1037,7 @@ export class NoteCreateService implements OnApplicationShutdown { // checkHibernation moved to HibernateUsersProcessorService } - r.exec(); + await r.exec(); } // checkHibernation moved to HibernateUsersProcessorService diff --git a/packages/backend/src/core/NoteDeleteService.ts b/packages/backend/src/core/NoteDeleteService.ts index c20b20c7ca..e9cf9bebef 100644 --- a/packages/backend/src/core/NoteDeleteService.ts +++ b/packages/backend/src/core/NoteDeleteService.ts @@ -158,7 +158,11 @@ export class NoteDeleteService { userId: user.id, }); - this.latestNoteService.handleDeletedNoteBG(note); + // Update the Latest Note index / following feed + this.latestNoteService.handleDeletedNoteDeferred(note); + for (const cascadingNote of cascadingNotes) { + this.latestNoteService.handleDeletedNote(cascadingNote); + } if (deleter && (note.userId !== deleter.id)) { const user = await this.usersRepository.findOneByOrFail({ id: note.userId }); @@ -174,14 +178,14 @@ export class NoteDeleteService { .map(n => n.uri) .filter((u): u is string => u != null); if (deletedUris.length > 0) { - this.apLogService.deleteObjectLogs(deletedUris) - .catch(err => this.logger.error(err, `Failed to delete AP logs for note '${note.uri}'`)); + trackPromise(this.apLogService.deleteObjectLogs(deletedUris) + .catch(err => this.logger.error(err, `Failed to delete AP logs for note '${note.uri}'`))); } } @bindThis - private decNotesCountOfUser(user: { id: MiUser['id']; }) { - this.usersRepository.createQueryBuilder().update() + private async decNotesCountOfUser(user: { id: MiUser['id']; }) { + await this.usersRepository.createQueryBuilder().update() .set({ updatedAt: this.timeService.date, notesCount: () => '"notesCount" - 1', diff --git a/packages/backend/src/core/NoteEditService.ts b/packages/backend/src/core/NoteEditService.ts index 21991e2966..0c595cbf20 100644 --- a/packages/backend/src/core/NoteEditService.ts +++ b/packages/backend/src/core/NoteEditService.ts @@ -14,7 +14,7 @@ import { extractCustomEmojisFromMfm } from '@/misc/extract-custom-emojis-from-mf import { extractHashtags } from '@/misc/extract-hashtags.js'; import type { IMentionedRemoteUsers } from '@/models/Note.js'; import { MiNote } from '@/models/Note.js'; -import type { NoteEditRepository, ChannelFollowingsRepository, ChannelsRepository, FollowingsRepository, InstancesRepository, MiFollowing, MiMeta, MutingsRepository, NotesRepository, NoteThreadMutingsRepository, UserListMembershipsRepository, UserProfilesRepository, UsersRepository, PollsRepository } from '@/models/_.js'; +import type { NoteEditsRepository, ChannelFollowingsRepository, ChannelsRepository, FollowingsRepository, InstancesRepository, MiFollowing, MiMeta, MutingsRepository, NotesRepository, NoteThreadMutingsRepository, UserListMembershipsRepository, UserProfilesRepository, UsersRepository, PollsRepository } from '@/models/_.js'; import type { MiDriveFile } from '@/models/DriveFile.js'; import type { MiApp } from '@/models/App.js'; import { concat } from '@/misc/prelude/array.js'; @@ -195,8 +195,8 @@ export class NoteEditService implements OnApplicationShutdown { @Inject(DI.channelFollowingsRepository) private channelFollowingsRepository: ChannelFollowingsRepository, - @Inject(DI.noteEditRepository) - private noteEditRepository: NoteEditRepository, + @Inject(DI.noteEditsRepository) + private noteEditsRepository: NoteEditsRepository, @Inject(DI.pollsRepository) private pollsRepository: PollsRepository, @@ -234,29 +234,29 @@ export class NoteEditService implements OnApplicationShutdown { throw new UnrecoverableError('edit failed: missing editid'); } - const oldnote = await this.notesRepository.findOneBy({ + const oldNote = await this.notesRepository.findOneBy({ id: editid, }); - if (oldnote == null) { + if (oldNote == null) { throw new UnrecoverableError(`edit failed for ${editid}: missing oldnote`); } - if (oldnote.userId !== user.id) { + if (oldNote.userId !== user.id) { throw new UnrecoverableError(`edit failed for ${editid}: user is not the note author`); } // we never want to change the replyId, so fetch the original "parent" - if (oldnote.replyId) { - data.reply = await this.notesRepository.findOneBy({ id: oldnote.replyId }); + if (oldNote.replyId) { + data.reply = await this.notesRepository.findOneBy({ id: oldNote.replyId }); } else { data.reply = undefined; } // changing visibility on an edit is ill-defined, let's try to // keep the same visibility as the original note - data.visibility = oldnote.visibility; - data.localOnly = oldnote.localOnly; + data.visibility = oldNote.visibility; + data.localOnly = oldNote.localOnly; // チャンネル外にリプライしたら対象のスコープに合わせる // (クライアントサイドでやっても良い処理だと思うけどとりあえずサーバーサイドで) @@ -463,46 +463,55 @@ export class NoteEditService implements OnApplicationShutdown { } const update: Partial = {}; - if (data.text !== undefined && data.text !== oldnote.text) { + if (data.text !== undefined && data.text !== oldNote.text) { update.text = data.text; } - if (data.cw !== undefined && data.cw !== oldnote.cw) { + if (data.cw !== undefined && data.cw !== oldNote.cw) { update.cw = data.cw; } - if (data.poll !== undefined && oldnote.hasPoll !== !!data.poll) { + if (data.poll !== undefined && oldNote.hasPoll !== !!data.poll) { update.hasPoll = !!data.poll; } - if (data.mandatoryCW !== undefined && oldnote.mandatoryCW !== data.mandatoryCW) { + if (data.mandatoryCW !== undefined && oldNote.mandatoryCW !== data.mandatoryCW) { update.mandatoryCW = data.mandatoryCW; } // TODO deep-compare files - const filesChanged = oldnote.fileIds.length || data.files?.length; + const filesChanged = oldNote.fileIds.length || data.files?.length; - const poll = await this.pollsRepository.findOneBy({ noteId: oldnote.id }); + const oldPoll = await this.pollsRepository.findOneBy({ noteId: oldNote.id }); - const oldPoll = poll ? { choices: poll.choices, multiple: poll.multiple, expiresAt: poll.expiresAt } : null; - const pollChanged = data.poll != null && JSON.stringify(data.poll) !== JSON.stringify(oldPoll); + const oldPollData = oldPoll ? { choices: oldPoll.choices, multiple: oldPoll.multiple, expiresAt: oldPoll.expiresAt } : null; + const pollChanged = + (data.poll == null && oldPoll != null) || + (data.poll != null && oldPoll == null) || + (data.poll != null && oldPoll != null && JSON.stringify(data.poll) !== JSON.stringify(oldPollData)); if (Object.keys(update).length > 0 || filesChanged || pollChanged) { - const exists = await this.noteEditRepository.findOneBy({ noteId: oldnote.id }); + const exists = await this.noteEditsRepository.findOneBy({ noteId: oldNote.id }); - await this.noteEditRepository.insert({ + await this.noteEditsRepository.insert({ id: this.idService.gen(), - noteId: oldnote.id, - oldText: oldnote.text || undefined, + userId: oldNote.userId, + noteId: oldNote.id, + renoteId: oldNote.renoteId, + replyId: oldNote.replyId, + visibility: oldNote.visibility, + text: oldNote.text || undefined, newText: update.text || undefined, - cw: update.cw || undefined, - fileIds: undefined, - oldDate: exists ? oldnote.updatedAt as Date : this.idService.parse(oldnote.id).date, + cw: oldNote.cw || undefined, + newCw: update.cw || undefined, + fileIds: oldNote.fileIds, + oldDate: exists ? oldNote.updatedAt as Date : this.idService.parse(oldNote.id).date, updatedAt: this.timeService.date, + hasPoll: oldPoll != null, }); const note = new MiNote({ - id: oldnote.id, + id: oldNote.id, updatedAt: data.updatedAt ? data.updatedAt : this.timeService.date, fileIds: data.files ? data.files.map(file => file.id) : [], - replyId: oldnote.replyId, + replyId: oldNote.replyId, renoteId: data.renote ? data.renote.id : null, channelId: data.channel ? data.channel.id : null, threadId: data.reply @@ -516,7 +525,7 @@ export class NoteEditService implements OnApplicationShutdown { cw: data.cw ?? null, tags: tags.map(tag => normalizeForSearch(tag)), emojis, - reactions: oldnote.reactions, + reactions: oldNote.reactions, userId: user.id, localOnly: data.localOnly!, reactionAcceptance: data.reactionAcceptance, @@ -535,7 +544,7 @@ export class NoteEditService implements OnApplicationShutdown { renoteUserId: data.renote ? data.renote.userId : null, renoteUserHost: data.renote ? data.renote.userHost : null, userHost: user.host, - reactionAndUserPairCache: oldnote.reactionAndUserPairCache, + reactionAndUserPairCache: oldNote.reactionAndUserPairCache, mandatoryCW: data.mandatoryCW, }); @@ -561,41 +570,44 @@ export class NoteEditService implements OnApplicationShutdown { if (pollChanged) { // Start transaction await this.db.transaction(async transactionalEntityManager => { - await transactionalEntityManager.update(MiNote, oldnote.id, note); + await transactionalEntityManager.update(MiNote, oldNote.id, note); - const poll = new MiPoll({ - noteId: note.id, - choices: data.poll!.choices, - expiresAt: data.poll!.expiresAt, - multiple: data.poll!.multiple, - votes: new Array(data.poll!.choices.length).fill(0), - noteVisibility: note.visibility, - userId: user.id, - userHost: user.host, - channelId: data.channel?.id ?? null, - }); + // Insert or update poll + if (data.poll) { + const poll = new MiPoll({ + noteId: note.id, + choices: data.poll.choices, + expiresAt: data.poll.expiresAt, + multiple: data.poll.multiple, + votes: new Array(data.poll.choices.length).fill(0), + noteVisibility: note.visibility, + userId: user.id, + userHost: user.host, + channelId: data.channel?.id ?? null, + }); - if (!oldnote.hasPoll) { - await transactionalEntityManager.insert(MiPoll, poll); - } else { - await transactionalEntityManager.update(MiPoll, oldnote.id, poll); + if (oldPoll) { + await transactionalEntityManager.update(MiPoll, { pollId: oldPoll.noteId }, poll); + } else { + await transactionalEntityManager.insert(MiPoll, poll); + } + // Delete poll + } else if (oldPoll) { + await transactionalEntityManager.delete(MiPoll, { pollId: oldPoll.noteId }); } }); } else { - await this.notesRepository.update(oldnote.id, note); + await this.notesRepository.update(oldNote.id, note); } // Re-fetch note to get the default values of null / unset fields. const edited = await this.notesRepository.findOneByOrFail({ id: note.id }); - // Update the Latest Note index / following feed - this.latestNoteService.handleUpdatedNoteBG(edited, oldnote); - await this.queueService.createPostNoteJob(note.id, silent, 'edit'); return edited; } else { - return oldnote; + return oldNote; } } @@ -606,7 +618,7 @@ export class NoteEditService implements OnApplicationShutdown { host: MiUser['host']; isBot: MiUser['isBot']; noindex: MiUser['noindex']; - }, data: Option, silent: boolean, tags: string[], mentionedUsers: MinimumUser[]) { + }, data: Option, silent: boolean, mentionedUsers: MinimumUser[]) { // Register host if (this.meta.enableStatsForFederatedInstances) { if (isRemoteUser(user)) { @@ -621,15 +633,15 @@ export class NoteEditService implements OnApplicationShutdown { } } - this.usersRepository.update({ id: user.id }, { updatedAt: this.timeService.date }); + await this.usersRepository.update({ id: user.id }, { updatedAt: this.timeService.date }); // ハッシュタグ更新 - this.pushToTl(note, user); + await this.pushToTl(note, user); if (data.poll && data.poll.expiresAt) { const delay = data.poll.expiresAt.getTime() - this.timeService.now; - this.queueService.endedPollNotificationQueue.remove(`pollEnd:${note.id}`); - this.queueService.endedPollNotificationQueue.add(note.id, { + await this.queueService.endedPollNotificationQueue.remove(`pollEnd:${note.id}`); + await this.queueService.endedPollNotificationQueue.add(note.id, { noteId: note.id, }, { jobId: `pollEnd_${note.id}`, @@ -648,9 +660,9 @@ export class NoteEditService implements OnApplicationShutdown { text: note.text ?? '', }); - this.roleService.addNoteToRoleTimeline(noteObj); + await this.roleService.addNoteToRoleTimeline(noteObj); - this.webhookService.enqueueUserWebhook(user.id, 'note', { note: noteObj }); + await this.webhookService.enqueueUserWebhook(user.id, 'note', { note: noteObj }); const nm = new NotificationManager(this.mutingsRepository, this.notificationService, user, note); @@ -673,16 +685,16 @@ export class NoteEditService implements OnApplicationShutdown { if (!isThreadMuted && !muted) { nm.push(data.reply.userId, 'edited'); this.globalEventService.publishMainStream(data.reply.userId, 'edited', noteObj); - this.webhookService.enqueueUserWebhook(data.reply.userId, 'reply', { note: noteObj }); + await this.webhookService.enqueueUserWebhook(data.reply.userId, 'reply', { note: noteObj }); } } } - nm.notify(); + await nm.notify(); //#region AP deliver if (!data.localOnly && isLocalUser(user)) { - trackTask(async () => { + await trackTask(async () => { const noteActivity = await this.apRendererService.renderNoteOrRenoteActivity(note, user, { renote: data.renote }); const dm = this.apDeliverManagerService.createDeliverManager(user, noteActivity); @@ -737,8 +749,8 @@ export class NoteEditService implements OnApplicationShutdown { } if (data.channel) { - this.channelsRepository.increment({ id: data.channel.id }, 'notesCount', 1); - this.channelsRepository.update(data.channel.id, { + await this.channelsRepository.increment({ id: data.channel.id }, 'notesCount', 1); + await this.channelsRepository.update(data.channel.id, { lastNotedAt: this.timeService.date, }); @@ -754,8 +766,11 @@ export class NoteEditService implements OnApplicationShutdown { }); } + // Update the Latest Note index / following feed + await this.latestNoteService.handleUpdatedNoteDeferred(note); + // Register to search database - if (!user.noindex) this.index(note); + if (!user.noindex) await this.index(note); } @bindThis @@ -776,10 +791,10 @@ export class NoteEditService implements OnApplicationShutdown { } @bindThis - private index(note: MiNote) { + private async index(note: MiNote) { if (note.text == null && note.cw == null) return; - this.searchService.indexNote(note); + await this.searchService.indexNote(note); } @bindThis @@ -909,7 +924,7 @@ export class NoteEditService implements OnApplicationShutdown { // checkHibernation moved to HibernateUsersProcessorService } - r.exec(); + await r.exec(); } // checkHibernation moved to HibernateUsersProcessorService diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 16b43ce4f7..95cc46c8cf 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -19,6 +19,7 @@ import { ApRequestCreator } from '@/core/activitypub/ApRequestService.js'; import { TimeService } from '@/global/TimeService.js'; import type { SystemWebhookPayload } from '@/core/SystemWebhookService.js'; import type { MiNote } from '@/models/Note.js'; +import type { MinimalNote } from '@/misc/is-renote.js'; import { type UserWebhookPayload } from './UserWebhookService.js'; import type { BackgroundTaskJobData, @@ -44,7 +45,6 @@ import type { } from './QueueModule.js'; import type httpSignature from '@peertube/http-signature'; import type * as Bull from 'bullmq'; -import type { MiUser } from '@/models/User.js'; export const QUEUE_TYPES = [ 'system', @@ -846,147 +846,93 @@ export class QueueService implements OnModuleInit { @bindThis public async createUpdateUserJob(userId: string) { - return await this.createBackgroundTask( - 'update-user', - { - type: 'update-user', - userId, - }, - { - id: `update-user:${userId}`, - }, - ); + return await this.createBackgroundTask({ type: 'update-user', userId }, userId); } @bindThis public async createUpdateFeaturedJob(userId: string) { - return await this.createBackgroundTask( - 'update-featured', - { - type: 'update-featured', - userId, - }, - { - id: `update-featured:${userId}`, - }, - ); + return await this.createBackgroundTask({ type: 'update-featured', userId }, userId); } @bindThis public async createUpdateInstanceJob(host: string) { - return await this.createBackgroundTask( - 'update-instance', - { - type: 'update-instance', - host, - }, - { - id: `update-instance:${host}`, - }, - ); + return await this.createBackgroundTask({ type: 'update-instance', host }, host); } @bindThis public async createPostDeliverJob(host: string, result: 'success' | 'temp-fail' | 'perm-fail') { - return await this.createBackgroundTask( - 'post-deliver', - { - type: 'post-deliver', - host, - result, - }, - ); + return await this.createBackgroundTask({ type: 'post-deliver', host, result }); } @bindThis public async createPostInboxJob(host: string) { - return await this.createBackgroundTask( - 'post-inbox', - { - type: 'post-inbox', - host, - }, - ); + return await this.createBackgroundTask({ type: 'post-inbox', host }); } @bindThis public async createPostNoteJob(noteId: string, silent: boolean, type: 'create' | 'edit') { - return await this.createBackgroundTask( - 'post-note', - { - type: 'post-note', - noteId, - silent, - edit: type === 'edit', - }, - { - id: `post-note:${noteId}:${type}`, - }, - ); + const edit = type === 'edit'; + const duplication = `${noteId}:${type}`; + + return await this.createBackgroundTask({ type: 'post-note', noteId, silent, edit }, duplication); } @bindThis public async createCheckHibernationJob(userId: string) { return await this.createBackgroundTask( - 'check-hibernation', - { - type: 'check-hibernation', - userId, - }, - { - id: `check-hibernation:${userId}`, - ttl: 1000 * 60 * 60 * 24, // This is a very heavy task, so only run once per day per user - }, + { type: 'check-hibernation', userId }, + + // This is a very heavy task, so only run once per day per user + { id: `check-hibernation:${userId}`, ttl: 1000 * 60 * 60 * 24 }, ); } @bindThis public async createUpdateUserTagsJob(userId: string) { - return await this.createBackgroundTask( - 'update-user-tags', - { - type: 'update-user-tags', - userId, - }, - { - id: `update-user-tags:${userId}`, - }, - ); + return await this.createBackgroundTask({ type: 'update-user-tags', userId }, userId); } @bindThis public async createUpdateNoteTagsJob(noteId: string) { - return await this.createBackgroundTask( - 'update-note-tags', - { - type: 'update-note-tags', - noteId, - }, - { - id: `update-note-tags:${noteId}`, - }, - ); + return await this.createBackgroundTask({ type: 'update-note-tags', noteId }, noteId); } @bindThis public async createDeleteFileJob(fileId: string, isExpired?: boolean, deleterId?: string) { - return await this.createBackgroundTask( - 'delete-file', - { - type: 'delete-file', - fileId, - isExpired, - deleterId, - }, - { - id: `delete-file:${fileId}`, - }, - ); + return await this.createBackgroundTask({ type: 'delete-file', fileId, isExpired, deleterId }, fileId); } - private async createBackgroundTask(name: string, data: BackgroundTaskJobData, duplication?: { id: string, ttl?: number }) { + @bindThis + public async createUpdateLatestNoteJob(note: MinimalNote) { + // Compact the note to avoid storing the entire thing in Redis, when all we need is minimal data for categorization + const packedNote: MinimalNote = { + id: note.id, + visibility: note.visibility, + userId: note.userId, + replyId: note.replyId, + renoteId: note.renoteId, + hasPoll: note.hasPoll, + text: note.text ? '1' : null, + cw: note.text ? '1' : null, + fileIds: note.fileIds.length > 0 ? ['1'] : [], + }; + + return await this.createBackgroundTask({ type: 'update-latest-note', note: packedNote }, note.id); + } + + @bindThis + public async createPostSuspendJob(userId: string) { + return await this.createBackgroundTask({ type: 'post-suspend', userId }, userId); + } + + @bindThis + public async createPostUnsuspendJob(userId: string) { + return await this.createBackgroundTask({ type: 'post-unsuspend', userId }, userId); + } + + private async createBackgroundTask(data: T, duplication?: string | { id: string, ttl?: number }) { return await this.backgroundTaskQueue.add( - name, + data.type, data, { removeOnComplete: { @@ -1006,7 +952,9 @@ export class QueueService implements OnModuleInit { }, // https://docs.bullmq.io/guide/jobs/deduplication - deduplication: duplication, + deduplication: typeof(duplication) === 'string' + ? { id: `${data.type}:${duplication}` } + : duplication, }, ); }; diff --git a/packages/backend/src/core/UserSuspendService.ts b/packages/backend/src/core/UserSuspendService.ts index 4e42c24383..395657434c 100644 --- a/packages/backend/src/core/UserSuspendService.ts +++ b/packages/backend/src/core/UserSuspendService.ts @@ -17,16 +17,10 @@ import { RelationshipJobData } from '@/queue/types.js'; import { ModerationLogService } from '@/core/ModerationLogService.js'; import { isSystemAccount } from '@/misc/is-system-account.js'; import { CacheService } from '@/core/CacheService.js'; -import { LoggerService } from '@/core/LoggerService.js'; -import type Logger from '@/logger.js'; -import { renderInlineError } from '@/misc/render-inline-error.js'; -import { trackPromise } from '@/misc/promise-tracker.js'; import { InternalEventService } from '@/global/InternalEventService.js'; @Injectable() export class UserSuspendService { - private readonly logger: Logger; - constructor( @Inject(DI.usersRepository) private usersRepository: UsersRepository, @@ -47,11 +41,7 @@ export class UserSuspendService { private moderationLogService: ModerationLogService, private readonly cacheService: CacheService, private readonly internalEventService: InternalEventService, - - loggerService: LoggerService, - ) { - this.logger = loggerService.getLogger('user-suspend'); - } + ) {} @bindThis public async suspend(user: MiUser, moderator: MiUser): Promise { @@ -69,10 +59,7 @@ export class UserSuspendService { userHost: user.host, }); - trackPromise((async () => { - await this.postSuspend(user); - await this.freezeAll(user); - })().catch(e => this.logger.error(`Error suspending user ${user.id}: ${renderInlineError(e)}`))); + await this.queueService.createPostSuspendJob(user.id); } @bindThis @@ -89,14 +76,11 @@ export class UserSuspendService { userHost: user.host, }); - trackPromise((async () => { - await this.postUnsuspend(user); - await this.unFreezeAll(user); - })().catch(e => this.logger.error(`Error un-suspending for user ${user.id}: ${renderInlineError(e)}`))); + await this.queueService.createPostUnsuspendJob(user.id); } @bindThis - private async postSuspend(user: { id: MiUser['id']; host: MiUser['host'] }): Promise { + public async postSuspend(user: MiUser): Promise { this.globalEventService.publishInternalEvent('userChangeSuspendedState', { id: user.id, isSuspended: true }); /* @@ -132,10 +116,12 @@ export class UserSuspendService { await this.queueService.deliverMany(user, content, queue); } + + await this.freezeAll(user); } @bindThis - private async postUnsuspend(user: MiUser): Promise { + public async postUnsuspend(user: MiUser): Promise { this.globalEventService.publishInternalEvent('userChangeSuspendedState', { id: user.id, isSuspended: false }); if (this.userEntityService.isLocalUser(user)) { @@ -162,6 +148,8 @@ export class UserSuspendService { await this.queueService.deliverMany(user, content, queue); } + + await this.unFreezeAll(user); } @bindThis diff --git a/packages/backend/src/misc/is-renote.ts b/packages/backend/src/misc/is-renote.ts index fcaafaf95a..cb88b64b8c 100644 --- a/packages/backend/src/misc/is-renote.ts +++ b/packages/backend/src/misc/is-renote.ts @@ -5,42 +5,51 @@ import type { MiNote } from '@/models/Note.js'; import type { Packed } from '@/misc/json-schema.js'; +import type { NoteEdit } from '@/models/NoteEdit.js'; // NoteEntityService.isPureRenote とよしなにリンク -type Renote = +export type Renote = MiNote & { renoteId: NonNullable }; -type Quote = +export type Quote = Renote & ({ text: NonNullable } | { cw: NonNullable } | { replyId: NonNullable - reply: NonNullable + reply: NonNullable // TODO this is wrong } | { hasPoll: true + } | { + fileIds: [string, ...string[]] }); -type PureRenote = +export type PureRenote = Renote & { text: null, cw: null, replyId: null, hasPoll: false, - fileIds: { - length: 0, - }, + fileIds: [], }; -export function isRenote(note: MiNote): note is Renote { +export function isRenote(note: MiNote): note is Renote; +export function isRenote(note: NoteEdit): note is RenoteEdit; +export function isRenote(note: MinimalNote): note is MinimalRenote; +export function isRenote(note: MiNote | NoteEdit | MinimalNote): note is Renote | RenoteEdit | MinimalRenote; +export function isRenote(note: MiNote | NoteEdit | MinimalNote): note is Renote | RenoteEdit | MinimalRenote { return note.renoteId != null; } -export function isQuote(note: Renote): note is Quote { +export function isQuote(note: Renote): note is Quote; +export function isQuote(note: RenoteEdit): note is QuoteEdit; +export function isQuote(note: MinimalNote): note is MinimalQuote; +export function isQuote(note: Renote | RenoteEdit | MinimalNote): note is Quote | QuoteEdit | MinimalQuote; +export function isQuote(note: Renote | RenoteEdit | MinimalNote): note is Quote | QuoteEdit | MinimalQuote { // NOTE: SYNC WITH NoteCreateService.isQuote return note.text != null || note.cw != null || @@ -49,7 +58,11 @@ export function isQuote(note: Renote): note is Quote { note.fileIds.length > 0; } -export function isPureRenote(note: MiNote): note is PureRenote { +export function isPureRenote(note: MiNote): note is PureRenote; +export function isPureRenote(note: NoteEdit): note is PureRenoteEdit; +export function isPureRenote(note: MinimalNote): note is MinimalPureRenote; +export function isPureRenote(note: MiNote | NoteEdit | MinimalNote): note is PureRenote | PureRenoteEdit | MinimalPureRenote; +export function isPureRenote(note: MiNote | NoteEdit | MinimalNote): note is PureRenote | PureRenoteEdit | MinimalPureRenote { return isRenote(note) && !isQuote(note); } @@ -68,15 +81,16 @@ type PackedQuote = } | { poll: NonNullable['poll']> } | { - fileIds: NonNullable['fileIds']> + fileIds: [string, ...string[]] }); type PackedPureRenote = PackedRenote & { - text: NonNullable['text']>; - cw: NonNullable['cw']>; - replyId: NonNullable['replyId']>; - poll: NonNullable['poll']>; - fileIds: NonNullable['fileIds']>; + text: null; + cw: null; + replyId: null; + reply: null; + poll: null; + fileIds: []; }; export function isRenotePacked(note: Packed<'Note'>): note is PackedRenote { @@ -94,3 +108,58 @@ export function isQuotePacked(note: PackedRenote): note is PackedQuote { export function isPackedPureRenote(note: Packed<'Note'>): note is PackedPureRenote { return isRenotePacked(note) && !isQuotePacked(note); } + +export type RenoteEdit = + NoteEdit & { + renoteId: NonNullable + }; + +export type QuoteEdit = + RenoteEdit & ({ + text: NonNullable + } | { + cw: NonNullable + } | { + replyId: NonNullable + } | { + hasPoll: true + } | { + fileIds: [string, ...string[]], + }); + +export type PureRenoteEdit = + RenoteEdit & { + text: null, + cw: null, + replyId: null, + reply: null, + hasPoll: false, + fileIds: [], + }; + +export type MinimalNote = Pick; + +export type MinimalRenote = MinimalNote & { + renoteId: string; +}; + +export type MinimalQuote = MinimalRenote & ({ + text: NonNullable +} | { + cw: NonNullable +} | { + replyId: NonNullable +} | { + hasPoll: true +} | { + fileIds: [string, ...string[]], +}); + +export type MinimalPureRenote = MinimalRenote & { + text: null, + cw: null, + replyId: null, + reply: null, + hasPoll: false, + fileIds: [], +}; diff --git a/packages/backend/src/models/LatestNote.ts b/packages/backend/src/models/LatestNote.ts index 37efb0d4b6..f33c84cc27 100644 --- a/packages/backend/src/models/LatestNote.ts +++ b/packages/backend/src/models/LatestNote.ts @@ -6,7 +6,7 @@ import { PrimaryColumn, Entity, JoinColumn, Column, ManyToOne } from 'typeorm'; import { MiUser } from '@/models/User.js'; import { MiNote } from '@/models/Note.js'; -import { isQuote, isRenote } from '@/misc/is-renote.js'; +import { isQuote, isRenote, MinimalNote } from '@/misc/is-renote.js'; /** * Maps a user to the most recent post by that user. @@ -76,7 +76,7 @@ export class SkLatestNote { /** * Generates a compound key matching a provided note. */ - static keyFor(note: MiNote) { + static keyFor(note: MinimalNote) { return { userId: note.userId, isPublic: note.visibility === 'public', @@ -88,7 +88,7 @@ export class SkLatestNote { /** * Checks if two notes would produce equivalent compound keys. */ - static areEquivalent(first: MiNote, second: MiNote): boolean { + static areEquivalent(first: MinimalNote, second: MinimalNote): boolean { const firstKey = SkLatestNote.keyFor(first); const secondKey = SkLatestNote.keyFor(second); diff --git a/packages/backend/src/models/NoteEdit.ts b/packages/backend/src/models/NoteEdit.ts index 449c974d52..9e2b516754 100644 --- a/packages/backend/src/models/NoteEdit.ts +++ b/packages/backend/src/models/NoteEdit.ts @@ -7,6 +7,8 @@ import { Entity, JoinColumn, Column, ManyToOne, PrimaryColumn, Index } from 'typ import { id } from './util/id.js'; import { MiNote } from './Note.js'; import type { MiDriveFile } from './DriveFile.js'; +import { MiUser } from '@/models/User.js'; +import { noteVisibilities } from '@/types.js'; @Entity() export class NoteEdit { @@ -26,17 +28,64 @@ export class NoteEdit { @JoinColumn() public note: MiNote | null; + // TODO data migration + @Column({ + ...id(), + comment: 'The ID of author.', + }) + public userId: MiUser['id']; + + @ManyToOne(type => MiUser, { + onDelete: 'CASCADE', + }) + @JoinColumn() + public user: MiUser | null; + + @Column({ + ...id(), + nullable: true, + comment: 'The ID of renote target. Will always be null for older edits', + }) + public renoteId: MiNote['id'] | null; + + @ManyToOne(() => MiNote, { + onDelete: 'CASCADE', + }) + @JoinColumn() + public renote: MiNote | null; + + @Column({ + ...id(), + nullable: true, + comment: 'The ID of reply target. Will always be null for older edits', + }) + public replyId: MiNote['id'] | null; + + @ManyToOne(() => MiNote, { + onDelete: 'CASCADE', + }) + @JoinColumn() + public reply: MiNote | null; + + @Column('enum', { enum: noteVisibilities }) + public visibility: typeof noteVisibilities[number]; + @Column('text', { nullable: true, }) public newText: string | null; - @Column('varchar', { - length: 512, + @Column('text', { nullable: true, + comment: 'Will always be null for older edits', }) public cw: string | null; + @Column('text', { + nullable: true, + }) + public newCw: string | null; + @Column({ ...id(), array: true, @@ -49,14 +98,21 @@ export class NoteEdit { }) public updatedAt: Date; + // TODO rename migration @Column('text', { nullable: true, }) - public oldText: string | null; + public text: string | null; @Column('timestamp with time zone', { comment: 'The old date from before the edit', nullable: true, }) public oldDate: Date | null; + + @Column('boolean', { + default: false, + comment: 'Whether this revision had a poll. Will always be false for older edits', + }) + public hasPoll: boolean; } diff --git a/packages/backend/src/queue/processors/BackgroundTaskProcessorService.ts b/packages/backend/src/queue/processors/BackgroundTaskProcessorService.ts index 721e28a73d..3d8339fa3b 100644 --- a/packages/backend/src/queue/processors/BackgroundTaskProcessorService.ts +++ b/packages/backend/src/queue/processors/BackgroundTaskProcessorService.ts @@ -5,7 +5,7 @@ import { Inject, Injectable } from '@nestjs/common'; import * as Bull from 'bullmq'; -import { BackgroundTaskJobData, CheckHibernationBackgroundTask, PostDeliverBackgroundTask, PostInboxBackgroundTask, PostNoteBackgroundTask, UpdateFeaturedBackgroundTask, UpdateInstanceBackgroundTask, UpdateUserTagsBackgroundTask, UpdateUserBackgroundTask, UpdateNoteTagsBackgroundTask, DeleteFileBackgroundTask } from '@/queue/types.js'; +import { BackgroundTaskJobData, CheckHibernationBackgroundTask, PostDeliverBackgroundTask, PostInboxBackgroundTask, PostNoteBackgroundTask, UpdateFeaturedBackgroundTask, UpdateInstanceBackgroundTask, UpdateUserTagsBackgroundTask, UpdateUserBackgroundTask, UpdateNoteTagsBackgroundTask, DeleteFileBackgroundTask, UpdateLatestNoteBackgroundTask, PostSuspendBackgroundTask, PostUnsuspendBackgroundTask } from '@/queue/types.js'; import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js'; import { QueueLoggerService } from '@/queue/QueueLoggerService.js'; import Logger from '@/logger.js'; @@ -19,11 +19,14 @@ import ApRequestChart from '@/core/chart/charts/ap-request.js'; import FederationChart from '@/core/chart/charts/federation.js'; import { UpdateInstanceQueue } from '@/core/UpdateInstanceQueue.js'; import { NoteCreateService } from '@/core/NoteCreateService.js'; -import type { DriveFilesRepository, NotesRepository } from '@/models/_.js'; +import type { DriveFilesRepository, NoteEditsRepository, NotesRepository } from '@/models/_.js'; import { MiUser } from '@/models/_.js'; import { NoteEditService } from '@/core/NoteEditService.js'; import { HashtagService } from '@/core/HashtagService.js'; import { DriveService } from '@/core/DriveService.js'; +import { LatestNoteService } from '@/core/LatestNoteService.js'; +import { trackTask } from '@/misc/promise-tracker.js'; +import { UserSuspendService } from '@/core/UserSuspendService.js'; @Injectable() export class BackgroundTaskProcessorService { @@ -39,6 +42,9 @@ export class BackgroundTaskProcessorService { @Inject(DI.driveFilesRepository) private readonly driveFilesRepository: DriveFilesRepository, + @Inject(DI.noteEditsRepository) + private readonly noteEditsRepository: NoteEditsRepository, + private readonly apPersonService: ApPersonService, private readonly cacheService: CacheService, private readonly federatedInstanceService: FederatedInstanceService, @@ -51,6 +57,8 @@ export class BackgroundTaskProcessorService { private readonly noteEditService: NoteEditService, private readonly hashtagService: HashtagService, private readonly driveService: DriveService, + private readonly latestNoteService: LatestNoteService, + private readonly userSuspendService: UserSuspendService, queueLoggerService: QueueLoggerService, ) { @@ -76,9 +84,15 @@ export class BackgroundTaskProcessorService { return await this.processPostNote(job.data); } else if (job.data.type === 'check-hibernation') { return await this.processCheckHibernation(job.data); - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition } else if (job.data.type === 'delete-file') { return await this.processDeleteFile(job.data); + } else if (job.data.type === 'update-latest-note') { + return await this.processUpdateLatestNote(job.data); + } else if (job.data.type === 'post-suspend') { + return await this.processPostSuspend(job.data); + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + } else if (job.data.type === 'post-unsuspend') { + return await this.processPostUnsuspend(job.data); } else { this.logger.warn(`Can't process unknown job type "${job.data}"; this is likely a bug. Full job data:`, job.data); throw new Error(`Unknown job type ${job.data}, see system logs for details`); @@ -201,12 +215,13 @@ export class BackgroundTaskProcessorService { const instance = await this.federatedInstanceService.fetchOrRegister(task.host); if (instance.isBlocked) return `Skipping post-inbox task: instance ${task.host} is blocked`; + // TODO move chart stuff out of background? // Update charts if (this.meta.enableChartsForFederatedInstances) { - await this.instanceChart.requestReceived(task.host); + this.instanceChart.requestReceived(task.host); } - await this.apRequestChart.inbox(); - await this.federationChart.inbox(task.host); + this.apRequestChart.inbox(); + this.federationChart.inbox(task.host); // Update instance metadata (deferred) await this.fetchInstanceMetadataService.fetchInstanceMetadataLazy(instance); @@ -229,9 +244,9 @@ export class BackgroundTaskProcessorService { const mentionedUsers = await this.cacheService.getUsers(note.mentions); if (task.edit) { - await this.noteEditService.postNoteEdited(note, user, note, task.silent, note.tags, Array.from(mentionedUsers.values())); + await this.noteEditService.postNoteEdited(note, user, note, task.silent, Array.from(mentionedUsers.values())); } else { - await this.noteCreateService.postNoteCreated(note, user, note, task.silent, note.tags, Array.from(mentionedUsers.values())); + await this.noteCreateService.postNoteCreated(note, user, note, task.silent, Array.from(mentionedUsers.values())); } return 'ok'; @@ -260,4 +275,50 @@ export class BackgroundTaskProcessorService { await this.driveService.deleteFileSync(file, task.isExpired, deleter); return 'ok'; } + + private async processUpdateLatestNote(task: UpdateLatestNoteBackgroundTask): Promise { + const note = await this.notesRepository.findOneBy({ id: task.note.id }); + + if (note) { + const lastEdit = await this.noteEditsRepository.findOne({ + where: { noteId: task.note.id }, + order: { id: 'desc' }, + }); + + if (lastEdit) { + // Update + await this.latestNoteService.handleUpdatedNote(lastEdit, note); + } else { + // Create + await this.latestNoteService.handleDeletedNote(note); + } + } else { + // Delete + await this.latestNoteService.handleDeletedNote(task.note); + } + + return 'ok'; + } + + private async processPostSuspend(task: PostSuspendBackgroundTask): Promise { + const user = await this.cacheService.findOptionalUserById(task.userId); + if (!user || user.isDeleted) return `Skipping post-suspend task: user ${task.userId} has been deleted`; + + await trackTask(async () => { + await this.userSuspendService.postSuspend(user); + }); + + return 'ok'; + } + + private async processPostUnsuspend(task: PostUnsuspendBackgroundTask): Promise { + const user = await this.cacheService.findOptionalUserById(task.userId); + if (!user || user.isDeleted) return `Skipping post-unsuspend task: user ${task.userId} has been deleted`; + + await trackTask(async () => { + await this.userSuspendService.postUnsuspend(user); + }); + + return 'ok'; + } } diff --git a/packages/backend/src/queue/types.ts b/packages/backend/src/queue/types.ts index 7f7f1b8a74..ed9bbdbadf 100644 --- a/packages/backend/src/queue/types.ts +++ b/packages/backend/src/queue/types.ts @@ -12,6 +12,7 @@ import type { MiWebhook, WebhookEventTypes } from '@/models/Webhook.js'; import type { IActivity } from '@/core/activitypub/type.js'; import type { SystemWebhookPayload } from '@/core/SystemWebhookService.js'; import type { UserWebhookPayload } from '@/core/UserWebhookService.js'; +import type { MinimalNote } from '@/misc/is-renote.js'; import type httpSignature from '@peertube/http-signature'; export type DeliverJobData = { @@ -179,7 +180,10 @@ export type BackgroundTaskJobData = PostInboxBackgroundTask | PostNoteBackgroundTask | CheckHibernationBackgroundTask | - DeleteFileBackgroundTask; + DeleteFileBackgroundTask | + UpdateLatestNoteBackgroundTask | + PostSuspendBackgroundTask | + PostUnsuspendBackgroundTask; export type UpdateUserBackgroundTask = { type: 'update-user'; @@ -235,3 +239,18 @@ export type DeleteFileBackgroundTask = { isExpired?: boolean; deleterId?: string; }; + +export type UpdateLatestNoteBackgroundTask = { + type: 'update-latest-note'; + note: MinimalNote; +}; + +export type PostSuspendBackgroundTask = { + type: 'post-suspend'; + userId: string; +}; + +export type PostUnsuspendBackgroundTask = { + type: 'post-unsuspend'; + userId: string; +}; From c9f2554b2f0dd6c9656c44ad0ffd862c9faaa027 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Thu, 19 Jun 2025 13:40:56 -0400 Subject: [PATCH 011/107] track federation-related promises to avoid data loss during restart --- .../backend/src/core/NotePiningService.ts | 5 ++-- .../backend/src/core/UserFollowingService.ts | 28 +++++++++---------- .../src/core/activitypub/ApResolverService.ts | 5 ++-- .../core/activitypub/models/ApNoteService.ts | 4 +-- .../queue/processors/InboxProcessorService.ts | 5 ++-- .../src/server/api/endpoints/i/update.ts | 3 +- .../server/api/endpoints/notes/polls/vote.ts | 3 +- 7 files changed, 29 insertions(+), 24 deletions(-) diff --git a/packages/backend/src/core/NotePiningService.ts b/packages/backend/src/core/NotePiningService.ts index 836db52e48..a510676ef9 100644 --- a/packages/backend/src/core/NotePiningService.ts +++ b/packages/backend/src/core/NotePiningService.ts @@ -18,6 +18,7 @@ import { ApDeliverManagerService } from '@/core/activitypub/ApDeliverManagerServ import { ApRendererService } from '@/core/activitypub/ApRendererService.js'; import { bindThis } from '@/decorators.js'; import { RoleService } from '@/core/RoleService.js'; +import { trackPromise } from '@/misc/promise-tracker.js'; import type { DataSource } from 'typeorm'; @Injectable() @@ -84,7 +85,7 @@ export class NotePiningService { // Deliver to remote followers if (this.userEntityService.isLocalUser(user) && !note.localOnly && ['public', 'home'].includes(note.visibility)) { - await this.deliverPinnedChange(user, note.id, true); + trackPromise(this.deliverPinnedChange(user, note.id, true)); } } @@ -112,7 +113,7 @@ export class NotePiningService { // Deliver to remote followers if (this.userEntityService.isLocalUser(user) && !note.localOnly && ['public', 'home'].includes(note.visibility)) { - await this.deliverPinnedChange(user, noteId, false); + trackPromise(this.deliverPinnedChange(user, noteId, false)); } } diff --git a/packages/backend/src/core/UserFollowingService.ts b/packages/backend/src/core/UserFollowingService.ts index 833ea97193..1d3eada189 100644 --- a/packages/backend/src/core/UserFollowingService.ts +++ b/packages/backend/src/core/UserFollowingService.ts @@ -31,6 +31,7 @@ import type { ThinUser } from '@/queue/types.js'; import { LoggerService } from '@/core/LoggerService.js'; import { InternalEventService } from '@/global/InternalEventService.js'; import type Logger from '../logger.js'; +import { trackPromise } from '@/misc/promise-tracker.js'; type Local = MiLocalUser | { id: MiLocalUser['id']; @@ -102,7 +103,7 @@ export class UserFollowingService implements OnModuleInit { @bindThis public async deliverAccept(follower: MiRemoteUser, followee: MiPartialLocalUser, requestId?: string) { const content = this.apRendererService.addContext(this.apRendererService.renderAccept(this.apRendererService.renderFollow(follower, followee, requestId), followee)); - this.queueService.deliver(followee, content, follower.inbox, false); + await this.queueService.deliver(followee, content, follower.inbox, false); } @bindThis @@ -152,7 +153,7 @@ export class UserFollowingService implements OnModuleInit { // すでにフォロー関係が存在している場合 if (this.userEntityService.isRemoteUser(follower) && this.userEntityService.isLocalUser(followee)) { // リモート → ローカル: acceptを送り返しておしまい - this.deliverAccept(follower, followee, requestId); + trackPromise(this.deliverAccept(follower, followee, requestId)); return; } if (this.userEntityService.isLocalUser(follower)) { @@ -206,7 +207,7 @@ export class UserFollowingService implements OnModuleInit { await this.insertFollowingDoc(followee, follower, silent, withReplies); if (this.userEntityService.isRemoteUser(follower) && this.userEntityService.isLocalUser(followee)) { - this.deliverAccept(follower, followee, requestId); + trackPromise(this.deliverAccept(follower, followee, requestId)); } } @@ -581,7 +582,7 @@ export class UserFollowingService implements OnModuleInit { await this.insertFollowingDoc(followee, follower, false, request.withReplies); if (this.userEntityService.isRemoteUser(follower) && this.userEntityService.isLocalUser(followee)) { - this.deliverAccept(follower, followee as MiPartialLocalUser, request.requestId ?? undefined); + trackPromise(this.deliverAccept(follower, followee as MiPartialLocalUser, request.requestId ?? undefined)); } this.userEntityService.pack(followee.id, followee, { @@ -595,14 +596,13 @@ export class UserFollowingService implements OnModuleInit { id: MiUser['id']; host: MiUser['host']; uri: MiUser['host']; inbox: MiUser['inbox']; sharedInbox: MiUser['sharedInbox']; }, ): Promise { - const requests = await this.followRequestsRepository.findBy({ + const requests = await this.followRequestsRepository.find({ where: { followeeId: user.id, - }); + }, relations: { + follower: true, + } }); - for (const request of requests) { - const follower = await this.usersRepository.findOneByOrFail({ id: request.followerId }); - this.acceptFollowRequest(user, follower); - } + await Promise.all(requests.map(request => this.acceptFollowRequest(user, request.follower as MiUser))); } /** @@ -611,7 +611,7 @@ export class UserFollowingService implements OnModuleInit { @bindThis public async rejectFollowRequest(user: Local, follower: Both): Promise { if (this.userEntityService.isRemoteUser(follower)) { - this.deliverReject(user, follower); + trackPromise(this.deliverReject(user, follower)); } await this.removeFollowRequest(user, follower); @@ -627,7 +627,7 @@ export class UserFollowingService implements OnModuleInit { @bindThis public async rejectFollow(user: Local, follower: Both): Promise { if (this.userEntityService.isRemoteUser(follower)) { - this.deliverReject(user, follower); + trackPromise(this.deliverReject(user, follower)); } await this.removeFollow(user, follower); @@ -696,7 +696,7 @@ export class UserFollowingService implements OnModuleInit { }); const content = this.apRendererService.addContext(this.apRendererService.renderReject(this.apRendererService.renderFollow(follower, followee, request?.requestId ?? undefined), followee)); - this.queueService.deliver(followee, content, follower.inbox, false); + await this.queueService.deliver(followee, content, follower.inbox, false); } /** @@ -720,7 +720,7 @@ export class UserFollowingService implements OnModuleInit { @bindThis public async isFollowing(followerId: MiUser['id'], followeeId: MiUser['id']) { - return this.cacheService.isFollowing(followerId, followeeId); + return await this.cacheService.isFollowing(followerId, followeeId); } @bindThis diff --git a/packages/backend/src/core/activitypub/ApResolverService.ts b/packages/backend/src/core/activitypub/ApResolverService.ts index f6178ace1a..506686cb46 100644 --- a/packages/backend/src/core/activitypub/ApResolverService.ts +++ b/packages/backend/src/core/activitypub/ApResolverService.ts @@ -23,6 +23,7 @@ import { IdentifiableError } from '@/misc/identifiable-error.js'; import { toArray } from '@/misc/prelude/array.js'; import { isPureRenote } from '@/misc/is-renote.js'; import { CacheService } from '@/core/CacheService.js'; +import { trackPromise } from '@/misc/promise-tracker.js'; import { AnyCollection, getApId, getNullableApId, IObjectWithId, isCollection, isCollectionOrOrderedCollection, isCollectionPage, isOrderedCollection, isOrderedCollectionPage } from './type.js'; import { ApDbResolverService } from './ApDbResolverService.js'; import { ApRendererService } from './ApRendererService.js'; @@ -269,8 +270,8 @@ export class Resolver { log.duration = calculateDurationSince(startTime); // Save or finalize asynchronously - this.apLogService.saveFetchLog(log) - .catch(err => this.logger.error('Failed to record AP object fetch:', err)); + trackPromise(this.apLogService.saveFetchLog(log) + .catch(err => this.logger.error('Failed to record AP object fetch:', err))); } } diff --git a/packages/backend/src/core/activitypub/models/ApNoteService.ts b/packages/backend/src/core/activitypub/models/ApNoteService.ts index 0eaf26de54..9b717af32d 100644 --- a/packages/backend/src/core/activitypub/models/ApNoteService.ts +++ b/packages/backend/src/core/activitypub/models/ApNoteService.ts @@ -277,7 +277,7 @@ export class ApNoteService implements OnModuleInit { return x; }) - .catch(async err => { + .catch(err => { this.logger.warn(`error ${renderInlineError(err)} fetching inReplyTo ${note.inReplyTo} for note ${entryUri}`); throw new IdentifiableError('1ebf0a96-2769-4973-a6c2-3dcbad409dff', `failed to create note ${entryUri}: could not fetch inReplyTo ${note.inReplyTo}`, true, err); }) @@ -456,7 +456,7 @@ export class ApNoteService implements OnModuleInit { return x; }) - .catch(async err => { + .catch(err => { this.logger.warn(`error ${renderInlineError(err)} fetching inReplyTo ${note.inReplyTo} for note ${entryUri}`); throw new IdentifiableError('1ebf0a96-2769-4973-a6c2-3dcbad409dff', `failed to update note ${entryUri}: could not fetch inReplyTo ${note.inReplyTo}`, true, err); }) diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index 1c7765fddf..1a2389dac4 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -34,6 +34,7 @@ import { TimeService } from '@/global/TimeService.js'; import { isRetryableError } from '@/misc/is-retryable-error.js'; import { renderInlineError } from '@/misc/render-inline-error.js'; import { QueueService } from '@/core/QueueService.js'; +import { trackPromise } from '@/misc/promise-tracker.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type { InboxJobData } from '../types.js'; @@ -103,8 +104,8 @@ export class InboxProcessorService implements OnApplicationShutdown { log.duration = calculateDurationSince(startTime); // Save or finalize asynchronously - this.apLogService.saveInboxLog(log) - .catch(err => this.logger.error('Failed to record AP activity:', err)); + trackPromise(this.apLogService.saveInboxLog(log) + .catch(err => this.logger.error('Failed to record AP activity:', err))); } } diff --git a/packages/backend/src/server/api/endpoints/i/update.ts b/packages/backend/src/server/api/endpoints/i/update.ts index fe3f180aca..90b83443e4 100644 --- a/packages/backend/src/server/api/endpoints/i/update.ts +++ b/packages/backend/src/server/api/endpoints/i/update.ts @@ -39,6 +39,7 @@ import { trackPromise } from '@/misc/promise-tracker.js'; import { QueueService } from '@/core/QueueService.js'; import { ApiLoggerService } from '../../ApiLoggerService.js'; import { ApiError } from '../../error.js'; +import { trackPromise } from '@/misc/promise-tracker.js'; export const meta = { tags: ['account'], @@ -643,7 +644,7 @@ export default class extends Endpoint { // eslint- // 鍵垢を解除したとき、溜まっていたフォローリクエストがあるならすべて承認 if (user.isLocked && ps.isLocked === false) { - await this.userFollowingService.acceptAllFollowRequests(user); + trackPromise(this.userFollowingService.acceptAllFollowRequests(user)); } // フォロワーにUpdateを配信 diff --git a/packages/backend/src/server/api/endpoints/notes/polls/vote.ts b/packages/backend/src/server/api/endpoints/notes/polls/vote.ts index f7a5db8739..5bc104aad3 100644 --- a/packages/backend/src/server/api/endpoints/notes/polls/vote.ts +++ b/packages/backend/src/server/api/endpoints/notes/polls/vote.ts @@ -16,6 +16,7 @@ import { GlobalEventService } from '@/core/GlobalEventService.js'; import { DI } from '@/di-symbols.js'; import { UserBlockingService } from '@/core/UserBlockingService.js'; import { TimeService } from '@/global/TimeService.js'; +import { trackPromise } from '@/misc/promise-tracker.js'; import { ApiError } from '../../../error.js'; export const meta = { @@ -176,7 +177,7 @@ export default class extends Endpoint { // eslint- } // リモートフォロワーにUpdate配信 - await this.pollService.deliverQuestionUpdate(note); + trackPromise(this.pollService.deliverQuestionUpdate(note)); }); } } From 41d62e45347d67cf795e4c496b9f271d1b3f548c Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Thu, 19 Jun 2025 13:41:14 -0400 Subject: [PATCH 012/107] fix notes/versions endpoint --- packages/backend/src/server/api/endpoints/notes/versions.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/server/api/endpoints/notes/versions.ts b/packages/backend/src/server/api/endpoints/notes/versions.ts index 1c6f9838f5..92a1f14dce 100644 --- a/packages/backend/src/server/api/endpoints/notes/versions.ts +++ b/packages/backend/src/server/api/endpoints/notes/versions.ts @@ -108,7 +108,7 @@ export default class extends Endpoint { // eslint- editArray.push({ oldDate: (edit.oldDate ?? edit.updatedAt).toISOString(), updatedAt: edit.updatedAt.toISOString(), - text: edit.oldText ?? edit.newText ?? null, + text: edit.text ?? null, }); } From 05f9d5d446e8c6c0841a64f65bb3167e063af430 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Thu, 19 Jun 2025 13:51:32 -0400 Subject: [PATCH 013/107] await more promises to improve stack traces --- .../backend/src/core/AnnouncementService.ts | 4 +-- packages/backend/src/core/AntennaService.ts | 2 +- .../src/core/AvatarDecorationService.ts | 2 +- packages/backend/src/core/ChatService.ts | 4 +-- .../src/core/ImageProcessingService.ts | 6 ++--- .../backend/src/core/NotificationService.ts | 4 +-- packages/backend/src/core/RoleService.ts | 2 +- packages/backend/src/core/S3Service.ts | 2 +- packages/backend/src/core/SearchService.ts | 4 +-- packages/backend/src/core/SponsorsService.ts | 4 +-- .../backend/src/core/UserSearchService.ts | 2 +- packages/backend/src/core/UserService.ts | 2 +- .../backend/src/core/UserWebhookService.ts | 2 +- .../src/core/activitypub/JsonLdService.ts | 4 +-- .../activitypub/models/ApPersonService.ts | 5 ++-- .../core/entities/BlockingEntityService.ts | 3 ++- .../src/core/entities/ChatEntityService.ts | 12 ++++----- .../src/core/entities/ClipEntityService.ts | 3 ++- .../core/entities/DriveFileEntityService.ts | 2 ++ .../src/core/entities/EmojiEntityService.ts | 2 +- .../src/core/entities/FlashEntityService.ts | 2 +- .../entities/FollowRequestEntityService.ts | 2 +- .../core/entities/FollowingEntityService.ts | 3 ++- .../core/entities/GalleryPostEntityService.ts | 3 ++- .../core/entities/InviteCodeEntityService.ts | 2 +- .../entities/ModerationLogEntityService.ts | 3 ++- .../src/core/entities/MutingEntityService.ts | 3 ++- .../src/core/entities/NoteEntityService.ts | 1 + .../entities/NoteReactionEntityService.ts | 2 +- .../src/core/entities/PageEntityService.ts | 5 ++-- .../entities/RenoteMutingEntityService.ts | 3 ++- .../src/core/entities/UserEntityService.ts | 9 ++++--- .../core/entities/UserListEntityService.ts | 2 +- .../processors/ImportNotesProcessorService.ts | 4 +-- .../backend/src/server/api/ApiCallService.ts | 25 +++++++++++-------- .../src/server/api/ApiServerService.ts | 2 +- .../src/server/api/SigninApiService.ts | 4 +-- .../notification-recipient/create.ts | 2 +- .../notification-recipient/list.ts | 2 +- .../notification-recipient/show.ts | 2 +- .../notification-recipient/update.ts | 2 +- .../api/endpoints/admin/captcha/current.ts | 2 +- .../admin/delete-all-files-of-a-user.ts | 2 +- .../api/endpoints/admin/drive/cleanup.ts | 2 +- .../server/api/endpoints/admin/emoji/add.ts | 2 +- .../server/api/endpoints/admin/emoji/copy.ts | 2 +- .../api/endpoints/admin/emoji/list-remote.ts | 2 +- .../server/api/endpoints/admin/emoji/list.ts | 2 +- .../admin/federation/delete-all-files.ts | 2 +- .../refresh-remote-instance-metadata.ts | 2 +- .../admin/federation/remove-all-following.ts | 2 +- .../server/api/endpoints/admin/queue/clear.ts | 4 +-- .../server/api/endpoints/admin/queue/jobs.ts | 2 +- .../api/endpoints/admin/queue/promote-jobs.ts | 4 +-- .../api/endpoints/admin/queue/queue-stats.ts | 2 +- .../api/endpoints/admin/queue/queues.ts | 2 +- .../api/endpoints/admin/queue/remove-job.ts | 2 +- .../api/endpoints/admin/queue/retry-job.ts | 2 +- .../api/endpoints/admin/queue/show-job.ts | 2 +- .../endpoints/admin/system-webhook/create.ts | 2 +- .../endpoints/admin/system-webhook/list.ts | 2 +- .../endpoints/admin/system-webhook/show.ts | 2 +- .../endpoints/admin/system-webhook/update.ts | 2 +- .../src/server/api/endpoints/announcements.ts | 2 +- .../api/endpoints/auth/session/userkey.ts | 2 +- .../api/endpoints/chat/messages/show.ts | 2 +- .../endpoints/chat/rooms/invitations/inbox.ts | 2 +- .../chat/rooms/invitations/outbox.ts | 2 +- .../api/endpoints/chat/rooms/joining.ts | 2 +- .../api/endpoints/chat/rooms/members.ts | 2 +- .../server/api/endpoints/chat/rooms/owned.ts | 2 +- .../server/api/endpoints/chat/rooms/show.ts | 2 +- .../server/api/endpoints/chat/rooms/update.ts | 2 +- .../api/endpoints/clips/my-favorites.ts | 2 +- .../api/endpoints/drive/folders/update.ts | 2 +- .../backend/src/server/api/endpoints/emoji.ts | 2 +- .../api/endpoints/export-custom-emojis.ts | 2 +- .../api/endpoints/federation/followers.ts | 2 +- .../api/endpoints/federation/following.ts | 2 +- .../server/api/endpoints/flash/my-likes.ts | 2 +- .../src/server/api/endpoints/hashtags/list.ts | 2 +- .../server/api/endpoints/i/export-antennas.ts | 2 +- .../server/api/endpoints/i/export-blocking.ts | 2 +- .../server/api/endpoints/i/export-clips.ts | 2 +- .../src/server/api/endpoints/i/export-data.ts | 2 +- .../api/endpoints/i/export-favorites.ts | 2 +- .../api/endpoints/i/export-following.ts | 2 +- .../src/server/api/endpoints/i/export-mute.ts | 2 +- .../server/api/endpoints/i/export-notes.ts | 2 +- .../api/endpoints/i/export-user-lists.ts | 2 +- .../src/server/api/endpoints/i/page-likes.ts | 2 +- .../api/endpoints/notes/schedule/list.ts | 2 +- .../api/endpoints/notifications/flush.ts | 2 +- .../notifications/mark-all-as-read.ts | 2 +- .../server/api/endpoints/reset-password.ts | 2 +- .../endpoints/users/lists/get-memberships.ts | 2 +- .../server/api/mastodon/MastodonConverters.ts | 2 +- 97 files changed, 143 insertions(+), 127 deletions(-) diff --git a/packages/backend/src/core/AnnouncementService.ts b/packages/backend/src/core/AnnouncementService.ts index 54496f9922..48c012da0f 100644 --- a/packages/backend/src/core/AnnouncementService.ts +++ b/packages/backend/src/core/AnnouncementService.ts @@ -218,9 +218,9 @@ export class AnnouncementService { announcementId: announcement.id, userId: me.id, }); - return this.announcementEntityService.pack({ ...announcement, isRead: read !== null }, me); + return await this.announcementEntityService.pack({ ...announcement, isRead: read !== null }, me); } else { - return this.announcementEntityService.pack(announcement, null); + return await this.announcementEntityService.pack(announcement, null); } } diff --git a/packages/backend/src/core/AntennaService.ts b/packages/backend/src/core/AntennaService.ts index 660c97dd6a..cc4b7dd719 100644 --- a/packages/backend/src/core/AntennaService.ts +++ b/packages/backend/src/core/AntennaService.ts @@ -107,7 +107,7 @@ export class AntennaService implements OnApplicationShutdown { this.globalEventService.publishAntennaStream(antenna.id, 'note', note); } - redisPipeline.exec(); + await redisPipeline.exec(); } // NOTE: フォローしているユーザーのノート、リストのユーザーのノート、グループのユーザーのノート指定はパフォーマンス上の理由で無効になっている diff --git a/packages/backend/src/core/AvatarDecorationService.ts b/packages/backend/src/core/AvatarDecorationService.ts index aedb1d6a80..0a586789f1 100644 --- a/packages/backend/src/core/AvatarDecorationService.ts +++ b/packages/backend/src/core/AvatarDecorationService.ts @@ -109,7 +109,7 @@ export class AvatarDecorationService implements OnApplicationShutdown { if (noCache) { this.cache.delete(); } - return this.cache.fetch(() => this.avatarDecorationsRepository.find()); + return await this.cache.fetch(() => this.avatarDecorationsRepository.find()); } @bindThis diff --git a/packages/backend/src/core/ChatService.ts b/packages/backend/src/core/ChatService.ts index cdecd41726..9132c166f9 100644 --- a/packages/backend/src/core/ChatService.ts +++ b/packages/backend/src/core/ChatService.ts @@ -605,12 +605,12 @@ export class ChatService { @bindThis public async findMyRoomById(ownerId: MiUser['id'], roomId: MiChatRoom['id']) { - return this.chatRoomsRepository.findOneBy({ id: roomId, ownerId: ownerId }); + return await this.chatRoomsRepository.findOneBy({ id: roomId, ownerId: ownerId }); } @bindThis public async findRoomById(roomId: MiChatRoom['id']) { - return this.chatRoomsRepository.findOne({ where: { id: roomId }, relations: ['owner'] }); + return await this.chatRoomsRepository.findOne({ where: { id: roomId }, relations: ['owner'] }); } @bindThis diff --git a/packages/backend/src/core/ImageProcessingService.ts b/packages/backend/src/core/ImageProcessingService.ts index 6f60475442..d042cf1f30 100644 --- a/packages/backend/src/core/ImageProcessingService.ts +++ b/packages/backend/src/core/ImageProcessingService.ts @@ -58,7 +58,7 @@ export class ImageProcessingService { */ @bindThis public async convertToWebp(path: string, width: number, height: number, options: sharp.WebpOptions = webpDefault): Promise { - return this.convertSharpToWebp(sharp(path), width, height, options); + return await this.convertSharpToWebp(sharp(path), width, height, options); } @bindThis @@ -100,7 +100,7 @@ export class ImageProcessingService { */ @bindThis public async convertToAvif(path: string, width: number, height: number, options: sharp.AvifOptions = avifDefault): Promise { - return this.convertSharpToAvif(sharp(path), width, height, options); + return await this.convertSharpToAvif(sharp(path), width, height, options); } @bindThis @@ -142,7 +142,7 @@ export class ImageProcessingService { */ @bindThis public async convertToPng(path: string, width: number, height: number): Promise { - return this.convertSharpToPng(sharp(path), width, height); + return await this.convertSharpToPng(sharp(path), width, height); } @bindThis diff --git a/packages/backend/src/core/NotificationService.ts b/packages/backend/src/core/NotificationService.ts index 9180dfa418..63d2011441 100644 --- a/packages/backend/src/core/NotificationService.ts +++ b/packages/backend/src/core/NotificationService.ts @@ -72,9 +72,9 @@ export class NotificationService implements OnApplicationShutdown { } @bindThis - private postReadAllNotifications(userId: MiUser['id']) { + private async postReadAllNotifications(userId: MiUser['id']) { this.globalEventService.publishMainStream(userId, 'readAllNotifications'); - this.pushNotificationService.pushNotification(userId, 'readAllNotifications', undefined); + await this.pushNotificationService.pushNotification(userId, 'readAllNotifications', undefined); } @bindThis diff --git a/packages/backend/src/core/RoleService.ts b/packages/backend/src/core/RoleService.ts index 6dd768c3c6..0b760560ca 100644 --- a/packages/backend/src/core/RoleService.ts +++ b/packages/backend/src/core/RoleService.ts @@ -751,7 +751,7 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { this.globalEventService.publishRoleTimelineStream(role.id, 'note', note); } - redisPipeline.exec(); + await redisPipeline.exec(); } @bindThis diff --git a/packages/backend/src/core/S3Service.ts b/packages/backend/src/core/S3Service.ts index 955d778015..30de5289f4 100644 --- a/packages/backend/src/core/S3Service.ts +++ b/packages/backend/src/core/S3Service.ts @@ -91,7 +91,7 @@ export class S3Service implements OnApplicationShutdown { @bindThis public async upload(input: PutObjectCommandInput) { const client = this.getS3Client(); - return new Upload({ + return await new Upload({ client, params: input, partSize: (client.config.endpoint && (await client.config.endpoint()).hostname === 'storage.googleapis.com') diff --git a/packages/backend/src/core/SearchService.ts b/packages/backend/src/core/SearchService.ts index 6e4dd9adca..04026865fb 100644 --- a/packages/backend/src/core/SearchService.ts +++ b/packages/backend/src/core/SearchService.ts @@ -256,10 +256,10 @@ export class SearchService { case 'sqlTsvector': { // ほとんど内容に差がないのでsqlLikeとsqlPgroongaを同じ処理にしている. // 今後の拡張で差が出る用であれば関数を分ける. - return this.searchNoteByLike(q, me, opts, pagination); + return await this.searchNoteByLike(q, me, opts, pagination); } case 'meilisearch': { - return this.searchNoteByMeiliSearch(q, me, opts, pagination); + return await this.searchNoteByMeiliSearch(q, me, opts, pagination); } default: { // eslint-disable-next-line @typescript-eslint/no-unused-vars diff --git a/packages/backend/src/core/SponsorsService.ts b/packages/backend/src/core/SponsorsService.ts index 23994b5761..454df4442e 100644 --- a/packages/backend/src/core/SponsorsService.ts +++ b/packages/backend/src/core/SponsorsService.ts @@ -94,12 +94,12 @@ export class SponsorsService { @bindThis public async instanceSponsors(forceUpdate: boolean) { if (forceUpdate) await this.cache.refresh('instance'); - return this.cache.fetch('instance'); + return await this.cache.fetch('instance'); } @bindThis public async sharkeySponsors(forceUpdate: boolean) { if (forceUpdate) await this.cache.refresh('sharkey'); - return this.cache.fetch('sharkey'); + return await this.cache.fetch('sharkey'); } } diff --git a/packages/backend/src/core/UserSearchService.ts b/packages/backend/src/core/UserSearchService.ts index 6c6d3a5280..9e2b5dfc64 100644 --- a/packages/backend/src/core/UserSearchService.ts +++ b/packages/backend/src/core/UserSearchService.ts @@ -99,7 +99,7 @@ export class UserSearchService { } } - return this.userEntityService.packMany<'UserLite' | 'UserDetailed'>( + return await this.userEntityService.packMany<'UserLite' | 'UserDetailed'>( [...resultSet].slice(0, limit), me, { schema: opts?.detail ? 'UserDetailed' : 'UserLite' }, diff --git a/packages/backend/src/core/UserService.ts b/packages/backend/src/core/UserService.ts index 34c343712c..69efb3a230 100644 --- a/packages/backend/src/core/UserService.ts +++ b/packages/backend/src/core/UserService.ts @@ -70,6 +70,6 @@ export class UserService { @bindThis public async notifySystemWebhook(user: MiUser, type: 'userCreated') { const packedUser = await this.userEntityService.pack(user, null, { schema: 'UserLite' }); - return this.systemWebhookService.enqueueSystemWebhook(type, packedUser); + return await this.systemWebhookService.enqueueSystemWebhook(type, packedUser); } } diff --git a/packages/backend/src/core/UserWebhookService.ts b/packages/backend/src/core/UserWebhookService.ts index 24a519bc5c..097482d6a5 100644 --- a/packages/backend/src/core/UserWebhookService.ts +++ b/packages/backend/src/core/UserWebhookService.ts @@ -94,7 +94,7 @@ export class UserWebhookService implements OnApplicationShutdown { ) { const webhooks = await this.getActiveWebhooks() .then(webhooks => webhooks.filter(webhook => webhook.userId === userId && webhook.on.includes(type))); - return Promise.all( + return await Promise.all( webhooks.map(webhook => { return this.queueService.userWebhookDeliver(webhook, type, content); }), diff --git a/packages/backend/src/core/activitypub/JsonLdService.ts b/packages/backend/src/core/activitypub/JsonLdService.ts index 8e14e0909f..194af63f5a 100644 --- a/packages/backend/src/core/activitypub/JsonLdService.ts +++ b/packages/backend/src/core/activitypub/JsonLdService.ts @@ -134,7 +134,7 @@ export class JsonLdService { const customLoader = this.getLoader(); // XXX: Importing jsonld dynamically since Jest frequently fails to import it statically // https://github.com/misskey-dev/misskey/pull/9894#discussion_r1103753595 - return (await import('jsonld')).default.compact(data, context, { + return await (await import('jsonld')).default.compact(data, context, { documentLoader: customLoader, }); } @@ -142,7 +142,7 @@ export class JsonLdService { @bindThis public async normalize(data: Document): Promise { const customLoader = this.getLoader(); - return (await import('jsonld')).default.normalize(data, { + return await (await import('jsonld')).default.normalize(data, { documentLoader: customLoader, }); } diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index f153d51c9d..a8cfdfe682 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -329,8 +329,9 @@ export class ApPersonService implements OnModuleInit { return user; } + // TODO fix these "any" types private async resolveAvatarAndBanner(user: MiRemoteUser, icon: any, image: any, bgimg: any): Promise>> { - const [avatar, banner, background] = await Promise.all([icon, image, bgimg].map(img => { + const [avatar, banner, background] = await Promise.all([icon, image, bgimg].map(async img => { // icon and image may be arrays // see https://www.w3.org/TR/activitystreams-vocabulary/#dfn-icon if (Array.isArray(img)) { @@ -343,7 +344,7 @@ export class ApPersonService implements OnModuleInit { return { id: null, url: null, blurhash: null }; } - return this.apImageService.resolveImage(user, img).catch(() => null); + return await this.apImageService.resolveImage(user, img).catch(() => null); })); if (((avatar != null && avatar.id != null) || (banner != null && banner.id != null)) diff --git a/packages/backend/src/core/entities/BlockingEntityService.ts b/packages/backend/src/core/entities/BlockingEntityService.ts index 1e699032e2..0ba7a0f2dd 100644 --- a/packages/backend/src/core/entities/BlockingEntityService.ts +++ b/packages/backend/src/core/entities/BlockingEntityService.ts @@ -35,6 +35,7 @@ export class BlockingEntityService { ): Promise> { const blocking = typeof src === 'object' ? src : await this.blockingsRepository.findOneByOrFail({ id: src }); + // noinspection ES6MissingAwait return await awaitAll({ id: blocking.id, createdAt: this.idService.parse(blocking.id).date.toISOString(), @@ -53,6 +54,6 @@ export class BlockingEntityService { const _blockees = blockings.map(({ blockee, blockeeId }) => blockee ?? blockeeId); const _userMap = await this.userEntityService.packMany(_blockees, me, { schema: 'UserDetailedNotMe' }) .then(users => new Map(users.map(u => [u.id, u]))); - return Promise.all(blockings.map(blocking => this.pack(blocking, me, { blockee: _userMap.get(blocking.blockeeId) }))); + return await Promise.all(blockings.map(blocking => this.pack(blocking, me, { blockee: _userMap.get(blocking.blockeeId) }))); } } diff --git a/packages/backend/src/core/entities/ChatEntityService.ts b/packages/backend/src/core/entities/ChatEntityService.ts index da112d5444..171a49ebcc 100644 --- a/packages/backend/src/core/entities/ChatEntityService.ts +++ b/packages/backend/src/core/entities/ChatEntityService.ts @@ -117,7 +117,7 @@ export class ChatEntityService { .then(rooms => new Map(rooms.map(r => [r.id, r]))), ]); - return Promise.all(messages.map(message => this.packMessageDetailed(message, me, { _hint_: { packedUsers, packedFiles, packedRooms } }))); + return await Promise.all(messages.map(message => this.packMessageDetailed(message, me, { _hint_: { packedUsers, packedFiles, packedRooms } }))); } @bindThis @@ -165,7 +165,7 @@ export class ChatEntityService { .then(files => new Map(files.map(f => [f.id, f]))), ]); - return Promise.all(messages.map(message => this.packMessageLiteFor1on1(message, { _hint_: { packedFiles } }))); + return await Promise.all(messages.map(message => this.packMessageLiteFor1on1(message, { _hint_: { packedFiles } }))); } @bindThis @@ -228,7 +228,7 @@ export class ChatEntityService { .then(files => new Map(files.map(f => [f.id, f]))), ]); - return Promise.all(messages.map(message => this.packMessageLiteForRoom(message, { _hint_: { packedFiles, packedUsers } }))); + return await Promise.all(messages.map(message => this.packMessageLiteForRoom(message, { _hint_: { packedFiles, packedUsers } }))); } @bindThis @@ -289,7 +289,7 @@ export class ChatEntityService { }).then(memberships => new Map(_rooms.map(r => [r.id, memberships.find(m => m.roomId === r.id)]))), ]); - return Promise.all(_rooms.map(room => this.packRoom(room, me, { _hint_: { packedOwners, memberships } }))); + return await Promise.all(_rooms.map(room => this.packRoom(room, me, { _hint_: { packedOwners, memberships } }))); } @bindThis @@ -322,7 +322,7 @@ export class ChatEntityService { ) { if (invitations.length === 0) return []; - return Promise.all(invitations.map(invitation => this.packRoomInvitation(invitation, me))); + return await Promise.all(invitations.map(invitation => this.packRoomInvitation(invitation, me))); } @bindThis @@ -371,6 +371,6 @@ export class ChatEntityService { .then(rooms => new Map(rooms.map(r => [r.id, r]))), ]); - return Promise.all(memberships.map(membership => this.packRoomMembership(membership, me, { ...options, _hint_: { packedUsers, packedRooms } }))); + return await Promise.all(memberships.map(membership => this.packRoomMembership(membership, me, { ...options, _hint_: { packedUsers, packedRooms } }))); } } diff --git a/packages/backend/src/core/entities/ClipEntityService.ts b/packages/backend/src/core/entities/ClipEntityService.ts index b700fe2efb..fba085767f 100644 --- a/packages/backend/src/core/entities/ClipEntityService.ts +++ b/packages/backend/src/core/entities/ClipEntityService.ts @@ -42,6 +42,7 @@ export class ClipEntityService { const meId = me ? me.id : null; const clip = typeof src === 'object' ? src : await this.clipsRepository.findOneByOrFail({ id: src }); + // noinspection ES6MissingAwait return await awaitAll({ id: clip.id, createdAt: this.idService.parse(clip.id).date.toISOString(), @@ -65,7 +66,7 @@ export class ClipEntityService { const _users = clips.map(({ user, userId }) => user ?? userId); const _userMap = await this.userEntityService.packMany(_users, me) .then(users => new Map(users.map(u => [u.id, u]))); - return Promise.all(clips.map(clip => this.pack(clip, me, { packedUser: _userMap.get(clip.userId) }))); + return await Promise.all(clips.map(clip => this.pack(clip, me, { packedUser: _userMap.get(clip.userId) }))); } } diff --git a/packages/backend/src/core/entities/DriveFileEntityService.ts b/packages/backend/src/core/entities/DriveFileEntityService.ts index a172f81eed..06197ff839 100644 --- a/packages/backend/src/core/entities/DriveFileEntityService.ts +++ b/packages/backend/src/core/entities/DriveFileEntityService.ts @@ -201,6 +201,7 @@ export class DriveFileEntityService implements OnModuleInit { const file = typeof src === 'object' ? src : await this.driveFilesRepository.findOneByOrFail({ id: src }); + // noinspection ES6MissingAwait return await awaitAll>({ id: file.id, createdAt: this.idService.parse(file.id).date.toISOString(), @@ -239,6 +240,7 @@ export class DriveFileEntityService implements OnModuleInit { const file = typeof src === 'object' ? src : await this.driveFilesRepository.findOneBy({ id: src }); if (file == null) return null; + // noinspection ES6MissingAwait return await awaitAll>({ id: file.id, createdAt: this.idService.parse(file.id).date.toISOString(), diff --git a/packages/backend/src/core/entities/EmojiEntityService.ts b/packages/backend/src/core/entities/EmojiEntityService.ts index 5f03df554c..9c3f0adfff 100644 --- a/packages/backend/src/core/entities/EmojiEntityService.ts +++ b/packages/backend/src/core/entities/EmojiEntityService.ts @@ -179,7 +179,7 @@ export class EmojiEntityService implements OnModuleInit { hintRoles = new Map(roles.map(x => [x.id, x])); } - return Promise.all(emojis.map(x => this.packDetailedAdmin(x, { roles: hintRoles }))); + return await Promise.all(emojis.map(x => this.packDetailedAdmin(x, { roles: hintRoles }))); } } diff --git a/packages/backend/src/core/entities/FlashEntityService.ts b/packages/backend/src/core/entities/FlashEntityService.ts index c2575e69aa..47617c5be7 100644 --- a/packages/backend/src/core/entities/FlashEntityService.ts +++ b/packages/backend/src/core/entities/FlashEntityService.ts @@ -77,7 +77,7 @@ export class FlashEntityService { .getRawMany<{ flashLike_flashId: string }>() .then(likes => [...new Set(likes.map(like => like.flashLike_flashId))]) : []; - return Promise.all( + return await Promise.all( flashes.map(flash => this.pack(flash, me, { packedUser: _userMap.get(flash.userId), likedFlashIds: _likedFlashIds, diff --git a/packages/backend/src/core/entities/FollowRequestEntityService.ts b/packages/backend/src/core/entities/FollowRequestEntityService.ts index 0101ec8aa7..05c69b4d56 100644 --- a/packages/backend/src/core/entities/FollowRequestEntityService.ts +++ b/packages/backend/src/core/entities/FollowRequestEntityService.ts @@ -50,7 +50,7 @@ export class FollowRequestEntityService { const _followees = requests.map(({ followee, followeeId }) => followee ?? followeeId); const _userMap = await this.userEntityService.packMany([..._followers, ..._followees], me) .then(users => new Map(users.map(u => [u.id, u]))); - return Promise.all( + return await Promise.all( requests.map(req => { const packedFollower = _userMap.get(req.followerId); const packedFollowee = _userMap.get(req.followeeId); diff --git a/packages/backend/src/core/entities/FollowingEntityService.ts b/packages/backend/src/core/entities/FollowingEntityService.ts index d54c954bf2..5645217ebc 100644 --- a/packages/backend/src/core/entities/FollowingEntityService.ts +++ b/packages/backend/src/core/entities/FollowingEntityService.ts @@ -139,6 +139,7 @@ export class FollowingEntityService { if (opts == null) opts = {}; + // noinspection ES6MissingAwait return await awaitAll({ id: following.id, createdAt: this.idService.parse(following.id).date.toISOString(), @@ -166,7 +167,7 @@ export class FollowingEntityService { const _followers = opts?.populateFollower ? followings.map(({ follower, followerId }) => follower ?? followerId) : []; const _userMap = await this.userEntityService.packMany([..._followees, ..._followers], me, { schema: 'UserDetailedNotMe' }) .then(users => new Map(users.map(u => [u.id, u]))); - return Promise.all( + return await Promise.all( followings.map(following => { const packedFollowee = opts?.populateFollowee ? _userMap.get(following.followeeId) : undefined; const packedFollower = opts?.populateFollower ? _userMap.get(following.followerId) : undefined; diff --git a/packages/backend/src/core/entities/GalleryPostEntityService.ts b/packages/backend/src/core/entities/GalleryPostEntityService.ts index 9746a4c1af..a276f1508c 100644 --- a/packages/backend/src/core/entities/GalleryPostEntityService.ts +++ b/packages/backend/src/core/entities/GalleryPostEntityService.ts @@ -42,6 +42,7 @@ export class GalleryPostEntityService { const meId = me ? me.id : null; const post = typeof src === 'object' ? src : await this.galleryPostsRepository.findOneByOrFail({ id: src }); + // noinspection ES6MissingAwait return await awaitAll({ id: post.id, createdAt: this.idService.parse(post.id).date.toISOString(), @@ -68,7 +69,7 @@ export class GalleryPostEntityService { const _users = posts.map(({ user, userId }) => user ?? userId); const _userMap = await this.userEntityService.packMany(_users, me) .then(users => new Map(users.map(u => [u.id, u]))); - return Promise.all(posts.map(post => this.pack(post, me, { packedUser: _userMap.get(post.userId) }))); + return await Promise.all(posts.map(post => this.pack(post, me, { packedUser: _userMap.get(post.userId) }))); } } diff --git a/packages/backend/src/core/entities/InviteCodeEntityService.ts b/packages/backend/src/core/entities/InviteCodeEntityService.ts index 5d3e823a2a..ceb666ac90 100644 --- a/packages/backend/src/core/entities/InviteCodeEntityService.ts +++ b/packages/backend/src/core/entities/InviteCodeEntityService.ts @@ -62,7 +62,7 @@ export class InviteCodeEntityService { const _usedBys = tickets.map(({ usedBy, usedById }) => usedBy ?? usedById).filter(x => x != null); const _userMap = await this.userEntityService.packMany([..._createdBys, ..._usedBys], me) .then(users => new Map(users.map(u => [u.id, u]))); - return Promise.all( + return await Promise.all( tickets.map(ticket => { const packedCreatedBy = ticket.createdById != null ? _userMap.get(ticket.createdById) : undefined; const packedUsedBy = ticket.usedById != null ? _userMap.get(ticket.usedById) : undefined; diff --git a/packages/backend/src/core/entities/ModerationLogEntityService.ts b/packages/backend/src/core/entities/ModerationLogEntityService.ts index bf1b2a002c..2d1de5cd46 100644 --- a/packages/backend/src/core/entities/ModerationLogEntityService.ts +++ b/packages/backend/src/core/entities/ModerationLogEntityService.ts @@ -34,6 +34,7 @@ export class ModerationLogEntityService { ) { const log = typeof src === 'object' ? src : await this.moderationLogsRepository.findOneByOrFail({ id: src }); + // noinspection ES6MissingAwait return await awaitAll({ id: log.id, createdAt: this.idService.parse(log.id).date.toISOString(), @@ -53,7 +54,7 @@ export class ModerationLogEntityService { const _users = reports.map(({ user, userId }) => user ?? userId); const _userMap = await this.userEntityService.packMany(_users, null, { schema: 'UserDetailedNotMe' }) .then(users => new Map(users.map(u => [u.id, u]))); - return Promise.all(reports.map(report => this.pack(report, { packedUser: _userMap.get(report.userId) }))); + return await Promise.all(reports.map(report => this.pack(report, { packedUser: _userMap.get(report.userId) }))); } } diff --git a/packages/backend/src/core/entities/MutingEntityService.ts b/packages/backend/src/core/entities/MutingEntityService.ts index d361a20271..7d8c3dcde2 100644 --- a/packages/backend/src/core/entities/MutingEntityService.ts +++ b/packages/backend/src/core/entities/MutingEntityService.ts @@ -36,6 +36,7 @@ export class MutingEntityService { ): Promise> { const muting = typeof src === 'object' ? src : await this.mutingsRepository.findOneByOrFail({ id: src }); + // noinspection ES6MissingAwait return await awaitAll({ id: muting.id, createdAt: this.idService.parse(muting.id).date.toISOString(), @@ -55,7 +56,7 @@ export class MutingEntityService { const _mutees = mutings.map(({ mutee, muteeId }) => mutee ?? muteeId); const _userMap = await this.userEntityService.packMany(_mutees, me, { schema: 'UserDetailedNotMe' }) .then(users => new Map(users.map(u => [u.id, u]))); - return Promise.all(mutings.map(muting => this.pack(muting, me, { packedMutee: _userMap.get(muting.muteeId) }))); + return await Promise.all(mutings.map(muting => this.pack(muting, me, { packedMutee: _userMap.get(muting.muteeId) }))); } } diff --git a/packages/backend/src/core/entities/NoteEntityService.ts b/packages/backend/src/core/entities/NoteEntityService.ts index 373f05332f..310e361736 100644 --- a/packages/backend/src/core/entities/NoteEntityService.ts +++ b/packages/backend/src/core/entities/NoteEntityService.ts @@ -592,6 +592,7 @@ export class NoteEntityService implements OnModuleInit { const bypassSilence = opts.bypassSilence || note.userId === meId; + // noinspection ES6MissingAwait const packed: Packed<'Note'> = await awaitAll({ id: note.id, threadId, diff --git a/packages/backend/src/core/entities/NoteReactionEntityService.ts b/packages/backend/src/core/entities/NoteReactionEntityService.ts index 2b0d69b261..1f6d52d20c 100644 --- a/packages/backend/src/core/entities/NoteReactionEntityService.ts +++ b/packages/backend/src/core/entities/NoteReactionEntityService.ts @@ -88,6 +88,6 @@ export class NoteReactionEntityService implements OnModuleInit { const _users = reactions.map(({ user, userId }) => user ?? userId); const _userMap = await this.userEntityService.packMany(_users, me) .then(users => new Map(users.map(u => [u.id, u]))); - return Promise.all(reactions.map(reaction => this.pack(reaction, me, opts, { packedUser: _userMap.get(reaction.userId) }))); + return await Promise.all(reactions.map(reaction => this.pack(reaction, me, opts, { packedUser: _userMap.get(reaction.userId) }))); } } diff --git a/packages/backend/src/core/entities/PageEntityService.ts b/packages/backend/src/core/entities/PageEntityService.ts index 46bf51bb6d..3106001676 100644 --- a/packages/backend/src/core/entities/PageEntityService.ts +++ b/packages/backend/src/core/entities/PageEntityService.ts @@ -83,11 +83,12 @@ export class PageEntityService { }; migrate(page.content); if (migrated) { - this.pagesRepository.update(page.id, { + await this.pagesRepository.update(page.id, { content: page.content, }); } + // noinspection ES6MissingAwait return await awaitAll({ id: page.id, createdAt: this.idService.parse(page.id).date.toISOString(), @@ -119,7 +120,7 @@ export class PageEntityService { const _users = pages.map(({ user, userId }) => user ?? userId); const _userMap = await this.userEntityService.packMany(_users, me) .then(users => new Map(users.map(u => [u.id, u]))); - return Promise.all(pages.map(page => this.pack(page, me, { packedUser: _userMap.get(page.userId) }))); + return await Promise.all(pages.map(page => this.pack(page, me, { packedUser: _userMap.get(page.userId) }))); } } diff --git a/packages/backend/src/core/entities/RenoteMutingEntityService.ts b/packages/backend/src/core/entities/RenoteMutingEntityService.ts index e4e154109a..199fc2366e 100644 --- a/packages/backend/src/core/entities/RenoteMutingEntityService.ts +++ b/packages/backend/src/core/entities/RenoteMutingEntityService.ts @@ -36,6 +36,7 @@ export class RenoteMutingEntityService { ): Promise> { const muting = typeof src === 'object' ? src : await this.renoteMutingsRepository.findOneByOrFail({ id: src }); + // noinspection ES6MissingAwait return await awaitAll({ id: muting.id, createdAt: this.idService.parse(muting.id).date.toISOString(), @@ -54,7 +55,7 @@ export class RenoteMutingEntityService { const _users = mutings.map(({ mutee, muteeId }) => mutee ?? muteeId); const _userMap = await this.userEntityService.packMany(_users, me, { schema: 'UserDetailedNotMe' }) .then(users => new Map(users.map(u => [u.id, u]))); - return Promise.all(mutings.map(muting => this.pack(muting, me, { packedMutee: _userMap.get(muting.muteeId) }))); + return await Promise.all(mutings.map(muting => this.pack(muting, me, { packedMutee: _userMap.get(muting.muteeId) }))); } } diff --git a/packages/backend/src/core/entities/UserEntityService.ts b/packages/backend/src/core/entities/UserEntityService.ts index 6e7f15610f..bacc3c3fde 100644 --- a/packages/backend/src/core/entities/UserEntityService.ts +++ b/packages/backend/src/core/entities/UserEntityService.ts @@ -485,7 +485,7 @@ export class UserEntityService implements OnModuleInit { if (user.avatarId != null && user.avatarUrl === null) { const avatar = await this.driveFilesRepository.findOneByOrFail({ id: user.avatarId }); user.avatarUrl = this.driveFileEntityService.getPublicUrl(avatar, 'avatar'); - this.usersRepository.update(user.id, { + await this.usersRepository.update(user.id, { avatarUrl: user.avatarUrl, avatarBlurhash: avatar.blurhash, }); @@ -493,7 +493,7 @@ export class UserEntityService implements OnModuleInit { if (user.bannerId != null && user.bannerUrl === null) { const banner = await this.driveFilesRepository.findOneByOrFail({ id: user.bannerId }); user.bannerUrl = this.driveFileEntityService.getPublicUrl(banner); - this.usersRepository.update(user.id, { + await this.usersRepository.update(user.id, { bannerUrl: user.bannerUrl, bannerBlurhash: banner.blurhash, }); @@ -501,7 +501,7 @@ export class UserEntityService implements OnModuleInit { if (user.backgroundId != null && user.backgroundUrl === null) { const background = await this.driveFilesRepository.findOneByOrFail({ id: user.backgroundId }); user.backgroundUrl = this.driveFileEntityService.getPublicUrl(background); - this.usersRepository.update(user.id, { + await this.usersRepository.update(user.id, { backgroundUrl: user.backgroundUrl, backgroundBlurhash: background.blurhash, }); @@ -581,6 +581,7 @@ export class UserEntityService implements OnModuleInit { const bypassSilence = isMe || (myFollowings ? myFollowings.has(user.id) : false); + // noinspection ES6MissingAwait const packed = { id: user.id, name: user.name, @@ -894,7 +895,7 @@ export class UserEntityService implements OnModuleInit { myFollowingsPromise, ]); - return Promise.all( + return await Promise.all( _users.map(u => this.pack( u, me, diff --git a/packages/backend/src/core/entities/UserListEntityService.ts b/packages/backend/src/core/entities/UserListEntityService.ts index 2722d52195..55a15dd07b 100644 --- a/packages/backend/src/core/entities/UserListEntityService.ts +++ b/packages/backend/src/core/entities/UserListEntityService.ts @@ -67,7 +67,7 @@ export class UserListEntityService { const _users = memberships.map(({ user, userId }) => user ?? userId); const _userMap = await this.userEntityService.packMany(_users) .then(users => new Map(users.map(u => [u.id, u]))); - return Promise.all(memberships.map(async x => ({ + return await Promise.all(memberships.map(async x => ({ id: x.id, createdAt: this.idService.parse(x.id).date.toISOString(), userId: x.userId, diff --git a/packages/backend/src/queue/processors/ImportNotesProcessorService.ts b/packages/backend/src/queue/processors/ImportNotesProcessorService.ts index 6f5e929673..a9d179d90e 100644 --- a/packages/backend/src/queue/processors/ImportNotesProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportNotesProcessorService.ts @@ -417,7 +417,7 @@ export class ImportNotesProcessorService { const hashtags = extractApHashtagObjects(toot.object.tag).map((x) => x.name).filter((x): x is string => x != null); try { - text = await this.mfmService.fromHtml(toot.object.content, hashtags); + text = this.mfmService.fromHtml(toot.object.content, hashtags); } catch (error) { text = undefined; } @@ -487,7 +487,7 @@ export class ImportNotesProcessorService { const hashtags = extractApHashtagObjects(post.object.tag).map((x) => x.name).filter((x): x is string => x != null); try { - text = await this.mfmService.fromHtml(post.object.content, hashtags); + text = this.mfmService.fromHtml(post.object.content, hashtags); } catch (error) { text = undefined; } diff --git a/packages/backend/src/server/api/ApiCallService.ts b/packages/backend/src/server/api/ApiCallService.ts index 6605783ff1..be78a63ebb 100644 --- a/packages/backend/src/server/api/ApiCallService.ts +++ b/packages/backend/src/server/api/ApiCallService.ts @@ -145,11 +145,11 @@ export class ApiCallService implements OnApplicationShutdown { } @bindThis - public handleRequest( + public async handleRequest( endpoint: IEndpoint & { exec: any }, request: FastifyRequest<{ Body: Record | undefined, Querystring: Record }>, reply: FastifyReply, - ): void { + ): Promise { // Tell crawlers not to index API endpoints. // https://developers.google.com/search/docs/crawling-indexing/block-indexing reply.header('X-Robots-Tag', 'noindex'); @@ -166,8 +166,8 @@ export class ApiCallService implements OnApplicationShutdown { reply.code(400); return; } - this.authenticateService.authenticate(token).then(([user, app]) => { - this.call(endpoint, user, app, body, null, request, reply).then((res) => { + await this.authenticateService.authenticate(token).then(async ([user, app]) => { + await this.call(endpoint, user, app, body, null, request, reply).then((res) => { if (request.method === 'GET' && endpoint.meta.cacheSec && !token && !user) { reply.header('Cache-Control', `public, max-age=${endpoint.meta.cacheSec}`); } @@ -177,7 +177,8 @@ export class ApiCallService implements OnApplicationShutdown { }); if (user) { - this.logIp(request, user); + // logIp records errors directly + this.logIp(request, user).catch(() => null); } }).catch(err => { this.#sendAuthenticationError(reply, err); @@ -225,8 +226,8 @@ export class ApiCallService implements OnApplicationShutdown { reply.code(400); return; } - this.authenticateService.authenticate(token).then(([user, app]) => { - this.call(endpoint, user, app, fields, { + await this.authenticateService.authenticate(token).then(async ([user, app]) => { + await this.call(endpoint, user, app, fields, { name: multipartData.filename, path: path, }, request, reply).then((res) => { @@ -237,7 +238,8 @@ export class ApiCallService implements OnApplicationShutdown { }); if (user) { - this.logIp(request, user); + // logIp records errors directly + this.logIp(request, user).catch(() => null); } }).catch(err => { cleanup(); @@ -268,7 +270,7 @@ export class ApiCallService implements OnApplicationShutdown { } @bindThis - private logIp(request: FastifyRequest, user: MiLocalUser) { + private async logIp(request: FastifyRequest, user: MiLocalUser) { if (!this.meta.enableIpLogging) return; const ip = request.ip; if (!ip) { @@ -285,12 +287,13 @@ export class ApiCallService implements OnApplicationShutdown { } try { - this.userIpsRepository.createQueryBuilder().insert().values({ + await this.userIpsRepository.createQueryBuilder().insert().values({ createdAt: this.timeService.date, userId: user.id, ip: ip, }).orIgnore(true).execute(); - } catch { + } catch (err) { + this.logger.warn(`Failed to save IP address ${ip} for user ${user.id}: ${renderInlineError(err)}`); } } } diff --git a/packages/backend/src/server/api/ApiServerService.ts b/packages/backend/src/server/api/ApiServerService.ts index 6ed139ad77..8900048de0 100644 --- a/packages/backend/src/server/api/ApiServerService.ts +++ b/packages/backend/src/server/api/ApiServerService.ts @@ -194,7 +194,7 @@ export class ApiServerService { }); if (token && token.session != null && !token.fetched) { - this.accessTokensRepository.update(token.id, { + await this.accessTokensRepository.update(token.id, { fetched: true, }); diff --git a/packages/backend/src/server/api/SigninApiService.ts b/packages/backend/src/server/api/SigninApiService.ts index a53fec88d0..6ff5ea13f1 100644 --- a/packages/backend/src/server/api/SigninApiService.ts +++ b/packages/backend/src/server/api/SigninApiService.ts @@ -243,7 +243,7 @@ export class SigninApiService { if (same) { if (profile.password!.startsWith('$2')) { const newHash = await argon2.hash(password); - this.userProfilesRepository.update(user.id, { + await this.userProfilesRepository.update(user.id, { password: newHash, }); } @@ -267,7 +267,7 @@ export class SigninApiService { try { if (profile.password!.startsWith('$2')) { const newHash = await argon2.hash(password); - this.userProfilesRepository.update(user.id, { + await this.userProfilesRepository.update(user.id, { password: newHash, }); } diff --git a/packages/backend/src/server/api/endpoints/admin/abuse-report/notification-recipient/create.ts b/packages/backend/src/server/api/endpoints/admin/abuse-report/notification-recipient/create.ts index bdfbcba518..35c1eb225a 100644 --- a/packages/backend/src/server/api/endpoints/admin/abuse-report/notification-recipient/create.ts +++ b/packages/backend/src/server/api/endpoints/admin/abuse-report/notification-recipient/create.ts @@ -116,7 +116,7 @@ export default class extends Endpoint { // eslint- me, ); - return this.abuseReportNotificationRecipientEntityService.pack(result); + return await this.abuseReportNotificationRecipientEntityService.pack(result); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/abuse-report/notification-recipient/list.ts b/packages/backend/src/server/api/endpoints/admin/abuse-report/notification-recipient/list.ts index dad9161a8a..f36d356205 100644 --- a/packages/backend/src/server/api/endpoints/admin/abuse-report/notification-recipient/list.ts +++ b/packages/backend/src/server/api/endpoints/admin/abuse-report/notification-recipient/list.ts @@ -49,7 +49,7 @@ export default class extends Endpoint { // eslint- ) { super(meta, paramDef, async (ps) => { const recipients = await this.abuseReportNotificationService.fetchRecipients({ method: ps.method }); - return this.abuseReportNotificationRecipientEntityService.packMany(recipients); + return await this.abuseReportNotificationRecipientEntityService.packMany(recipients); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/abuse-report/notification-recipient/show.ts b/packages/backend/src/server/api/endpoints/admin/abuse-report/notification-recipient/show.ts index 557798f946..b2e5e197e3 100644 --- a/packages/backend/src/server/api/endpoints/admin/abuse-report/notification-recipient/show.ts +++ b/packages/backend/src/server/api/endpoints/admin/abuse-report/notification-recipient/show.ts @@ -58,7 +58,7 @@ export default class extends Endpoint { // eslint- throw new ApiError(meta.errors.noSuchRecipient); } - return this.abuseReportNotificationRecipientEntityService.pack(recipients[0]); + return await this.abuseReportNotificationRecipientEntityService.pack(recipients[0]); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/abuse-report/notification-recipient/update.ts b/packages/backend/src/server/api/endpoints/admin/abuse-report/notification-recipient/update.ts index bd4b485217..c3d88412b3 100644 --- a/packages/backend/src/server/api/endpoints/admin/abuse-report/notification-recipient/update.ts +++ b/packages/backend/src/server/api/endpoints/admin/abuse-report/notification-recipient/update.ts @@ -122,7 +122,7 @@ export default class extends Endpoint { // eslint- me, ); - return this.abuseReportNotificationRecipientEntityService.pack(result); + return await this.abuseReportNotificationRecipientEntityService.pack(result); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/captcha/current.ts b/packages/backend/src/server/api/endpoints/admin/captcha/current.ts index 7cc1bc675a..d620f937b1 100644 --- a/packages/backend/src/server/api/endpoints/admin/captcha/current.ts +++ b/packages/backend/src/server/api/endpoints/admin/captcha/current.ts @@ -75,7 +75,7 @@ export default class extends Endpoint { // eslint- private captchaService: CaptchaService, ) { super(meta, paramDef, async () => { - return this.captchaService.get(); + return await this.captchaService.get(); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/delete-all-files-of-a-user.ts b/packages/backend/src/server/api/endpoints/admin/delete-all-files-of-a-user.ts index 2ace85062a..f2c8ac7d42 100644 --- a/packages/backend/src/server/api/endpoints/admin/delete-all-files-of-a-user.ts +++ b/packages/backend/src/server/api/endpoints/admin/delete-all-files-of-a-user.ts @@ -50,7 +50,7 @@ export default class extends Endpoint { // eslint- }); for (const file of files) { - this.driveService.deleteFile(file); + await this.driveService.deleteFile(file); } }); } diff --git a/packages/backend/src/server/api/endpoints/admin/drive/cleanup.ts b/packages/backend/src/server/api/endpoints/admin/drive/cleanup.ts index f5d20366cf..3ca2c348a4 100644 --- a/packages/backend/src/server/api/endpoints/admin/drive/cleanup.ts +++ b/packages/backend/src/server/api/endpoints/admin/drive/cleanup.ts @@ -43,7 +43,7 @@ export default class extends Endpoint { // eslint- }); for (const file of files) { - this.driveService.deleteFile(file); + await this.driveService.deleteFile(file); } }); } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/add.ts b/packages/backend/src/server/api/endpoints/admin/emoji/add.ts index 5ef8307df0..130d2dd2cc 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/add.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/add.ts @@ -106,7 +106,7 @@ export default class extends Endpoint { // eslint- roleIdsThatCanBeUsedThisEmojiAsReaction: ps.roleIdsThatCanBeUsedThisEmojiAsReaction ?? [], }, me); - return this.emojiEntityService.packDetailed(emoji); + return await this.emojiEntityService.packDetailed(emoji); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts b/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts index a7d88954d9..7765a762d9 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts @@ -100,7 +100,7 @@ export default class extends Endpoint { // eslint- roleIdsThatCanBeUsedThisEmojiAsReaction: emoji.roleIdsThatCanBeUsedThisEmojiAsReaction, }, me); - return this.emojiEntityService.packDetailed(addedEmoji); + return await this.emojiEntityService.packDetailed(addedEmoji); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts b/packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts index 7f4ba083cf..0af5295ecb 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts @@ -108,7 +108,7 @@ export default class extends Endpoint { // eslint- .limit(ps.limit) .getMany(); - return this.emojiEntityService.packDetailedMany(emojis); + return await this.emojiEntityService.packDetailedMany(emojis); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/list.ts b/packages/backend/src/server/api/endpoints/admin/emoji/list.ts index b1b8e63d2f..9a5826d13d 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/list.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/list.ts @@ -111,7 +111,7 @@ export default class extends Endpoint { // eslint- emojis = await q.take(ps.limit).skip(ps.offset ?? 0).getMany(); } - return this.emojiEntityService.packDetailedMany(emojis); + return await this.emojiEntityService.packDetailedMany(emojis); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/federation/delete-all-files.ts b/packages/backend/src/server/api/endpoints/admin/federation/delete-all-files.ts index 89fd4be99c..b98d0a7163 100644 --- a/packages/backend/src/server/api/endpoints/admin/federation/delete-all-files.ts +++ b/packages/backend/src/server/api/endpoints/admin/federation/delete-all-files.ts @@ -45,7 +45,7 @@ export default class extends Endpoint { // eslint- }); for (const file of files) { - this.driveService.deleteFile(file); + await this.driveService.deleteFile(file); } }); } diff --git a/packages/backend/src/server/api/endpoints/admin/federation/refresh-remote-instance-metadata.ts b/packages/backend/src/server/api/endpoints/admin/federation/refresh-remote-instance-metadata.ts index 556e291025..d6f38601ea 100644 --- a/packages/backend/src/server/api/endpoints/admin/federation/refresh-remote-instance-metadata.ts +++ b/packages/backend/src/server/api/endpoints/admin/federation/refresh-remote-instance-metadata.ts @@ -42,7 +42,7 @@ export default class extends Endpoint { // eslint- throw new Error('instance not found'); } - this.fetchInstanceMetadataService.fetchInstanceMetadata(instance, true); + await this.fetchInstanceMetadataService.fetchInstanceMetadata(instance, true); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/federation/remove-all-following.ts b/packages/backend/src/server/api/endpoints/admin/federation/remove-all-following.ts index e5d85e1d57..8a66e004d7 100644 --- a/packages/backend/src/server/api/endpoints/admin/federation/remove-all-following.ts +++ b/packages/backend/src/server/api/endpoints/admin/federation/remove-all-following.ts @@ -57,7 +57,7 @@ export default class extends Endpoint { // eslint- host: ps.host, }); - this.queueService.createUnfollowJob(pairs.map(p => ({ from: p[0], to: p[1], silent: true }))); + await this.queueService.createUnfollowJob(pairs.map(p => ({ from: p[0], to: p[1], silent: true }))); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/queue/clear.ts b/packages/backend/src/server/api/endpoints/admin/queue/clear.ts index 81cb4b8119..c10e200f86 100644 --- a/packages/backend/src/server/api/endpoints/admin/queue/clear.ts +++ b/packages/backend/src/server/api/endpoints/admin/queue/clear.ts @@ -32,9 +32,9 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - this.queueService.queueClear(ps.queue, ps.state); + await this.queueService.queueClear(ps.queue, ps.state); - this.moderationLogService.log(me, 'clearQueue'); + await this.moderationLogService.log(me, 'clearQueue'); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/queue/jobs.ts b/packages/backend/src/server/api/endpoints/admin/queue/jobs.ts index aba68376ad..45b8f4ffb2 100644 --- a/packages/backend/src/server/api/endpoints/admin/queue/jobs.ts +++ b/packages/backend/src/server/api/endpoints/admin/queue/jobs.ts @@ -32,7 +32,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - return this.queueService.queueGetJobs(ps.queue, ps.state, ps.search); + return await this.queueService.queueGetJobs(ps.queue, ps.state, ps.search); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/queue/promote-jobs.ts b/packages/backend/src/server/api/endpoints/admin/queue/promote-jobs.ts index d22385e261..6ea49bdb7b 100644 --- a/packages/backend/src/server/api/endpoints/admin/queue/promote-jobs.ts +++ b/packages/backend/src/server/api/endpoints/admin/queue/promote-jobs.ts @@ -31,9 +31,9 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - this.queueService.queuePromoteJobs(ps.queue); + await this.queueService.queuePromoteJobs(ps.queue); - this.moderationLogService.log(me, 'promoteQueue'); + await this.moderationLogService.log(me, 'promoteQueue'); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/queue/queue-stats.ts b/packages/backend/src/server/api/endpoints/admin/queue/queue-stats.ts index 10ce48332a..eafc1998db 100644 --- a/packages/backend/src/server/api/endpoints/admin/queue/queue-stats.ts +++ b/packages/backend/src/server/api/endpoints/admin/queue/queue-stats.ts @@ -30,7 +30,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - return this.queueService.queueGetQueue(ps.queue); + return await this.queueService.queueGetQueue(ps.queue); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/queue/queues.ts b/packages/backend/src/server/api/endpoints/admin/queue/queues.ts index 3a38275f60..83def8bf27 100644 --- a/packages/backend/src/server/api/endpoints/admin/queue/queues.ts +++ b/packages/backend/src/server/api/endpoints/admin/queue/queues.ts @@ -29,7 +29,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - return this.queueService.queueGetQueues(); + return await this.queueService.queueGetQueues(); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/queue/remove-job.ts b/packages/backend/src/server/api/endpoints/admin/queue/remove-job.ts index 2c73f689d0..e2ca37c75d 100644 --- a/packages/backend/src/server/api/endpoints/admin/queue/remove-job.ts +++ b/packages/backend/src/server/api/endpoints/admin/queue/remove-job.ts @@ -32,7 +32,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - this.queueService.queueRemoveJob(ps.queue, ps.jobId); + await this.queueService.queueRemoveJob(ps.queue, ps.jobId); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/queue/retry-job.ts b/packages/backend/src/server/api/endpoints/admin/queue/retry-job.ts index b2603128f8..517f99e67f 100644 --- a/packages/backend/src/server/api/endpoints/admin/queue/retry-job.ts +++ b/packages/backend/src/server/api/endpoints/admin/queue/retry-job.ts @@ -32,7 +32,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - this.queueService.queueRetryJob(ps.queue, ps.jobId); + await this.queueService.queueRetryJob(ps.queue, ps.jobId); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/queue/show-job.ts b/packages/backend/src/server/api/endpoints/admin/queue/show-job.ts index 63747b5540..c931d57cb2 100644 --- a/packages/backend/src/server/api/endpoints/admin/queue/show-job.ts +++ b/packages/backend/src/server/api/endpoints/admin/queue/show-job.ts @@ -32,7 +32,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - return this.queueService.queueGetJob(ps.queue, ps.jobId); + return await this.queueService.queueGetJob(ps.queue, ps.jobId); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/system-webhook/create.ts b/packages/backend/src/server/api/endpoints/admin/system-webhook/create.ts index 28071e7a33..7dc5f1fd39 100644 --- a/packages/backend/src/server/api/endpoints/admin/system-webhook/create.ts +++ b/packages/backend/src/server/api/endpoints/admin/system-webhook/create.ts @@ -79,7 +79,7 @@ export default class extends Endpoint { // eslint- me, ); - return this.systemWebhookEntityService.pack(result); + return await this.systemWebhookEntityService.pack(result); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/system-webhook/list.ts b/packages/backend/src/server/api/endpoints/admin/system-webhook/list.ts index 7a440a774e..c776459c5e 100644 --- a/packages/backend/src/server/api/endpoints/admin/system-webhook/list.ts +++ b/packages/backend/src/server/api/endpoints/admin/system-webhook/list.ts @@ -54,7 +54,7 @@ export default class extends Endpoint { // eslint- isActive: ps.isActive, on: ps.on, }); - return this.systemWebhookEntityService.packMany(webhooks); + return await this.systemWebhookEntityService.packMany(webhooks); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/system-webhook/show.ts b/packages/backend/src/server/api/endpoints/admin/system-webhook/show.ts index 75862c96a7..024c00e59e 100644 --- a/packages/backend/src/server/api/endpoints/admin/system-webhook/show.ts +++ b/packages/backend/src/server/api/endpoints/admin/system-webhook/show.ts @@ -56,7 +56,7 @@ export default class extends Endpoint { // eslint- throw new ApiError(meta.errors.noSuchSystemWebhook); } - return this.systemWebhookEntityService.pack(webhooks[0]); + return await this.systemWebhookEntityService.pack(webhooks[0]); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/system-webhook/update.ts b/packages/backend/src/server/api/endpoints/admin/system-webhook/update.ts index 8d68bb8f87..72f7df2ad0 100644 --- a/packages/backend/src/server/api/endpoints/admin/system-webhook/update.ts +++ b/packages/backend/src/server/api/endpoints/admin/system-webhook/update.ts @@ -85,7 +85,7 @@ export default class extends Endpoint { // eslint- me, ); - return this.systemWebhookEntityService.pack(result); + return await this.systemWebhookEntityService.pack(result); }); } } diff --git a/packages/backend/src/server/api/endpoints/announcements.ts b/packages/backend/src/server/api/endpoints/announcements.ts index 08528ce826..c21d358ead 100644 --- a/packages/backend/src/server/api/endpoints/announcements.ts +++ b/packages/backend/src/server/api/endpoints/announcements.ts @@ -70,7 +70,7 @@ export default class extends Endpoint { // eslint- const announcements = await query.limit(ps.limit).getMany(); - return this.announcementEntityService.packMany(announcements, me); + return await this.announcementEntityService.packMany(announcements, me); }); } } diff --git a/packages/backend/src/server/api/endpoints/auth/session/userkey.ts b/packages/backend/src/server/api/endpoints/auth/session/userkey.ts index 1027eeb4d4..a80a0705d3 100644 --- a/packages/backend/src/server/api/endpoints/auth/session/userkey.ts +++ b/packages/backend/src/server/api/endpoints/auth/session/userkey.ts @@ -113,7 +113,7 @@ export default class extends Endpoint { // eslint- }); // Delete session - this.authSessionsRepository.delete(session.id); + await this.authSessionsRepository.delete(session.id); return { accessToken: accessToken.token, diff --git a/packages/backend/src/server/api/endpoints/chat/messages/show.ts b/packages/backend/src/server/api/endpoints/chat/messages/show.ts index 9a2bbb8742..950666deb9 100644 --- a/packages/backend/src/server/api/endpoints/chat/messages/show.ts +++ b/packages/backend/src/server/api/endpoints/chat/messages/show.ts @@ -59,7 +59,7 @@ export default class extends Endpoint { // eslint- if (message.fromUserId !== me.id && message.toUserId !== me.id && !(await this.roleService.isModerator(me))) { throw new ApiError(meta.errors.noSuchMessage); } - return this.chatEntityService.packMessageDetailed(message, me); + return await this.chatEntityService.packMessageDetailed(message, me); }); } } diff --git a/packages/backend/src/server/api/endpoints/chat/rooms/invitations/inbox.ts b/packages/backend/src/server/api/endpoints/chat/rooms/invitations/inbox.ts index 8a02d1c704..40688b026d 100644 --- a/packages/backend/src/server/api/endpoints/chat/rooms/invitations/inbox.ts +++ b/packages/backend/src/server/api/endpoints/chat/rooms/invitations/inbox.ts @@ -50,7 +50,7 @@ export default class extends Endpoint { // eslint- await this.chatService.checkChatAvailability(me.id, 'read'); const invitations = await this.chatService.getReceivedRoomInvitationsWithPagination(me.id, ps.limit, ps.sinceId, ps.untilId); - return this.chatEntityService.packRoomInvitations(invitations, me); + return await this.chatEntityService.packRoomInvitations(invitations, me); }); } } diff --git a/packages/backend/src/server/api/endpoints/chat/rooms/invitations/outbox.ts b/packages/backend/src/server/api/endpoints/chat/rooms/invitations/outbox.ts index 0702ba086c..61e6402cab 100644 --- a/packages/backend/src/server/api/endpoints/chat/rooms/invitations/outbox.ts +++ b/packages/backend/src/server/api/endpoints/chat/rooms/invitations/outbox.ts @@ -63,7 +63,7 @@ export default class extends Endpoint { // eslint- } const invitations = await this.chatService.getSentRoomInvitationsWithPagination(ps.roomId, ps.limit, ps.sinceId, ps.untilId); - return this.chatEntityService.packRoomInvitations(invitations, me); + return await this.chatEntityService.packRoomInvitations(invitations, me); }); } } diff --git a/packages/backend/src/server/api/endpoints/chat/rooms/joining.ts b/packages/backend/src/server/api/endpoints/chat/rooms/joining.ts index ba9242c762..c9a5288527 100644 --- a/packages/backend/src/server/api/endpoints/chat/rooms/joining.ts +++ b/packages/backend/src/server/api/endpoints/chat/rooms/joining.ts @@ -51,7 +51,7 @@ export default class extends Endpoint { // eslint- const memberships = await this.chatService.getMyMemberships(me.id, ps.limit, ps.sinceId, ps.untilId); - return this.chatEntityService.packRoomMemberships(memberships, me, { + return await this.chatEntityService.packRoomMemberships(memberships, me, { populateUser: false, populateRoom: true, }); diff --git a/packages/backend/src/server/api/endpoints/chat/rooms/members.ts b/packages/backend/src/server/api/endpoints/chat/rooms/members.ts index f5ffa21d32..388513e939 100644 --- a/packages/backend/src/server/api/endpoints/chat/rooms/members.ts +++ b/packages/backend/src/server/api/endpoints/chat/rooms/members.ts @@ -67,7 +67,7 @@ export default class extends Endpoint { // eslint- const memberships = await this.chatService.getRoomMembershipsWithPagination(room.id, ps.limit, ps.sinceId, ps.untilId); - return this.chatEntityService.packRoomMemberships(memberships, me, { + return await this.chatEntityService.packRoomMemberships(memberships, me, { populateUser: true, populateRoom: false, }); diff --git a/packages/backend/src/server/api/endpoints/chat/rooms/owned.ts b/packages/backend/src/server/api/endpoints/chat/rooms/owned.ts index accf7e1bee..d9cd60a39e 100644 --- a/packages/backend/src/server/api/endpoints/chat/rooms/owned.ts +++ b/packages/backend/src/server/api/endpoints/chat/rooms/owned.ts @@ -50,7 +50,7 @@ export default class extends Endpoint { // eslint- await this.chatService.checkChatAvailability(me.id, 'read'); const rooms = await this.chatService.getOwnedRoomsWithPagination(me.id, ps.limit, ps.sinceId, ps.untilId); - return this.chatEntityService.packRooms(rooms, me); + return await this.chatEntityService.packRooms(rooms, me); }); } } diff --git a/packages/backend/src/server/api/endpoints/chat/rooms/show.ts b/packages/backend/src/server/api/endpoints/chat/rooms/show.ts index 50da210d81..e4bc07cc0c 100644 --- a/packages/backend/src/server/api/endpoints/chat/rooms/show.ts +++ b/packages/backend/src/server/api/endpoints/chat/rooms/show.ts @@ -54,7 +54,7 @@ export default class extends Endpoint { // eslint- throw new ApiError(meta.errors.noSuchRoom); } - return this.chatEntityService.packRoom(room, me); + return await this.chatEntityService.packRoom(room, me); }); } } diff --git a/packages/backend/src/server/api/endpoints/chat/rooms/update.ts b/packages/backend/src/server/api/endpoints/chat/rooms/update.ts index 0cd62cb040..423cff1227 100644 --- a/packages/backend/src/server/api/endpoints/chat/rooms/update.ts +++ b/packages/backend/src/server/api/endpoints/chat/rooms/update.ts @@ -61,7 +61,7 @@ export default class extends Endpoint { // eslint- description: ps.description, }); - return this.chatEntityService.packRoom(updated, me); + return await this.chatEntityService.packRoom(updated, me); }); } } diff --git a/packages/backend/src/server/api/endpoints/clips/my-favorites.ts b/packages/backend/src/server/api/endpoints/clips/my-favorites.ts index 1f9b24e6c9..9278e0578d 100644 --- a/packages/backend/src/server/api/endpoints/clips/my-favorites.ts +++ b/packages/backend/src/server/api/endpoints/clips/my-favorites.ts @@ -56,7 +56,7 @@ export default class extends Endpoint { // eslint- const favorites = await query .getMany(); - return this.clipEntityService.packMany(favorites.map(x => x.clip!), me); + return await this.clipEntityService.packMany(favorites.map(x => x.clip!), me); }); } } diff --git a/packages/backend/src/server/api/endpoints/drive/folders/update.ts b/packages/backend/src/server/api/endpoints/drive/folders/update.ts index 8d51d09ea6..06beb82c7a 100644 --- a/packages/backend/src/server/api/endpoints/drive/folders/update.ts +++ b/packages/backend/src/server/api/endpoints/drive/folders/update.ts @@ -125,7 +125,7 @@ export default class extends Endpoint { // eslint- } // Update - this.driveFoldersRepository.update(folder.id, { + await this.driveFoldersRepository.update(folder.id, { name: folder.name, parentId: folder.parentId, }); diff --git a/packages/backend/src/server/api/endpoints/emoji.ts b/packages/backend/src/server/api/endpoints/emoji.ts index caef5d1528..7b1be89453 100644 --- a/packages/backend/src/server/api/endpoints/emoji.ts +++ b/packages/backend/src/server/api/endpoints/emoji.ts @@ -53,7 +53,7 @@ export default class extends Endpoint { // eslint- super(meta, paramDef, async (ps, me) => { const emoji = await this.customEmojiService.emojisByKeyCache.fetch(ps.name); - return this.emojiEntityService.packDetailed(emoji); + return await this.emojiEntityService.packDetailed(emoji); }); } } diff --git a/packages/backend/src/server/api/endpoints/export-custom-emojis.ts b/packages/backend/src/server/api/endpoints/export-custom-emojis.ts index 5ff099524d..53c157c5cd 100644 --- a/packages/backend/src/server/api/endpoints/export-custom-emojis.ts +++ b/packages/backend/src/server/api/endpoints/export-custom-emojis.ts @@ -29,7 +29,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - this.queueService.createExportCustomEmojisJob(me); + await this.queueService.createExportCustomEmojisJob(me); }); } } diff --git a/packages/backend/src/server/api/endpoints/federation/followers.ts b/packages/backend/src/server/api/endpoints/federation/followers.ts index 9add00ccde..ddf9cf0cbd 100644 --- a/packages/backend/src/server/api/endpoints/federation/followers.ts +++ b/packages/backend/src/server/api/endpoints/federation/followers.ts @@ -50,7 +50,7 @@ export default class extends Endpoint { // eslint- private followingEntityService: FollowingEntityService, ) { super(meta, paramDef, async (ps, me) => { - return this.followingEntityService.getFollowers(me, ps); + return await this.followingEntityService.getFollowers(me, ps); }); } } diff --git a/packages/backend/src/server/api/endpoints/federation/following.ts b/packages/backend/src/server/api/endpoints/federation/following.ts index 849bb61fb4..2ae3a0d485 100644 --- a/packages/backend/src/server/api/endpoints/federation/following.ts +++ b/packages/backend/src/server/api/endpoints/federation/following.ts @@ -50,7 +50,7 @@ export default class extends Endpoint { // eslint- private followingEntityService: FollowingEntityService, ) { super(meta, paramDef, async (ps, me) => { - return this.followingEntityService.getFollowing(me, ps); + return await this.followingEntityService.getFollowing(me, ps); }); } } diff --git a/packages/backend/src/server/api/endpoints/flash/my-likes.ts b/packages/backend/src/server/api/endpoints/flash/my-likes.ts index 22eae381da..baaf5ba845 100644 --- a/packages/backend/src/server/api/endpoints/flash/my-likes.ts +++ b/packages/backend/src/server/api/endpoints/flash/my-likes.ts @@ -72,7 +72,7 @@ export default class extends Endpoint { // eslint- .limit(ps.limit) .getMany(); - return this.flashLikeEntityService.packMany(likes, me); + return await this.flashLikeEntityService.packMany(likes, me); }); } } diff --git a/packages/backend/src/server/api/endpoints/hashtags/list.ts b/packages/backend/src/server/api/endpoints/hashtags/list.ts index b49c907432..b59d25b18b 100644 --- a/packages/backend/src/server/api/endpoints/hashtags/list.ts +++ b/packages/backend/src/server/api/endpoints/hashtags/list.ts @@ -91,7 +91,7 @@ export default class extends Endpoint { // eslint- const tags = await query.limit(ps.limit).getMany(); - return this.hashtagEntityService.packMany(tags); + return await this.hashtagEntityService.packMany(tags); }); } } diff --git a/packages/backend/src/server/api/endpoints/i/export-antennas.ts b/packages/backend/src/server/api/endpoints/i/export-antennas.ts index 77fb4a895f..c7ea520fb2 100644 --- a/packages/backend/src/server/api/endpoints/i/export-antennas.ts +++ b/packages/backend/src/server/api/endpoints/i/export-antennas.ts @@ -29,7 +29,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - this.queueService.createExportAntennasJob(me); + await this.queueService.createExportAntennasJob(me); }); } } diff --git a/packages/backend/src/server/api/endpoints/i/export-blocking.ts b/packages/backend/src/server/api/endpoints/i/export-blocking.ts index 7573018bec..84ad687cd6 100644 --- a/packages/backend/src/server/api/endpoints/i/export-blocking.ts +++ b/packages/backend/src/server/api/endpoints/i/export-blocking.ts @@ -29,7 +29,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - this.queueService.createExportBlockingJob(me); + await this.queueService.createExportBlockingJob(me); }); } } diff --git a/packages/backend/src/server/api/endpoints/i/export-clips.ts b/packages/backend/src/server/api/endpoints/i/export-clips.ts index 10d1fdac73..9611a32a29 100644 --- a/packages/backend/src/server/api/endpoints/i/export-clips.ts +++ b/packages/backend/src/server/api/endpoints/i/export-clips.ts @@ -29,7 +29,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - this.queueService.createExportClipsJob(me); + await this.queueService.createExportClipsJob(me); }); } } diff --git a/packages/backend/src/server/api/endpoints/i/export-data.ts b/packages/backend/src/server/api/endpoints/i/export-data.ts index d9a1e087b9..9465ac30ce 100644 --- a/packages/backend/src/server/api/endpoints/i/export-data.ts +++ b/packages/backend/src/server/api/endpoints/i/export-data.ts @@ -29,7 +29,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - this.queueService.createExportAccountDataJob(me); + await this.queueService.createExportAccountDataJob(me); }); } } diff --git a/packages/backend/src/server/api/endpoints/i/export-favorites.ts b/packages/backend/src/server/api/endpoints/i/export-favorites.ts index 5e03f70170..d482618769 100644 --- a/packages/backend/src/server/api/endpoints/i/export-favorites.ts +++ b/packages/backend/src/server/api/endpoints/i/export-favorites.ts @@ -29,7 +29,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - this.queueService.createExportFavoritesJob(me); + await this.queueService.createExportFavoritesJob(me); }); } } diff --git a/packages/backend/src/server/api/endpoints/i/export-following.ts b/packages/backend/src/server/api/endpoints/i/export-following.ts index 2e5ba14737..774704ba2a 100644 --- a/packages/backend/src/server/api/endpoints/i/export-following.ts +++ b/packages/backend/src/server/api/endpoints/i/export-following.ts @@ -32,7 +32,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - this.queueService.createExportFollowingJob(me, ps.excludeMuting, ps.excludeInactive); + await this.queueService.createExportFollowingJob(me, ps.excludeMuting, ps.excludeInactive); }); } } diff --git a/packages/backend/src/server/api/endpoints/i/export-mute.ts b/packages/backend/src/server/api/endpoints/i/export-mute.ts index 0384cf142b..902cea1ac2 100644 --- a/packages/backend/src/server/api/endpoints/i/export-mute.ts +++ b/packages/backend/src/server/api/endpoints/i/export-mute.ts @@ -29,7 +29,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - this.queueService.createExportMuteJob(me); + await this.queueService.createExportMuteJob(me); }); } } diff --git a/packages/backend/src/server/api/endpoints/i/export-notes.ts b/packages/backend/src/server/api/endpoints/i/export-notes.ts index db4e78f667..a3d7d2c0d9 100644 --- a/packages/backend/src/server/api/endpoints/i/export-notes.ts +++ b/packages/backend/src/server/api/endpoints/i/export-notes.ts @@ -29,7 +29,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - this.queueService.createExportNotesJob(me); + await this.queueService.createExportNotesJob(me); }); } } diff --git a/packages/backend/src/server/api/endpoints/i/export-user-lists.ts b/packages/backend/src/server/api/endpoints/i/export-user-lists.ts index 6cd662102c..72ebb4f16e 100644 --- a/packages/backend/src/server/api/endpoints/i/export-user-lists.ts +++ b/packages/backend/src/server/api/endpoints/i/export-user-lists.ts @@ -29,7 +29,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - this.queueService.createExportUserListsJob(me); + await this.queueService.createExportUserListsJob(me); }); } } diff --git a/packages/backend/src/server/api/endpoints/i/page-likes.ts b/packages/backend/src/server/api/endpoints/i/page-likes.ts index 19baa9726d..53a9b0f3b9 100644 --- a/packages/backend/src/server/api/endpoints/i/page-likes.ts +++ b/packages/backend/src/server/api/endpoints/i/page-likes.ts @@ -72,7 +72,7 @@ export default class extends Endpoint { // eslint- .limit(ps.limit) .getMany(); - return this.pageLikeEntityService.packMany(likes, me); + return await this.pageLikeEntityService.packMany(likes, me); }); } } diff --git a/packages/backend/src/server/api/endpoints/notes/schedule/list.ts b/packages/backend/src/server/api/endpoints/notes/schedule/list.ts index cbf3a961c0..374a4248d1 100644 --- a/packages/backend/src/server/api/endpoints/notes/schedule/list.ts +++ b/packages/backend/src/server/api/endpoints/notes/schedule/list.ts @@ -151,7 +151,7 @@ export default class extends Endpoint { // eslint- const note = await this.notesRepository.findOneBy({ id }); if (note) { note.reactionAndUserPairCache ??= []; - return this.noteEntityService.pack(note, me); + return await this.noteEntityService.pack(note, me); } } return null; diff --git a/packages/backend/src/server/api/endpoints/notifications/flush.ts b/packages/backend/src/server/api/endpoints/notifications/flush.ts index ab78435b89..ed619d5dd2 100644 --- a/packages/backend/src/server/api/endpoints/notifications/flush.ts +++ b/packages/backend/src/server/api/endpoints/notifications/flush.ts @@ -33,7 +33,7 @@ export default class extends Endpoint { // eslint- private notificationService: NotificationService, ) { super(meta, paramDef, async (ps, me) => { - this.notificationService.flushAllNotifications(me.id); + await this.notificationService.flushAllNotifications(me.id); }); } } diff --git a/packages/backend/src/server/api/endpoints/notifications/mark-all-as-read.ts b/packages/backend/src/server/api/endpoints/notifications/mark-all-as-read.ts index bc83f8d794..4dfe0b92b7 100644 --- a/packages/backend/src/server/api/endpoints/notifications/mark-all-as-read.ts +++ b/packages/backend/src/server/api/endpoints/notifications/mark-all-as-read.ts @@ -33,7 +33,7 @@ export default class extends Endpoint { // eslint- private notificationService: NotificationService, ) { super(meta, paramDef, async (ps, me) => { - this.notificationService.readAllNotification(me.id, true); + await this.notificationService.readAllNotification(me.id, true); }); } } diff --git a/packages/backend/src/server/api/endpoints/reset-password.ts b/packages/backend/src/server/api/endpoints/reset-password.ts index 813550bbcd..dd4e387a20 100644 --- a/packages/backend/src/server/api/endpoints/reset-password.ts +++ b/packages/backend/src/server/api/endpoints/reset-password.ts @@ -67,7 +67,7 @@ export default class extends Endpoint { // eslint- password: hash, }); - this.passwordResetRequestsRepository.delete(req.id); + await this.passwordResetRequestsRepository.delete(req.id); }); } } diff --git a/packages/backend/src/server/api/endpoints/users/lists/get-memberships.ts b/packages/backend/src/server/api/endpoints/users/lists/get-memberships.ts index 559f08b654..142176deb2 100644 --- a/packages/backend/src/server/api/endpoints/users/lists/get-memberships.ts +++ b/packages/backend/src/server/api/endpoints/users/lists/get-memberships.ts @@ -108,7 +108,7 @@ export default class extends Endpoint { .limit(ps.limit) .getMany(); - return this.userListEntityService.packMembershipsMany(memberships); + return await this.userListEntityService.packMembershipsMany(memberships); }); } } diff --git a/packages/backend/src/server/api/mastodon/MastodonConverters.ts b/packages/backend/src/server/api/mastodon/MastodonConverters.ts index 6b0283bf55..69c7817021 100644 --- a/packages/backend/src/server/api/mastodon/MastodonConverters.ts +++ b/packages/backend/src/server/api/mastodon/MastodonConverters.ts @@ -175,7 +175,7 @@ export class MastodonConverters { const bioText = profile?.description && this.mfmService.toMastoApiHtml(mfm.parse(profile.description)); - return awaitAll({ + return await awaitAll({ id: account.id, username: user.username, acct: acct, From 6d8d91ba8c5b718987b3fdb7e0406b9b58f6ce3d Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Thu, 19 Jun 2025 13:51:48 -0400 Subject: [PATCH 014/107] fix type error in DriveService.ts --- packages/backend/src/core/DriveService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts index 468c63a9cc..28e79d1c1d 100644 --- a/packages/backend/src/core/DriveService.ts +++ b/packages/backend/src/core/DriveService.ts @@ -207,7 +207,7 @@ export class DriveService { //#region Uploads this.registerLogger.debug(`uploading original: ${key}`); - const uploads = [ + const uploads: Promise[] = [ this.upload(key, fs.createReadStream(path), type, null, name), ]; From db4cd29b748c5bace1f382929a6e927fc0752954 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Thu, 19 Jun 2025 13:52:06 -0400 Subject: [PATCH 015/107] fix non-Error exceptions thrown from FetchInstanceMetadataService.ts --- packages/backend/src/core/FetchInstanceMetadataService.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/backend/src/core/FetchInstanceMetadataService.ts b/packages/backend/src/core/FetchInstanceMetadataService.ts index 855ecd7553..8453758c39 100644 --- a/packages/backend/src/core/FetchInstanceMetadataService.ts +++ b/packages/backend/src/core/FetchInstanceMetadataService.ts @@ -187,10 +187,7 @@ export class FetchInstanceMetadataService { throw new Error('No nodeinfo link provided'); } - const info = await this.httpRequestService.getJson(link.href) - .catch(err => { - throw err.statusCode ?? err.message; - }); + const info = await this.httpRequestService.getJson(link.href); this.logger.debug(`Successfuly fetched nodeinfo of ${instance.host}`); From a4a8329f691cb03543ba989c90c3ae8bceb49062 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Thu, 19 Jun 2025 13:52:42 -0400 Subject: [PATCH 016/107] optionally use note.user relation in PollService.deliverQuestionUpdate --- packages/backend/src/core/PollService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/core/PollService.ts b/packages/backend/src/core/PollService.ts index e93cc7ba4c..b5544db9d4 100644 --- a/packages/backend/src/core/PollService.ts +++ b/packages/backend/src/core/PollService.ts @@ -92,7 +92,7 @@ export class PollService { public async deliverQuestionUpdate(note: MiNote) { if (note.localOnly) return; - const user = await this.usersRepository.findOneBy({ id: note.userId }); + const user = note.user ?? await this.usersRepository.findOneBy({ id: note.userId }); if (user == null) throw new Error('note not found'); if (isLocalUser(user)) { From dad9944e7ee64ef5b28850a331f4436459a5c0a5 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Thu, 19 Jun 2025 13:53:00 -0400 Subject: [PATCH 017/107] a few more awaits --- packages/backend/src/models/_.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/models/_.ts b/packages/backend/src/models/_.ts index e362230d7e..071945646a 100644 --- a/packages/backend/src/models/_.ts +++ b/packages/backend/src/models/_.ts @@ -118,12 +118,12 @@ export const miRepository = { if (opt.replication) { const queryRunner = this.manager.connection.createQueryRunner('master'); try { - return this.insertOneImpl(entity, findOptions, queryRunner); + return await this.insertOneImpl(entity, findOptions, queryRunner); } finally { await queryRunner.release(); } } else { - return this.insertOneImpl(entity, findOptions); + return await this.insertOneImpl(entity, findOptions); } }, async insertOneImpl(entity, findOptions?, queryRunner?) { From a45e264574e042f415943842356e11b7be0bdc87 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Thu, 19 Jun 2025 13:53:33 -0400 Subject: [PATCH 018/107] rename noteEditRepository to noteEditsRepository, matching other repository names --- packages/backend/src/di-symbols.ts | 2 +- packages/backend/src/models/RepositoryModule.ts | 2 +- packages/backend/src/models/_.ts | 2 +- .../src/queue/processors/DeleteAccountProcessorService.ts | 8 ++++---- packages/backend/src/server/api/GetterService.ts | 8 ++++---- .../backend/src/server/api/endpoints/notes/unrenote.ts | 7 ++++--- .../backend/src/server/api/mastodon/MastodonConverters.ts | 8 ++++---- 7 files changed, 19 insertions(+), 18 deletions(-) diff --git a/packages/backend/src/di-symbols.ts b/packages/backend/src/di-symbols.ts index e2c73562c8..d34431cdea 100644 --- a/packages/backend/src/di-symbols.ts +++ b/packages/backend/src/di-symbols.ts @@ -93,7 +93,7 @@ export const DI = { chatRoomsRepository: Symbol('chatRoomsRepository'), chatRoomMembershipsRepository: Symbol('chatRoomMembershipsRepository'), chatRoomInvitationsRepository: Symbol('chatRoomInvitationsRepository'), - noteEditRepository: Symbol('noteEditRepository'), + noteEditsRepository: Symbol('noteEditsRepository'), bubbleGameRecordsRepository: Symbol('bubbleGameRecordsRepository'), reversiGamesRepository: Symbol('reversiGamesRepository'), noteScheduleRepository: Symbol('noteScheduleRepository'), diff --git a/packages/backend/src/models/RepositoryModule.ts b/packages/backend/src/models/RepositoryModule.ts index 5e0154fe50..476765cc37 100644 --- a/packages/backend/src/models/RepositoryModule.ts +++ b/packages/backend/src/models/RepositoryModule.ts @@ -519,7 +519,7 @@ const $userMemosRepository: Provider = { }; const $noteEditRepository: Provider = { - provide: DI.noteEditRepository, + provide: DI.noteEditsRepository, useFactory: (db: DataSource) => db.getRepository(NoteEdit), inject: [DI.db], }; diff --git a/packages/backend/src/models/_.ts b/packages/backend/src/models/_.ts index 071945646a..1e742b0bf8 100644 --- a/packages/backend/src/models/_.ts +++ b/packages/backend/src/models/_.ts @@ -326,5 +326,5 @@ export type ChatRoomInvitationsRepository = Repository & M export type ChatApprovalsRepository = Repository & MiRepository; export type BubbleGameRecordsRepository = Repository & MiRepository; export type ReversiGamesRepository = Repository & MiRepository; -export type NoteEditRepository = Repository & MiRepository; +export type NoteEditsRepository = Repository & MiRepository; export type NoteScheduleRepository = Repository & MiRepository; diff --git a/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts b/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts index 06a4b7ab7f..d4a32f6249 100644 --- a/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts +++ b/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts @@ -6,7 +6,7 @@ import { Inject, Injectable } from '@nestjs/common'; import { In, MoreThan } from 'typeorm'; import { DI } from '@/di-symbols.js'; -import type { DriveFilesRepository, NoteReactionsRepository, NotesRepository, UserProfilesRepository, UsersRepository, NoteScheduleRepository, MiNoteSchedule, FollowingsRepository, FollowRequestsRepository, BlockingsRepository, MutingsRepository, ClipsRepository, ClipNotesRepository, LatestNotesRepository, NoteEditRepository, NoteFavoritesRepository, PollVotesRepository, PollsRepository, SigninsRepository, UserIpsRepository, RegistryItemsRepository } from '@/models/_.js'; +import type { DriveFilesRepository, NoteReactionsRepository, NotesRepository, UserProfilesRepository, UsersRepository, NoteScheduleRepository, MiNoteSchedule, FollowingsRepository, FollowRequestsRepository, BlockingsRepository, MutingsRepository, ClipsRepository, ClipNotesRepository, LatestNotesRepository, NoteEditsRepository, NoteFavoritesRepository, PollVotesRepository, PollsRepository, SigninsRepository, UserIpsRepository, RegistryItemsRepository } from '@/models/_.js'; import type Logger from '@/logger.js'; import { DriveService } from '@/core/DriveService.js'; import type { MiDriveFile } from '@/models/DriveFile.js'; @@ -69,8 +69,8 @@ export class DeleteAccountProcessorService { @Inject(DI.latestNotesRepository) private readonly latestNotesRepository: LatestNotesRepository, - @Inject(DI.noteEditRepository) - private readonly noteEditRepository: NoteEditRepository, + @Inject(DI.noteEditsRepository) + private readonly noteEditsRepository: NoteEditsRepository, @Inject(DI.noteFavoritesRepository) private readonly noteFavoritesRepository: NoteFavoritesRepository, @@ -318,7 +318,7 @@ export class DeleteAccountProcessorService { const ids = notes.map(note => note.id); - await this.noteEditRepository.delete({ + await this.noteEditsRepository.delete({ noteId: In(ids), }); await this.notesRepository.delete({ diff --git a/packages/backend/src/server/api/GetterService.ts b/packages/backend/src/server/api/GetterService.ts index edf70c0185..47b4e2f1bc 100644 --- a/packages/backend/src/server/api/GetterService.ts +++ b/packages/backend/src/server/api/GetterService.ts @@ -5,7 +5,7 @@ import { Inject, Injectable } from '@nestjs/common'; import { DI } from '@/di-symbols.js'; -import type { NotesRepository, UsersRepository, NoteEditRepository } from '@/models/_.js'; +import type { NotesRepository, UsersRepository, NoteEditsRepository } from '@/models/_.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; import type { MiLocalUser, MiRemoteUser, MiUser } from '@/models/User.js'; import { isRemoteUser, isLocalUser } from '@/models/User.js'; @@ -22,8 +22,8 @@ export class GetterService { @Inject(DI.notesRepository) private notesRepository: NotesRepository, - @Inject(DI.noteEditRepository) - private noteEditRepository: NoteEditRepository, + @Inject(DI.noteEditsRepository) + private noteEditsRepository: NoteEditsRepository, private readonly cacheService: CacheService, ) { @@ -59,7 +59,7 @@ export class GetterService { */ @bindThis public async getEdits(noteId: MiNote['id']) { - const edits = await this.noteEditRepository.findBy({ noteId: noteId }).catch(() => { + const edits = await this.noteEditsRepository.findBy({ noteId: noteId }).catch(() => { throw new IdentifiableError('9725d0ce-ba28-4dde-95a7-2cbb2c15de24', `Note ${noteId} does not exist`); }); diff --git a/packages/backend/src/server/api/endpoints/notes/unrenote.ts b/packages/backend/src/server/api/endpoints/notes/unrenote.ts index 4a3ca8b656..5c5bd740dc 100644 --- a/packages/backend/src/server/api/endpoints/notes/unrenote.ts +++ b/packages/backend/src/server/api/endpoints/notes/unrenote.ts @@ -10,6 +10,7 @@ import { Endpoint } from '@/server/api/endpoint-base.js'; import { NoteDeleteService } from '@/core/NoteDeleteService.js'; import { DI } from '@/di-symbols.js'; import { GetterService } from '@/server/api/GetterService.js'; +import { isQuote, Renote } from '@/misc/is-renote.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -64,14 +65,14 @@ export default class extends Endpoint { // eslint- const renotes = await this.notesRepository.findBy({ userId: me.id, renoteId: note.id, - }); + }) as Renote[]; // TODO inline this into the above query for (const note of renotes) { if (ps.quote) { - if (note.text) this.noteDeleteService.delete(me, note, false); + if (isQuote(note)) await this.noteDeleteService.delete(me, note, false); } else { - if (!note.text) this.noteDeleteService.delete(me, note, false); + if (!isQuote(note)) await this.noteDeleteService.delete(me, note, false); } } }); diff --git a/packages/backend/src/server/api/mastodon/MastodonConverters.ts b/packages/backend/src/server/api/mastodon/MastodonConverters.ts index 69c7817021..963bde3726 100644 --- a/packages/backend/src/server/api/mastodon/MastodonConverters.ts +++ b/packages/backend/src/server/api/mastodon/MastodonConverters.ts @@ -13,7 +13,7 @@ import { MfmService } from '@/core/MfmService.js'; import type { Config } from '@/config.js'; import { IMentionedRemoteUsers, MiNote } from '@/models/Note.js'; import type { MiLocalUser, MiUser } from '@/models/User.js'; -import type { NoteEditRepository, UserProfilesRepository } from '@/models/_.js'; +import type { NoteEditsRepository, UserProfilesRepository } from '@/models/_.js'; import { awaitAll } from '@/misc/prelude/await-all.js'; import { CustomEmojiService } from '@/core/CustomEmojiService.js'; import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js'; @@ -60,8 +60,8 @@ export class MastodonConverters { @Inject(DI.userProfilesRepository) private readonly userProfilesRepository: UserProfilesRepository, - @Inject(DI.noteEditRepository) - private readonly noteEditRepository: NoteEditRepository, + @Inject(DI.noteEditsRepository) + private readonly noteEditsRepository: NoteEditsRepository, private readonly mfmService: MfmService, private readonly getterService: GetterService, @@ -214,7 +214,7 @@ export class MastodonConverters { const noteUser = await this.getUser(note.userId); const noteInstance = noteUser.instance ?? (noteUser.host ? await this.federatedInstanceService.fetch(noteUser.host) : null); const account = await this.convertAccount(noteUser); - const edits = await this.noteEditRepository.find({ where: { noteId: note.id }, order: { id: 'ASC' } }); + const edits = await this.noteEditsRepository.find({ where: { noteId: note.id }, order: { id: 'ASC' } }); const history: StatusEdit[] = []; const mentionedRemoteUsers = JSON.parse(note.mentionedRemoteUsers); From ec6ae4c939a100d0e4ce94d259a879ac90237ba1 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Thu, 19 Jun 2025 16:13:16 -0400 Subject: [PATCH 019/107] fix typo and correctly call handleCreatedNote in BackgroundTaskProcessorService.ts --- .../src/queue/processors/BackgroundTaskProcessorService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/queue/processors/BackgroundTaskProcessorService.ts b/packages/backend/src/queue/processors/BackgroundTaskProcessorService.ts index 3d8339fa3b..8e655fc08f 100644 --- a/packages/backend/src/queue/processors/BackgroundTaskProcessorService.ts +++ b/packages/backend/src/queue/processors/BackgroundTaskProcessorService.ts @@ -290,7 +290,7 @@ export class BackgroundTaskProcessorService { await this.latestNoteService.handleUpdatedNote(lastEdit, note); } else { // Create - await this.latestNoteService.handleDeletedNote(note); + await this.latestNoteService.handleCreatedNote(note); } } else { // Delete From 4501ecb293c390b918c58c5409ab7d04de8c7023 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Thu, 19 Jun 2025 17:12:06 -0400 Subject: [PATCH 020/107] implement DeleteApLogsBackgroundTask --- packages/backend/src/core/ApLogService.ts | 17 ++++++++-- .../backend/src/core/NoteDeleteService.ts | 5 ++- packages/backend/src/core/QueueService.ts | 5 +++ .../BackgroundTaskProcessorService.ts | 31 ++++++++++++++----- .../DeleteAccountProcessorService.ts | 9 ++---- packages/backend/src/queue/types.ts | 9 +++++- 6 files changed, 57 insertions(+), 19 deletions(-) diff --git a/packages/backend/src/core/ApLogService.ts b/packages/backend/src/core/ApLogService.ts index 89837a60d2..391d90b95a 100644 --- a/packages/backend/src/core/ApLogService.ts +++ b/packages/backend/src/core/ApLogService.ts @@ -14,7 +14,9 @@ import { JsonValue } from '@/misc/json-value.js'; import { UtilityService } from '@/core/UtilityService.js'; import { TimeService } from '@/global/TimeService.js'; import { IdService } from '@/core/IdService.js'; -import { IActivity, IObject } from './activitypub/type.js'; +import { IActivity, IObject } from '@/core/activitypub/type.js'; +import { bindThis } from '@/decorators.js'; +import { QueueService } from '@/core/QueueService.js'; @Injectable() export class ApLogService { @@ -23,7 +25,7 @@ export class ApLogService { private readonly config: Config, @Inject(DI.apContextsRepository) - private apContextsRepository: ApContextsRepository, + private readonly apContextsRepository: ApContextsRepository, @Inject(DI.apInboxLogsRepository) private readonly apInboxLogsRepository: ApInboxLogsRepository, @@ -34,6 +36,7 @@ export class ApLogService { private readonly utilityService: UtilityService, private readonly idService: IdService, private readonly timeService: TimeService, + private readonly queueService: QueueService, ) {} /** @@ -123,6 +126,16 @@ export class ApLogService { .execute(); } + @bindThis + public async deleteObjectLogsDeferred(objectUris: string | string[]): Promise { + await this.queueService.createDeleteApLogsJob('object', objectUris); + } + + @bindThis + public async deleteInboxLogsDeferred(userIds: string | string[]): Promise { + await this.queueService.createDeleteApLogsJob('inbox', userIds); + } + /** * Deletes all logged copies of an object or objects * @param objectUris URIs / AP IDs of the objects to delete diff --git a/packages/backend/src/core/NoteDeleteService.ts b/packages/backend/src/core/NoteDeleteService.ts index e9cf9bebef..d780e5167d 100644 --- a/packages/backend/src/core/NoteDeleteService.ts +++ b/packages/backend/src/core/NoteDeleteService.ts @@ -161,7 +161,7 @@ export class NoteDeleteService { // Update the Latest Note index / following feed this.latestNoteService.handleDeletedNoteDeferred(note); for (const cascadingNote of cascadingNotes) { - this.latestNoteService.handleDeletedNote(cascadingNote); + this.latestNoteService.handleDeletedNoteDeferred(cascadingNote); } if (deleter && (note.userId !== deleter.id)) { @@ -178,8 +178,7 @@ export class NoteDeleteService { .map(n => n.uri) .filter((u): u is string => u != null); if (deletedUris.length > 0) { - trackPromise(this.apLogService.deleteObjectLogs(deletedUris) - .catch(err => this.logger.error(err, `Failed to delete AP logs for note '${note.uri}'`))); + await this.apLogService.deleteObjectLogsDeferred(deletedUris); } } diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 95cc46c8cf..99911f38a7 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -930,6 +930,11 @@ export class QueueService implements OnModuleInit { return await this.createBackgroundTask({ type: 'post-unsuspend', userId }, userId); } + @bindThis + public async createDeleteApLogsJob(dataType: 'inbox' | 'object', data: string | string[]) { + return await this.createBackgroundTask({ type: 'delete-ap-logs', dataType, data }); + } + private async createBackgroundTask(data: T, duplication?: string | { id: string, ttl?: number }) { return await this.backgroundTaskQueue.add( data.type, diff --git a/packages/backend/src/queue/processors/BackgroundTaskProcessorService.ts b/packages/backend/src/queue/processors/BackgroundTaskProcessorService.ts index 8e655fc08f..826b2af193 100644 --- a/packages/backend/src/queue/processors/BackgroundTaskProcessorService.ts +++ b/packages/backend/src/queue/processors/BackgroundTaskProcessorService.ts @@ -5,7 +5,7 @@ import { Inject, Injectable } from '@nestjs/common'; import * as Bull from 'bullmq'; -import { BackgroundTaskJobData, CheckHibernationBackgroundTask, PostDeliverBackgroundTask, PostInboxBackgroundTask, PostNoteBackgroundTask, UpdateFeaturedBackgroundTask, UpdateInstanceBackgroundTask, UpdateUserTagsBackgroundTask, UpdateUserBackgroundTask, UpdateNoteTagsBackgroundTask, DeleteFileBackgroundTask, UpdateLatestNoteBackgroundTask, PostSuspendBackgroundTask, PostUnsuspendBackgroundTask } from '@/queue/types.js'; +import { BackgroundTaskJobData, CheckHibernationBackgroundTask, PostDeliverBackgroundTask, PostInboxBackgroundTask, PostNoteBackgroundTask, UpdateFeaturedBackgroundTask, UpdateInstanceBackgroundTask, UpdateUserTagsBackgroundTask, UpdateUserBackgroundTask, UpdateNoteTagsBackgroundTask, DeleteFileBackgroundTask, UpdateLatestNoteBackgroundTask, PostSuspendBackgroundTask, PostUnsuspendBackgroundTask, DeleteApLogsBackgroundTask } from '@/queue/types.js'; import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js'; import { QueueLoggerService } from '@/queue/QueueLoggerService.js'; import Logger from '@/logger.js'; @@ -27,6 +27,7 @@ import { DriveService } from '@/core/DriveService.js'; import { LatestNoteService } from '@/core/LatestNoteService.js'; import { trackTask } from '@/misc/promise-tracker.js'; import { UserSuspendService } from '@/core/UserSuspendService.js'; +import { ApLogService } from '@/core/ApLogService.js'; @Injectable() export class BackgroundTaskProcessorService { @@ -59,6 +60,7 @@ export class BackgroundTaskProcessorService { private readonly driveService: DriveService, private readonly latestNoteService: LatestNoteService, private readonly userSuspendService: UserSuspendService, + private readonly apLogService: ApLogService, queueLoggerService: QueueLoggerService, ) { @@ -90,9 +92,11 @@ export class BackgroundTaskProcessorService { return await this.processUpdateLatestNote(job.data); } else if (job.data.type === 'post-suspend') { return await this.processPostSuspend(job.data); - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition } else if (job.data.type === 'post-unsuspend') { return await this.processPostUnsuspend(job.data); + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + } else if (job.data.type === 'delete-ap-logs') { + return await this.processDeleteApLogs(job.data); } else { this.logger.warn(`Can't process unknown job type "${job.data}"; this is likely a bug. Full job data:`, job.data); throw new Error(`Unknown job type ${job.data}, see system logs for details`); @@ -199,14 +203,14 @@ export class BackgroundTaskProcessorService { // Update charts if (this.meta.enableChartsForFederatedInstances) { - await this.instanceChart.requestSent(task.host, success); + this.instanceChart.requestSent(task.host, success); } if (success) { - await this.apRequestChart.deliverSucc(); + this.apRequestChart.deliverSucc(); } else { - await this.apRequestChart.deliverFail(); + this.apRequestChart.deliverFail(); } - await this.federationChart.deliverd(task.host, success); + this.federationChart.deliverd(task.host, success); return 'ok'; } @@ -215,7 +219,6 @@ export class BackgroundTaskProcessorService { const instance = await this.federatedInstanceService.fetchOrRegister(task.host); if (instance.isBlocked) return `Skipping post-inbox task: instance ${task.host} is blocked`; - // TODO move chart stuff out of background? // Update charts if (this.meta.enableChartsForFederatedInstances) { this.instanceChart.requestReceived(task.host); @@ -321,4 +324,18 @@ export class BackgroundTaskProcessorService { return 'ok'; } + + private async processDeleteApLogs(task: DeleteApLogsBackgroundTask): Promise { + if (task.dataType === 'object') { + await this.apLogService.deleteObjectLogs(task.data); + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + } else if (task.dataType === 'inbox') { + await this.apLogService.deleteInboxLogs(task.data); + } else { + this.logger.warn(`Can't process unknown data type "${task.dataType}"; this is likely a bug. Full task data:`, task); + throw new Error(`Unknown task type ${task.dataType}, see system logs for details`); + } + + return 'ok'; + } } diff --git a/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts b/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts index d4a32f6249..7132b5841b 100644 --- a/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts +++ b/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts @@ -332,8 +332,7 @@ export class DeleteAccountProcessorService { // Delete note AP logs const noteUris = notes.map(n => n.uri).filter(u => !!u) as string[]; if (noteUris.length > 0) { - await this.apLogService.deleteObjectLogs(noteUris) - .catch(err => this.logger.error(err, `Failed to delete AP logs for notes of user '${user.uri ?? user.id}'`)); + await this.apLogService.deleteObjectLogsDeferred(noteUris); } } @@ -371,12 +370,10 @@ export class DeleteAccountProcessorService { { // Delete actor logs if (user.uri) { - await this.apLogService.deleteObjectLogs(user.uri) - .catch(err => this.logger.error(err, `Failed to delete AP logs for user '${user.uri}'`)); + await this.apLogService.deleteObjectLogsDeferred(user.uri); } - await this.apLogService.deleteInboxLogs(user.id) - .catch(err => this.logger.error(err, `Failed to delete AP logs for user '${user.uri}'`)); + await this.apLogService.deleteInboxLogsDeferred(user.id); this.logger.info('All AP logs deleted'); } diff --git a/packages/backend/src/queue/types.ts b/packages/backend/src/queue/types.ts index ed9bbdbadf..cb31e5a3e1 100644 --- a/packages/backend/src/queue/types.ts +++ b/packages/backend/src/queue/types.ts @@ -183,7 +183,8 @@ export type BackgroundTaskJobData = DeleteFileBackgroundTask | UpdateLatestNoteBackgroundTask | PostSuspendBackgroundTask | - PostUnsuspendBackgroundTask; + PostUnsuspendBackgroundTask | + DeleteApLogsBackgroundTask; export type UpdateUserBackgroundTask = { type: 'update-user'; @@ -254,3 +255,9 @@ export type PostUnsuspendBackgroundTask = { type: 'post-unsuspend'; userId: string; }; + +export type DeleteApLogsBackgroundTask = { + type: 'delete-ap-logs'; + dataType: 'inbox' | 'object'; + data: string | string[]; +}; From e587e376e5242f310c24fb7eaf38719c1fef32ac Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Thu, 19 Jun 2025 17:15:13 -0400 Subject: [PATCH 021/107] fix account deletion not triggering background tasks for cascaded note deletions --- .../DeleteAccountProcessorService.ts | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts b/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts index 7132b5841b..b95b89d6ce 100644 --- a/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts +++ b/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts @@ -4,9 +4,9 @@ */ import { Inject, Injectable } from '@nestjs/common'; -import { In, MoreThan } from 'typeorm'; +import { In, IsNull, MoreThan } from 'typeorm'; import { DI } from '@/di-symbols.js'; -import type { DriveFilesRepository, NoteReactionsRepository, NotesRepository, UserProfilesRepository, UsersRepository, NoteScheduleRepository, MiNoteSchedule, FollowingsRepository, FollowRequestsRepository, BlockingsRepository, MutingsRepository, ClipsRepository, ClipNotesRepository, LatestNotesRepository, NoteEditsRepository, NoteFavoritesRepository, PollVotesRepository, PollsRepository, SigninsRepository, UserIpsRepository, RegistryItemsRepository } from '@/models/_.js'; +import type { DriveFilesRepository, NoteReactionsRepository, NotesRepository, UserProfilesRepository, UsersRepository, NoteScheduleRepository, MiNoteSchedule, FollowingsRepository, FollowRequestsRepository, BlockingsRepository, MutingsRepository, ClipsRepository, ClipNotesRepository, LatestNotesRepository, NoteEditsRepository, NoteFavoritesRepository, PollVotesRepository, PollsRepository, SigninsRepository, UserIpsRepository, RegistryItemsRepository, MiUser } from '@/models/_.js'; import type Logger from '@/logger.js'; import { DriveService } from '@/core/DriveService.js'; import type { MiDriveFile } from '@/models/DriveFile.js'; @@ -19,6 +19,7 @@ import { ApLogService } from '@/core/ApLogService.js'; import { ReactionService } from '@/core/ReactionService.js'; import { QueueService } from '@/core/QueueService.js'; import { CacheService } from '@/core/CacheService.js'; +import { NoteDeleteService } from '@/core/NoteDeleteService.js'; import { QueueLoggerService } from '@/queue/QueueLoggerService.js'; import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js'; import * as Acct from '@/misc/acct.js'; @@ -99,6 +100,7 @@ export class DeleteAccountProcessorService { private readonly apLogService: ApLogService, private readonly cacheService: CacheService, private readonly apPersonService: ApPersonService, + private readonly noteDeleteService: NoteDeleteService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('delete-account'); } @@ -293,11 +295,12 @@ export class DeleteAccountProcessorService { const notes = await this.notesRepository.find({ where: { userId: user.id, + replyId: IsNull(), ...(cursor ? { id: MoreThan(cursor) } : {}), }, take: 100, order: { - id: 1, + id: 'desc', }, }) as MiNote[]; @@ -318,6 +321,16 @@ export class DeleteAccountProcessorService { const ids = notes.map(note => note.id); + const replies = await this.notesRepository.find({ + where: { replyId: In(ids) }, + relations: { user: true }, + }); + + // Delete replies through the usual service to ensure we get all "cascading notes" logic. + for (const reply of replies) { + await this.noteDeleteService.delete(reply.user as MiUser, reply); + } + await this.noteEditsRepository.delete({ noteId: In(ids), }); From 693f8d52eeaa516d0716451dd8c254c356f5365d Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Thu, 19 Jun 2025 17:24:33 -0400 Subject: [PATCH 022/107] implement MarkUserUpdatedBackgroundTask --- .../backend/src/core/NoteCreateService.ts | 2 +- .../backend/src/core/NoteDeleteService.ts | 13 +++------ packages/backend/src/core/NoteEditService.ts | 2 +- packages/backend/src/core/QueueService.ts | 5 ++++ packages/backend/src/core/ReactionService.ts | 7 +++-- .../BackgroundTaskProcessorService.ts | 28 +++++++++++++++++-- packages/backend/src/queue/types.ts | 8 +++++- 7 files changed, 47 insertions(+), 18 deletions(-) diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 9b7ef57d9c..e7f55f10ad 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -611,7 +611,7 @@ export class NoteCreateService implements OnApplicationShutdown { // Increment notes count (user) await this.incNotesCountOfUser(user); } else { - await this.usersRepository.update({ id: user.id }, { updatedAt: this.timeService.date }); + await this.queueService.createMarkUserUpdatedJob(user.id); } await this.pushToTl(note, user); diff --git a/packages/backend/src/core/NoteDeleteService.ts b/packages/backend/src/core/NoteDeleteService.ts index d780e5167d..82959f0782 100644 --- a/packages/backend/src/core/NoteDeleteService.ts +++ b/packages/backend/src/core/NoteDeleteService.ts @@ -28,12 +28,10 @@ import { ApLogService } from '@/core/ApLogService.js'; import type Logger from '@/logger.js'; import { TimeService } from '@/global/TimeService.js'; import { trackPromise } from '@/misc/promise-tracker.js'; -import { LoggerService } from '@/core/LoggerService.js'; +import { QueueService } from '@/core/QueueService.js'; @Injectable() export class NoteDeleteService { - private readonly logger: Logger; - constructor( @Inject(DI.config) private config: Config, @@ -63,11 +61,8 @@ export class NoteDeleteService { private latestNoteService: LatestNoteService, private readonly apLogService: ApLogService, private readonly timeService: TimeService, - - loggerService: LoggerService, - ) { - this.logger = loggerService.getLogger('note-delete-service'); - } + private readonly queueService: QueueService, + ) {} /** * 投稿を削除します。 @@ -131,7 +126,7 @@ export class NoteDeleteService { // Decrement notes count (user) this.decNotesCountOfUser(user); } else { - this.usersRepository.update({ id: user.id }, { updatedAt: this.timeService.date }); + await this.queueService.createMarkUserUpdatedJob(user.id); } if (this.meta.enableStatsForFederatedInstances) { diff --git a/packages/backend/src/core/NoteEditService.ts b/packages/backend/src/core/NoteEditService.ts index 0c595cbf20..11b22c0798 100644 --- a/packages/backend/src/core/NoteEditService.ts +++ b/packages/backend/src/core/NoteEditService.ts @@ -633,7 +633,7 @@ export class NoteEditService implements OnApplicationShutdown { } } - await this.usersRepository.update({ id: user.id }, { updatedAt: this.timeService.date }); + await this.queueService.createMarkUserUpdatedJob(user.id); // ハッシュタグ更新 await this.pushToTl(note, user); diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 99911f38a7..9cd9754f21 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -935,6 +935,11 @@ export class QueueService implements OnModuleInit { return await this.createBackgroundTask({ type: 'delete-ap-logs', dataType, data }); } + @bindThis + public async createMarkUserUpdatedJob(userId: string) { + return await this.createBackgroundTask({ type: 'mark-user-updated', userId }, userId); + } + private async createBackgroundTask(data: T, duplication?: string | { id: string, ttl?: number }) { return await this.backgroundTaskQueue.add( data.type, diff --git a/packages/backend/src/core/ReactionService.ts b/packages/backend/src/core/ReactionService.ts index 27a6d7b514..8f642e014e 100644 --- a/packages/backend/src/core/ReactionService.ts +++ b/packages/backend/src/core/ReactionService.ts @@ -33,6 +33,7 @@ import { PER_NOTE_REACTION_USER_PAIR_CACHE_MAX } from '@/const.js'; import { CacheService } from '@/core/CacheService.js'; import { NoteVisibilityService } from '@/core/NoteVisibilityService.js'; import { TimeService } from '@/global/TimeService.js'; +import { QueueService } from '@/core/QueueService.js'; import type { DataSource } from 'typeorm'; const FALLBACK = '\u2764'; @@ -110,6 +111,7 @@ export class ReactionService implements OnModuleInit { private readonly cacheService: CacheService, private readonly noteVisibilityService: NoteVisibilityService, private readonly timeService: TimeService, + private readonly queueService: QueueService, ) { } @@ -224,7 +226,7 @@ export class ReactionService implements OnModuleInit { .execute(); } - this.usersRepository.update({ id: user.id }, { updatedAt: this.timeService.date }); + await this.queueService.createMarkUserUpdatedJob(user.id); // 30%の確率、セルフではない、3日以内に投稿されたノートの場合ハイライト用ランキング更新 if ( @@ -340,8 +342,7 @@ export class ReactionService implements OnModuleInit { .execute(); } - // TODO update caches - this.usersRepository.update({ id: user.id }, { updatedAt: this.timeService.date }); + await this.queueService.createMarkUserUpdatedJob(user.id); this.globalEventService.publishNoteStream(note.id, 'unreacted', { reaction: this.decodeReaction(exist.reaction).reaction, diff --git a/packages/backend/src/queue/processors/BackgroundTaskProcessorService.ts b/packages/backend/src/queue/processors/BackgroundTaskProcessorService.ts index 826b2af193..14c1523c3e 100644 --- a/packages/backend/src/queue/processors/BackgroundTaskProcessorService.ts +++ b/packages/backend/src/queue/processors/BackgroundTaskProcessorService.ts @@ -5,7 +5,7 @@ import { Inject, Injectable } from '@nestjs/common'; import * as Bull from 'bullmq'; -import { BackgroundTaskJobData, CheckHibernationBackgroundTask, PostDeliverBackgroundTask, PostInboxBackgroundTask, PostNoteBackgroundTask, UpdateFeaturedBackgroundTask, UpdateInstanceBackgroundTask, UpdateUserTagsBackgroundTask, UpdateUserBackgroundTask, UpdateNoteTagsBackgroundTask, DeleteFileBackgroundTask, UpdateLatestNoteBackgroundTask, PostSuspendBackgroundTask, PostUnsuspendBackgroundTask, DeleteApLogsBackgroundTask } from '@/queue/types.js'; +import { BackgroundTaskJobData, CheckHibernationBackgroundTask, PostDeliverBackgroundTask, PostInboxBackgroundTask, PostNoteBackgroundTask, UpdateFeaturedBackgroundTask, UpdateInstanceBackgroundTask, UpdateUserTagsBackgroundTask, UpdateUserBackgroundTask, UpdateNoteTagsBackgroundTask, DeleteFileBackgroundTask, UpdateLatestNoteBackgroundTask, PostSuspendBackgroundTask, PostUnsuspendBackgroundTask, DeleteApLogsBackgroundTask, MarkUserUpdatedBackgroundTask } from '@/queue/types.js'; import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js'; import { QueueLoggerService } from '@/queue/QueueLoggerService.js'; import Logger from '@/logger.js'; @@ -19,7 +19,7 @@ import ApRequestChart from '@/core/chart/charts/ap-request.js'; import FederationChart from '@/core/chart/charts/federation.js'; import { UpdateInstanceQueue } from '@/core/UpdateInstanceQueue.js'; import { NoteCreateService } from '@/core/NoteCreateService.js'; -import type { DriveFilesRepository, NoteEditsRepository, NotesRepository } from '@/models/_.js'; +import type { DriveFilesRepository, NoteEditsRepository, NotesRepository, UsersRepository } from '@/models/_.js'; import { MiUser } from '@/models/_.js'; import { NoteEditService } from '@/core/NoteEditService.js'; import { HashtagService } from '@/core/HashtagService.js'; @@ -28,6 +28,7 @@ import { LatestNoteService } from '@/core/LatestNoteService.js'; import { trackTask } from '@/misc/promise-tracker.js'; import { UserSuspendService } from '@/core/UserSuspendService.js'; import { ApLogService } from '@/core/ApLogService.js'; +import { InternalEventService } from '@/core/InternalEventService.js'; @Injectable() export class BackgroundTaskProcessorService { @@ -46,6 +47,9 @@ export class BackgroundTaskProcessorService { @Inject(DI.noteEditsRepository) private readonly noteEditsRepository: NoteEditsRepository, + @Inject(DI.usersRepository) + private readonly usersRepository: UsersRepository, + private readonly apPersonService: ApPersonService, private readonly cacheService: CacheService, private readonly federatedInstanceService: FederatedInstanceService, @@ -61,6 +65,7 @@ export class BackgroundTaskProcessorService { private readonly latestNoteService: LatestNoteService, private readonly userSuspendService: UserSuspendService, private readonly apLogService: ApLogService, + private readonly internalEventService: InternalEventService, queueLoggerService: QueueLoggerService, ) { @@ -94,9 +99,11 @@ export class BackgroundTaskProcessorService { return await this.processPostSuspend(job.data); } else if (job.data.type === 'post-unsuspend') { return await this.processPostUnsuspend(job.data); - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition } else if (job.data.type === 'delete-ap-logs') { return await this.processDeleteApLogs(job.data); + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + } else if (job.data.type === 'mark-user-updated') { + return await this.processMarkUserUpdated(job.data); } else { this.logger.warn(`Can't process unknown job type "${job.data}"; this is likely a bug. Full job data:`, job.data); throw new Error(`Unknown job type ${job.data}, see system logs for details`); @@ -338,4 +345,19 @@ export class BackgroundTaskProcessorService { return 'ok'; } + + private async processMarkUserUpdated(task: MarkUserUpdatedBackgroundTask): Promise { + const user = await this.cacheService.findOptionalUserById(task.userId); + if (!user || user.isDeleted) return `Skipping post-unsuspend task: user ${task.userId} has been deleted`; + + await this.usersRepository.update({ id: user.id }, { updatedAt: new Date() }); + + if (user.host == null) { + await this.internalEventService.emit('localUserUpdated', { id: user.id }); + } else { + await this.internalEventService.emit('remoteUserUpdated', { id: user.id }); + } + + return 'ok'; + } } diff --git a/packages/backend/src/queue/types.ts b/packages/backend/src/queue/types.ts index cb31e5a3e1..671500a275 100644 --- a/packages/backend/src/queue/types.ts +++ b/packages/backend/src/queue/types.ts @@ -184,7 +184,8 @@ export type BackgroundTaskJobData = UpdateLatestNoteBackgroundTask | PostSuspendBackgroundTask | PostUnsuspendBackgroundTask | - DeleteApLogsBackgroundTask; + DeleteApLogsBackgroundTask | + MarkUserUpdatedBackgroundTask; export type UpdateUserBackgroundTask = { type: 'update-user'; @@ -261,3 +262,8 @@ export type DeleteApLogsBackgroundTask = { dataType: 'inbox' | 'object'; data: string | string[]; }; + +export type MarkUserUpdatedBackgroundTask = { + type: 'mark-user-updated'; + userId: string; +}; From cf138a148d42d48aed1ed18f3e3fb62b84d337f6 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Thu, 19 Jun 2025 17:43:27 -0400 Subject: [PATCH 023/107] make NoteDeleteService.delete safe to call inside a loop --- .../backend/src/core/NoteDeleteService.ts | 82 +++++++++---------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/packages/backend/src/core/NoteDeleteService.ts b/packages/backend/src/core/NoteDeleteService.ts index 82959f0782..ddee49dcf9 100644 --- a/packages/backend/src/core/NoteDeleteService.ts +++ b/packages/backend/src/core/NoteDeleteService.ts @@ -22,13 +22,13 @@ import { ApDeliverManagerService } from '@/core/activitypub/ApDeliverManagerServ import { bindThis } from '@/decorators.js'; import { SearchService } from '@/core/SearchService.js'; import { ModerationLogService } from '@/core/ModerationLogService.js'; -import { isQuote, isRenote } from '@/misc/is-renote.js'; +import { isPureRenote } from '@/misc/is-renote.js'; import { LatestNoteService } from '@/core/LatestNoteService.js'; import { ApLogService } from '@/core/ApLogService.js'; -import type Logger from '@/logger.js'; import { TimeService } from '@/global/TimeService.js'; -import { trackPromise } from '@/misc/promise-tracker.js'; +import { trackTask } from '@/misc/promise-tracker.js'; import { QueueService } from '@/core/QueueService.js'; +import { CacheService } from '@/core/CacheService.js'; @Injectable() export class NoteDeleteService { @@ -62,6 +62,7 @@ export class NoteDeleteService { private readonly apLogService: ApLogService, private readonly timeService: TimeService, private readonly queueService: QueueService, + private readonly cacheService: CacheService, ) {} /** @@ -70,41 +71,36 @@ export class NoteDeleteService { * @param note 投稿 */ async delete(user: { id: MiUser['id']; uri: MiUser['uri']; host: MiUser['host']; isBot: MiUser['isBot']; }, note: MiNote, quiet = false, deleter?: MiUser) { + // This kicks off lots of things that can run in parallel, but we should still wait for completion to ensure consistent state and to avoid task flood when calling in a loop. + const promises: Promise[] = []; + const deletedAt = this.timeService.date; const cascadingNotes = await this.findCascadingNotes(note); if (note.replyId) { - await this.notesRepository.decrement({ id: note.replyId }, 'repliesCount', 1); + promises.push(this.notesRepository.decrement({ id: note.replyId }, 'repliesCount', 1)); } - if (note.renoteId && note.text == null && !note.hasPoll && (note.fileIds == null || note.fileIds.length === 0)) { - await this.notesRepository.findOneBy({ id: note.renoteId }).then(async (renote) => { - if (!renote) return; - if (renote.userId !== user.id) await this.notesRepository.decrement({ id: renote.id }, 'renoteCount', 1); - }); + if (isPureRenote(note)) { + promises.push(this.notesRepository.decrement({ id: note.renoteId }, 'renoteCount', 1)); } if (!quiet) { - this.globalEventService.publishNoteStream(note.id, 'deleted', { + promises.push(this.globalEventService.publishNoteStream(note.id, 'deleted', { deletedAt: deletedAt, - }); + })); //#region ローカルの投稿なら削除アクティビティを配送 if (isLocalUser(user) && !note.localOnly) { - let renote: MiNote | null = null; - - // if deleted note is renote - if (isRenote(note) && !isQuote(note)) { - renote = await this.notesRepository.findOneBy({ - id: note.renoteId, - }); - } + const renote = isPureRenote(note) + ? await this.notesRepository.findOneBy({ id: note.renoteId }) + : null; const content = this.apRendererService.addContext(renote ? this.apRendererService.renderUndo(this.apRendererService.renderAnnounce(renote.uri ?? `${this.config.url}/notes/${renote.id}`, note), user) : this.apRendererService.renderDelete(this.apRendererService.renderTombstone(`${this.config.url}/notes/${note.id}`), user)); - trackPromise(this.deliverToConcerned(user, note, content)); + promises.push(this.deliverToConcerned(user, note, content)); } // also deliver delete activity to cascaded notes @@ -113,7 +109,7 @@ export class NoteDeleteService { if (!cascadingNote.user) continue; if (!isLocalUser(cascadingNote.user)) continue; const content = this.apRendererService.addContext(this.apRendererService.renderDelete(this.apRendererService.renderTombstone(`${this.config.url}/notes/${cascadingNote.id}`), cascadingNote.user)); - trackPromise(this.deliverToConcerned(cascadingNote.user, cascadingNote, content)); + promises.push(this.deliverToConcerned(cascadingNote.user, cascadingNote, content)); } //#endregion @@ -122,59 +118,63 @@ export class NoteDeleteService { this.perUserNotesChart.update(user, note, false); } - if (!isRenote(note) || isQuote(note)) { + if (!isPureRenote(note)) { // Decrement notes count (user) - this.decNotesCountOfUser(user); + promises.push(this.decNotesCountOfUser(user)); } else { - await this.queueService.createMarkUserUpdatedJob(user.id); + promises.push(this.queueService.createMarkUserUpdatedJob(user.id)); } if (this.meta.enableStatsForFederatedInstances) { if (isRemoteUser(user)) { - this.federatedInstanceService.fetchOrRegister(user.host).then(async i => { - if (note.renoteId && note.text || !note.renoteId) { - this.instancesRepository.decrement({ id: i.id }, 'notesCount', 1); - } - if (this.meta.enableChartsForFederatedInstances) { - this.instanceChart.updateNote(i.host, note, false); - } - }); + if (!isPureRenote(note)) { + const i = await this.federatedInstanceService.fetchOrRegister(user.host); + promises.push(this.instancesRepository.decrement({ id: i.id }, 'notesCount', 1)); + } + if (this.meta.enableChartsForFederatedInstances) { + this.instanceChart.updateNote(user.host, note, false); + } } } } for (const cascadingNote of cascadingNotes) { - this.searchService.unindexNote(cascadingNote); + promises.push(this.searchService.unindexNote(cascadingNote)); } - this.searchService.unindexNote(note); + promises.push(this.searchService.unindexNote(note)); + // Don't put this in the promise array, since it needs to happen before the next section await this.notesRepository.delete({ id: note.id, userId: user.id, }); - // Update the Latest Note index / following feed - this.latestNoteService.handleDeletedNoteDeferred(note); + // Update the Latest Note index / following feed *after* note is deleted + promises.push(this.latestNoteService.handleDeletedNoteDeferred(note)); for (const cascadingNote of cascadingNotes) { - this.latestNoteService.handleDeletedNoteDeferred(cascadingNote); + promises.push(this.latestNoteService.handleDeletedNoteDeferred(cascadingNote)); } if (deleter && (note.userId !== deleter.id)) { - const user = await this.usersRepository.findOneByOrFail({ id: note.userId }); - this.moderationLogService.log(deleter, 'deleteNote', { + const user = await this.cacheService.findUserById(note.userId); + promises.push(this.moderationLogService.log(deleter, 'deleteNote', { noteId: note.id, noteUserId: note.userId, noteUserUsername: user.username, noteUserHost: user.host, - }); + })); } const deletedUris = [note, ...cascadingNotes] .map(n => n.uri) .filter((u): u is string => u != null); if (deletedUris.length > 0) { - await this.apLogService.deleteObjectLogsDeferred(deletedUris); + promises.push(this.apLogService.deleteObjectLogsDeferred(deletedUris)); } + + await trackTask(async () => { + await Promise.allSettled(promises); + }); } @bindThis From 2bca84048db0e046b2935920f11e4a70619fd933 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Tue, 24 Jun 2025 11:46:38 -0400 Subject: [PATCH 024/107] fix postNoteCreated and postNoteEdited not working properly due to missing Data options --- packages/backend/src/core/NoteCreateService.ts | 8 +------- packages/backend/src/core/NoteEditService.ts | 8 +------- .../processors/BackgroundTaskProcessorService.ts | 15 +++++++++++---- 3 files changed, 13 insertions(+), 18 deletions(-) diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index e7f55f10ad..0b0babfafd 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -574,13 +574,7 @@ export class NoteCreateService implements OnApplicationShutdown { } @bindThis - public async postNoteCreated(note: MiNote, user: MiUser & { - id: MiUser['id']; - username: MiUser['username']; - host: MiUser['host']; - isBot: MiUser['isBot']; - noindex: MiUser['noindex']; - }, data: Option, silent: boolean, mentionedUsers: MinimumUser[]) { + public async postNoteCreated(note: MiNote, user: MiUser, data: MiNote & { poll: MiPoll | null }, silent: boolean, mentionedUsers: MinimumUser[]) { this.notesChart.update(note, true); if (note.visibility !== 'specified' && (this.meta.enableChartsForRemoteUser || (user.host == null))) { this.perUserNotesChart.update(user, note, true); diff --git a/packages/backend/src/core/NoteEditService.ts b/packages/backend/src/core/NoteEditService.ts index 11b22c0798..dc2990c64f 100644 --- a/packages/backend/src/core/NoteEditService.ts +++ b/packages/backend/src/core/NoteEditService.ts @@ -612,13 +612,7 @@ export class NoteEditService implements OnApplicationShutdown { } @bindThis - public async postNoteEdited(note: MiNote, user: MiUser & { - id: MiUser['id']; - username: MiUser['username']; - host: MiUser['host']; - isBot: MiUser['isBot']; - noindex: MiUser['noindex']; - }, data: Option, silent: boolean, mentionedUsers: MinimumUser[]) { + public async postNoteEdited(note: MiNote, user: MiUser, data: MiNote & { poll: MiPoll | null }, silent: boolean, mentionedUsers: MinimumUser[]) { // Register host if (this.meta.enableStatsForFederatedInstances) { if (isRemoteUser(user)) { diff --git a/packages/backend/src/queue/processors/BackgroundTaskProcessorService.ts b/packages/backend/src/queue/processors/BackgroundTaskProcessorService.ts index 14c1523c3e..3d82dfa89b 100644 --- a/packages/backend/src/queue/processors/BackgroundTaskProcessorService.ts +++ b/packages/backend/src/queue/processors/BackgroundTaskProcessorService.ts @@ -19,7 +19,7 @@ import ApRequestChart from '@/core/chart/charts/ap-request.js'; import FederationChart from '@/core/chart/charts/federation.js'; import { UpdateInstanceQueue } from '@/core/UpdateInstanceQueue.js'; import { NoteCreateService } from '@/core/NoteCreateService.js'; -import type { DriveFilesRepository, NoteEditsRepository, NotesRepository, UsersRepository } from '@/models/_.js'; +import type { DriveFilesRepository, NoteEditsRepository, NotesRepository, PollsRepository, UsersRepository } from '@/models/_.js'; import { MiUser } from '@/models/_.js'; import { NoteEditService } from '@/core/NoteEditService.js'; import { HashtagService } from '@/core/HashtagService.js'; @@ -50,6 +50,9 @@ export class BackgroundTaskProcessorService { @Inject(DI.usersRepository) private readonly usersRepository: UsersRepository, + @Inject(DI.pollsRepository) + private readonly pollsRepository: PollsRepository, + private readonly apPersonService: ApPersonService, private readonly cacheService: CacheService, private readonly federatedInstanceService: FederatedInstanceService, @@ -246,17 +249,21 @@ export class BackgroundTaskProcessorService { } private async processPostNote(task: PostNoteBackgroundTask): Promise { - const note = await this.notesRepository.findOneBy({ id: task.noteId }); + const note = await this.notesRepository.findOne({ + where: { id: task.noteId }, + relations: { user: true, renote: true, reply: true, channel: true }, + }); if (!note) return `Skipping post-note task: note ${task.noteId} has been deleted`; const user = await this.cacheService.findUserById(note.userId); if (user.isSuspended) return `Skipping post-note task: note ${task.noteId}'s user ${note.userId} is suspended`; const mentionedUsers = await this.cacheService.getUsers(note.mentions); + const poll = await this.pollsRepository.findOneBy({ noteId: note.id }); if (task.edit) { - await this.noteEditService.postNoteEdited(note, user, note, task.silent, Array.from(mentionedUsers.values())); + await this.noteEditService.postNoteEdited(note, user, { ...note, poll }, task.silent, Array.from(mentionedUsers.values())); } else { - await this.noteCreateService.postNoteCreated(note, user, note, task.silent, Array.from(mentionedUsers.values())); + await this.noteCreateService.postNoteCreated(note, user, { ...note, poll }, task.silent, Array.from(mentionedUsers.values())); } return 'ok'; From d06b2353b5c0a1c948891de1fcd29d19dad829fd Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Tue, 24 Jun 2025 11:47:01 -0400 Subject: [PATCH 025/107] make renote count increment and decrement the same way --- packages/backend/src/core/NoteCreateService.ts | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 0b0babfafd..c4d5c17803 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -644,7 +644,7 @@ export class NoteCreateService implements OnApplicationShutdown { }); } - if (this.isRenote(data) && !this.isQuote(data) && data.renote.userId !== user.id && !user.isBot) { + if (this.isPureRenote(data) && data.renote.userId !== user.id && !user.isBot) { await this.incRenoteCount(data.renote, user); } @@ -822,12 +822,7 @@ export class NoteCreateService implements OnApplicationShutdown { @bindThis private async incRenoteCount(renote: MiNote, user: MiUser) { - await this.notesRepository.createQueryBuilder().update() - .set({ - renoteCount: () => '"renoteCount" + 1', - }) - .where('id = :id', { id: renote.id }) - .execute(); + await this.notesRepository.increment({ id: renote.id }, 'renoteCount', 1); // 30%の確率、3日以内に投稿されたノートの場合ハイライト用ランキング更新 if (user.isExplorable && Math.random() < 0.3 && (this.timeService.now - this.idService.parse(renote.id).date.getTime()) < 1000 * 60 * 60 * 24 * 3) { From cad059f039f26b185a00bde0ec6b0c8c1fb1e79d Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Tue, 24 Jun 2025 11:47:21 -0400 Subject: [PATCH 026/107] fix cascaded note deletion and make it more efficient --- .../backend/src/core/NoteDeleteService.ts | 92 +++++++++++++++---- 1 file changed, 74 insertions(+), 18 deletions(-) diff --git a/packages/backend/src/core/NoteDeleteService.ts b/packages/backend/src/core/NoteDeleteService.ts index ddee49dcf9..21dce62744 100644 --- a/packages/backend/src/core/NoteDeleteService.ts +++ b/packages/backend/src/core/NoteDeleteService.ts @@ -79,10 +79,18 @@ export class NoteDeleteService { if (note.replyId) { promises.push(this.notesRepository.decrement({ id: note.replyId }, 'repliesCount', 1)); + } else if (isPureRenote(note)) { + promises.push(this.notesRepository.decrement({ id: note.renoteId }, 'renoteCount', 1)); } - if (isPureRenote(note)) { - promises.push(this.notesRepository.decrement({ id: note.renoteId }, 'renoteCount', 1)); + const cascadeReplies = cascadingNotes.filter(cascade => cascade.replyId != null); + const cascadeRenotes = cascadingNotes.filter(cascade => cascade.renoteId != null); + + if (cascadeReplies.length > 0) { + promises.push(this.notesRepository.decrement({ id: In(cascadeReplies.map(cascade => cascade.replyId)) }, 'repliesCount', 1)); + } + if (cascadeRenotes.length > 0) { + promises.push(this.notesRepository.decrement({ id: In(cascadeRenotes.map(cascade => cascade.renoteId)) }, 'renoteCount', 1)); } if (!quiet) { @@ -90,6 +98,12 @@ export class NoteDeleteService { deletedAt: deletedAt, })); + for (const cascade of cascadingNotes) { + promises.push(this.globalEventService.publishNoteStream(cascade.id, 'deleted', { + deletedAt: deletedAt, + })); + } + //#region ローカルの投稿なら削除アクティビティを配送 if (isLocalUser(user) && !note.localOnly) { const renote = isPureRenote(note) @@ -118,6 +132,13 @@ export class NoteDeleteService { this.perUserNotesChart.update(user, note, false); } + for (const cascade of cascadingNotes) { + this.notesChart.update(cascade, false); + if (this.meta.enableChartsForRemoteUser || (cascade.user.host == null)) { + this.perUserNotesChart.update(cascade.user, cascade, false); + } + } + if (!isPureRenote(note)) { // Decrement notes count (user) promises.push(this.decNotesCountOfUser(user)); @@ -125,6 +146,13 @@ export class NoteDeleteService { promises.push(this.queueService.createMarkUserUpdatedJob(user.id)); } + for (const cascade of cascadingNotes) { + if (!isPureRenote(cascade)) { + promises.push(this.decNotesCountOfUser(cascade.user)); + } + // Don't mark cascaded user as updated (active) + } + if (this.meta.enableStatsForFederatedInstances) { if (isRemoteUser(user)) { if (!isPureRenote(note)) { @@ -135,6 +163,18 @@ export class NoteDeleteService { this.instanceChart.updateNote(user.host, note, false); } } + + for (const cascade of cascadingNotes) { + if (this.userEntityService.isRemoteUser(cascade.user)) { + if (!isPureRenote(cascade)) { + const i = await this.federatedInstanceService.fetchOrRegister(cascade.user.host); + promises.push(this.instancesRepository.decrement({ id: i.id }, 'notesCount', 1)); + } + if (this.meta.enableChartsForFederatedInstances) { + this.instanceChart.updateNote(cascade.user.host, cascade, false); + } + } + } } } @@ -189,26 +229,42 @@ export class NoteDeleteService { } @bindThis - private async findCascadingNotes(note: MiNote): Promise { - const recursive = async (noteId: string): Promise => { - const query = this.notesRepository.createQueryBuilder('note') - .where('note.replyId = :noteId', { noteId }) - .orWhere(new Brackets(q => { - q.where('note.renoteId = :noteId', { noteId }) - .andWhere('note.text IS NOT NULL'); - })) - .leftJoinAndSelect('note.user', 'user'); - const replies = await query.getMany(); + private async findCascadingNotes(note: MiNote): Promise<(MiNote & { user: MiUser })[]> { + const cascadingNotes: MiNote[] = []; - return [ - replies, - ...await Promise.all(replies.map(reply => recursive(reply.id))), - ].flat(); + /** + * Finds all replies, quotes, and renotes of the given list of notes. + * These are the notes that will be CASCADE deleted when the origin note is deleted. + * + * This works by operating in "layers" that radiate out from the origin note like a web. + * The process is roughly like this: + * 1. Find all immediate replies and renotes of the origin. + * 2. Find all immediate replies and renotes of the results from step one. + * 3. Repeat until step 2 returns no new results. + * 4. Collect all the step 2 results; those are the set of all cascading notes. + */ + const cascade = async (layer: MiNote[]): Promise => { + const layerIds = layer.map(layer => layer.id); + const refs = await this.notesRepository.find({ + where: [ + { replyId: In(layerIds) }, + { renoteId: In(layerIds) }, + ], + relations: { user: true }, + }); + + // Stop when we reach the end of all threads + if (refs.length === 0) return; + + cascadingNotes.push(...refs); + await cascade(refs); }; - const cascadingNotes: MiNote[] = await recursive(note.id); + // Start with the origin, which should *not* be in the result set! + await cascade([note]); - return cascadingNotes; + // Type cast is safe - we load the relation above. + return cascadingNotes as (MiNote & { user: MiUser })[]; } @bindThis From c3ae0c7e8c2eab162516651427a742364e0aea99 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Wed, 25 Jun 2025 11:33:29 -0400 Subject: [PATCH 027/107] fix API schema for admin/queue/jobs endpoint --- .../server/api/endpoints/admin/queue/jobs.ts | 70 +++++++++++++++++++ packages/misskey-js/src/autogen/endpoint.ts | 3 +- packages/misskey-js/src/autogen/entities.ts | 1 + 3 files changed, 73 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/server/api/endpoints/admin/queue/jobs.ts b/packages/backend/src/server/api/endpoints/admin/queue/jobs.ts index 45b8f4ffb2..be85a50990 100644 --- a/packages/backend/src/server/api/endpoints/admin/queue/jobs.ts +++ b/packages/backend/src/server/api/endpoints/admin/queue/jobs.ts @@ -14,6 +14,76 @@ export const meta = { requireCredential: true, requireModerator: true, kind: 'read:admin:queue', + + res: { + type: 'array', + nullable: false, optional: false, + items: { + type: 'object', + nullable: false, optional: false, + properties: { + id: { + type: 'string', + nullable: false, optional: true, + }, + name: { + type: 'string', + nullable: false, optional: false, + }, + data: { + type: 'object', + nullable: true, optional: true, + additionalProperties: true, + }, + opts: { + type: 'object', + nullable: false, optional: false, + additionalProperties: true, + }, + timestamp: { + type: 'number', + nullable: false, optional: false, + }, + processedOn: { + type: 'number', + nullable: false, optional: true, + }, + processedBy: { + type: 'string', + nullable: false, optional: true, + }, + finishedOn: { + type: 'number', + nullable: false, optional: true, + }, + progress: {}, + attempts: { + type: 'number', + nullable: false, optional: false, + }, + delay: { + type: 'number', + nullable: false, optional: false, + }, + failedReason: { + type: 'string', + nullable: false, optional: true, + }, + stackTrace: { + type: 'array', + nullable: false, optional: true, + items: { + type: 'string', + }, + }, + returnValue: {}, + isFailed: { + type: 'boolean', + nullable: false, optional: true, + }, + }, + }, + }, } as const; export const paramDef = { diff --git a/packages/misskey-js/src/autogen/endpoint.ts b/packages/misskey-js/src/autogen/endpoint.ts index 831a868792..2a83b219cb 100644 --- a/packages/misskey-js/src/autogen/endpoint.ts +++ b/packages/misskey-js/src/autogen/endpoint.ts @@ -87,6 +87,7 @@ import type { AdminQueueDeliverDelayedResponse, AdminQueueInboxDelayedResponse, AdminQueueJobsRequest, + AdminQueueJobsResponse, AdminQueuePromoteJobsRequest, AdminQueueQueueStatsRequest, AdminQueueRemoveJobRequest, @@ -739,7 +740,7 @@ export type Endpoints = { 'admin/queue/clear': { req: AdminQueueClearRequest; res: EmptyResponse }; 'admin/queue/deliver-delayed': { req: EmptyRequest; res: AdminQueueDeliverDelayedResponse }; 'admin/queue/inbox-delayed': { req: EmptyRequest; res: AdminQueueInboxDelayedResponse }; - 'admin/queue/jobs': { req: AdminQueueJobsRequest; res: EmptyResponse }; + 'admin/queue/jobs': { req: AdminQueueJobsRequest; res: AdminQueueJobsResponse }; 'admin/queue/promote-jobs': { req: AdminQueuePromoteJobsRequest; res: EmptyResponse }; 'admin/queue/queue-stats': { req: AdminQueueQueueStatsRequest; res: EmptyResponse }; 'admin/queue/queues': { req: EmptyRequest; res: EmptyResponse }; diff --git a/packages/misskey-js/src/autogen/entities.ts b/packages/misskey-js/src/autogen/entities.ts index e1cb079a74..91329fddaa 100644 --- a/packages/misskey-js/src/autogen/entities.ts +++ b/packages/misskey-js/src/autogen/entities.ts @@ -90,6 +90,7 @@ export type AdminQueueClearRequest = operations['admin___queue___clear']['reques export type AdminQueueDeliverDelayedResponse = operations['admin___queue___deliver-delayed']['responses']['200']['content']['application/json']; export type AdminQueueInboxDelayedResponse = operations['admin___queue___inbox-delayed']['responses']['200']['content']['application/json']; export type AdminQueueJobsRequest = operations['admin___queue___jobs']['requestBody']['content']['application/json']; +export type AdminQueueJobsResponse = operations['admin___queue___jobs']['responses']['200']['content']['application/json']; export type AdminQueuePromoteJobsRequest = operations['admin___queue___promote-jobs']['requestBody']['content']['application/json']; export type AdminQueueQueueStatsRequest = operations['admin___queue___queue-stats']['requestBody']['content']['application/json']; export type AdminQueueRemoveJobRequest = operations['admin___queue___remove-job']['requestBody']['content']['application/json']; From 428c9258a1e2ae6246f67d480d7d63af79e9bcc8 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Wed, 25 Jun 2025 11:33:45 -0400 Subject: [PATCH 028/107] remove unused API and websocket calls from admin dashboard --- .../frontend/src/pages/admin/overview.vue | 102 +----------------- 1 file changed, 1 insertion(+), 101 deletions(-) diff --git a/packages/frontend/src/pages/admin/overview.vue b/packages/frontend/src/pages/admin/overview.vue index caa888b51d..9844391e11 100644 --- a/packages/frontend/src/pages/admin/overview.vue +++ b/packages/frontend/src/pages/admin/overview.vue @@ -65,8 +65,7 @@ SPDX-License-Identifier: AGPL-3.0-only