diff --git a/.config/ci.yml b/.config/ci.yml index 8543205b17..d5f288fc05 100644 --- a/.config/ci.yml +++ b/.config/ci.yml @@ -253,21 +253,42 @@ id: 'aidx' # Number of worker processes #clusterLimit: 1 +# +-------------------------+ +# | Job concurrency options | +# +-------------------------+ +# +### Available options: +# [type]JobConcurrency - limits the number jobs that can run at the same time. +# Sharkey will allow this many jobs of this type *per worker process*. +# [type]JobPerSec - limits the total number of jobs that may complete within a single second. +# If this limit is exceeded, then Sharkey will pause this type of job until the next second. +# [type]JobMaxAttempts - limits the number of times that a job is allowed to fail and re-try before it's permanently stopped. +# If this limit is exceeded, then the job is considered "failed" and recorded for debugging. +# +### Job types: +# inbox - processes ActivityPub messages (AKA "Activities") received from remote instances. +# All inbound activities are queued and processed in chronological order by this job. +# deliver - processes ActivityPub messages (AKA "Activities") being set to remote instances. +# All outbound activities are queued and processed in chronological order by this job. +# relationship - processes user-to-user tasks including follow/unfollow, block/unblock, account migrations, and all follow import jobs. +# background - processes background synchronization tasks that need to happen soon (but not immediately), such as remote user updates and instance metadata updates. + # Job concurrency per worker -# deliverJobConcurrency: 128 -# inboxJobConcurrency: 16 -# relashionshipJobConcurrency: 16 -# What's relashionshipJob?: -# Follow, unfollow, block and unblock(ings) while following-imports, etc. or account migrations. +#deliverJobConcurrency: 128 +#inboxJobConcurrency: 16 +#relationshipJobConcurrency: 16 +#backgroundJobConcurrency: 32 # Job rate limiter -# deliverJobPerSec: 128 -# inboxJobPerSec: 32 -# relashionshipJobPerSec: 64 +#deliverJobPerSec: 128 +#inboxJobPerSec: 32 +#relationshipJobPerSec: 64 +#backgroundJobPerSec: 256 # Job attempts -# deliverJobMaxAttempts: 12 -# inboxJobMaxAttempts: 8 +#deliverJobMaxAttempts: 12 +#inboxJobMaxAttempts: 8 +#backgroundJobMaxAttempts: 8 # Local address used for outgoing requests #outgoingAddress: 127.0.0.1 diff --git a/.config/cypress-devcontainer.yml b/.config/cypress-devcontainer.yml index f705d06d45..ba6d51959c 100644 --- a/.config/cypress-devcontainer.yml +++ b/.config/cypress-devcontainer.yml @@ -223,17 +223,42 @@ id: 'aidx' # Number of worker processes #clusterLimit: 1 +# +-------------------------+ +# | Job concurrency options | +# +-------------------------+ +# +### Available options: +# [type]JobConcurrency - limits the number jobs that can run at the same time. +# Sharkey will allow this many jobs of this type *per worker process*. +# [type]JobPerSec - limits the total number of jobs that may complete within a single second. +# If this limit is exceeded, then Sharkey will pause this type of job until the next second. +# [type]JobMaxAttempts - limits the number of times that a job is allowed to fail and re-try before it's permanently stopped. +# If this limit is exceeded, then the job is considered "failed" and recorded for debugging. +# +### Job types: +# inbox - processes ActivityPub messages (AKA "Activities") received from remote instances. +# All inbound activities are queued and processed in chronological order by this job. +# deliver - processes ActivityPub messages (AKA "Activities") being set to remote instances. +# All outbound activities are queued and processed in chronological order by this job. +# relationship - processes user-to-user tasks including follow/unfollow, block/unblock, account migrations, and all follow import jobs. +# background - processes background synchronization tasks that need to happen soon (but not immediately), such as remote user updates and instance metadata updates. + # Job concurrency per worker -# deliverJobConcurrency: 128 -# inboxJobConcurrency: 16 +#deliverJobConcurrency: 128 +#inboxJobConcurrency: 16 +#relationshipJobConcurrency: 16 +#backgroundJobConcurrency: 32 # Job rate limiter -# deliverJobPerSec: 128 -# inboxJobPerSec: 32 +#deliverJobPerSec: 128 +#inboxJobPerSec: 32 +#relationshipJobPerSec: 64 +#backgroundJobPerSec: 256 # Job attempts -# deliverJobMaxAttempts: 12 -# inboxJobMaxAttempts: 8 +#deliverJobMaxAttempts: 12 +#inboxJobMaxAttempts: 8 +#backgroundJobMaxAttempts: 8 # IP address family used for outgoing request (ipv4, ipv6 or dual) #outgoingAddressFamily: ipv4 diff --git a/.config/docker_example.yml b/.config/docker_example.yml index 5905e3deed..8ca8d7ff50 100644 --- a/.config/docker_example.yml +++ b/.config/docker_example.yml @@ -307,21 +307,42 @@ id: 'aidx' # Number of worker processes #clusterLimit: 1 +# +-------------------------+ +# | Job concurrency options | +# +-------------------------+ +# +### Available options: +# [type]JobConcurrency - limits the number jobs that can run at the same time. +# Sharkey will allow this many jobs of this type *per worker process*. +# [type]JobPerSec - limits the total number of jobs that may complete within a single second. +# If this limit is exceeded, then Sharkey will pause this type of job until the next second. +# [type]JobMaxAttempts - limits the number of times that a job is allowed to fail and re-try before it's permanently stopped. +# If this limit is exceeded, then the job is considered "failed" and recorded for debugging. +# +### Job types: +# inbox - processes ActivityPub messages (AKA "Activities") received from remote instances. +# All inbound activities are queued and processed in chronological order by this job. +# deliver - processes ActivityPub messages (AKA "Activities") being set to remote instances. +# All outbound activities are queued and processed in chronological order by this job. +# relationship - processes user-to-user tasks including follow/unfollow, block/unblock, account migrations, and all follow import jobs. +# background - processes background synchronization tasks that need to happen soon (but not immediately), such as remote user updates and instance metadata updates. + # Job concurrency per worker #deliverJobConcurrency: 128 #inboxJobConcurrency: 16 #relationshipJobConcurrency: 16 -# What's relationshipJob?: -# Follow, unfollow, block and unblock(ings) while following-imports, etc. or account migrations. +#backgroundJobConcurrency: 32 # Job rate limiter #deliverJobPerSec: 128 #inboxJobPerSec: 32 #relationshipJobPerSec: 64 +#backgroundJobPerSec: 256 # Job attempts #deliverJobMaxAttempts: 12 #inboxJobMaxAttempts: 8 +#backgroundJobMaxAttempts: 8 # Local address used for outgoing requests #outgoingAddress: 127.0.0.1 diff --git a/.config/example.yml b/.config/example.yml index cffc333d14..6fa3f02026 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -310,21 +310,42 @@ id: 'aidx' # Number of worker processes #clusterLimit: 1 +# +-------------------------+ +# | Job concurrency options | +# +-------------------------+ +# +### Available options: +# [type]JobConcurrency - limits the number jobs that can run at the same time. +# Sharkey will allow this many jobs of this type *per worker process*. +# [type]JobPerSec - limits the total number of jobs that may complete within a single second. +# If this limit is exceeded, then Sharkey will pause this type of job until the next second. +# [type]JobMaxAttempts - limits the number of times that a job is allowed to fail and re-try before it's permanently stopped. +# If this limit is exceeded, then the job is considered "failed" and recorded for debugging. +# +### Job types: +# inbox - processes ActivityPub messages (AKA "Activities") received from remote instances. +# All inbound activities are queued and processed in chronological order by this job. +# deliver - processes ActivityPub messages (AKA "Activities") being set to remote instances. +# All outbound activities are queued and processed in chronological order by this job. +# relationship - processes user-to-user tasks including follow/unfollow, block/unblock, account migrations, and all follow import jobs. +# background - processes background synchronization tasks that need to happen soon (but not immediately), such as remote user updates and instance metadata updates. + # Job concurrency per worker #deliverJobConcurrency: 128 #inboxJobConcurrency: 16 #relationshipJobConcurrency: 16 -# What's relationshipJob?: -# Follow, unfollow, block and unblock(ings) while following-imports, etc. or account migrations. +#backgroundJobConcurrency: 32 # Job rate limiter #deliverJobPerSec: 128 #inboxJobPerSec: 32 #relationshipJobPerSec: 64 +#backgroundJobPerSec: 256 # Job attempts #deliverJobMaxAttempts: 12 #inboxJobMaxAttempts: 8 +#backgroundJobMaxAttempts: 8 # Local address used for outgoing requests #outgoingAddress: 127.0.0.1 diff --git a/locales/index.d.ts b/locales/index.d.ts index 8aa409ca60..9f5a238cc9 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -13669,6 +13669,10 @@ export interface Locale extends ILocale { * Are you sure you want to restart this account migration? */ "restartMigrationConfirm": string; + /** + * Background queue + */ + "backgroundQueue": string; } declare const locales: { [lang: string]: Locale; diff --git a/packages/backend/migration/1748990662839-fix-IDX_instance_host_key.js b/packages/backend/migration/1748990662839-fix-IDX_instance_host_key.js index fc6d303743..e423ecd1b6 100644 --- a/packages/backend/migration/1748990662839-fix-IDX_instance_host_key.js +++ b/packages/backend/migration/1748990662839-fix-IDX_instance_host_key.js @@ -4,6 +4,8 @@ */ export class FixIDXInstanceHostKey1748990662839 { + name = 'FixIDXInstanceHostKey1748990662839'; + async up(queryRunner) { // must include host for index-only scans: https://www.postgresql.org/docs/current/indexes-index-only-scans.html await queryRunner.query(`DROP INDEX "public"."IDX_instance_host_key"`); diff --git a/packages/backend/migration/1748991828473-create-IDX_note_for_timelines.js b/packages/backend/migration/1748991828473-create-IDX_note_for_timelines.js index 2ea7fe95d2..54debcee27 100644 --- a/packages/backend/migration/1748991828473-create-IDX_note_for_timelines.js +++ b/packages/backend/migration/1748991828473-create-IDX_note_for_timelines.js @@ -4,6 +4,8 @@ */ export class CreateIDXNoteForTimelines1748991828473 { + name = 'CreateIDXNoteForTimelines1748991828473'; + async up(queryRunner) { await queryRunner.query(` create index "IDX_note_for_timelines" diff --git a/packages/backend/migration/1748992017688-create-IDX_instance_host_filters.js b/packages/backend/migration/1748992017688-create-IDX_instance_host_filters.js index 76cf16a6de..24b2b1894f 100644 --- a/packages/backend/migration/1748992017688-create-IDX_instance_host_filters.js +++ b/packages/backend/migration/1748992017688-create-IDX_instance_host_filters.js @@ -4,6 +4,8 @@ */ export class CreateIDXInstanceHostFilters1748992017688 { + name = 'CreateIDXInstanceHostFilters1748992017688'; + async up(queryRunner) { await queryRunner.query(` create index "IDX_instance_host_filters" diff --git a/packages/backend/migration/1748992128683-create-statistics.js b/packages/backend/migration/1748992128683-create-statistics.js index 5d08868536..daa50332ff 100644 --- a/packages/backend/migration/1748992128683-create-statistics.js +++ b/packages/backend/migration/1748992128683-create-statistics.js @@ -4,6 +4,8 @@ */ export class CreateStatistics1748992128683 { + name = 'CreateStatistics1748992128683'; + async up(queryRunner) { await queryRunner.query(`CREATE STATISTICS "STTS_instance_isBlocked_isBubbled" (mcv) ON "isBlocked", "isBubbled" FROM "instance"`); await queryRunner.query(`CREATE STATISTICS "STTS_instance_isBlocked_isSilenced" (mcv) ON "isBlocked", "isSilenced" FROM "instance"`); diff --git a/packages/backend/migration/1749097536193-fix-IDX_note_for_timeline.js b/packages/backend/migration/1749097536193-fix-IDX_note_for_timeline.js index 9a651e5871..57c5579110 100644 --- a/packages/backend/migration/1749097536193-fix-IDX_note_for_timeline.js +++ b/packages/backend/migration/1749097536193-fix-IDX_note_for_timeline.js @@ -4,6 +4,8 @@ */ export class FixIDXNoteForTimeline1749097536193 { + name = 'FixIDXNoteForTimeline1749097536193'; + async up(queryRunner) { await queryRunner.query('drop index "IDX_note_for_timelines"'); await queryRunner.query(` diff --git a/packages/backend/migration/1749267016885-remove-IDX_instance_host_filters.js b/packages/backend/migration/1749267016885-remove-IDX_instance_host_filters.js index d0a4e4f91e..4236399a6e 100644 --- a/packages/backend/migration/1749267016885-remove-IDX_instance_host_filters.js +++ b/packages/backend/migration/1749267016885-remove-IDX_instance_host_filters.js @@ -4,6 +4,8 @@ */ export class RemoveIDXInstanceHostFilters1749267016885 { + name = 'RemoveIDXInstanceHostFilters1749267016885'; + async up(queryRunner) { await queryRunner.query(`DROP INDEX IF EXISTS "IDX_instance_host_filters"`); } diff --git a/packages/backend/migration/1750217001651-enable-instance-HOT-updates.js b/packages/backend/migration/1750217001651-enable-instance-HOT-updates.js new file mode 100644 index 0000000000..12b5f65ac2 --- /dev/null +++ b/packages/backend/migration/1750217001651-enable-instance-HOT-updates.js @@ -0,0 +1,26 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +// https://www.cybertec-postgresql.com/en/hot-updates-in-postgresql-for-better-performance/ + +/** + * @typedef {import('typeorm').MigrationInterface} MigrationInterface + */ + +/** + * @class + * @implements {MigrationInterface} + */ +export class EnableInstanceHOTUpdates1750217001651 { + name = 'EnableInstanceHOTUpdates1750217001651'; + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "instance" SET (fillfactor = 50)`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "instance" SET (fillfactor = 100)`); + } +} diff --git a/packages/backend/migration/1750353421706-more-note_edit-columns.js b/packages/backend/migration/1750353421706-more-note_edit-columns.js new file mode 100644 index 0000000000..3bc24b1acc --- /dev/null +++ b/packages/backend/migration/1750353421706-more-note_edit-columns.js @@ -0,0 +1,75 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class MoreNoteEditColumns1750353421706 { + name = 'MoreNoteEditColumns1750353421706' + + async up(queryRunner) { + // Update column types + await queryRunner.query(`ALTER TABLE "note_edit" ALTER COLUMN "cw" TYPE text USING "cw"::text`); + + // Rename columns + await queryRunner.query(`ALTER TABLE "note_edit" RENAME COLUMN "oldText" TO "text"`); + await queryRunner.query(`ALTER TABLE "note_edit" RENAME COLUMN "cw" TO "newCw"`); + + // Add new fields + await queryRunner.query(`ALTER TABLE "note_edit" ADD "userId" character varying(32)`); + await queryRunner.query(`COMMENT ON COLUMN "note_edit"."userId" IS 'The ID of author.'`); + await queryRunner.query(`ALTER TABLE "note_edit" ADD CONSTRAINT "FK_7f1ded0f6e8a5bef701b7e698ab" FOREIGN KEY ("userId") REFERENCES "user"("id") ON DELETE CASCADE ON UPDATE NO ACTION`); + + await queryRunner.query(`ALTER TABLE "note_edit" ADD "renoteId" character varying(32)`); + await queryRunner.query(`COMMENT ON COLUMN "note_edit"."renoteId" IS 'The ID of renote target. Will always be null for older edits'`); + await queryRunner.query(`ALTER TABLE "note_edit" ADD CONSTRAINT "FK_d3003e5256bcbfad6c3588835c0" FOREIGN KEY ("renoteId") REFERENCES "note"("id") ON DELETE CASCADE ON UPDATE NO ACTION`); + + await queryRunner.query(`ALTER TABLE "note_edit" ADD "replyId" character varying(32)`); + await queryRunner.query(`COMMENT ON COLUMN "note_edit"."replyId" IS 'The ID of reply target. Will always be null for older edits'`); + await queryRunner.query(`ALTER TABLE "note_edit" ADD CONSTRAINT "FK_f34b53ab9b39774ca014972ad84" FOREIGN KEY ("replyId") REFERENCES "note"("id") ON DELETE CASCADE ON UPDATE NO ACTION`); + + await queryRunner.query(`ALTER TABLE "note_edit" ADD "visibility" "public"."note_visibility_enum"`); + + await queryRunner.query(`ALTER TABLE "note_edit" ADD "cw" text`); + await queryRunner.query(`COMMENT ON COLUMN "note_edit"."cw" IS 'Will always be null for older edits'`); + + await queryRunner.query(`ALTER TABLE "note_edit" ADD "hasPoll" boolean NOT NULL DEFAULT false`); + await queryRunner.query(`COMMENT ON COLUMN "note_edit"."hasPoll" IS 'Whether this revision had a poll. Will always be false for older edits'`); + + // Populate non-nullable fields + await queryRunner.query(` + UPDATE "note_edit" "e" + SET + "visibility" = "n"."visibility", + "userId" = "n"."userId" + FROM "note" "n" + WHERE "n"."id" = "e"."noteId" + `); + await queryRunner.query(`ALTER TABLE "note_edit" ALTER COLUMN "visibility" SET NOT NULL`); + await queryRunner.query(`ALTER TABLE "note_edit" ALTER COLUMN "userId" SET NOT NULL`); + } + + async down(queryRunner) { + // Drop new columns + await queryRunner.query(`ALTER TABLE "note_edit" DROP COLUMN "visibility"`); + await queryRunner.query(`ALTER TABLE "note_edit" DROP COLUMN "hasPoll"`); + await queryRunner.query(`ALTER TABLE "note_edit" DROP COLUMN "cw"`); + + await queryRunner.query(`ALTER TABLE "note_edit" DROP COLUMN "userId"`); + + await queryRunner.query(`ALTER TABLE "note_edit" DROP CONSTRAINT "FK_f34b53ab9b39774ca014972ad84"`); + await queryRunner.query(`ALTER TABLE "note_edit" DROP COLUMN "replyId"`); + + await queryRunner.query(`ALTER TABLE "note_edit" DROP CONSTRAINT "FK_d3003e5256bcbfad6c3588835c0"`); + await queryRunner.query(`ALTER TABLE "note_edit" DROP COLUMN "renoteId"`); + + await queryRunner.query(`ALTER TABLE "note_edit" DROP CONSTRAINT "FK_7f1ded0f6e8a5bef701b7e698ab"`); + await queryRunner.query(`ALTER TABLE "note_edit" DROP COLUMN "userId"`); + + // Rename new columns + await queryRunner.query(`ALTER TABLE "note_edit" RENAME COLUMN "text" TO "oldText"`); + await queryRunner.query(`ALTER TABLE "note_edit" RENAME COLUMN "newCw" TO "cw"`); + + // Revert column types + await queryRunner.query(`ALTER TABLE "note_edit" ALTER COLUMN "cw" TYPE varchar(512) USING "cw"::varchar(512)`); + } +} diff --git a/packages/backend/migration/1758129782800-add-user-lastFetchedFeaturedAt.js b/packages/backend/migration/1758129782800-add-user-lastFetchedFeaturedAt.js new file mode 100644 index 0000000000..fc3713cec9 --- /dev/null +++ b/packages/backend/migration/1758129782800-add-user-lastFetchedFeaturedAt.js @@ -0,0 +1,14 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class AddUserLastFetchedFeaturedAt1758129782800 { + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "user" ADD "lastFetchedFeaturedAt" DATE`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "user" DROP COLUMN "lastFetchedFeaturedAt"`); + } +} diff --git a/packages/backend/migration/1758136690898-fix-user-lastFetchedFeaturedAt-type.js b/packages/backend/migration/1758136690898-fix-user-lastFetchedFeaturedAt-type.js new file mode 100644 index 0000000000..4a0d767eb9 --- /dev/null +++ b/packages/backend/migration/1758136690898-fix-user-lastFetchedFeaturedAt-type.js @@ -0,0 +1,14 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class FixUserLastFetchedFeaturedAtType1758136690898 { + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "user" ALTER COLUMN "lastFetchedFeaturedAt" TYPE TIMESTAMP WITH TIME ZONE`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "user" ALTER COLUMN "lastFetchedFeaturedAt" TYPE DATE`); + } +} diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 5607f50eb7..be3892e942 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -111,11 +111,14 @@ type Source = { deliverJobConcurrency?: number; inboxJobConcurrency?: number; relationshipJobConcurrency?: number; + backgroundJobConcurrency?: number; deliverJobPerSec?: number; inboxJobPerSec?: number; relationshipJobPerSec?: number; + backgroundJobPerSec?: number; deliverJobMaxAttempts?: number; inboxJobMaxAttempts?: number; + backgroundJobMaxAttempts?: number; mediaDirectory?: string; mediaProxy?: string; @@ -272,11 +275,14 @@ export type Config = { deliverJobConcurrency: number | undefined; inboxJobConcurrency: number | undefined; relationshipJobConcurrency: number | undefined; + backgroundJobConcurrency: number | undefined; deliverJobPerSec: number | undefined; inboxJobPerSec: number | undefined; relationshipJobPerSec: number | undefined; + backgroundJobPerSec: number | undefined; deliverJobMaxAttempts: number | undefined; inboxJobMaxAttempts: number | undefined; + backgroundJobMaxAttempts: number | undefined; proxyRemoteFiles: boolean | undefined; customMOTD: string[] | undefined; signToActivityPubGet: boolean; @@ -475,11 +481,14 @@ export function loadConfig(loggerService: LoggerService): Config { deliverJobConcurrency: config.deliverJobConcurrency, inboxJobConcurrency: config.inboxJobConcurrency, relationshipJobConcurrency: config.relationshipJobConcurrency, + backgroundJobConcurrency: config.backgroundJobConcurrency, deliverJobPerSec: config.deliverJobPerSec, inboxJobPerSec: config.inboxJobPerSec, relationshipJobPerSec: config.relationshipJobPerSec, + backgroundJobPerSec: config.backgroundJobPerSec, deliverJobMaxAttempts: config.deliverJobMaxAttempts, inboxJobMaxAttempts: config.inboxJobMaxAttempts, + backgroundJobMaxAttempts: config.backgroundJobMaxAttempts, proxyRemoteFiles: config.proxyRemoteFiles, customMOTD: config.customMOTD, signToActivityPubGet: config.signToActivityPubGet ?? true, diff --git a/packages/backend/src/core/AnnouncementService.ts b/packages/backend/src/core/AnnouncementService.ts index 54496f9922..48c012da0f 100644 --- a/packages/backend/src/core/AnnouncementService.ts +++ b/packages/backend/src/core/AnnouncementService.ts @@ -218,9 +218,9 @@ export class AnnouncementService { announcementId: announcement.id, userId: me.id, }); - return this.announcementEntityService.pack({ ...announcement, isRead: read !== null }, me); + return await this.announcementEntityService.pack({ ...announcement, isRead: read !== null }, me); } else { - return this.announcementEntityService.pack(announcement, null); + return await this.announcementEntityService.pack(announcement, null); } } diff --git a/packages/backend/src/core/AntennaService.ts b/packages/backend/src/core/AntennaService.ts index 660c97dd6a..fcb3650c5c 100644 --- a/packages/backend/src/core/AntennaService.ts +++ b/packages/backend/src/core/AntennaService.ts @@ -18,13 +18,16 @@ import type { AntennasRepository, UserListMembershipsRepository } from '@/models import type { MiAntenna } from '@/models/Antenna.js'; import type { MiNote } from '@/models/Note.js'; import type { MiUser } from '@/models/User.js'; +import { InternalEventService } from '@/global/InternalEventService.js'; +import { promiseMap } from '@/misc/promise-map.js'; import { CacheService } from './CacheService.js'; import type { OnApplicationShutdown } from '@nestjs/common'; @Injectable() export class AntennaService implements OnApplicationShutdown { + // TODO implement QuantumSingleCache then replace this private antennasFetched: boolean; - private antennas: MiAntenna[]; + private antennas: Map; constructor( @Inject(DI.redisForTimelines) @@ -43,9 +46,10 @@ export class AntennaService implements OnApplicationShutdown { private utilityService: UtilityService, private globalEventService: GlobalEventService, private fanoutTimelineService: FanoutTimelineService, + private readonly internalEventService: InternalEventService, ) { this.antennasFetched = false; - this.antennas = []; + this.antennas = new Map(); this.redisForSub.on('message', this.onRedisMessage); } @@ -58,35 +62,16 @@ export class AntennaService implements OnApplicationShutdown { const { type, body } = obj.message as GlobalEvents['internal']['payload']; switch (type) { case 'antennaCreated': - this.antennas.push({ // TODO: このあたりのデシリアライズ処理は各modelファイル内に関数としてexportしたい + case 'antennaUpdated': + this.antennas.set(body.id, { // TODO: このあたりのデシリアライズ処理は各modelファイル内に関数としてexportしたい ...body, lastUsedAt: new Date(body.lastUsedAt), user: null, // joinなカラムは通常取ってこないので userList: null, // joinなカラムは通常取ってこないので }); break; - case 'antennaUpdated': { - const idx = this.antennas.findIndex(a => a.id === body.id); - if (idx >= 0) { - this.antennas[idx] = { // TODO: このあたりのデシリアライズ処理は各modelファイル内に関数としてexportしたい - ...body, - lastUsedAt: new Date(body.lastUsedAt), - user: null, // joinなカラムは通常取ってこないので - userList: null, // joinなカラムは通常取ってこないので - }; - } else { - // サーバ起動時にactiveじゃなかった場合、リストに持っていないので追加する必要あり - this.antennas.push({ // TODO: このあたりのデシリアライズ処理は各modelファイル内に関数としてexportしたい - ...body, - lastUsedAt: new Date(body.lastUsedAt), - user: null, // joinなカラムは通常取ってこないので - userList: null, // joinなカラムは通常取ってこないので - }); - } - } - break; case 'antennaDeleted': - this.antennas = this.antennas.filter(a => a.id !== body.id); + this.antennas.delete(body.id); break; default: break; @@ -94,10 +79,27 @@ export class AntennaService implements OnApplicationShutdown { } } + @bindThis + public async updateAntenna(id: string, data: Partial) { + await this.antennasRepository.update({ id }, data); + + const antenna = this.antennas.get(id) ?? await this.antennasRepository.findOneBy({ id }); + if (antenna) { + // This will be handled above to save result + await this.internalEventService.emit('antennaUpdated', { + ...antenna, + ...data, + }); + } + } + @bindThis public async addNoteToAntennas(note: MiNote, noteUser: { id: MiUser['id']; username: string; host: string | null; isBot: boolean; }): Promise { const antennas = await this.getAntennas(); - const antennasWithMatchResult = await Promise.all(antennas.map(antenna => this.checkHitAntenna(antenna, note, noteUser).then(hit => [antenna, hit] as const))); + const antennasWithMatchResult = await promiseMap(antennas, async antenna => { + const hit = await this.checkHitAntenna(antenna, note, noteUser); + return [antenna, hit] as const; + }); const matchedAntennas = antennasWithMatchResult.filter(([, hit]) => hit).map(([antenna]) => antenna); const redisPipeline = this.redisForTimelines.pipeline(); @@ -107,7 +109,7 @@ export class AntennaService implements OnApplicationShutdown { this.globalEventService.publishAntennaStream(antenna.id, 'note', note); } - redisPipeline.exec(); + await redisPipeline.exec(); } // NOTE: フォローしているユーザーのノート、リストのユーザーのノート、グループのユーザーのノート指定はパフォーマンス上の理由で無効になっている @@ -212,13 +214,14 @@ export class AntennaService implements OnApplicationShutdown { @bindThis public async getAntennas() { if (!this.antennasFetched) { - this.antennas = await this.antennasRepository.findBy({ + const allAntennas = await this.antennasRepository.findBy({ isActive: true, }); + this.antennas = new Map(allAntennas.map(a => [a.id, a])); this.antennasFetched = true; } - return this.antennas; + return Array.from(this.antennas.values()); } @bindThis diff --git a/packages/backend/src/core/ApLogService.ts b/packages/backend/src/core/ApLogService.ts index 89837a60d2..391d90b95a 100644 --- a/packages/backend/src/core/ApLogService.ts +++ b/packages/backend/src/core/ApLogService.ts @@ -14,7 +14,9 @@ import { JsonValue } from '@/misc/json-value.js'; import { UtilityService } from '@/core/UtilityService.js'; import { TimeService } from '@/global/TimeService.js'; import { IdService } from '@/core/IdService.js'; -import { IActivity, IObject } from './activitypub/type.js'; +import { IActivity, IObject } from '@/core/activitypub/type.js'; +import { bindThis } from '@/decorators.js'; +import { QueueService } from '@/core/QueueService.js'; @Injectable() export class ApLogService { @@ -23,7 +25,7 @@ export class ApLogService { private readonly config: Config, @Inject(DI.apContextsRepository) - private apContextsRepository: ApContextsRepository, + private readonly apContextsRepository: ApContextsRepository, @Inject(DI.apInboxLogsRepository) private readonly apInboxLogsRepository: ApInboxLogsRepository, @@ -34,6 +36,7 @@ export class ApLogService { private readonly utilityService: UtilityService, private readonly idService: IdService, private readonly timeService: TimeService, + private readonly queueService: QueueService, ) {} /** @@ -123,6 +126,16 @@ export class ApLogService { .execute(); } + @bindThis + public async deleteObjectLogsDeferred(objectUris: string | string[]): Promise { + await this.queueService.createDeleteApLogsJob('object', objectUris); + } + + @bindThis + public async deleteInboxLogsDeferred(userIds: string | string[]): Promise { + await this.queueService.createDeleteApLogsJob('inbox', userIds); + } + /** * Deletes all logged copies of an object or objects * @param objectUris URIs / AP IDs of the objects to delete diff --git a/packages/backend/src/core/AvatarDecorationService.ts b/packages/backend/src/core/AvatarDecorationService.ts index aedb1d6a80..0a586789f1 100644 --- a/packages/backend/src/core/AvatarDecorationService.ts +++ b/packages/backend/src/core/AvatarDecorationService.ts @@ -109,7 +109,7 @@ export class AvatarDecorationService implements OnApplicationShutdown { if (noCache) { this.cache.delete(); } - return this.cache.fetch(() => this.avatarDecorationsRepository.find()); + return await this.cache.fetch(() => this.avatarDecorationsRepository.find()); } @bindThis diff --git a/packages/backend/src/core/ChatService.ts b/packages/backend/src/core/ChatService.ts index cdecd41726..9132c166f9 100644 --- a/packages/backend/src/core/ChatService.ts +++ b/packages/backend/src/core/ChatService.ts @@ -605,12 +605,12 @@ export class ChatService { @bindThis public async findMyRoomById(ownerId: MiUser['id'], roomId: MiChatRoom['id']) { - return this.chatRoomsRepository.findOneBy({ id: roomId, ownerId: ownerId }); + return await this.chatRoomsRepository.findOneBy({ id: roomId, ownerId: ownerId }); } @bindThis public async findRoomById(roomId: MiChatRoom['id']) { - return this.chatRoomsRepository.findOne({ where: { id: roomId }, relations: ['owner'] }); + return await this.chatRoomsRepository.findOne({ where: { id: roomId }, relations: ['owner'] }); } @bindThis diff --git a/packages/backend/src/core/ClipService.ts b/packages/backend/src/core/ClipService.ts index 35b962cc04..6bb6b78eea 100644 --- a/packages/backend/src/core/ClipService.ts +++ b/packages/backend/src/core/ClipService.ts @@ -11,6 +11,7 @@ import { bindThis } from '@/decorators.js'; import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js'; import { RoleService } from '@/core/RoleService.js'; import { IdService } from '@/core/IdService.js'; +import { CollapsedQueueService } from '@/core/CollapsedQueueService.js'; import type { MiLocalUser } from '@/models/User.js'; import { TimeService } from '@/global/TimeService.js'; @@ -35,6 +36,7 @@ export class ClipService { private roleService: RoleService, private idService: IdService, private readonly timeService: TimeService, + private readonly collapsedQueueService: CollapsedQueueService, ) { } @@ -130,7 +132,7 @@ export class ClipService { lastClippedAt: this.timeService.date, }); - this.notesRepository.increment({ id: noteId }, 'clippedCount', 1); + await this.collapsedQueueService.updateNoteQueue.enqueue(noteId, { clippedCountDelta: 1 }); } @bindThis @@ -155,6 +157,6 @@ export class ClipService { clipId: clip.id, }); - this.notesRepository.decrement({ id: noteId }, 'clippedCount', 1); + await this.collapsedQueueService.updateNoteQueue.enqueue(noteId, { clippedCountDelta: -1 }); } } diff --git a/packages/backend/src/core/CollapsedQueueService.ts b/packages/backend/src/core/CollapsedQueueService.ts new file mode 100644 index 0000000000..faf1cee524 --- /dev/null +++ b/packages/backend/src/core/CollapsedQueueService.ts @@ -0,0 +1,383 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; +import { LoggerService } from '@/core/LoggerService.js'; +import type Logger from '@/logger.js'; +import { CollapsedQueue } from '@/misc/collapsed-queue.js'; +import { renderInlineError } from '@/misc/render-inline-error.js'; +import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; +import { EnvService } from '@/global/EnvService.js'; +import { bindThis } from '@/decorators.js'; +import { InternalEventService } from '@/global/InternalEventService.js'; +import type { UsersRepository, NotesRepository, AccessTokensRepository, MiAntenna, FollowingsRepository } from '@/models/_.js'; +import { DI } from '@/di-symbols.js'; +import { AntennaService } from '@/core/AntennaService.js'; +import { CacheService } from '@/core/CacheService.js'; +import { TimeService } from '@/global/TimeService.js'; + +export type UpdateInstanceJob = { + latestRequestReceivedAt?: Date, + notRespondingSince?: Date | null, + shouldUnsuspend?: boolean, + shouldSuspendGone?: boolean, + shouldSuspendNotResponding?: boolean, + notesCountDelta?: number, + usersCountDelta?: number, + followingCountDelta?: number, + followersCountDelta?: number, +}; + +export type UpdateUserJob = { + updatedAt?: Date, + lastActiveDate?: Date, + notesCountDelta?: number, + followingCountDelta?: number, + followersCountDelta?: number, +}; + +export type UpdateNoteJob = { + repliesCountDelta?: number; + renoteCountDelta?: number; + clippedCountDelta?: number; +}; + +export type UpdateAccessTokenJob = { + lastUsedAt: Date; +}; + +export type UpdateAntennaJob = { + isActive: boolean, + lastUsedAt?: Date, +}; + +@Injectable() +export class CollapsedQueueService implements OnApplicationShutdown { + // Moved from InboxProcessorService + public readonly updateInstanceQueue: CollapsedQueue; + + // Moved from NoteCreateService, NoteEditService, and NoteDeleteService + public readonly updateUserQueue: CollapsedQueue; + + public readonly updateNoteQueue: CollapsedQueue; + public readonly updateAccessTokenQueue: CollapsedQueue; + public readonly updateAntennaQueue: CollapsedQueue; + + private readonly logger: Logger; + + constructor( + @Inject(DI.usersRepository) + private readonly usersRepository: UsersRepository, + + @Inject(DI.notesRepository) + private readonly notesRepository: NotesRepository, + + @Inject(DI.accessTokensRepository) + private readonly accessTokensRepository: AccessTokensRepository, + + @Inject(DI.followingsRepository) + private readonly followingsRepository: FollowingsRepository, + + private readonly federatedInstanceService: FederatedInstanceService, + private readonly envService: EnvService, + private readonly internalEventService: InternalEventService, + private readonly antennaService: AntennaService, + private readonly cacheService: CacheService, + private readonly timeService: TimeService, + + loggerService: LoggerService, + ) { + this.logger = loggerService.getLogger('collapsed-queue'); + + const fiveMinuteInterval = this.envService.env.NODE_ENV !== 'test' ? 60 * 1000 * 5 : 0; + const oneMinuteInterval = this.envService.env.NODE_ENV !== 'test' ? 60 * 1000 : 0; + + this.updateInstanceQueue = new CollapsedQueue( + this.internalEventService, + this.timeService, + 'updateInstance', + fiveMinuteInterval, + (oldJob, newJob) => ({ + latestRequestReceivedAt: maxDate(oldJob.latestRequestReceivedAt, newJob.latestRequestReceivedAt), + notRespondingSince: maxDate(oldJob.notRespondingSince, newJob.notRespondingSince), + shouldUnsuspend: oldJob.shouldUnsuspend || newJob.shouldUnsuspend, + shouldSuspendGone: oldJob.shouldSuspendGone || newJob.shouldSuspendGone, + shouldSuspendNotResponding: oldJob.shouldSuspendNotResponding || newJob.shouldSuspendNotResponding, + notesCountDelta: (oldJob.notesCountDelta ?? 0) + (newJob.notesCountDelta ?? 0), + usersCountDelta: (oldJob.usersCountDelta ?? 0) + (newJob.usersCountDelta ?? 0), + followingCountDelta: (oldJob.followingCountDelta ?? 0) + (newJob.followingCountDelta ?? 0), + followersCountDelta: (oldJob.followersCountDelta ?? 0) + (newJob.followersCountDelta ?? 0), + }), + async (id, job) => { + // Have to check this because all properties are optional + if ( + job.latestRequestReceivedAt || + job.notRespondingSince !== undefined || + job.shouldSuspendNotResponding || + job.shouldSuspendGone || + job.shouldUnsuspend || + job.notesCountDelta || + job.usersCountDelta || + job.followingCountDelta || + job.followersCountDelta + ) { + await this.federatedInstanceService.update(id, { + // Direct update if defined + latestRequestReceivedAt: job.latestRequestReceivedAt, + + // null (responding) > Date (not responding) + notRespondingSince: job.latestRequestReceivedAt + ? null + : job.notRespondingSince, + + // false (responding) > true (not responding) + isNotResponding: job.latestRequestReceivedAt + ? false + : job.notRespondingSince + ? true + : undefined, + + // gone > none > auto + suspensionState: job.shouldSuspendGone + ? 'goneSuspended' + : job.shouldUnsuspend + ? 'none' + : job.shouldSuspendNotResponding + ? 'autoSuspendedForNotResponding' + : undefined, + + // Increment if defined + notesCount: job.notesCountDelta ? () => `"notesCount" + ${job.notesCountDelta}` : undefined, + usersCount: job.usersCountDelta ? () => `"usersCount" + ${job.usersCountDelta}` : undefined, + followingCount: job.followingCountDelta ? () => `"followingCount" + ${job.followingCountDelta}` : undefined, + followersCount: job.followersCountDelta ? () => `"followersCount" + ${job.followersCountDelta}` : undefined, + }); + } + }, + { + onError: this.onQueueError, + concurrency: 2, // Low concurrency, this table is slow for some reason + redisParser: data => ({ + ...data, + latestRequestReceivedAt: data.latestRequestReceivedAt != null + ? new Date(data.latestRequestReceivedAt) + : data.latestRequestReceivedAt, + notRespondingSince: data.notRespondingSince != null + ? new Date(data.notRespondingSince) + : data.notRespondingSince, + }), + }, + ); + + this.updateUserQueue = new CollapsedQueue( + this.internalEventService, + this.timeService, + 'updateUser', + oneMinuteInterval, + (oldJob, newJob) => ({ + updatedAt: maxDate(oldJob.updatedAt, newJob.updatedAt), + lastActiveDate: maxDate(oldJob.lastActiveDate, newJob.lastActiveDate), + notesCountDelta: (oldJob.notesCountDelta ?? 0) + (newJob.notesCountDelta ?? 0), + followingCountDelta: (oldJob.followingCountDelta ?? 0) + (newJob.followingCountDelta ?? 0), + followersCountDelta: (oldJob.followersCountDelta ?? 0) + (newJob.followersCountDelta ?? 0), + }), + async (id, job) => { + // Have to check this because all properties are optional + if (job.updatedAt || job.lastActiveDate || job.notesCountDelta || job.followingCountDelta || job.followersCountDelta) { + // Updating the user should implicitly mark them as active + const lastActiveDate = job.lastActiveDate ?? job.updatedAt; + const isWakingUp = lastActiveDate && (await this.cacheService.findUserById(id)).isHibernated; + + // Update user before the hibernation cache, because the latter may refresh from DB + await this.usersRepository.update({ id }, { + updatedAt: job.updatedAt, + lastActiveDate, + isHibernated: isWakingUp ? false : undefined, + notesCount: job.notesCountDelta ? () => `"notesCount" + ${job.notesCountDelta}` : undefined, + followingCount: job.followingCountDelta ? () => `"followingCount" + ${job.followingCountDelta}` : undefined, + followersCount: job.followersCountDelta ? () => `"followersCount" + ${job.followersCountDelta}` : undefined, + }); + await this.internalEventService.emit('userUpdated', { id }); + + // Wake up hibernated users + if (isWakingUp) { + await this.followingsRepository.update({ followerId: id }, { isFollowerHibernated: false }); + await this.cacheService.hibernatedUserCache.set(id, false); + } + } + }, + { + onError: this.onQueueError, + concurrency: 4, // High concurrency - this queue gets a lot of activity + redisParser: data => ({ + ...data, + updatedAt: data.updatedAt != null + ? new Date(data.updatedAt) + : data.updatedAt, + lastActiveDate: data.lastActiveDate != null + ? new Date(data.lastActiveDate) + : data.lastActiveDate, + }), + }, + ); + + this.updateNoteQueue = new CollapsedQueue( + this.internalEventService, + this.timeService, + 'updateNote', + oneMinuteInterval, + (oldJob, newJob) => ({ + repliesCountDelta: (oldJob.repliesCountDelta ?? 0) + (newJob.repliesCountDelta ?? 0), + renoteCountDelta: (oldJob.renoteCountDelta ?? 0) + (newJob.renoteCountDelta ?? 0), + clippedCountDelta: (oldJob.clippedCountDelta ?? 0) + (newJob.clippedCountDelta ?? 0), + }), + async (id, job) => { + // Have to check this because all properties are optional + if (job.repliesCountDelta || job.renoteCountDelta || job.clippedCountDelta) { + await this.notesRepository.update({ id }, { + repliesCount: job.repliesCountDelta ? () => `"repliesCount" + ${job.repliesCountDelta}` : undefined, + renoteCount: job.renoteCountDelta ? () => `"renoteCount" + ${job.renoteCountDelta}` : undefined, + clippedCount: job.clippedCountDelta ? () => `"clippedCount" + ${job.clippedCountDelta}` : undefined, + }); + } + }, + { + onError: this.onQueueError, + concurrency: 4, // High concurrency - this queue gets a lot of activity + }, + ); + + this.updateAccessTokenQueue = new CollapsedQueue( + this.internalEventService, + this.timeService, + 'updateAccessToken', + fiveMinuteInterval, + (oldJob, newJob) => ({ + lastUsedAt: maxDate(oldJob.lastUsedAt, newJob.lastUsedAt), + }), + async (id, job) => await this.accessTokensRepository.update({ id }, { + lastUsedAt: job.lastUsedAt, + }), + { + onError: this.onQueueError, + concurrency: 2, + redisParser: data => ({ + ...data, + lastUsedAt: new Date(data.lastUsedAt), + }), + }, + ); + + this.updateAntennaQueue = new CollapsedQueue( + this.internalEventService, + this.timeService, + 'updateAntenna', + fiveMinuteInterval, + (oldJob, newJob) => ({ + isActive: oldJob.isActive || newJob.isActive, + lastUsedAt: maxDate(oldJob.lastUsedAt, newJob.lastUsedAt), + }), + async (id, job) => await this.antennaService.updateAntenna(id, { + isActive: job.isActive, + lastUsedAt: job.lastUsedAt, + }), + { + onError: this.onQueueError, + concurrency: 4, + redisParser: data => ({ + ...data, + lastUsedAt: data.lastUsedAt != null + ? new Date(data.lastUsedAt) + : data.lastUsedAt, + }), + }, + ); + + this.internalEventService.on('userChangeDeletedState', this.onUserDeleted); + this.internalEventService.on('antennaDeleted', this.onAntennaDeleted); + this.internalEventService.on('antennaUpdated', this.onAntennaDeleted); + } + + @bindThis + private async performQueue(queue: CollapsedQueue): Promise { + try { + const results = await queue.performAllNow(); + + const [succeeded, failed] = results.reduce((counts, result) => { + counts[result ? 0 : 1]++; + return counts; + }, [0, 0]); + + this.logger.debug(`Persistence completed for ${queue.name}: ${succeeded} succeeded and ${failed} failed`); + } catch (err) { + this.logger.error(`Persistence failed for ${queue.name}: ${renderInlineError(err)}`); + } + } + + @bindThis + private onQueueError(queue: CollapsedQueue, error: unknown): void { + this.logger.error(`Error persisting ${queue.name}: ${renderInlineError(error)}`); + } + + @bindThis + private async onUserDeleted(data: { id: string, isDeleted: boolean }) { + if (data.isDeleted) { + await this.updateUserQueue.delete(data.id); + } + } + + @bindThis + private async onAntennaDeleted(data: MiAntenna) { + await this.updateAntennaQueue.delete(data.id); + } + + @bindThis + async dispose() { + this.internalEventService.off('userChangeDeletedState', this.onUserDeleted); + this.internalEventService.off('antennaDeleted', this.onAntennaDeleted); + this.internalEventService.off('antennaUpdated', this.onAntennaDeleted); + + this.logger.info('Persisting all collapsed queues...'); + + await this.performQueue(this.updateInstanceQueue); + await this.performQueue(this.updateUserQueue); + await this.performQueue(this.updateNoteQueue); + await this.performQueue(this.updateAccessTokenQueue); + await this.performQueue(this.updateAntennaQueue); + + this.logger.info('Persistence complete.'); + } + + async onApplicationShutdown() { + await this.dispose(); + } +} + +function maxDate(first: Date | undefined, second: Date): Date; +function maxDate(first: Date, second: Date | undefined): Date; +function maxDate(first: Date | undefined, second: Date | undefined): Date | undefined; +function maxDate(first: Date | null | undefined, second: Date | null | undefined): Date | null | undefined; + +function maxDate(first: Date | null | undefined, second: Date | null | undefined): Date | null | undefined { + if (first !== undefined && second !== undefined) { + if (first != null && second != null) { + if (first.getTime() > second.getTime()) { + return first; + } else { + return second; + } + } else { + // Null is considered infinitely in the future, and is therefore newer than any date. + return null; + } + } else if (first !== undefined) { + return first; + } else if (second !== undefined) { + return second; + } else { + // Undefined in considered infinitely in the past, and is therefore older than any date. + return undefined; + } +} diff --git a/packages/backend/src/core/CoreModule.ts b/packages/backend/src/core/CoreModule.ts index 11119c52a3..a75c3b3abf 100644 --- a/packages/backend/src/core/CoreModule.ts +++ b/packages/backend/src/core/CoreModule.ts @@ -17,7 +17,7 @@ import { WebhookTestService } from '@/core/WebhookTestService.js'; import { FlashService } from '@/core/FlashService.js'; import { ApUtilityService } from '@/core/activitypub/ApUtilityService.js'; import { ApLogService } from '@/core/ApLogService.js'; -import { UpdateInstanceQueue } from '@/core/UpdateInstanceQueue.js'; +import { CollapsedQueueService } from '@/core/CollapsedQueueService.js'; import { InstanceStatsService } from '@/core/InstanceStatsService.js'; import { NoteVisibilityService } from '@/core/NoteVisibilityService.js'; import { AccountMoveService } from './AccountMoveService.js'; @@ -218,7 +218,7 @@ const $UserRenoteMutingService: Provider = { provide: 'UserRenoteMutingService', const $UserSearchService: Provider = { provide: 'UserSearchService', useExisting: UserSearchService }; const $UserSuspendService: Provider = { provide: 'UserSuspendService', useExisting: UserSuspendService }; const $UserAuthService: Provider = { provide: 'UserAuthService', useExisting: UserAuthService }; -const $UpdateInstanceQueue: Provider = { provide: 'UpdateInstanceQueue', useExisting: UpdateInstanceQueue }; +const $CollapsedQueueService: Provider = { provide: 'CollapsedQueueService', useExisting: CollapsedQueueService }; const $VideoProcessingService: Provider = { provide: 'VideoProcessingService', useExisting: VideoProcessingService }; const $UserWebhookService: Provider = { provide: 'UserWebhookService', useExisting: UserWebhookService }; const $SystemWebhookService: Provider = { provide: 'SystemWebhookService', useExisting: SystemWebhookService }; @@ -377,7 +377,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp UserSearchService, UserSuspendService, UserAuthService, - UpdateInstanceQueue, + CollapsedQueueService, VideoProcessingService, UserWebhookService, SystemWebhookService, @@ -531,7 +531,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp $UserSearchService, $UserSuspendService, $UserAuthService, - $UpdateInstanceQueue, + $CollapsedQueueService, $VideoProcessingService, $UserWebhookService, $SystemWebhookService, @@ -686,7 +686,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp UserSearchService, UserSuspendService, UserAuthService, - UpdateInstanceQueue, + CollapsedQueueService, VideoProcessingService, UserWebhookService, SystemWebhookService, @@ -839,7 +839,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp $UserSearchService, $UserSuspendService, $UserAuthService, - $UpdateInstanceQueue, + $CollapsedQueueService, $VideoProcessingService, $UserWebhookService, $SystemWebhookService, diff --git a/packages/backend/src/core/CustomEmojiService.ts b/packages/backend/src/core/CustomEmojiService.ts index dff6560378..15adf0faae 100644 --- a/packages/backend/src/core/CustomEmojiService.ts +++ b/packages/backend/src/core/CustomEmojiService.ts @@ -23,6 +23,7 @@ import { DriveService } from '@/core/DriveService.js'; import { CacheManagementService, type ManagedQuantumKVCache } from '@/global/CacheManagementService.js'; import { TimeService } from '@/global/TimeService.js'; import { LoggerService } from '@/core/LoggerService.js'; +import { promiseMap } from '@/misc/promise-map.js'; import { isRetryableSymbol } from '@/misc/is-retryable-error.js'; import type Logger from '@/logger.js'; import { KeyNotFoundError } from '@/misc/errors/KeyNotFoundError.js'; @@ -577,7 +578,7 @@ export class CustomEmojiService { */ @bindThis public async populateEmojis(emojiNames: string[], noteUserHost: string | null): Promise> { - const emojis = await Promise.all(emojiNames.map(x => this.populateEmoji(x, noteUserHost))); + const emojis = await promiseMap(emojiNames, async x => await this.populateEmoji(x, noteUserHost), { limit: 4 }); const res = {} as Record; for (let i = 0; i < emojiNames.length; i++) { const resolvedEmoji = emojis[i]; diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts index 94289f237d..11e34a7952 100644 --- a/packages/backend/src/core/DriveService.ts +++ b/packages/backend/src/core/DriveService.ts @@ -207,7 +207,7 @@ export class DriveService { //#region Uploads this.registerLogger.debug(`uploading original: ${key}`); - const uploads = [ + const uploads: Promise[] = [ this.upload(key, fs.createReadStream(path), type, null, name), ]; @@ -470,7 +470,7 @@ export class DriveService { for (const fileId of exceedFileIds) { const file = await this.driveFilesRepository.findOneBy({ id: fileId }); if (file == null) continue; - this.deleteFile(file, true); + await this.deleteFile(file, true); } } @@ -718,14 +718,14 @@ export class DriveService { if (values.isSensitive !== undefined && values.isSensitive !== file.isSensitive) { const user = file.userId ? await this.usersRepository.findOneByOrFail({ id: file.userId }) : null; if (values.isSensitive) { - this.moderationLogService.log(updater, 'markSensitiveDriveFile', { + await this.moderationLogService.log(updater, 'markSensitiveDriveFile', { fileId: file.id, fileUserId: file.userId, fileUserUsername: user?.username ?? null, fileUserHost: user?.host ?? null, }); } else { - this.moderationLogService.log(updater, 'unmarkSensitiveDriveFile', { + await this.moderationLogService.log(updater, 'unmarkSensitiveDriveFile', { fileId: file.id, fileUserId: file.userId, fileUserUsername: user?.username ?? null, @@ -740,29 +740,7 @@ export class DriveService { @bindThis public async deleteFile(file: MiDriveFile, isExpired = false, deleter?: { id: string }) { - if (file.storedInternal) { - this.deleteLocalFile(file.accessKey!); - - if (file.thumbnailUrl) { - this.deleteLocalFile(file.thumbnailAccessKey!); - } - - if (file.webpublicUrl) { - this.deleteLocalFile(file.webpublicAccessKey!); - } - } else if (!file.isLink) { - this.queueService.createDeleteObjectStorageFileJob(file.accessKey!); - - if (file.thumbnailUrl) { - this.queueService.createDeleteObjectStorageFileJob(file.thumbnailAccessKey!); - } - - if (file.webpublicUrl) { - this.queueService.createDeleteObjectStorageFileJob(file.webpublicAccessKey!); - } - } - - this.deletePostProcess(file, isExpired, deleter); + await this.queueService.createDeleteFileJob(file.id, isExpired, deleter?.id); } @bindThis @@ -793,14 +771,14 @@ export class DriveService { await Promise.all(promises); - this.deletePostProcess(file, isExpired, deleter); + await this.deletePostProcess(file, isExpired, deleter); } @bindThis private async deletePostProcess(file: MiDriveFile, isExpired = false, deleter?: { id: string }) { // リモートファイル期限切れ削除後は直リンクにする if (isExpired && file.userHost !== null && file.uri != null) { - this.driveFilesRepository.update(file.id, { + await this.driveFilesRepository.update(file.id, { isLink: true, url: file.uri, thumbnailUrl: null, @@ -812,7 +790,7 @@ export class DriveService { webpublicAccessKey: 'webpublic-' + randomUUID(), }); } else { - this.driveFilesRepository.delete(file.id); + await this.driveFilesRepository.delete(file.id); } this.driveChart.update(file, false); @@ -831,7 +809,7 @@ export class DriveService { if (deleter && await this.roleService.isModerator(deleter) && (file.userId !== deleter.id)) { const user = file.userId ? await this.usersRepository.findOneByOrFail({ id: file.userId }) : null; - this.moderationLogService.log(deleter, 'deleteDriveFile', { + await this.moderationLogService.log(deleter, 'deleteDriveFile', { fileId: file.id, fileUserId: file.userId, fileUserUsername: user?.username ?? null, diff --git a/packages/backend/src/core/FetchInstanceMetadataService.ts b/packages/backend/src/core/FetchInstanceMetadataService.ts index d288c5d231..d929e4bb4e 100644 --- a/packages/backend/src/core/FetchInstanceMetadataService.ts +++ b/packages/backend/src/core/FetchInstanceMetadataService.ts @@ -17,6 +17,7 @@ import { bindThis } from '@/decorators.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; import { TimeService } from '@/global/TimeService.js'; import { renderInlineError } from '@/misc/render-inline-error.js'; +import { QueueService } from '@/core/QueueService.js'; import type { CheerioAPI } from 'cheerio/slim'; type NodeInfo = { @@ -50,6 +51,7 @@ export class FetchInstanceMetadataService { private redisClient: Redis.Redis, private readonly timeService: TimeService, + private readonly queueService: QueueService, ) { this.logger = this.loggerService.getLogger('metadata', 'cyan'); } @@ -73,8 +75,21 @@ export class FetchInstanceMetadataService { return this.redisClient.del(`fetchInstanceMetadata:mutex:v2:${host}`); } + /** + * Schedules a deferred update on the background task worker. + * Duplicate updates are automatically skipped. + */ + @bindThis + public async fetchInstanceMetadataLazy(instance: MiInstance): Promise { + if (!instance.isBlocked) { + await this.queueService.createUpdateInstanceJob(instance.host); + } + } + @bindThis public async fetchInstanceMetadata(instance: MiInstance, force = false): Promise { + if (instance.isBlocked) return; + const host = instance.host; // finallyでunlockされてしまうのでtry内でロックチェックをしない @@ -110,25 +125,30 @@ export class FetchInstanceMetadataService { this.getDescription(info, dom, manifest).catch(() => null), ]); - this.logger.debug(`Successfuly fetched metadata of ${instance.host}`); + this.logger.debug(`Successfully fetched metadata of ${instance.host}`); const updates = { infoUpdatedAt: this.timeService.date, } as Record; if (info) { - updates.softwareName = typeof info.software?.name === 'string' ? info.software.name.toLowerCase() : '?'; - updates.softwareVersion = info.software?.version; - updates.openRegistrations = info.openRegistrations; - updates.maintainerName = info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.name ?? null) : null : null; - updates.maintainerEmail = info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.email ?? null) : null : null; + const softwareName = typeof info.software?.name === 'string' ? info.software.name.toLowerCase() : '?'; + if (softwareName !== instance.softwareName) updates.softwareName = softwareName; + const softwareVersion = typeof info.software?.version === 'string' ? info.software.version.toLowerCase() : '?'; + if (softwareVersion !== instance.softwareVersion) updates.softwareVersion = softwareVersion; + if (info.openRegistrations !== instance.openRegistrations) updates.openRegistrations = info.openRegistrations; + const maintainerName = info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.name ?? null) : null : null; + if (maintainerName !== instance.maintainerName) updates.maintainerName = maintainerName; + const maintainerEmail = info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.email ?? null) : null : null; + if (maintainerEmail !== instance.maintainerEmail) updates.maintainerEmail = maintainerEmail; } - if (name) updates.name = name; - if (description) updates.description = description; - if (icon ?? favicon) updates.iconUrl = (icon && !icon.includes('data:image/png;base64')) ? icon : favicon; - if (favicon) updates.faviconUrl = favicon; - if (themeColor) updates.themeColor = themeColor; + if (name !== instance.name) updates.name = name; + if (description !== instance.description) updates.description = description; + const iconUrl = (icon && !icon.includes('data:image/png;base64')) ? icon : favicon; + if (iconUrl !== instance.iconUrl) updates.iconUrl = iconUrl; + if (favicon !== instance.faviconUrl) updates.faviconUrl = favicon; + if (themeColor !== instance.themeColor) updates.themeColor = themeColor; await this.federatedInstanceService.update(instance.id, updates); @@ -169,10 +189,7 @@ export class FetchInstanceMetadataService { throw new Error('No nodeinfo link provided'); } - const info = await this.httpRequestService.getJson(link.href) - .catch(err => { - throw err.statusCode ?? err.message; - }); + const info = await this.httpRequestService.getJson(link.href); this.logger.debug(`Successfuly fetched nodeinfo of ${instance.host}`); diff --git a/packages/backend/src/core/GlobalEventService.ts b/packages/backend/src/core/GlobalEventService.ts index 2be5a580da..eeba5d6bae 100644 --- a/packages/backend/src/core/GlobalEventService.ts +++ b/packages/backend/src/core/GlobalEventService.ts @@ -277,6 +277,8 @@ export interface InternalEventTypes { userListMemberBulkRemoved: { userListIds: MiUserList['id'][]; memberId: MiUser['id']; }; quantumCacheUpdated: { name: string, keys: string[] }; quantumCacheReset: { name: string }; + collapsedQueueDefer: { name: string, key: string, deferred: boolean }; + collapsedQueueEnqueue: { name: string, key: string, value: unknown }; } type EventTypesToEventPayload = EventUnionFromDictionary>>; diff --git a/packages/backend/src/core/HashtagService.ts b/packages/backend/src/core/HashtagService.ts index 0035c4b0d5..b9945d34b3 100644 --- a/packages/backend/src/core/HashtagService.ts +++ b/packages/backend/src/core/HashtagService.ts @@ -59,7 +59,7 @@ export class HashtagService { tag = normalizeForSearch(tag); // TODO: サンプリング - this.updateHashtagsRanking(tag, user.id); + await this.updateHashtagsRanking(tag, user.id); const index = await this.hashtagsRepository.findOneBy({ name: tag }); @@ -119,11 +119,11 @@ export class HashtagService { if (Object.keys(set).length > 0) { q.set(set); - q.execute(); + await q.execute(); } } else { if (isUserAttached) { - this.hashtagsRepository.insert({ + await this.hashtagsRepository.insert({ id: this.idService.gen(), name: tag, mentionedUserIds: [], @@ -140,7 +140,7 @@ export class HashtagService { attachedRemoteUsersCount: isRemoteUser(user) ? 1 : 0, } as MiHashtag); } else { - this.hashtagsRepository.insert({ + await this.hashtagsRepository.insert({ id: this.idService.gen(), name: tag, mentionedUserIds: [user.id], @@ -174,7 +174,7 @@ export class HashtagService { const exist = await this.redisClient.sismember(`hashtagUsers:${hashtag}`, userId); if (exist === 1) return; - this.featuredService.updateHashtagsRanking(hashtag, 1); + await this.featuredService.updateHashtagsRanking(hashtag, 1); const redisPipeline = this.redisClient.pipeline(); @@ -193,7 +193,7 @@ export class HashtagService { 'NX', // "NX -- Set expiry only when the key has no expiry" = 有効期限がないときだけ設定 ); - redisPipeline.exec(); + await redisPipeline.exec(); } @bindThis diff --git a/packages/backend/src/core/ImageProcessingService.ts b/packages/backend/src/core/ImageProcessingService.ts index 6f60475442..d042cf1f30 100644 --- a/packages/backend/src/core/ImageProcessingService.ts +++ b/packages/backend/src/core/ImageProcessingService.ts @@ -58,7 +58,7 @@ export class ImageProcessingService { */ @bindThis public async convertToWebp(path: string, width: number, height: number, options: sharp.WebpOptions = webpDefault): Promise { - return this.convertSharpToWebp(sharp(path), width, height, options); + return await this.convertSharpToWebp(sharp(path), width, height, options); } @bindThis @@ -100,7 +100,7 @@ export class ImageProcessingService { */ @bindThis public async convertToAvif(path: string, width: number, height: number, options: sharp.AvifOptions = avifDefault): Promise { - return this.convertSharpToAvif(sharp(path), width, height, options); + return await this.convertSharpToAvif(sharp(path), width, height, options); } @bindThis @@ -142,7 +142,7 @@ export class ImageProcessingService { */ @bindThis public async convertToPng(path: string, width: number, height: number): Promise { - return this.convertSharpToPng(sharp(path), width, height); + return await this.convertSharpToPng(sharp(path), width, height); } @bindThis diff --git a/packages/backend/src/core/LatestNoteService.ts b/packages/backend/src/core/LatestNoteService.ts index 63f973c6c6..e714b50320 100644 --- a/packages/backend/src/core/LatestNoteService.ts +++ b/packages/backend/src/core/LatestNoteService.ts @@ -1,18 +1,14 @@ import { Inject, Injectable } from '@nestjs/common'; import { Not } from 'typeorm'; -import { MiNote } from '@/models/Note.js'; -import { isPureRenote } from '@/misc/is-renote.js'; +import { isPureRenote, MinimalNote } from '@/misc/is-renote.js'; import { SkLatestNote } from '@/models/LatestNote.js'; import { DI } from '@/di-symbols.js'; -import type { LatestNotesRepository, NotesRepository } from '@/models/_.js'; -import { LoggerService } from '@/core/LoggerService.js'; -import Logger from '@/logger.js'; -import { QueryService } from './QueryService.js'; +import type { LatestNotesRepository, MiNote, NotesRepository } from '@/models/_.js'; +import { QueryService } from '@/core/QueryService.js'; +import { QueueService } from '@/core/QueueService.js'; @Injectable() export class LatestNoteService { - private readonly logger: Logger; - constructor( @Inject(DI.notesRepository) private readonly notesRepository: NotesRepository, @@ -21,19 +17,23 @@ export class LatestNoteService { private readonly latestNotesRepository: LatestNotesRepository, private readonly queryService: QueryService, - loggerService: LoggerService, - ) { - this.logger = loggerService.getLogger('LatestNoteService'); + private readonly queueService: QueueService, + ) {} + + async handleUpdatedNoteDeferred(note: MiNote): Promise { + await this.queueService.createUpdateLatestNoteJob(note); } - handleUpdatedNoteBG(before: MiNote, after: MiNote): void { - this - .handleUpdatedNote(before, after) - .catch(err => this.logger.error('Unhandled exception while updating latest_note (after update):', err)); + async handleCreatedNoteDeferred(note: MiNote): Promise { + await this.queueService.createUpdateLatestNoteJob(note); } - async handleUpdatedNote(before: MiNote, after: MiNote): Promise { - // If the key didn't change, then there's nothing to update + async handleDeletedNoteDeferred(note: MiNote): Promise { + await this.queueService.createUpdateLatestNoteJob(note); + } + + async handleUpdatedNote(before: MinimalNote, after: MinimalNote): Promise { + // If the key didn't change, then there's nothing to update. if (SkLatestNote.areEquivalent(before, after)) return; // Simulate update as delete + create @@ -41,13 +41,7 @@ export class LatestNoteService { await this.handleCreatedNote(after); } - handleCreatedNoteBG(note: MiNote): void { - this - .handleCreatedNote(note) - .catch(err => this.logger.error('Unhandled exception while updating latest_note (after create):', err)); - } - - async handleCreatedNote(note: MiNote): Promise { + async handleCreatedNote(note: MinimalNote): Promise { // Ignore DMs. // Followers-only posts are *included*, as this table is used to back the "following" feed. if (note.visibility === 'specified') return; @@ -71,13 +65,7 @@ export class LatestNoteService { await this.latestNotesRepository.upsert(latestNote, ['userId', 'isPublic', 'isReply', 'isQuote']); } - handleDeletedNoteBG(note: MiNote): void { - this - .handleDeletedNote(note) - .catch(err => this.logger.error('Unhandled exception while updating latest_note (after delete):', err)); - } - - async handleDeletedNote(note: MiNote): Promise { + async handleDeletedNote(note: MinimalNote): Promise { // If it's a DM, then it can't possibly be the latest note so we can safely skip this. if (note.visibility === 'specified') return; diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index fd55c33bfb..da1d23100b 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -59,6 +59,8 @@ import { CollapsedQueue } from '@/misc/collapsed-queue.js'; import { CacheService } from '@/core/CacheService.js'; import { TimeService } from '@/global/TimeService.js'; import { NoteVisibilityService } from '@/core/NoteVisibilityService.js'; +import { CollapsedQueueService } from '@/core/CollapsedQueueService.js'; +import { promiseMap } from '@/misc/promise-map.js'; type NotificationType = 'reply' | 'renote' | 'quote' | 'mention'; @@ -154,7 +156,6 @@ export type PureRenoteOption = Option & { renote: MiNote } & ({ text?: null } | @Injectable() export class NoteCreateService implements OnApplicationShutdown { #shutdownController = new AbortController(); - private updateNotesCountQueue: CollapsedQueue; constructor( @Inject(DI.config) @@ -226,8 +227,8 @@ export class NoteCreateService implements OnApplicationShutdown { private latestNoteService: LatestNoteService, private readonly timeService: TimeService, private readonly noteVisibilityService: NoteVisibilityService, + private readonly collapsedQueueService: CollapsedQueueService, ) { - this.updateNotesCountQueue = new CollapsedQueue(this.timeService, process.env.NODE_ENV !== 'test' ? 60 * 1000 * 5 : 0, this.collapseNotesCount, this.performUpdateNotesCount); } @bindThis @@ -458,10 +459,7 @@ export class NoteCreateService implements OnApplicationShutdown { const note = await this.insertNote(user, data, tags, emojis, mentionedUsers); - setImmediate('post created', { signal: this.#shutdownController.signal }).then( - () => this.postNoteCreated(note, user, data, silent, tags!, mentionedUsers!), - () => { /* aborted, ignore this */ }, - ); + await this.queueService.createPostNoteJob(note.id, silent, 'create'); return note; } @@ -474,7 +472,7 @@ export class NoteCreateService implements OnApplicationShutdown { isBot: MiUser['isBot']; noindex: MiUser['noindex']; }, data: Option): Promise { - return this.create(user, data, true); + return await this.create(user, data, true); } @bindThis @@ -577,13 +575,7 @@ export class NoteCreateService implements OnApplicationShutdown { } @bindThis - private async postNoteCreated(note: MiNote, user: MiUser & { - id: MiUser['id']; - username: MiUser['username']; - host: MiUser['host']; - isBot: MiUser['isBot']; - noindex: MiUser['noindex']; - }, data: Option, silent: boolean, tags: string[], mentionedUsers: MinimumUser[]) { + public async postNoteCreated(note: MiNote, user: MiUser, data: MiNote & { poll: MiPoll | null }, silent: boolean, mentionedUsers: MinimumUser[]) { this.notesChart.update(note, true); if (note.visibility !== 'specified' && (this.meta.enableChartsForRemoteUser || (user.host == null))) { this.perUserNotesChart.update(user, note, true); @@ -594,7 +586,7 @@ export class NoteCreateService implements OnApplicationShutdown { if (isRemoteUser(user)) { this.federatedInstanceService.fetchOrRegister(user.host).then(async i => { if (!this.isRenote(note) || this.isQuote(note)) { - this.updateNotesCountQueue.enqueue(i.id, 1); + await this.collapsedQueueService.updateInstanceQueue.enqueue(i.id, { notesCountDelta: 1 }); } if (this.meta.enableChartsForFederatedInstances) { this.instanceChart.updateNote(i.host, note, true); @@ -606,26 +598,26 @@ export class NoteCreateService implements OnApplicationShutdown { // ハッシュタグ更新 if (data.visibility === 'public' || data.visibility === 'home') { if (!user.isBot || this.meta.enableBotTrending) { - this.hashtagService.updateHashtags(user, tags); + await this.queueService.createUpdateNoteTagsJob(note.id); } } if (!this.isRenote(note) || this.isQuote(note)) { // Increment notes count (user) - this.incNotesCountOfUser(user); - } else { - this.usersRepository.update({ id: user.id }, { updatedAt: this.timeService.date }); + await this.collapsedQueueService.updateUserQueue.enqueue(user.id, { notesCountDelta: 1 }); } - this.pushToTl(note, user); + await this.collapsedQueueService.updateUserQueue.enqueue(user.id, { updatedAt: this.timeService.date }); - this.antennaService.addNoteToAntennas({ + await this.pushToTl(note, user); + + await this.antennaService.addNoteToAntennas({ ...note, channel: data.channel ?? null, }, user); if (data.reply) { - this.saveReply(data.reply, note); + await this.collapsedQueueService.updateNoteQueue.enqueue(data.reply.id, { repliesCountDelta: 1 }); } if (data.reply == null) { @@ -653,13 +645,14 @@ export class NoteCreateService implements OnApplicationShutdown { }); } - if (this.isRenote(data) && !this.isQuote(data) && data.renote.userId !== user.id && !user.isBot) { - this.incRenoteCount(data.renote, user); + if (this.isPureRenote(data)) { + await this.collapsedQueueService.updateNoteQueue.enqueue(data.renote.id, { renoteCountDelta: 1 }); + await this.incRenoteCount(data.renote, user); } if (data.poll && data.poll.expiresAt) { const delay = data.poll.expiresAt.getTime() - this.timeService.now; - this.queueService.endedPollNotificationQueue.add(note.id, { + await this.queueService.endedPollNotificationQueue.add(note.id, { noteId: note.id, }, { jobId: `pollEnd_${note.id}`, @@ -683,9 +676,9 @@ export class NoteCreateService implements OnApplicationShutdown { this.globalEventService.publishNotesStream(noteObj); - this.roleService.addNoteToRoleTimeline(noteObj); + await this.roleService.addNoteToRoleTimeline(noteObj); - this.webhookService.enqueueUserWebhook(user.id, 'note', { note: noteObj }); + await this.webhookService.enqueueUserWebhook(user.id, 'note', { note: noteObj }); const nm = new NotificationManager(this.mutingsRepository, this.notificationService, user, note); @@ -714,7 +707,7 @@ export class NoteCreateService implements OnApplicationShutdown { if (!isThreadMuted && !muted) { nm.push(data.reply.userId, 'reply'); this.globalEventService.publishMainStream(data.reply.userId, 'reply', noteObj); - this.webhookService.enqueueUserWebhook(data.reply.userId, 'reply', { note: noteObj }); + await this.webhookService.enqueueUserWebhook(data.reply.userId, 'reply', { note: noteObj }); } } } @@ -745,15 +738,15 @@ export class NoteCreateService implements OnApplicationShutdown { // Publish event if ((user.id !== data.renote.userId) && data.renote.userHost === null) { this.globalEventService.publishMainStream(data.renote.userId, 'renote', noteObj); - this.webhookService.enqueueUserWebhook(data.renote.userId, 'renote', { note: noteObj }); + await this.webhookService.enqueueUserWebhook(data.renote.userId, 'renote', { note: noteObj }); } } - nm.notify(); + await nm.notify(); //#region AP deliver if (!data.localOnly && isLocalUser(user)) { - trackTask(async () => { + await trackTask(async () => { const noteActivity = await this.apRendererService.renderNoteOrRenoteActivity(note, user, { renote: data.renote }); const dm = this.apDeliverManagerService.createDeliverManager(user, noteActivity); @@ -790,12 +783,12 @@ export class NoteCreateService implements OnApplicationShutdown { } if (data.channel) { - this.channelsRepository.increment({ id: data.channel.id }, 'notesCount', 1); - this.channelsRepository.update(data.channel.id, { + await this.channelsRepository.increment({ id: data.channel.id }, 'notesCount', 1); + await this.channelsRepository.update(data.channel.id, { lastNotedAt: this.timeService.date, }); - this.notesRepository.countBy({ + await this.notesRepository.countBy({ userId: user.id, channelId: data.channel.id, }).then(count => { @@ -808,10 +801,10 @@ export class NoteCreateService implements OnApplicationShutdown { } // Update the Latest Note index / following feed - this.latestNoteService.handleCreatedNoteBG(note); + await this.latestNoteService.handleCreatedNoteDeferred(note); // Register to search database - if (!user.noindex) this.index(note); + if (!user.noindex) await this.index(note); } /** @@ -829,14 +822,11 @@ export class NoteCreateService implements OnApplicationShutdown { */ readonly isQuote = isQuote; + // Note: does not increment the count! used only for featured rankings. @bindThis private async incRenoteCount(renote: MiNote, user: MiUser) { - await this.notesRepository.createQueryBuilder().update() - .set({ - renoteCount: () => '"renoteCount" + 1', - }) - .where('id = :id', { id: renote.id }) - .execute(); + // Moved down from the containing block + if (renote.userId === user.id || user.isBot) return; // 30%の確率、3日以内に投稿されたノートの場合ハイライト用ランキング更新 if (user.isExplorable && Math.random() < 0.3 && (this.timeService.now - this.idService.parse(renote.id).date.getTime()) < 1000 * 60 * 60 * 24 * 3) { @@ -844,12 +834,12 @@ export class NoteCreateService implements OnApplicationShutdown { if (policies.canTrend) { if (renote.channelId != null) { if (renote.replyId == null) { - this.featuredService.updateInChannelNotesRanking(renote.channelId, renote, 5); + await this.featuredService.updateInChannelNotesRanking(renote.channelId, renote, 5); } } else { if (renote.visibility === 'public' && renote.userHost == null && renote.replyId == null) { - this.featuredService.updateGlobalNotesRanking(renote, 5); - this.featuredService.updatePerUserNotesRanking(renote.userId, renote, 5); + await this.featuredService.updateGlobalNotesRanking(renote, 5); + await this.featuredService.updatePerUserNotesRanking(renote.userId, renote, 5); } } } @@ -883,7 +873,7 @@ export class NoteCreateService implements OnApplicationShutdown { }); this.globalEventService.publishMainStream(u.id, 'mention', detailPackedNote); - this.webhookService.enqueueUserWebhook(u.id, 'mention', { note: detailPackedNote }); + await this.webhookService.enqueueUserWebhook(u.id, 'mention', { note: detailPackedNote }); // Create notification nm.push(u.id, 'mention'); @@ -891,43 +881,23 @@ export class NoteCreateService implements OnApplicationShutdown { } @bindThis - private saveReply(reply: MiNote, note: MiNote) { - this.notesRepository.increment({ id: reply.id }, 'repliesCount', 1); - } - - @bindThis - private index(note: MiNote) { + private async index(note: MiNote) { if (note.text == null && note.cw == null) return; - this.searchService.indexNote(note); + await this.searchService.indexNote(note); } @bindThis - private incNotesCountOfUser(user: { id: MiUser['id']; }) { - this.usersRepository.createQueryBuilder().update() - .set({ - updatedAt: this.timeService.date, - notesCount: () => '"notesCount" + 1', - }) - .where('id = :id', { id: user.id }) - .execute(); - } + public async extractMentionedUsers(user: { host: MiUser['host']; }, tokens: mfm.MfmNode[]): Promise { + if (tokens == null || tokens.length === 0) return []; - @bindThis - private async extractMentionedUsers(user: { host: MiUser['host']; }, tokens: mfm.MfmNode[]): Promise { - if (tokens == null) return []; + const allMentions = extractMentions(tokens); + const mentions = new Map(allMentions.map(m => [`${m.username.toLowerCase()}@${m.host?.toLowerCase()}`, m])); - const mentions = extractMentions(tokens); - let mentionedUsers = (await Promise.all(mentions.map(m => - this.remoteUserResolveService.resolveUser(m.username, m.host ?? user.host).catch(() => null), - ))).filter(x => x != null); + const allMentionedUsers = await promiseMap(mentions.values(), async m => await this.remoteUserResolveService.resolveUser(m.username, m.host ?? user.host).catch(() => null), { limit: 2 }); + const mentionedUsers = new Map(allMentionedUsers.filter(u => u != null).map(u => [u.id, u])); - // Drop duplicate users - mentionedUsers = mentionedUsers.filter((u, i, self) => - i === self.findIndex(u2 => u.id === u2.id), - ); - - return mentionedUsers; + return Array.from(mentionedUsers.values()); } @bindThis @@ -1040,7 +1010,7 @@ export class NoteCreateService implements OnApplicationShutdown { // checkHibernation moved to HibernateUsersProcessorService } - r.exec(); + await r.exec(); } // checkHibernation moved to HibernateUsersProcessorService @@ -1062,20 +1032,11 @@ export class NoteCreateService implements OnApplicationShutdown { return false; } - @bindThis - private collapseNotesCount(oldValue: number, newValue: number) { - return oldValue + newValue; - } - - @bindThis - private async performUpdateNotesCount(id: MiNote['id'], incrBy: number) { - await this.instancesRepository.increment({ id: id }, 'notesCount', incrBy); - } + // collapseNotesCount moved to CollapsedQueueService @bindThis public async dispose(): Promise { this.#shutdownController.abort(); - await this.updateNotesCountQueue.performAllNow(); } @bindThis @@ -1100,8 +1061,8 @@ export class NoteCreateService implements OnApplicationShutdown { // Instance cannot quote if (user.host) { - const instance = await this.federatedInstanceService.fetch(user.host); - if (instance?.rejectQuotes) { + const instance = await this.federatedInstanceService.fetchOrRegister(user.host); + if (instance.rejectQuotes) { (data as Option).renote = null; (data.processErrors ??= []).push('quoteUnavailable'); } diff --git a/packages/backend/src/core/NoteDeleteService.ts b/packages/backend/src/core/NoteDeleteService.ts index c20b20c7ca..6316e886c8 100644 --- a/packages/backend/src/core/NoteDeleteService.ts +++ b/packages/backend/src/core/NoteDeleteService.ts @@ -22,18 +22,15 @@ import { ApDeliverManagerService } from '@/core/activitypub/ApDeliverManagerServ import { bindThis } from '@/decorators.js'; import { SearchService } from '@/core/SearchService.js'; import { ModerationLogService } from '@/core/ModerationLogService.js'; -import { isQuote, isRenote } from '@/misc/is-renote.js'; +import { isPureRenote } from '@/misc/is-renote.js'; import { LatestNoteService } from '@/core/LatestNoteService.js'; import { ApLogService } from '@/core/ApLogService.js'; -import type Logger from '@/logger.js'; import { TimeService } from '@/global/TimeService.js'; -import { trackPromise } from '@/misc/promise-tracker.js'; -import { LoggerService } from '@/core/LoggerService.js'; +import { trackTask } from '@/misc/promise-tracker.js'; +import { CollapsedQueueService } from '@/core/CollapsedQueueService.js'; @Injectable() export class NoteDeleteService { - private readonly logger: Logger; - constructor( @Inject(DI.config) private config: Config, @@ -63,53 +60,56 @@ export class NoteDeleteService { private latestNoteService: LatestNoteService, private readonly apLogService: ApLogService, private readonly timeService: TimeService, - - loggerService: LoggerService, - ) { - this.logger = loggerService.getLogger('note-delete-service'); - } + private readonly collapsedQueueService: CollapsedQueueService, + ) {} /** * 投稿を削除します。 - * @param user 投稿者 - * @param note 投稿 */ - async delete(user: { id: MiUser['id']; uri: MiUser['uri']; host: MiUser['host']; isBot: MiUser['isBot']; }, note: MiNote, quiet = false, deleter?: MiUser) { + async delete(user: MiUser, note: MiNote, deleter?: MiUser, immediate = false) { + // This kicks off lots of things that can run in parallel, but we should still wait for completion to ensure consistent state and to avoid task flood when calling in a loop. + const promises: Promise[] = []; + const deletedAt = this.timeService.date; const cascadingNotes = await this.findCascadingNotes(note); if (note.replyId) { - await this.notesRepository.decrement({ id: note.replyId }, 'repliesCount', 1); + await this.collapsedQueueService.updateNoteQueue.enqueue(note.replyId, { repliesCountDelta: -1 }); + } else if (isPureRenote(note)) { + await this.collapsedQueueService.updateNoteQueue.enqueue(note.renoteId, { renoteCountDelta: -1 }); } - if (note.renoteId && note.text == null && !note.hasPoll && (note.fileIds == null || note.fileIds.length === 0)) { - await this.notesRepository.findOneBy({ id: note.renoteId }).then(async (renote) => { - if (!renote) return; - if (renote.userId !== user.id) await this.notesRepository.decrement({ id: renote.id }, 'renoteCount', 1); - }); + for (const cascade of cascadingNotes) { + if (cascade.replyId) { + await this.collapsedQueueService.updateNoteQueue.enqueue(cascade.replyId, { repliesCountDelta: -1 }); + } else if (isPureRenote(cascade)) { + await this.collapsedQueueService.updateNoteQueue.enqueue(cascade.renoteId, { renoteCountDelta: -1 }); + } } - if (!quiet) { - this.globalEventService.publishNoteStream(note.id, 'deleted', { + // Braces preserved to avoid merge conflicts + { + promises.push(this.globalEventService.publishNoteStream(note.id, 'deleted', { deletedAt: deletedAt, - }); + })); + + for (const cascade of cascadingNotes) { + promises.push(this.globalEventService.publishNoteStream(cascade.id, 'deleted', { + deletedAt: deletedAt, + })); + } //#region ローカルの投稿なら削除アクティビティを配送 if (isLocalUser(user) && !note.localOnly) { - let renote: MiNote | null = null; - - // if deleted note is renote - if (isRenote(note) && !isQuote(note)) { - renote = await this.notesRepository.findOneBy({ - id: note.renoteId, - }); - } + const renote = isPureRenote(note) + ? await this.notesRepository.findOneBy({ id: note.renoteId }) + : null; const content = this.apRendererService.addContext(renote ? this.apRendererService.renderUndo(this.apRendererService.renderAnnounce(renote.uri ?? `${this.config.url}/notes/${renote.id}`, note), user) : this.apRendererService.renderDelete(this.apRendererService.renderTombstone(`${this.config.url}/notes/${note.id}`), user)); - trackPromise(this.deliverToConcerned(user, note, content)); + promises.push(this.deliverToConcerned(user, note, content)); } // also deliver delete activity to cascaded notes @@ -118,7 +118,7 @@ export class NoteDeleteService { if (!cascadingNote.user) continue; if (!isLocalUser(cascadingNote.user)) continue; const content = this.apRendererService.addContext(this.apRendererService.renderDelete(this.apRendererService.renderTombstone(`${this.config.url}/notes/${cascadingNote.id}`), cascadingNote.user)); - trackPromise(this.deliverToConcerned(cascadingNote.user, cascadingNote, content)); + promises.push(this.deliverToConcerned(cascadingNote.user, cascadingNote, content)); } //#endregion @@ -127,90 +127,142 @@ export class NoteDeleteService { this.perUserNotesChart.update(user, note, false); } - if (!isRenote(note) || isQuote(note)) { + for (const cascade of cascadingNotes) { + this.notesChart.update(cascade, false); + if (this.meta.enableChartsForRemoteUser || (cascade.user.host == null)) { + this.perUserNotesChart.update(cascade.user, cascade, false); + } + } + + if (!isPureRenote(note)) { // Decrement notes count (user) - this.decNotesCountOfUser(user); - } else { - this.usersRepository.update({ id: user.id }, { updatedAt: this.timeService.date }); + await this.collapsedQueueService.updateUserQueue.enqueue(user.id, { notesCountDelta: -1 }); + } + + await this.collapsedQueueService.updateUserQueue.enqueue(user.id, { updatedAt: this.timeService.date }); + + for (const cascade of cascadingNotes) { + if (!isPureRenote(cascade)) { + await this.collapsedQueueService.updateUserQueue.enqueue(cascade.user.id, { notesCountDelta: -1 }); + } + // Don't mark cascaded user as updated (active) } if (this.meta.enableStatsForFederatedInstances) { if (isRemoteUser(user)) { - this.federatedInstanceService.fetchOrRegister(user.host).then(async i => { - if (note.renoteId && note.text || !note.renoteId) { - this.instancesRepository.decrement({ id: i.id }, 'notesCount', 1); + if (!isPureRenote(note)) { + const i = await this.federatedInstanceService.fetchOrRegister(user.host); + await this.collapsedQueueService.updateInstanceQueue.enqueue(i.id, { notesCountDelta: -1 }); + } + if (this.meta.enableChartsForFederatedInstances) { + this.instanceChart.updateNote(user.host, note, false); + } + } + + for (const cascade of cascadingNotes) { + if (isRemoteUser(cascade.user)) { + if (!isPureRenote(cascade)) { + const i = await this.federatedInstanceService.fetchOrRegister(cascade.user.host); + await this.collapsedQueueService.updateInstanceQueue.enqueue(i.id, { notesCountDelta: -1 }); } if (this.meta.enableChartsForFederatedInstances) { - this.instanceChart.updateNote(i.host, note, false); + this.instanceChart.updateNote(cascade.user.host, cascade, false); } - }); + } } } } for (const cascadingNote of cascadingNotes) { - this.searchService.unindexNote(cascadingNote); + promises.push(this.searchService.unindexNote(cascadingNote)); } - this.searchService.unindexNote(note); + promises.push(this.searchService.unindexNote(note)); + // Don't put this in the promise array, since it needs to happen before the next section await this.notesRepository.delete({ id: note.id, userId: user.id, }); - this.latestNoteService.handleDeletedNoteBG(note); + // Update the Latest Note index / following feed *after* note is deleted + promises.push(immediate + ? this.latestNoteService.handleDeletedNote(note) + : this.latestNoteService.handleDeletedNoteDeferred(note)); + for (const cascadingNote of cascadingNotes) { + promises.push(immediate + ? this.latestNoteService.handleDeletedNote(cascadingNote) + : this.latestNoteService.handleDeletedNoteDeferred(cascadingNote)); + } - if (deleter && (note.userId !== deleter.id)) { - const user = await this.usersRepository.findOneByOrFail({ id: note.userId }); - this.moderationLogService.log(deleter, 'deleteNote', { + if (deleter && (user.id !== deleter.id)) { + promises.push(this.moderationLogService.log(deleter, 'deleteNote', { noteId: note.id, noteUserId: note.userId, noteUserUsername: user.username, noteUserHost: user.host, - }); + })); } const deletedUris = [note, ...cascadingNotes] .map(n => n.uri) .filter((u): u is string => u != null); if (deletedUris.length > 0) { - this.apLogService.deleteObjectLogs(deletedUris) - .catch(err => this.logger.error(err, `Failed to delete AP logs for note '${note.uri}'`)); + promises.push(immediate + ? this.apLogService.deleteObjectLogs(deletedUris) + : this.apLogService.deleteObjectLogsDeferred(deletedUris)); } + + await trackTask(async () => { + await Promise.allSettled(promises); + + // This is deferred to make sure we don't race the enqueue() calls + if (immediate) { + await Promise.allSettled([ + this.collapsedQueueService.updateNoteQueue.performAllNow(), + this.collapsedQueueService.updateUserQueue.performAllNow(), + this.collapsedQueueService.updateInstanceQueue.performAllNow(), + ]); + } + }); } @bindThis - private decNotesCountOfUser(user: { id: MiUser['id']; }) { - this.usersRepository.createQueryBuilder().update() - .set({ - updatedAt: this.timeService.date, - notesCount: () => '"notesCount" - 1', - }) - .where('id = :id', { id: user.id }) - .execute(); - } + private async findCascadingNotes(note: MiNote): Promise<(MiNote & { user: MiUser })[]> { + const cascadingNotes: MiNote[] = []; - @bindThis - private async findCascadingNotes(note: MiNote): Promise { - const recursive = async (noteId: string): Promise => { - const query = this.notesRepository.createQueryBuilder('note') - .where('note.replyId = :noteId', { noteId }) - .orWhere(new Brackets(q => { - q.where('note.renoteId = :noteId', { noteId }) - .andWhere('note.text IS NOT NULL'); - })) - .leftJoinAndSelect('note.user', 'user'); - const replies = await query.getMany(); + /** + * Finds all replies, quotes, and renotes of the given list of notes. + * These are the notes that will be CASCADE deleted when the origin note is deleted. + * + * This works by operating in "layers" that radiate out from the origin note like a web. + * The process is roughly like this: + * 1. Find all immediate replies and renotes of the origin. + * 2. Find all immediate replies and renotes of the results from step one. + * 3. Repeat until step 2 returns no new results. + * 4. Collect all the step 2 results; those are the set of all cascading notes. + */ + const cascade = async (layer: MiNote[]): Promise => { + const layerIds = layer.map(layer => layer.id); + const refs = await this.notesRepository.find({ + where: [ + { replyId: In(layerIds) }, + { renoteId: In(layerIds) }, + ], + relations: { user: true }, + }); - return [ - replies, - ...await Promise.all(replies.map(reply => recursive(reply.id))), - ].flat(); + // Stop when we reach the end of all threads + if (refs.length === 0) return; + + cascadingNotes.push(...refs); + await cascade(refs); }; - const cascadingNotes: MiNote[] = await recursive(note.id); + // Start with the origin, which should *not* be in the result set! + await cascade([note]); - return cascadingNotes; + // Type cast is safe - we load the relation above. + return cascadingNotes as (MiNote & { user: MiUser })[]; } @bindThis diff --git a/packages/backend/src/core/NoteEditService.ts b/packages/backend/src/core/NoteEditService.ts index 20afc4e63c..daf78c359c 100644 --- a/packages/backend/src/core/NoteEditService.ts +++ b/packages/backend/src/core/NoteEditService.ts @@ -3,18 +3,16 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { setImmediate } from 'node:timers/promises'; import * as mfm from 'mfm-js'; -import { DataSource, In, IsNull, LessThan } from 'typeorm'; +import { DataSource, In } from 'typeorm'; import * as Redis from 'ioredis'; import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import { UnrecoverableError } from 'bullmq'; -import { extractMentions } from '@/misc/extract-mentions.js'; import { extractCustomEmojisFromMfm } from '@/misc/extract-custom-emojis-from-mfm.js'; import { extractHashtags } from '@/misc/extract-hashtags.js'; import type { IMentionedRemoteUsers } from '@/models/Note.js'; import { MiNote } from '@/models/Note.js'; -import type { NoteEditRepository, ChannelFollowingsRepository, ChannelsRepository, FollowingsRepository, InstancesRepository, MiFollowing, MiMeta, MutingsRepository, NotesRepository, NoteThreadMutingsRepository, UserListMembershipsRepository, UserProfilesRepository, UsersRepository, PollsRepository } from '@/models/_.js'; +import type { NoteEditsRepository, ChannelFollowingsRepository, ChannelsRepository, FollowingsRepository, InstancesRepository, MiMeta, MutingsRepository, NotesRepository, NoteThreadMutingsRepository, UserListMembershipsRepository, UserProfilesRepository, UsersRepository, PollsRepository } from '@/models/_.js'; import type { MiDriveFile } from '@/models/DriveFile.js'; import type { MiApp } from '@/models/App.js'; import { concat } from '@/misc/prelude/array.js'; @@ -50,11 +48,11 @@ import { trackTask } from '@/misc/promise-tracker.js'; import { isUserRelated } from '@/misc/is-user-related.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; import { LatestNoteService } from '@/core/LatestNoteService.js'; -import { CollapsedQueue } from '@/misc/collapsed-queue.js'; import { NoteCreateService } from '@/core/NoteCreateService.js'; import { TimeService } from '@/global/TimeService.js'; import { NoteVisibilityService } from '@/core/NoteVisibilityService.js'; import { isPureRenote } from '@/misc/is-renote.js'; +import { CollapsedQueueService } from '@/core/CollapsedQueueService.js'; type NotificationType = 'reply' | 'renote' | 'quote' | 'mention' | 'edited'; @@ -150,7 +148,6 @@ export type Option = { @Injectable() export class NoteEditService implements OnApplicationShutdown { #shutdownController = new AbortController(); - private updateNotesCountQueue: CollapsedQueue; constructor( @Inject(DI.config) @@ -195,8 +192,8 @@ export class NoteEditService implements OnApplicationShutdown { @Inject(DI.channelFollowingsRepository) private channelFollowingsRepository: ChannelFollowingsRepository, - @Inject(DI.noteEditRepository) - private noteEditRepository: NoteEditRepository, + @Inject(DI.noteEditsRepository) + private noteEditsRepository: NoteEditsRepository, @Inject(DI.pollsRepository) private pollsRepository: PollsRepository, @@ -224,8 +221,8 @@ export class NoteEditService implements OnApplicationShutdown { private noteCreateService: NoteCreateService, private readonly timeService: TimeService, private readonly noteVisibilityService: NoteVisibilityService, + private readonly collapsedQueueService: CollapsedQueueService, ) { - this.updateNotesCountQueue = new CollapsedQueue(this.timeService, process.env.NODE_ENV !== 'test' ? 60 * 1000 * 5 : 0, this.collapseNotesCount, this.performUpdateNotesCount); } @bindThis @@ -234,29 +231,29 @@ export class NoteEditService implements OnApplicationShutdown { throw new UnrecoverableError('edit failed: missing editid'); } - const oldnote = await this.notesRepository.findOneBy({ + const oldNote = await this.notesRepository.findOneBy({ id: editid, }); - if (oldnote == null) { + if (oldNote == null) { throw new UnrecoverableError(`edit failed for ${editid}: missing oldnote`); } - if (oldnote.userId !== user.id) { + if (oldNote.userId !== user.id) { throw new UnrecoverableError(`edit failed for ${editid}: user is not the note author`); } // we never want to change the replyId, so fetch the original "parent" - if (oldnote.replyId) { - data.reply = await this.notesRepository.findOneBy({ id: oldnote.replyId }); + if (oldNote.replyId) { + data.reply = await this.notesRepository.findOneBy({ id: oldNote.replyId }); } else { data.reply = undefined; } // changing visibility on an edit is ill-defined, let's try to // keep the same visibility as the original note - data.visibility = oldnote.visibility; - data.localOnly = oldnote.localOnly; + data.visibility = oldNote.visibility; + data.localOnly = oldNote.localOnly; // チャンネル外にリプライしたら対象のスコープに合わせる // (クライアントサイドでやっても良い処理だと思うけどとりあえずサーバーサイドで) @@ -354,12 +351,12 @@ export class NoteEditService implements OnApplicationShutdown { } // Check for recursion - if (data.renote.id === oldnote.id) { - throw new IdentifiableError('33510210-8452-094c-6227-4a6c05d99f02', `edit failed for ${oldnote.id}: note cannot quote itself`); + if (data.renote.id === oldNote.id) { + throw new IdentifiableError('33510210-8452-094c-6227-4a6c05d99f02', `edit failed for ${oldNote.id}: note cannot quote itself`); } for (let nextRenoteId = data.renote.renoteId; nextRenoteId != null;) { - if (nextRenoteId === oldnote.id) { - throw new IdentifiableError('ea93b7c2-3d6c-4e10-946b-00d50b1a75cb', `edit failed for ${oldnote.id}: note cannot quote a quote of itself`); + if (nextRenoteId === oldNote.id) { + throw new IdentifiableError('ea93b7c2-3d6c-4e10-946b-00d50b1a75cb', `edit failed for ${oldNote.id}: note cannot quote a quote of itself`); } // TODO create something like threadId but for quotes, that way we don't need full recursion @@ -432,7 +429,7 @@ export class NoteEditService implements OnApplicationShutdown { emojis = data.apEmojis ?? extractCustomEmojisFromMfm(combinedTokens); - mentionedUsers = data.apMentions ?? await this.extractMentionedUsers(user, combinedTokens); + mentionedUsers = data.apMentions ?? await this.noteCreateService.extractMentionedUsers(user, combinedTokens); } // if the host is media-silenced, custom emojis are not allowed @@ -463,46 +460,52 @@ export class NoteEditService implements OnApplicationShutdown { } const update: Partial = {}; - if (data.text !== undefined && data.text !== oldnote.text) { + if (data.text !== undefined && data.text !== oldNote.text) { update.text = data.text; } - if (data.cw !== undefined && data.cw !== oldnote.cw) { + if (data.cw !== undefined && data.cw !== oldNote.cw) { update.cw = data.cw; } - if (data.poll !== undefined && oldnote.hasPoll !== !!data.poll) { + if (data.poll !== undefined && oldNote.hasPoll !== !!data.poll) { update.hasPoll = !!data.poll; } - if (data.mandatoryCW !== undefined && oldnote.mandatoryCW !== data.mandatoryCW) { + if (data.mandatoryCW !== undefined && oldNote.mandatoryCW !== data.mandatoryCW) { update.mandatoryCW = data.mandatoryCW; } // TODO deep-compare files - const filesChanged = oldnote.fileIds.length || data.files?.length; + const filesChanged = oldNote.fileIds.length || data.files?.length; - const poll = await this.pollsRepository.findOneBy({ noteId: oldnote.id }); - - const oldPoll = poll ? { choices: poll.choices, multiple: poll.multiple, expiresAt: poll.expiresAt } : null; - const pollChanged = data.poll != null && JSON.stringify(data.poll) !== JSON.stringify(oldPoll); + const oldPoll = await this.pollsRepository.findOneBy({ noteId: oldNote.id }); + const oldPollData = oldPoll ? { choices: oldPoll.choices, multiple: oldPoll.multiple, expiresAt: oldPoll.expiresAt?.toISOString() ?? null } : null; + const newPollData = data.poll ? { choices: data.poll.choices, multiple: data.poll.multiple, expiresAt: data.poll.expiresAt ?? null } : null; + const pollChanged = data.poll !== undefined && JSON.stringify(oldPollData) !== JSON.stringify(newPollData); if (Object.keys(update).length > 0 || filesChanged || pollChanged) { - const exists = await this.noteEditRepository.findOneBy({ noteId: oldnote.id }); + const exists = await this.noteEditsRepository.findOneBy({ noteId: oldNote.id }); - await this.noteEditRepository.insert({ + await this.noteEditsRepository.insert({ id: this.idService.gen(), - noteId: oldnote.id, - oldText: oldnote.text || undefined, + userId: oldNote.userId, + noteId: oldNote.id, + renoteId: oldNote.renoteId, + replyId: oldNote.replyId, + visibility: oldNote.visibility, + text: oldNote.text || undefined, newText: update.text || undefined, - cw: update.cw || undefined, - fileIds: undefined, - oldDate: exists ? oldnote.updatedAt as Date : this.idService.parse(oldnote.id).date, + cw: oldNote.cw || undefined, + newCw: update.cw || undefined, + fileIds: oldNote.fileIds, + oldDate: exists ? oldNote.updatedAt as Date : this.idService.parse(oldNote.id).date, updatedAt: this.timeService.date, + hasPoll: oldPoll != null, }); const note = new MiNote({ - id: oldnote.id, + id: oldNote.id, updatedAt: data.updatedAt ? data.updatedAt : this.timeService.date, fileIds: data.files ? data.files.map(file => file.id) : [], - replyId: oldnote.replyId, + replyId: oldNote.replyId, renoteId: data.renote ? data.renote.id : null, channelId: data.channel ? data.channel.id : null, threadId: data.reply @@ -516,7 +519,7 @@ export class NoteEditService implements OnApplicationShutdown { cw: data.cw ?? null, tags: tags.map(tag => normalizeForSearch(tag)), emojis, - reactions: oldnote.reactions, + reactions: oldNote.reactions, userId: user.id, localOnly: data.localOnly!, reactionAcceptance: data.reactionAcceptance, @@ -535,7 +538,7 @@ export class NoteEditService implements OnApplicationShutdown { renoteUserId: data.renote ? data.renote.userId : null, renoteUserHost: data.renote ? data.renote.userHost : null, userHost: user.host, - reactionAndUserPairCache: oldnote.reactionAndUserPairCache, + reactionAndUserPairCache: oldNote.reactionAndUserPairCache, mandatoryCW: data.mandatoryCW, }); @@ -561,58 +564,55 @@ export class NoteEditService implements OnApplicationShutdown { if (pollChanged) { // Start transaction await this.db.transaction(async transactionalEntityManager => { - await transactionalEntityManager.update(MiNote, oldnote.id, note); + await transactionalEntityManager.update(MiNote, oldNote.id, note); - const poll = new MiPoll({ - noteId: note.id, - choices: data.poll!.choices, - expiresAt: data.poll!.expiresAt, - multiple: data.poll!.multiple, - votes: new Array(data.poll!.choices.length).fill(0), - noteVisibility: note.visibility, - userId: user.id, - userHost: user.host, - channelId: data.channel?.id ?? null, - }); + // Insert or update poll + if (data.poll) { + const poll = new MiPoll({ + noteId: note.id, + choices: data.poll.choices, + expiresAt: data.poll.expiresAt, + multiple: data.poll.multiple, + votes: new Array(data.poll.choices.length).fill(0), + noteVisibility: note.visibility, + userId: user.id, + userHost: user.host, + channelId: data.channel?.id ?? null, + }); - if (!oldnote.hasPoll) { - await transactionalEntityManager.insert(MiPoll, poll); - } else { - await transactionalEntityManager.update(MiPoll, oldnote.id, poll); + if (oldPoll) { + await transactionalEntityManager.update(MiPoll, { noteId: oldPoll.noteId }, poll); + } else { + await transactionalEntityManager.insert(MiPoll, poll); + } + // Delete poll + } else if (oldPoll) { + await transactionalEntityManager.delete(MiPoll, { noteId: oldPoll.noteId }); } }); } else { - await this.notesRepository.update(oldnote.id, note); + await this.notesRepository.update(oldNote.id, note); } // Re-fetch note to get the default values of null / unset fields. const edited = await this.notesRepository.findOneByOrFail({ id: note.id }); - setImmediate('post edited', { signal: this.#shutdownController.signal }).then( - () => this.postNoteEdited(edited, oldnote, user, data, silent, tags!, mentionedUsers!), - () => { /* aborted, ignore this */ }, - ); + await this.queueService.createPostNoteJob(note.id, silent, 'edit'); return edited; } else { - return oldnote; + return oldNote; } } @bindThis - private async postNoteEdited(note: MiNote, oldNote: MiNote, user: MiUser & { - id: MiUser['id']; - username: MiUser['username']; - host: MiUser['host']; - isBot: MiUser['isBot']; - noindex: MiUser['noindex']; - }, data: Option, silent: boolean, tags: string[], mentionedUsers: MinimumUser[]) { + public async postNoteEdited(note: MiNote, user: MiUser, data: MiNote & { poll: MiPoll | null }, silent: boolean, mentionedUsers: MinimumUser[]) { // Register host if (this.meta.enableStatsForFederatedInstances) { if (isRemoteUser(user)) { this.federatedInstanceService.fetchOrRegister(user.host).then(async i => { if (note.renote && note.text || !note.renote) { - this.updateNotesCountQueue.enqueue(i.id, 1); + await this.collapsedQueueService.updateInstanceQueue.enqueue(i.id, { notesCountDelta: 1 }); } if (this.meta.enableChartsForFederatedInstances) { this.instanceChart.updateNote(i.host, note, true); @@ -621,15 +621,15 @@ export class NoteEditService implements OnApplicationShutdown { } } - this.usersRepository.update({ id: user.id }, { updatedAt: this.timeService.date }); + await this.collapsedQueueService.updateUserQueue.enqueue(user.id, { updatedAt: this.timeService.date }); // ハッシュタグ更新 - this.pushToTl(note, user); + await this.pushToTl(note, user); if (data.poll && data.poll.expiresAt) { const delay = data.poll.expiresAt.getTime() - this.timeService.now; - this.queueService.endedPollNotificationQueue.remove(`pollEnd:${note.id}`); - this.queueService.endedPollNotificationQueue.add(note.id, { + await this.queueService.endedPollNotificationQueue.remove(`pollEnd:${note.id}`); + await this.queueService.endedPollNotificationQueue.add(note.id, { noteId: note.id, }, { jobId: `pollEnd_${note.id}`, @@ -648,9 +648,9 @@ export class NoteEditService implements OnApplicationShutdown { text: note.text ?? '', }); - this.roleService.addNoteToRoleTimeline(noteObj); + await this.roleService.addNoteToRoleTimeline(noteObj); - this.webhookService.enqueueUserWebhook(user.id, 'note', { note: noteObj }); + await this.webhookService.enqueueUserWebhook(user.id, 'note', { note: noteObj }); const nm = new NotificationManager(this.mutingsRepository, this.notificationService, user, note); @@ -673,16 +673,16 @@ export class NoteEditService implements OnApplicationShutdown { if (!isThreadMuted && !muted) { nm.push(data.reply.userId, 'edited'); this.globalEventService.publishMainStream(data.reply.userId, 'edited', noteObj); - this.webhookService.enqueueUserWebhook(data.reply.userId, 'reply', { note: noteObj }); + await this.webhookService.enqueueUserWebhook(data.reply.userId, 'reply', { note: noteObj }); } } } - nm.notify(); + await nm.notify(); //#region AP deliver if (!data.localOnly && isLocalUser(user)) { - trackTask(async () => { + await trackTask(async () => { const noteActivity = await this.apRendererService.renderNoteOrRenoteActivity(note, user, { renote: data.renote }); const dm = this.apDeliverManagerService.createDeliverManager(user, noteActivity); @@ -737,8 +737,8 @@ export class NoteEditService implements OnApplicationShutdown { } if (data.channel) { - this.channelsRepository.increment({ id: data.channel.id }, 'notesCount', 1); - this.channelsRepository.update(data.channel.id, { + await this.channelsRepository.increment({ id: data.channel.id }, 'notesCount', 1); + await this.channelsRepository.update(data.channel.id, { lastNotedAt: this.timeService.date, }); @@ -755,10 +755,10 @@ export class NoteEditService implements OnApplicationShutdown { } // Update the Latest Note index / following feed - this.latestNoteService.handleUpdatedNoteBG(oldNote, note); + await this.latestNoteService.handleUpdatedNoteDeferred(note); // Register to search database - if (!user.noindex) this.index(note); + if (!user.noindex) await this.index(note); } @bindThis @@ -779,27 +779,10 @@ export class NoteEditService implements OnApplicationShutdown { } @bindThis - private index(note: MiNote) { + private async index(note: MiNote) { if (note.text == null && note.cw == null) return; - this.searchService.indexNote(note); - } - - @bindThis - private async extractMentionedUsers(user: { host: MiUser['host']; }, tokens: mfm.MfmNode[]): Promise { - if (tokens == null) return []; - - const mentions = extractMentions(tokens); - let mentionedUsers = (await Promise.all(mentions.map(m => - this.remoteUserResolveService.resolveUser(m.username, m.host ?? user.host).catch(() => null), - ))).filter(x => x !== null) as MiUser[]; - - // Drop duplicate users - mentionedUsers = mentionedUsers.filter((u, i, self) => - i === self.findIndex(u2 => u.id === u2.id), - ); - - return mentionedUsers; + await this.searchService.indexNote(note); } @bindThis @@ -912,25 +895,14 @@ export class NoteEditService implements OnApplicationShutdown { // checkHibernation moved to HibernateUsersProcessorService } - r.exec(); + await r.exec(); } // checkHibernation moved to HibernateUsersProcessorService - @bindThis - private collapseNotesCount(oldValue: number, newValue: number) { - return oldValue + newValue; - } - - @bindThis - private async performUpdateNotesCount(id: MiNote['id'], incrBy: number) { - await this.instancesRepository.increment({ id: id }, 'notesCount', incrBy); - } - @bindThis public async dispose(): Promise { this.#shutdownController.abort(); - await this.updateNotesCountQueue.performAllNow(); } @bindThis diff --git a/packages/backend/src/core/NotePiningService.ts b/packages/backend/src/core/NotePiningService.ts index 836db52e48..a510676ef9 100644 --- a/packages/backend/src/core/NotePiningService.ts +++ b/packages/backend/src/core/NotePiningService.ts @@ -18,6 +18,7 @@ import { ApDeliverManagerService } from '@/core/activitypub/ApDeliverManagerServ import { ApRendererService } from '@/core/activitypub/ApRendererService.js'; import { bindThis } from '@/decorators.js'; import { RoleService } from '@/core/RoleService.js'; +import { trackPromise } from '@/misc/promise-tracker.js'; import type { DataSource } from 'typeorm'; @Injectable() @@ -84,7 +85,7 @@ export class NotePiningService { // Deliver to remote followers if (this.userEntityService.isLocalUser(user) && !note.localOnly && ['public', 'home'].includes(note.visibility)) { - await this.deliverPinnedChange(user, note.id, true); + trackPromise(this.deliverPinnedChange(user, note.id, true)); } } @@ -112,7 +113,7 @@ export class NotePiningService { // Deliver to remote followers if (this.userEntityService.isLocalUser(user) && !note.localOnly && ['public', 'home'].includes(note.visibility)) { - await this.deliverPinnedChange(user, noteId, false); + trackPromise(this.deliverPinnedChange(user, noteId, false)); } } diff --git a/packages/backend/src/core/NotificationService.ts b/packages/backend/src/core/NotificationService.ts index 9180dfa418..63d2011441 100644 --- a/packages/backend/src/core/NotificationService.ts +++ b/packages/backend/src/core/NotificationService.ts @@ -72,9 +72,9 @@ export class NotificationService implements OnApplicationShutdown { } @bindThis - private postReadAllNotifications(userId: MiUser['id']) { + private async postReadAllNotifications(userId: MiUser['id']) { this.globalEventService.publishMainStream(userId, 'readAllNotifications'); - this.pushNotificationService.pushNotification(userId, 'readAllNotifications', undefined); + await this.pushNotificationService.pushNotification(userId, 'readAllNotifications', undefined); } @bindThis diff --git a/packages/backend/src/core/PollService.ts b/packages/backend/src/core/PollService.ts index e93cc7ba4c..b5544db9d4 100644 --- a/packages/backend/src/core/PollService.ts +++ b/packages/backend/src/core/PollService.ts @@ -92,7 +92,7 @@ export class PollService { public async deliverQuestionUpdate(note: MiNote) { if (note.localOnly) return; - const user = await this.usersRepository.findOneBy({ id: note.userId }); + const user = note.user ?? await this.usersRepository.findOneBy({ id: note.userId }); if (user == null) throw new Error('note not found'); if (isLocalUser(user)) { diff --git a/packages/backend/src/core/QueueModule.ts b/packages/backend/src/core/QueueModule.ts index 2f594394a6..078c6002e8 100644 --- a/packages/backend/src/core/QueueModule.ts +++ b/packages/backend/src/core/QueueModule.ts @@ -20,6 +20,7 @@ import { UserWebhookDeliverJobData, SystemWebhookDeliverJobData, ScheduleNotePostJobData, + BackgroundTaskJobData, } from '../queue/types.js'; import type { Provider } from '@nestjs/common'; @@ -33,6 +34,7 @@ export type ObjectStorageQueue = Bull.Queue; export type UserWebhookDeliverQueue = Bull.Queue; export type SystemWebhookDeliverQueue = Bull.Queue; export type ScheduleNotePostQueue = Bull.Queue; +export type BackgroundTaskQueue = Bull.Queue; const $system: Provider = { provide: 'queue:system', @@ -94,6 +96,12 @@ const $scheduleNotePost: Provider = { inject: [DI.config], }; +const $backgroundTask: Provider = { + provide: 'queue:backgroundTask', + useFactory: (config: Config) => new Bull.Queue(QUEUE.BACKGROUND_TASK, baseQueueOptions(config, QUEUE.BACKGROUND_TASK)), + inject: [DI.config], +}; + @Module({ imports: [ ], @@ -108,6 +116,7 @@ const $scheduleNotePost: Provider = { $userWebhookDeliver, $systemWebhookDeliver, $scheduleNotePost, + $backgroundTask, ], exports: [ $system, @@ -120,6 +129,7 @@ const $scheduleNotePost: Provider = { $userWebhookDeliver, $systemWebhookDeliver, $scheduleNotePost, + $backgroundTask, ], }) export class QueueModule implements OnApplicationShutdown { @@ -136,6 +146,7 @@ export class QueueModule implements OnApplicationShutdown { @Inject('queue:userWebhookDeliver') public userWebhookDeliverQueue: UserWebhookDeliverQueue, @Inject('queue:systemWebhookDeliver') public systemWebhookDeliverQueue: SystemWebhookDeliverQueue, @Inject('queue:scheduleNotePost') public scheduleNotePostQueue: ScheduleNotePostQueue, + @Inject('queue:backgroundTask') public readonly backgroundTaskQueue: BackgroundTaskQueue, ) {} public async dispose(): Promise { @@ -155,6 +166,7 @@ export class QueueModule implements OnApplicationShutdown { this.userWebhookDeliverQueue.close(), this.systemWebhookDeliverQueue.close(), this.scheduleNotePostQueue.close(), + this.backgroundTaskQueue.close(), ]).then(res => { for (const result of res) { if (result.status === 'rejected') { diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 9fd646e655..15260fe272 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -19,8 +19,10 @@ import { ApRequestCreator } from '@/core/activitypub/ApRequestService.js'; import { TimeService } from '@/global/TimeService.js'; import type { SystemWebhookPayload } from '@/core/SystemWebhookService.js'; import type { MiNote } from '@/models/Note.js'; +import type { MinimalNote } from '@/misc/is-renote.js'; import { type UserWebhookPayload } from './UserWebhookService.js'; import type { + BackgroundTaskJobData, DbJobData, DeliverJobData, RelationshipJobData, @@ -39,6 +41,7 @@ import type { SystemWebhookDeliverQueue, UserWebhookDeliverQueue, ScheduleNotePostQueue, + BackgroundTaskQueue, } from './QueueModule.js'; import type httpSignature from '@peertube/http-signature'; import type * as Bull from 'bullmq'; @@ -54,6 +57,7 @@ export const QUEUE_TYPES = [ 'userWebhookDeliver', 'systemWebhookDeliver', 'scheduleNotePost', + 'backgroundTask', ] as const; @Injectable() @@ -72,6 +76,7 @@ export class QueueService implements OnModuleInit { @Inject('queue:userWebhookDeliver') public userWebhookDeliverQueue: UserWebhookDeliverQueue, @Inject('queue:systemWebhookDeliver') public systemWebhookDeliverQueue: SystemWebhookDeliverQueue, @Inject('queue:scheduleNotePost') public ScheduleNotePostQueue: ScheduleNotePostQueue, + @Inject('queue:backgroundTask') public readonly backgroundTaskQueue: BackgroundTaskQueue, private readonly timeService: TimeService, ) {} @@ -839,6 +844,107 @@ export class QueueService implements OnModuleInit { }); } + @bindThis + public async createUpdateUserJob(userId: string) { + return await this.createBackgroundTask({ type: 'update-user', userId }, userId); + } + + @bindThis + public async createUpdateFeaturedJob(userId: string) { + return await this.createBackgroundTask({ type: 'update-featured', userId }, userId); + } + + @bindThis + public async createUpdateInstanceJob(host: string) { + return await this.createBackgroundTask({ type: 'update-instance', host }, host); + } + + @bindThis + public async createPostDeliverJob(host: string, result: 'success' | 'temp-fail' | 'perm-fail') { + return await this.createBackgroundTask({ type: 'post-deliver', host, result }); + } + + @bindThis + public async createPostInboxJob(host: string) { + return await this.createBackgroundTask({ type: 'post-inbox', host }); + } + + @bindThis + public async createPostNoteJob(noteId: string, silent: boolean, type: 'create' | 'edit') { + const edit = type === 'edit'; + const duplication = `${noteId}_${type}`; + + return await this.createBackgroundTask({ type: 'post-note', noteId, silent, edit }, duplication); + } + + @bindThis + public async createUpdateUserTagsJob(userId: string) { + return await this.createBackgroundTask({ type: 'update-user-tags', userId }, userId); + } + + @bindThis + public async createUpdateNoteTagsJob(noteId: string) { + return await this.createBackgroundTask({ type: 'update-note-tags', noteId }, noteId); + } + + @bindThis + public async createDeleteFileJob(fileId: string, isExpired?: boolean, deleterId?: string) { + return await this.createBackgroundTask({ type: 'delete-file', fileId, isExpired, deleterId }, fileId); + } + + @bindThis + public async createUpdateLatestNoteJob(note: MinimalNote) { + // Compact the note to avoid storing the entire thing in Redis, when all we need is minimal data for categorization + const minimizedNote: MinimalNote = { + id: note.id, + visibility: note.visibility, + userId: note.userId, + replyId: note.replyId, + renoteId: note.renoteId, + hasPoll: note.hasPoll, + text: note.text ? '1' : null, + cw: note.text ? '1' : null, + fileIds: note.fileIds.length > 0 ? ['1'] : [], + }; + + return await this.createBackgroundTask({ type: 'update-latest-note', note: minimizedNote }, note.id); + } + + @bindThis + public async createPostSuspendJob(userId: string) { + return await this.createBackgroundTask({ type: 'post-suspend', userId }, userId); + } + + @bindThis + public async createPostUnsuspendJob(userId: string) { + return await this.createBackgroundTask({ type: 'post-unsuspend', userId }, userId); + } + + @bindThis + public async createDeleteApLogsJob(dataType: 'inbox' | 'object', data: string | string[]) { + return await this.createBackgroundTask({ type: 'delete-ap-logs', dataType, data }); + } + + private async createBackgroundTask(data: T, duplication?: string | { id: string, ttl?: number }) { + return await this.backgroundTaskQueue.add( + data.type, + data, + { + // https://docs.bullmq.io/guide/retrying-failing-jobs#custom-back-off-strategies + attempts: this.config.backgroundJobMaxAttempts ?? 8, + backoff: { + // Resolves to QueueProcessorService::HttpRelatedBackoff() + type: 'custom', + }, + + // https://docs.bullmq.io/guide/jobs/deduplication + deduplication: typeof(duplication) === 'string' + ? { id: `${data.type}_${duplication}` } + : duplication, + }, + ); + }; + /** * @see UserWebhookDeliverJobData * @see UserWebhookDeliverProcessorService @@ -927,6 +1033,7 @@ export class QueueService implements OnModuleInit { case 'userWebhookDeliver': return this.userWebhookDeliverQueue; case 'systemWebhookDeliver': return this.systemWebhookDeliverQueue; case 'scheduleNotePost': return this.ScheduleNotePostQueue; + case 'backgroundTask': return this.backgroundTaskQueue; default: throw new Error(`Unrecognized queue type: ${type}`); } } diff --git a/packages/backend/src/core/ReactionService.ts b/packages/backend/src/core/ReactionService.ts index d3ab48e3ff..b038acc0e4 100644 --- a/packages/backend/src/core/ReactionService.ts +++ b/packages/backend/src/core/ReactionService.ts @@ -33,6 +33,7 @@ import { PER_NOTE_REACTION_USER_PAIR_CACHE_MAX } from '@/const.js'; import { CacheService } from '@/core/CacheService.js'; import { NoteVisibilityService } from '@/core/NoteVisibilityService.js'; import { TimeService } from '@/global/TimeService.js'; +import { CollapsedQueueService } from '@/core/CollapsedQueueService.js'; import type { DataSource } from 'typeorm'; const FALLBACK = '\u2764'; @@ -110,6 +111,7 @@ export class ReactionService implements OnModuleInit { private readonly cacheService: CacheService, private readonly noteVisibilityService: NoteVisibilityService, private readonly timeService: TimeService, + private readonly collapsedQueueService: CollapsedQueueService, ) { } @@ -119,7 +121,7 @@ export class ReactionService implements OnModuleInit { } @bindThis - public async create(user: { id: MiUser['id']; host: MiUser['host']; isBot: MiUser['isBot'] }, note: MiNote, _reaction?: string | null) { + public async create(user: MiUser, note: MiNote, _reaction?: string | null) { // Check blocking if (note.userId !== user.id) { const blocked = await this.userBlockingService.checkBlocked(note.userId, user.id); @@ -224,7 +226,7 @@ export class ReactionService implements OnModuleInit { .execute(); } - this.usersRepository.update({ id: user.id }, { updatedAt: this.timeService.date }); + await this.collapsedQueueService.updateUserQueue.enqueue(user.id, { updatedAt: this.timeService.date }); // 30%の確率、セルフではない、3日以内に投稿されたノートの場合ハイライト用ランキング更新 if ( @@ -289,16 +291,18 @@ export class ReactionService implements OnModuleInit { const content = this.apRendererService.addContext(await this.apRendererService.renderLike(record, note)); const dm = this.apDeliverManagerService.createDeliverManager(user, content); if (note.userHost !== null) { - const reactee = await this.usersRepository.findOneBy({ id: note.userId }); + const reactee = await this.cacheService.findRemoteUserById(note.userId); dm.addDirectRecipe(reactee as MiRemoteUser); } if (['public', 'home', 'followers'].includes(note.visibility)) { dm.addFollowersRecipe(); } else if (note.visibility === 'specified') { - const visibleUsers = await Promise.all(note.visibleUserIds.map(id => this.usersRepository.findOneBy({ id }))); - for (const u of visibleUsers.filter(u => u && isRemoteUser(u))) { - dm.addDirectRecipe(u as MiRemoteUser); + const visibleUsers = await this.cacheService.findUsersById(note.visibleUserIds); + for (const u of visibleUsers.values()) { + if (isRemoteUser(u)) { + dm.addDirectRecipe(u as MiRemoteUser); + } } } @@ -308,7 +312,7 @@ export class ReactionService implements OnModuleInit { } @bindThis - public async delete(user: { id: MiUser['id']; host: MiUser['host']; isBot: MiUser['isBot']; }, note: MiNote, exist?: MiNoteReaction | null) { + public async delete(user: MiUser, note: MiNote, exist?: MiNoteReaction | null) { // if already unreacted exist ??= await this.noteReactionsRepository.findOneBy({ noteId: note.id, @@ -340,7 +344,7 @@ export class ReactionService implements OnModuleInit { .execute(); } - this.usersRepository.update({ id: user.id }, { updatedAt: this.timeService.date }); + await this.collapsedQueueService.updateUserQueue.enqueue(user.id, { updatedAt: this.timeService.date }); this.globalEventService.publishNoteStream(note.id, 'unreacted', { reaction: this.decodeReaction(exist.reaction).reaction, @@ -352,7 +356,7 @@ export class ReactionService implements OnModuleInit { const content = this.apRendererService.addContext(this.apRendererService.renderUndo(await this.apRendererService.renderLike(exist, note), user)); const dm = this.apDeliverManagerService.createDeliverManager(user, content); if (note.userHost !== null) { - const reactee = await this.usersRepository.findOneBy({ id: note.userId }); + const reactee = await this.cacheService.findRemoteUserById(note.userId); dm.addDirectRecipe(reactee as MiRemoteUser); } dm.addFollowersRecipe(); diff --git a/packages/backend/src/core/RemoteUserResolveService.ts b/packages/backend/src/core/RemoteUserResolveService.ts index 3d21cb92a9..d239fb637d 100644 --- a/packages/backend/src/core/RemoteUserResolveService.ts +++ b/packages/backend/src/core/RemoteUserResolveService.ts @@ -8,7 +8,7 @@ import chalk from 'chalk'; import { IsNull } from 'typeorm'; import { DI } from '@/di-symbols.js'; import type { UsersRepository } from '@/models/_.js'; -import type { MiLocalUser, MiRemoteUser } from '@/models/User.js'; +import type { MiUser, MiLocalUser, MiRemoteUser } from '@/models/User.js'; import type { Config } from '@/config.js'; import type Logger from '@/logger.js'; import { UtilityService } from '@/core/UtilityService.js'; @@ -59,7 +59,7 @@ export class RemoteUserResolveService { const acct = Acct.toString({ username, host }); // username+host -> acct (handle) // Try fetch from DB - let user = await this.cacheService.findUserByAcct(acct).catch(() => null); // Error is expected if the user doesn't exist yet + let user: MiUser | null | undefined = await this.cacheService.findOptionalUserByAcct(acct); // Opportunistically update remote users if (user != null && isRemoteUser(user)) { diff --git a/packages/backend/src/core/ReversiService.ts b/packages/backend/src/core/ReversiService.ts index 2edba73677..388ebf61e2 100644 --- a/packages/backend/src/core/ReversiService.ts +++ b/packages/backend/src/core/ReversiService.ts @@ -587,6 +587,7 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit { updatedAt: parsed.user1.updatedAt != null ? new Date(parsed.user1.updatedAt) : null, lastActiveDate: parsed.user1.lastActiveDate != null ? new Date(parsed.user1.lastActiveDate) : null, lastFetchedAt: parsed.user1.lastFetchedAt != null ? new Date(parsed.user1.lastFetchedAt) : null, + lastFetchedFeaturedAt: parsed.user1.lastFetchedFeaturedAt != null ? new Date(parsed.user1.lastFetchedFeaturedAt) : null, movedAt: parsed.user1.movedAt != null ? new Date(parsed.user1.movedAt) : null, instance: null, userProfile: null, @@ -599,6 +600,7 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit { updatedAt: parsed.user2.updatedAt != null ? new Date(parsed.user2.updatedAt) : null, lastActiveDate: parsed.user2.lastActiveDate != null ? new Date(parsed.user2.lastActiveDate) : null, lastFetchedAt: parsed.user2.lastFetchedAt != null ? new Date(parsed.user2.lastFetchedAt) : null, + lastFetchedFeaturedAt: parsed.user2.lastFetchedFeaturedAt != null ? new Date(parsed.user2.lastFetchedFeaturedAt) : null, movedAt: parsed.user2.movedAt != null ? new Date(parsed.user2.movedAt) : null, instance: null, userProfile: null, diff --git a/packages/backend/src/core/RoleService.ts b/packages/backend/src/core/RoleService.ts index 97212b4ac7..d13f1a99af 100644 --- a/packages/backend/src/core/RoleService.ts +++ b/packages/backend/src/core/RoleService.ts @@ -844,7 +844,7 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { this.globalEventService.publishRoleTimelineStream(role.id, 'note', note); } - redisPipeline.exec(); + await redisPipeline.exec(); } @bindThis diff --git a/packages/backend/src/core/S3Service.ts b/packages/backend/src/core/S3Service.ts index 955d778015..30de5289f4 100644 --- a/packages/backend/src/core/S3Service.ts +++ b/packages/backend/src/core/S3Service.ts @@ -91,7 +91,7 @@ export class S3Service implements OnApplicationShutdown { @bindThis public async upload(input: PutObjectCommandInput) { const client = this.getS3Client(); - return new Upload({ + return await new Upload({ client, params: input, partSize: (client.config.endpoint && (await client.config.endpoint()).hostname === 'storage.googleapis.com') diff --git a/packages/backend/src/core/SearchService.ts b/packages/backend/src/core/SearchService.ts index 6e4dd9adca..04026865fb 100644 --- a/packages/backend/src/core/SearchService.ts +++ b/packages/backend/src/core/SearchService.ts @@ -256,10 +256,10 @@ export class SearchService { case 'sqlTsvector': { // ほとんど内容に差がないのでsqlLikeとsqlPgroongaを同じ処理にしている. // 今後の拡張で差が出る用であれば関数を分ける. - return this.searchNoteByLike(q, me, opts, pagination); + return await this.searchNoteByLike(q, me, opts, pagination); } case 'meilisearch': { - return this.searchNoteByMeiliSearch(q, me, opts, pagination); + return await this.searchNoteByMeiliSearch(q, me, opts, pagination); } default: { // eslint-disable-next-line @typescript-eslint/no-unused-vars diff --git a/packages/backend/src/core/SponsorsService.ts b/packages/backend/src/core/SponsorsService.ts index 551768bdc9..454df4442e 100644 --- a/packages/backend/src/core/SponsorsService.ts +++ b/packages/backend/src/core/SponsorsService.ts @@ -61,6 +61,7 @@ export class SponsorsService { } try { + // TODO use HTTP service const backers = await fetch(`${this.meta.donationUrl}/members/users.json`).then((response) => response.json() as Promise); // Merge both together into one array and make sure it only has Active subscriptions @@ -76,6 +77,7 @@ export class SponsorsService { @bindThis private async fetchSharkeySponsors(): Promise { try { + // TODO use HTTP service const backers = await fetch('https://opencollective.com/sharkey/tiers/backer/all.json').then((response) => response.json() as Promise); const sponsorsOC = await fetch('https://opencollective.com/sharkey/tiers/sponsor/all.json').then((response) => response.json() as Promise); @@ -92,12 +94,12 @@ export class SponsorsService { @bindThis public async instanceSponsors(forceUpdate: boolean) { if (forceUpdate) await this.cache.refresh('instance'); - return this.cache.fetch('instance'); + return await this.cache.fetch('instance'); } @bindThis public async sharkeySponsors(forceUpdate: boolean) { if (forceUpdate) await this.cache.refresh('sharkey'); - return this.cache.fetch('sharkey'); + return await this.cache.fetch('sharkey'); } } diff --git a/packages/backend/src/core/UpdateInstanceQueue.ts b/packages/backend/src/core/UpdateInstanceQueue.ts deleted file mode 100644 index c136241344..0000000000 --- a/packages/backend/src/core/UpdateInstanceQueue.ts +++ /dev/null @@ -1,54 +0,0 @@ -/* - * SPDX-FileCopyrightText: syuilo and misskey-project - * SPDX-License-Identifier: AGPL-3.0-only - */ - -import { Injectable, OnApplicationShutdown } from '@nestjs/common'; -import { CollapsedQueue } from '@/misc/collapsed-queue.js'; -import { bindThis } from '@/decorators.js'; -import { MiNote } from '@/models/Note.js'; -import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; -import { TimeService } from '@/global/TimeService.js'; - -type UpdateInstanceJob = { - latestRequestReceivedAt: Date, - shouldUnsuspend: boolean, -}; - -// Moved from InboxProcessorService to allow access from ApInboxService -@Injectable() -export class UpdateInstanceQueue extends CollapsedQueue implements OnApplicationShutdown { - constructor( - private readonly federatedInstanceService: FederatedInstanceService, - timeService: TimeService, - ) { - super(timeService, process.env.NODE_ENV !== 'test' ? 60 * 1000 * 5 : 0, (id, job) => this.collapseUpdateInstanceJobs(id, job), (id, job) => this.performUpdateInstance(id, job)); - } - - @bindThis - private collapseUpdateInstanceJobs(oldJob: UpdateInstanceJob, newJob: UpdateInstanceJob) { - const latestRequestReceivedAt = oldJob.latestRequestReceivedAt < newJob.latestRequestReceivedAt - ? newJob.latestRequestReceivedAt - : oldJob.latestRequestReceivedAt; - const shouldUnsuspend = oldJob.shouldUnsuspend || newJob.shouldUnsuspend; - return { - latestRequestReceivedAt, - shouldUnsuspend, - }; - } - - @bindThis - private async performUpdateInstance(id: string, job: UpdateInstanceJob) { - await this.federatedInstanceService.update(id, { - latestRequestReceivedAt: this.timeService.date, - isNotResponding: false, - // もしサーバーが死んでるために配信が止まっていた場合には自動的に復活させてあげる - suspensionState: job.shouldUnsuspend ? 'none' : undefined, - }); - } - - @bindThis - async onApplicationShutdown() { - await this.performAllNow(); - } -} diff --git a/packages/backend/src/core/UserFollowingService.ts b/packages/backend/src/core/UserFollowingService.ts index 833ea97193..a17006f57c 100644 --- a/packages/backend/src/core/UserFollowingService.ts +++ b/packages/backend/src/core/UserFollowingService.ts @@ -30,6 +30,8 @@ import { UtilityService } from '@/core/UtilityService.js'; import type { ThinUser } from '@/queue/types.js'; import { LoggerService } from '@/core/LoggerService.js'; import { InternalEventService } from '@/global/InternalEventService.js'; +import { trackPromise } from '@/misc/promise-tracker.js'; +import { CollapsedQueueService } from '@/core/CollapsedQueueService.js'; import type Logger from '../logger.js'; type Local = MiLocalUser | { @@ -88,6 +90,7 @@ export class UserFollowingService implements OnModuleInit { private perUserFollowingChart: PerUserFollowingChart, private instanceChart: InstanceChart, private readonly internalEventService: InternalEventService, + private readonly collapsedQueueService: CollapsedQueueService, loggerService: LoggerService, ) { @@ -102,7 +105,7 @@ export class UserFollowingService implements OnModuleInit { @bindThis public async deliverAccept(follower: MiRemoteUser, followee: MiPartialLocalUser, requestId?: string) { const content = this.apRendererService.addContext(this.apRendererService.renderAccept(this.apRendererService.renderFollow(follower, followee, requestId), followee)); - this.queueService.deliver(followee, content, follower.inbox, false); + await this.queueService.deliver(followee, content, follower.inbox, false); } @bindThis @@ -152,7 +155,7 @@ export class UserFollowingService implements OnModuleInit { // すでにフォロー関係が存在している場合 if (this.userEntityService.isRemoteUser(follower) && this.userEntityService.isLocalUser(followee)) { // リモート → ローカル: acceptを送り返しておしまい - this.deliverAccept(follower, followee, requestId); + trackPromise(this.deliverAccept(follower, followee, requestId)); return; } if (this.userEntityService.isLocalUser(follower)) { @@ -206,7 +209,7 @@ export class UserFollowingService implements OnModuleInit { await this.insertFollowingDoc(followee, follower, silent, withReplies); if (this.userEntityService.isRemoteUser(follower) && this.userEntityService.isLocalUser(followee)) { - this.deliverAccept(follower, followee, requestId); + trackPromise(this.deliverAccept(follower, followee, requestId)); } } @@ -285,24 +288,22 @@ export class UserFollowingService implements OnModuleInit { // Neither followee nor follower has moved. if (!followeeUser.movedToUri && !followerUser.movedToUri) { //#region Increment counts - await Promise.all([ - this.usersRepository.increment({ id: follower.id }, 'followingCount', 1), - this.usersRepository.increment({ id: followee.id }, 'followersCount', 1), - ]); + await this.collapsedQueueService.updateUserQueue.enqueue(follower.id, { followingCountDelta: 1 }); + await this.collapsedQueueService.updateUserQueue.enqueue(followee.id, { followersCountDelta: 1 }); //#endregion //#region Update instance stats if (this.meta.enableStatsForFederatedInstances) { if (this.userEntityService.isRemoteUser(follower) && this.userEntityService.isLocalUser(followee)) { this.federatedInstanceService.fetchOrRegister(follower.host).then(async i => { - this.instancesRepository.increment({ id: i.id }, 'followingCount', 1); + await this.collapsedQueueService.updateInstanceQueue.enqueue(i.id, { followingCountDelta: 1 }); if (this.meta.enableChartsForFederatedInstances) { this.instanceChart.updateFollowing(i.host, true); } }); } else if (this.userEntityService.isLocalUser(follower) && this.userEntityService.isRemoteUser(followee)) { this.federatedInstanceService.fetchOrRegister(followee.host).then(async i => { - this.instancesRepository.increment({ id: i.id }, 'followersCount', 1); + await this.collapsedQueueService.updateInstanceQueue.enqueue(i.id, { followersCountDelta: 1 }); if (this.meta.enableChartsForFederatedInstances) { this.instanceChart.updateFollowers(i.host, true); } @@ -397,24 +398,22 @@ export class UserFollowingService implements OnModuleInit { // Neither followee nor follower has moved. if (!follower.movedToUri && !followee.movedToUri) { //#region Decrement following / followers counts - await Promise.all([ - this.usersRepository.decrement({ id: follower.id }, 'followingCount', 1), - this.usersRepository.decrement({ id: followee.id }, 'followersCount', 1), - ]); + await this.collapsedQueueService.updateUserQueue.enqueue(follower.id, { followingCountDelta: -1 }); + await this.collapsedQueueService.updateUserQueue.enqueue(followee.id, { followersCountDelta: -1 }); //#endregion //#region Update instance stats if (this.meta.enableStatsForFederatedInstances) { if (this.userEntityService.isRemoteUser(follower) && this.userEntityService.isLocalUser(followee)) { this.federatedInstanceService.fetchOrRegister(follower.host).then(async i => { - this.instancesRepository.decrement({ id: i.id }, 'followingCount', 1); + await this.collapsedQueueService.updateInstanceQueue.enqueue(i.id, { followingCountDelta: -1 }); if (this.meta.enableChartsForFederatedInstances) { this.instanceChart.updateFollowing(i.host, false); } }); } else if (this.userEntityService.isLocalUser(follower) && this.userEntityService.isRemoteUser(followee)) { this.federatedInstanceService.fetchOrRegister(followee.host).then(async i => { - this.instancesRepository.decrement({ id: i.id }, 'followersCount', 1); + await this.collapsedQueueService.updateInstanceQueue.enqueue(i.id, { followersCountDelta: -1 }); if (this.meta.enableChartsForFederatedInstances) { this.instanceChart.updateFollowers(i.host, false); } @@ -581,7 +580,7 @@ export class UserFollowingService implements OnModuleInit { await this.insertFollowingDoc(followee, follower, false, request.withReplies); if (this.userEntityService.isRemoteUser(follower) && this.userEntityService.isLocalUser(followee)) { - this.deliverAccept(follower, followee as MiPartialLocalUser, request.requestId ?? undefined); + trackPromise(this.deliverAccept(follower, followee as MiPartialLocalUser, request.requestId ?? undefined)); } this.userEntityService.pack(followee.id, followee, { @@ -595,14 +594,13 @@ export class UserFollowingService implements OnModuleInit { id: MiUser['id']; host: MiUser['host']; uri: MiUser['host']; inbox: MiUser['inbox']; sharedInbox: MiUser['sharedInbox']; }, ): Promise { - const requests = await this.followRequestsRepository.findBy({ + const requests = await this.followRequestsRepository.find({ where: { followeeId: user.id, - }); + }, relations: { + follower: true, + } }); - for (const request of requests) { - const follower = await this.usersRepository.findOneByOrFail({ id: request.followerId }); - this.acceptFollowRequest(user, follower); - } + await Promise.all(requests.map(request => this.acceptFollowRequest(user, request.follower as MiUser))); } /** @@ -611,7 +609,7 @@ export class UserFollowingService implements OnModuleInit { @bindThis public async rejectFollowRequest(user: Local, follower: Both): Promise { if (this.userEntityService.isRemoteUser(follower)) { - this.deliverReject(user, follower); + trackPromise(this.deliverReject(user, follower)); } await this.removeFollowRequest(user, follower); @@ -627,7 +625,7 @@ export class UserFollowingService implements OnModuleInit { @bindThis public async rejectFollow(user: Local, follower: Both): Promise { if (this.userEntityService.isRemoteUser(follower)) { - this.deliverReject(user, follower); + trackPromise(this.deliverReject(user, follower)); } await this.removeFollow(user, follower); @@ -696,7 +694,7 @@ export class UserFollowingService implements OnModuleInit { }); const content = this.apRendererService.addContext(this.apRendererService.renderReject(this.apRendererService.renderFollow(follower, followee, request?.requestId ?? undefined), followee)); - this.queueService.deliver(followee, content, follower.inbox, false); + await this.queueService.deliver(followee, content, follower.inbox, false); } /** @@ -720,7 +718,7 @@ export class UserFollowingService implements OnModuleInit { @bindThis public async isFollowing(followerId: MiUser['id'], followeeId: MiUser['id']) { - return this.cacheService.isFollowing(followerId, followeeId); + return await this.cacheService.isFollowing(followerId, followeeId); } @bindThis diff --git a/packages/backend/src/core/UserSearchService.ts b/packages/backend/src/core/UserSearchService.ts index 6c6d3a5280..9e2b5dfc64 100644 --- a/packages/backend/src/core/UserSearchService.ts +++ b/packages/backend/src/core/UserSearchService.ts @@ -99,7 +99,7 @@ export class UserSearchService { } } - return this.userEntityService.packMany<'UserLite' | 'UserDetailed'>( + return await this.userEntityService.packMany<'UserLite' | 'UserDetailed'>( [...resultSet].slice(0, limit), me, { schema: opts?.detail ? 'UserDetailed' : 'UserLite' }, diff --git a/packages/backend/src/core/UserService.ts b/packages/backend/src/core/UserService.ts index 34c343712c..4e04183523 100644 --- a/packages/backend/src/core/UserService.ts +++ b/packages/backend/src/core/UserService.ts @@ -10,7 +10,7 @@ import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; import { SystemWebhookService } from '@/core/SystemWebhookService.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; -import { CacheService } from '@/core/CacheService.js'; +import { CollapsedQueueService } from '@/core/CollapsedQueueService.js'; import { TimeService } from '@/global/TimeService.js'; @Injectable() @@ -22,43 +22,14 @@ export class UserService { private followingsRepository: FollowingsRepository, private systemWebhookService: SystemWebhookService, private userEntityService: UserEntityService, - private readonly cacheService: CacheService, + private readonly collapsedQueueService: CollapsedQueueService, private readonly timeService: TimeService, ) { } @bindThis public async updateLastActiveDate(user: MiUser): Promise { - if (user.isHibernated) { - const result = await this.usersRepository.createQueryBuilder().update() - .set({ - lastActiveDate: this.timeService.date, - }) - .where('id = :id', { id: user.id }) - .returning('*') - .execute() - .then((response) => { - return response.raw[0]; - }); - const wokeUp = result.isHibernated; - if (wokeUp) { - await Promise.all([ - this.usersRepository.update(user.id, { - isHibernated: false, - }), - this.followingsRepository.update({ - followerId: user.id, - }, { - isFollowerHibernated: false, - }), - this.cacheService.hibernatedUserCache.set(user.id, false), - ]); - } - } else { - this.usersRepository.update(user.id, { - lastActiveDate: this.timeService.date, - }); - } + await this.collapsedQueueService.updateUserQueue.enqueue(user.id, { lastActiveDate: this.timeService.date }); } /** @@ -70,6 +41,6 @@ export class UserService { @bindThis public async notifySystemWebhook(user: MiUser, type: 'userCreated') { const packedUser = await this.userEntityService.pack(user, null, { schema: 'UserLite' }); - return this.systemWebhookService.enqueueSystemWebhook(type, packedUser); + return await this.systemWebhookService.enqueueSystemWebhook(type, packedUser); } } diff --git a/packages/backend/src/core/UserSuspendService.ts b/packages/backend/src/core/UserSuspendService.ts index 4e42c24383..395657434c 100644 --- a/packages/backend/src/core/UserSuspendService.ts +++ b/packages/backend/src/core/UserSuspendService.ts @@ -17,16 +17,10 @@ import { RelationshipJobData } from '@/queue/types.js'; import { ModerationLogService } from '@/core/ModerationLogService.js'; import { isSystemAccount } from '@/misc/is-system-account.js'; import { CacheService } from '@/core/CacheService.js'; -import { LoggerService } from '@/core/LoggerService.js'; -import type Logger from '@/logger.js'; -import { renderInlineError } from '@/misc/render-inline-error.js'; -import { trackPromise } from '@/misc/promise-tracker.js'; import { InternalEventService } from '@/global/InternalEventService.js'; @Injectable() export class UserSuspendService { - private readonly logger: Logger; - constructor( @Inject(DI.usersRepository) private usersRepository: UsersRepository, @@ -47,11 +41,7 @@ export class UserSuspendService { private moderationLogService: ModerationLogService, private readonly cacheService: CacheService, private readonly internalEventService: InternalEventService, - - loggerService: LoggerService, - ) { - this.logger = loggerService.getLogger('user-suspend'); - } + ) {} @bindThis public async suspend(user: MiUser, moderator: MiUser): Promise { @@ -69,10 +59,7 @@ export class UserSuspendService { userHost: user.host, }); - trackPromise((async () => { - await this.postSuspend(user); - await this.freezeAll(user); - })().catch(e => this.logger.error(`Error suspending user ${user.id}: ${renderInlineError(e)}`))); + await this.queueService.createPostSuspendJob(user.id); } @bindThis @@ -89,14 +76,11 @@ export class UserSuspendService { userHost: user.host, }); - trackPromise((async () => { - await this.postUnsuspend(user); - await this.unFreezeAll(user); - })().catch(e => this.logger.error(`Error un-suspending for user ${user.id}: ${renderInlineError(e)}`))); + await this.queueService.createPostUnsuspendJob(user.id); } @bindThis - private async postSuspend(user: { id: MiUser['id']; host: MiUser['host'] }): Promise { + public async postSuspend(user: MiUser): Promise { this.globalEventService.publishInternalEvent('userChangeSuspendedState', { id: user.id, isSuspended: true }); /* @@ -132,10 +116,12 @@ export class UserSuspendService { await this.queueService.deliverMany(user, content, queue); } + + await this.freezeAll(user); } @bindThis - private async postUnsuspend(user: MiUser): Promise { + public async postUnsuspend(user: MiUser): Promise { this.globalEventService.publishInternalEvent('userChangeSuspendedState', { id: user.id, isSuspended: false }); if (this.userEntityService.isLocalUser(user)) { @@ -162,6 +148,8 @@ export class UserSuspendService { await this.queueService.deliverMany(user, content, queue); } + + await this.unFreezeAll(user); } @bindThis diff --git a/packages/backend/src/core/UserWebhookService.ts b/packages/backend/src/core/UserWebhookService.ts index 24a519bc5c..097482d6a5 100644 --- a/packages/backend/src/core/UserWebhookService.ts +++ b/packages/backend/src/core/UserWebhookService.ts @@ -94,7 +94,7 @@ export class UserWebhookService implements OnApplicationShutdown { ) { const webhooks = await this.getActiveWebhooks() .then(webhooks => webhooks.filter(webhook => webhook.userId === userId && webhook.on.includes(type))); - return Promise.all( + return await Promise.all( webhooks.map(webhook => { return this.queueService.userWebhookDeliver(webhook, type, content); }), diff --git a/packages/backend/src/core/activitypub/ApAudienceService.ts b/packages/backend/src/core/activitypub/ApAudienceService.ts index 15c4546063..e6c00aa5e7 100644 --- a/packages/backend/src/core/activitypub/ApAudienceService.ts +++ b/packages/backend/src/core/activitypub/ApAudienceService.ts @@ -8,6 +8,7 @@ import promiseLimit from 'promise-limit'; import type { MiRemoteUser, MiUser } from '@/models/User.js'; import { concat, unique } from '@/misc/prelude/array.js'; import { bindThis } from '@/decorators.js'; +import { promiseMap } from '@/misc/promise-map.js'; import { getApIds } from './type.js'; import { ApPersonService } from './models/ApPersonService.js'; import type { ApObject } from './type.js'; @@ -37,10 +38,12 @@ export class ApAudienceService { const others = unique(concat([toGroups.other, ccGroups.other])); - const limit = promiseLimit(2); - const mentionedUsers = (await Promise.all( - others.map(id => limit(() => this.apPersonService.resolvePerson(id, resolver).catch(() => null))), - )).filter(x => x != null); + const resolved = await promiseMap(others, async x => { + return await this.apPersonService.resolvePerson(x, resolver).catch(() => null) as MiUser | null; + }, { + limit: 2, + }); + const mentionedUsers = resolved.filter(x => x != null); // If no audience is specified, then assume public if ( diff --git a/packages/backend/src/core/activitypub/ApDbResolverService.ts b/packages/backend/src/core/activitypub/ApDbResolverService.ts index a5259b1c4a..aba526b19b 100644 --- a/packages/backend/src/core/activitypub/ApDbResolverService.ts +++ b/packages/backend/src/core/activitypub/ApDbResolverService.ts @@ -92,10 +92,9 @@ export class ApDbResolverService implements OnApplicationShutdown { key: MiUserPublickey; } | null> { const key = await this.apPersonService.findPublicKeyByKeyId(keyId); - if (key == null) return null; - const user = await this.cacheService.findUserById(key.userId).catch(() => null) as MiRemoteUser | null; + const user = await this.cacheService.findOptionalRemoteUserById(key.userId); if (user == null) return null; if (user.isDeleted) return null; diff --git a/packages/backend/src/core/activitypub/ApInboxService.ts b/packages/backend/src/core/activitypub/ApInboxService.ts index 91eba793d6..3cd742533f 100644 --- a/packages/backend/src/core/activitypub/ApInboxService.ts +++ b/packages/backend/src/core/activitypub/ApInboxService.ts @@ -33,10 +33,6 @@ import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; import { fromTuple } from '@/misc/from-tuple.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; import { renderInlineError } from '@/misc/render-inline-error.js'; -import InstanceChart from '@/core/chart/charts/instance.js'; -import FederationChart from '@/core/chart/charts/federation.js'; -import { FetchInstanceMetadataService } from '@/core/FetchInstanceMetadataService.js'; -import { UpdateInstanceQueue } from '@/core/UpdateInstanceQueue.js'; import { CacheService } from '@/core/CacheService.js'; import { NoteVisibilityService } from '@/core/NoteVisibilityService.js'; import { TimeService } from '@/global/TimeService.js'; @@ -97,10 +93,6 @@ export class ApInboxService { private queueService: QueueService, private globalEventService: GlobalEventService, private readonly federatedInstanceService: FederatedInstanceService, - private readonly fetchInstanceMetadataService: FetchInstanceMetadataService, - private readonly instanceChart: InstanceChart, - private readonly federationChart: FederationChart, - private readonly updateInstanceQueue: UpdateInstanceQueue, private readonly cacheService: CacheService, private readonly noteVisibilityService: NoteVisibilityService, private readonly timeService: TimeService, @@ -115,7 +107,7 @@ export class ApInboxService { const results = [] as [string, string | void][]; resolver ??= this.apResolverService.createResolver(); - const items = await resolver.resolveCollectionItems(activity); + const items = await resolver.resolveCollectionItems(activity, true, getNullableApId(activity) ?? undefined); for (let i = 0; i < items.length; i++) { const act = items[i]; if (act.id != null) { @@ -153,11 +145,10 @@ export class ApInboxService { // ついでにリモートユーザーの情報が古かったら更新しておく if (actor.uri) { if (actor.lastFetchedAt == null || this.timeService.now - actor.lastFetchedAt.getTime() > 1000 * 60 * 60 * 24) { - setImmediate(() => { + { // 同一ユーザーの情報を再度処理するので、使用済みのresolverを再利用してはいけない - this.apPersonService.updatePerson(actor.uri) - .catch(err => this.logger.error(`Failed to update person: ${renderInlineError(err)}`)); - }); + await this.apPersonService.updatePersonLazy(actor); + } } } return result; @@ -424,42 +415,14 @@ export class ApInboxService { } // Update stats (adapted from InboxProcessorService) - this.federationChart.inbox(actor.host).then(); - process.nextTick(async () => { - const i = await (this.meta.enableStatsForFederatedInstances - ? this.federatedInstanceService.fetchOrRegister(actor.host) - : this.federatedInstanceService.fetch(actor.host)); - - if (i == null) return; - - this.updateInstanceQueue.enqueue(i.id, { - latestRequestReceivedAt: this.timeService.date, - shouldUnsuspend: i.suspensionState === 'autoSuspendedForNotResponding', - }); - - if (this.meta.enableChartsForFederatedInstances) { - this.instanceChart.requestReceived(i.host).then(); - } - - this.fetchInstanceMetadataService.fetchInstanceMetadata(i).then(); - }); + await this.queueService.createPostInboxJob(actor.host); // Process it! - return await this.performOneActivity(actor, activity, resolver) - .finally(() => { - // Update user (adapted from performActivity) - if (actor.lastFetchedAt == null || this.timeService.now - actor.lastFetchedAt.getTime() > 1000 * 60 * 60 * 24) { - setImmediate(() => { - // Don't re-use the resolver, or it may throw recursion errors. - // Instead, create a new resolver with an appropriately-reduced recursion limit. - const subResolver = this.apResolverService.createResolver({ - recursionLimit: resolver.getRecursionLimit() - resolver.getHistory().length, - }); - this.apPersonService.updatePerson(actor.uri, subResolver) - .catch(err => this.logger.error(`Failed to update person: ${renderInlineError(err)}`)); - }); - } - }); + try { + return await this.performOneActivity(actor, activity, resolver); + } finally { + await this.apPersonService.updatePersonLazy(actor); + } } @bindThis diff --git a/packages/backend/src/core/activitypub/ApResolverService.ts b/packages/backend/src/core/activitypub/ApResolverService.ts index f6178ace1a..2db7208967 100644 --- a/packages/backend/src/core/activitypub/ApResolverService.ts +++ b/packages/backend/src/core/activitypub/ApResolverService.ts @@ -5,7 +5,6 @@ import { Inject, Injectable } from '@nestjs/common'; import { IsNull, Not } from 'typeorm'; -import promiseLimit from 'promise-limit'; import type { MiLocalUser, MiRemoteUser } from '@/models/User.js'; import type { NotesRepository, PollsRepository, NoteReactionsRepository, UsersRepository, FollowRequestsRepository, MiMeta, SkApFetchLog } from '@/models/_.js'; import type { Config } from '@/config.js'; @@ -23,6 +22,9 @@ import { IdentifiableError } from '@/misc/identifiable-error.js'; import { toArray } from '@/misc/prelude/array.js'; import { isPureRenote } from '@/misc/is-renote.js'; import { CacheService } from '@/core/CacheService.js'; +import { promiseMap } from '@/misc/promise-map.js'; +import { trackPromise } from '@/misc/promise-tracker.js'; +import { renderInlineError } from '@/misc/render-inline-error.js'; import { AnyCollection, getApId, getNullableApId, IObjectWithId, isCollection, isCollectionOrOrderedCollection, isCollectionPage, isOrderedCollection, isOrderedCollectionPage } from './type.js'; import { ApDbResolverService } from './ApDbResolverService.js'; import { ApRendererService } from './ApRendererService.js'; @@ -68,27 +70,21 @@ export class Resolver { return this.recursionLimit; } - public async resolveCollection(value: string | IObjectWithId, allowAnonymous?: boolean, sentFromUri?: string): Promise; - public async resolveCollection(value: string | IObject, allowAnonymous: boolean | undefined, sentFromUri: string): Promise; - public async resolveCollection(value: string | IObject, allowAnonymous?: boolean, sentFromUri?: string): Promise; @bindThis public async resolveCollection(value: string | IObject, allowAnonymous?: boolean, sentFromUri?: string): Promise { - const collection = typeof value === 'string' - ? sentFromUri - ? await this.secureResolve(value, sentFromUri, allowAnonymous) - : await this.resolve(value, allowAnonymous) - : value; // TODO try and remove this eventually, as it's a major security foot-gun + const collection = sentFromUri + ? await this.secureResolve(value, sentFromUri, allowAnonymous) + : allowAnonymous + ? await this.resolveAnonymous(value) + : await this.resolve(value, allowAnonymous); if (isCollectionOrOrderedCollection(collection)) { return collection; } else { - throw new IdentifiableError('f100eccf-f347-43fb-9b45-96a0831fb635', `collection ${getApId(value)} has unsupported type: ${collection.type}`); + throw new IdentifiableError('f100eccf-f347-43fb-9b45-96a0831fb635', `collection ${getNullableApId(value)} has unsupported type: ${collection.type}`); } } - public async resolveCollectionItems(collection: IAnonymousObject, limit?: number | null, allowAnonymousItems?: true, concurrency?: number): Promise; - public async resolveCollectionItems(collection: string | IObjectWithId, limit?: number | null, allowAnonymousItems?: boolean, concurrency?: number): Promise; - public async resolveCollectionItems(collection: string | IObject, limit?: number | null, allowAnonymousItems?: boolean, concurrency?: number): Promise; /** * Recursively resolves items from a collection. * Stops when reaching the resolution limit or an optional item limit - whichever is lower. @@ -96,11 +92,13 @@ export class Resolver { * Malformed collections (mixing Ordered and un-Ordered types) are also supported. * @param collection Collection to resolve from - can be a URL or object of any supported collection type. * @param limit Maximum number of items to resolve. If null or undefined (default), then items will be resolved until reaching the recursion limit. - * @param allowAnonymousItems If true, collection items can be anonymous (lack an ID). If false (default), then an error is thrown when reaching an item without ID. + * @param allowAnonymous If true, collection items can be anonymous (lack an ID). If false (default), then an error is thrown when reaching an item without ID. + * @param sentFromUri If collection is an object, this is the URI where it was sent from. * @param concurrency Maximum number of items to resolve at once. (default: 4) + * @param ignoreErrors If true (default), inaccessible items will be skipped instead of causing an exception. Inaccessible collections will still throw. */ @bindThis - public async resolveCollectionItems(collection: string | IObject, limit?: number | null, allowAnonymousItems?: boolean, concurrency = 4): Promise { + public async resolveCollectionItems(collection: string | IObject, allowAnonymous = false, sentFromUri?: string, limit?: number | null, concurrency = 4, ignoreErrors = true): Promise { const resolvedItems: IObject[] = []; // This is pulled up to avoid code duplication below @@ -108,11 +106,10 @@ export class Resolver { const sentFrom = current.id; const itemArr = toArray(items); const itemLimit = limit ?? Number.MAX_SAFE_INTEGER; - const allowAnonymous = allowAnonymousItems ?? false; - await this.resolveItemArray(itemArr, sentFrom, itemLimit, concurrency, allowAnonymous, resolvedItems); + await this.resolveItemArray(itemArr, sentFrom, itemLimit, concurrency, allowAnonymous, resolvedItems, ignoreErrors); }; - let current: AnyCollection | null = await this.resolveCollection(collection); + let current: AnyCollection | null = await this.resolveCollection(collection, allowAnonymous, sentFromUri); do { // Iterate all items in the current page if (current.items) { @@ -130,10 +127,10 @@ export class Resolver { current = null; } else if (isCollection(current) || isOrderedCollection(current)) { // Continue to first page - current = current.first ? await this.resolveCollection(current.first, true, current.id) : null; + current = current.first ? await this.resolveCollection(current.first, allowAnonymous, current.id) : null; } else if (isCollectionPage(current) || isOrderedCollectionPage(current)) { // Continue to next page - current = current.next ? await this.resolveCollection(current.next, true, current.id) : null; + current = current.next ? await this.resolveCollection(current.next, allowAnonymous, current.id) : null; } else { // Stop in all other conditions current = null; @@ -143,17 +140,12 @@ export class Resolver { return resolvedItems; } - private async resolveItemArray(source: (string | IObject)[], sentFrom: undefined, itemLimit: number, concurrency: number, allowAnonymousItems: true, destination: IAnonymousObject[]): Promise; - private async resolveItemArray(source: (string | IObject)[], sentFrom: string, itemLimit: number, concurrency: number, allowAnonymousItems: boolean, destination: IObjectWithId[]): Promise; - private async resolveItemArray(source: (string | IObject)[], sentFrom: string | undefined, itemLimit: number, concurrency: number, allowAnonymousItems: boolean, destination: IObject[]): Promise; - private async resolveItemArray(source: (string | IObject)[], sentFrom: string | undefined, itemLimit: number, concurrency: number, allowAnonymousItems: boolean, destination: IObject[]): Promise { + private async resolveItemArray(source: (string | IObject)[], sentFrom: string | undefined, itemLimit: number, concurrency: number, allowAnonymousItems: boolean, destination: IObject[], ignoreErrors?: boolean): Promise { const recursionLimit = this.recursionLimit - this.history.size; const batchLimit = Math.min(source.length, recursionLimit, itemLimit); - const limiter = promiseLimit(concurrency); - const batch = await Promise.all(source - .slice(0, batchLimit) - .map(item => limiter(async () => { + const batch = await promiseMap(source.slice(0, batchLimit), async item => { + try { if (sentFrom) { // Use secureResolve to avoid re-fetching items that were included inline. return await this.secureResolve(item, sentFrom, allowAnonymousItems); @@ -164,9 +156,22 @@ export class Resolver { const id = getApId(item); return await this.resolve(id); } - }))); + } catch (err) { + if (ignoreErrors) { + this.logger.warn(`Ignoring error in collection item ${getNullableApId(item)}: ${renderInlineError(err)}`); + return null; + } else { + throw err; + } + } + }, { + limit: concurrency, + }); - destination.push(...batch); + // Items will be null if a request fails and ignoreErrors is true + const batchItems = batch.filter(item => item != null); + + destination.push(...batchItems); }; /** @@ -269,8 +274,8 @@ export class Resolver { log.duration = calculateDurationSince(startTime); // Save or finalize asynchronously - this.apLogService.saveFetchLog(log) - .catch(err => this.logger.error('Failed to record AP object fetch:', err)); + trackPromise(this.apLogService.saveFetchLog(log) + .catch(err => this.logger.error('Failed to record AP object fetch:', err))); } } diff --git a/packages/backend/src/core/activitypub/JsonLdService.ts b/packages/backend/src/core/activitypub/JsonLdService.ts index 8e14e0909f..194af63f5a 100644 --- a/packages/backend/src/core/activitypub/JsonLdService.ts +++ b/packages/backend/src/core/activitypub/JsonLdService.ts @@ -134,7 +134,7 @@ export class JsonLdService { const customLoader = this.getLoader(); // XXX: Importing jsonld dynamically since Jest frequently fails to import it statically // https://github.com/misskey-dev/misskey/pull/9894#discussion_r1103753595 - return (await import('jsonld')).default.compact(data, context, { + return await (await import('jsonld')).default.compact(data, context, { documentLoader: customLoader, }); } @@ -142,7 +142,7 @@ export class JsonLdService { @bindThis public async normalize(data: Document): Promise { const customLoader = this.getLoader(); - return (await import('jsonld')).default.normalize(data, { + return await (await import('jsonld')).default.normalize(data, { documentLoader: customLoader, }); } diff --git a/packages/backend/src/core/activitypub/models/ApMentionService.ts b/packages/backend/src/core/activitypub/models/ApMentionService.ts index 2cd151fa04..1aab5894d3 100644 --- a/packages/backend/src/core/activitypub/models/ApMentionService.ts +++ b/packages/backend/src/core/activitypub/models/ApMentionService.ts @@ -12,6 +12,7 @@ import { isMention } from '../type.js'; import { Resolver } from '../ApResolverService.js'; import { ApPersonService } from './ApPersonService.js'; import type { IObject, IApMention } from '../type.js'; +import { promiseMap } from '@/misc/promise-map.js'; @Injectable() export class ApMentionService { @@ -24,12 +25,13 @@ export class ApMentionService { public async extractApMentions(tags: IObject | IObject[] | null | undefined, resolver: Resolver): Promise { const hrefs = unique(this.extractApMentionObjects(tags).map(x => x.href)); - const limit = promiseLimit(2); - const mentionedUsers = (await Promise.all( - hrefs.map(x => limit(() => this.apPersonService.resolvePerson(x, resolver).catch(() => null))), - )).filter(x => x != null); + const mentionedUsers = await promiseMap(hrefs, async x => { + return await this.apPersonService.resolvePerson(x, resolver).catch(() => null) as MiUser | null; + }, { + limit: 2, + }); - return mentionedUsers; + return mentionedUsers.filter(resolved => resolved != null); } @bindThis diff --git a/packages/backend/src/core/activitypub/models/ApNoteService.ts b/packages/backend/src/core/activitypub/models/ApNoteService.ts index 0eaf26de54..e1c71c3b6d 100644 --- a/packages/backend/src/core/activitypub/models/ApNoteService.ts +++ b/packages/backend/src/core/activitypub/models/ApNoteService.ts @@ -6,7 +6,6 @@ import { Inject, Injectable, OnModuleInit } from '@nestjs/common'; import { In } from 'typeorm'; import { UnrecoverableError } from 'bullmq'; -import promiseLimit from 'promise-limit'; import { ModuleRef } from '@nestjs/core'; import { DI } from '@/di-symbols.js'; import type { UsersRepository, PollsRepository, EmojisRepository, NotesRepository, MiMeta } from '@/models/_.js'; @@ -32,6 +31,7 @@ import { renderInlineError } from '@/misc/render-inline-error.js'; import { extractMediaFromHtml } from '@/core/activitypub/misc/extract-media-from-html.js'; import { extractMediaFromMfm } from '@/core/activitypub/misc/extract-media-from-mfm.js'; import { getContentByType } from '@/core/activitypub/misc/get-content-by-type.js'; +import { promiseMap } from '@/misc/promise-map.js'; import { trackPromise } from '@/misc/promise-tracker.js'; import { CustomEmojiService, encodeEmojiKey, isValidEmojiName } from '@/core/CustomEmojiService.js'; import { TimeService } from '@/global/TimeService.js'; @@ -277,7 +277,7 @@ export class ApNoteService implements OnModuleInit { return x; }) - .catch(async err => { + .catch(err => { this.logger.warn(`error ${renderInlineError(err)} fetching inReplyTo ${note.inReplyTo} for note ${entryUri}`); throw new IdentifiableError('1ebf0a96-2769-4973-a6c2-3dcbad409dff', `failed to create note ${entryUri}: could not fetch inReplyTo ${note.inReplyTo}`, true, err); }) @@ -456,7 +456,7 @@ export class ApNoteService implements OnModuleInit { return x; }) - .catch(async err => { + .catch(err => { this.logger.warn(`error ${renderInlineError(err)} fetching inReplyTo ${note.inReplyTo} for note ${entryUri}`); throw new IdentifiableError('1ebf0a96-2769-4973-a6c2-3dcbad409dff', `failed to update note ${entryUri}: could not fetch inReplyTo ${note.inReplyTo}`, true, err); }) @@ -583,8 +583,8 @@ export class ApNoteService implements OnModuleInit { const emojiKeys = eomjiTags.map(tag => encodeEmojiKey({ name: tag.name, host })); const existingEmojis = await this.customEmojiService.emojisByKeyCache.fetchMany(emojiKeys); - return await Promise.all(eomjiTags.map(async tag => { - const name = tag.name; + return await promiseMap(eomjiTags, async tag => { + const name = tag.name.replaceAll(':', ''); tag.icon = toSingle(tag.icon); const exists = existingEmojis.values.find(x => x.name === name); @@ -627,7 +627,9 @@ export class ApNoteService implements OnModuleInit { // _misskey_license が存在しなければ `null` license: (tag._misskey_license?.freeText ?? null), }); - })); + }, { + limit: 4, + }); } /** @@ -691,7 +693,7 @@ export class ApNoteService implements OnModuleInit { } }; - const results = await Promise.all(Array.from(quoteUris).map(u => resolveQuote(u))); + const results = await promiseMap(quoteUris, async u => resolveQuote(u), { limit: 2 }); // Success - return the quote const quote = results.find(r => typeof(r) === 'object'); @@ -753,14 +755,10 @@ export class ApNoteService implements OnModuleInit { // Resolve all files w/ concurrency 2. // This prevents one big file from blocking the others. - const limiter = promiseLimit(2); - const results = await Promise - .all(Array - .from(attachments.values()) - .map(attach => limiter(async () => { - attach.sensitive ??= note.sensitive; - return await this.resolveImage(actor, attach); - }))); + const results = await promiseMap(attachments.values(), async attach => { + attach.sensitive ??= note.sensitive; + return await this.resolveImage(actor, attach); + }, { limit: 2 }); // Process results let hasFileError = false; diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index 4ce0e8db3f..0ee4a75e2a 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -24,7 +24,6 @@ import type { MiNote } from '@/models/Note.js'; import { IdService } from '@/core/IdService.js'; import type { MfmService } from '@/core/MfmService.js'; import { toArray } from '@/misc/prelude/array.js'; -import type { GlobalEventService } from '@/core/GlobalEventService.js'; import type { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; import type { FetchInstanceMetadataService } from '@/core/FetchInstanceMetadataService.js'; import { MiUserProfile } from '@/models/UserProfile.js'; @@ -45,9 +44,12 @@ import { TimeService } from '@/global/TimeService.js'; import { verifyFieldLinks } from '@/misc/verify-field-link.js'; import { isRetryableError } from '@/misc/is-retryable-error.js'; import { renderInlineError } from '@/misc/render-inline-error.js'; -import { IdentifiableError } from '@/misc/identifiable-error.js'; +import { errorCodes, IdentifiableError } from '@/misc/identifiable-error.js'; import { QueueService } from '@/core/QueueService.js'; -import { getApId, getApType, isActor, isCollection, isCollectionOrOrderedCollection, isPropertyValue } from '../type.js'; +import { InternalEventService } from '@/global/InternalEventService.js'; +import { CollapsedQueueService } from '@/core/CollapsedQueueService.js'; +import { promiseMap } from '@/misc/promise-map.js'; +import { getApId, getApType, getNullableApId, isActor, isPost, isPropertyValue } from '../type.js'; import { ApLoggerService } from '../ApLoggerService.js'; import { extractApHashtags } from './tag.js'; import type { OnModuleInit } from '@nestjs/common'; @@ -72,7 +74,6 @@ export class ApPersonService implements OnModuleInit { private readonly publicKeyByUserIdCache: ManagedQuantumKVCache; private driveFileEntityService: DriveFileEntityService; - private globalEventService: GlobalEventService; private federatedInstanceService: FederatedInstanceService; private fetchInstanceMetadataService: FetchInstanceMetadataService; private cacheService: CacheService; @@ -86,6 +87,7 @@ export class ApPersonService implements OnModuleInit { private instanceChart: InstanceChart; private accountMoveService: AccountMoveService; private logger: Logger; + private idService: IdService; constructor( private moduleRef: ModuleRef, @@ -120,9 +122,10 @@ export class ApPersonService implements OnModuleInit { private readonly cacheManagementService: CacheManagementService, private readonly utilityService: UtilityService, private readonly apUtilityService: ApUtilityService, - private readonly idService: IdService, private readonly timeService: TimeService, private readonly queueService: QueueService, + private readonly collapsedQueueService: CollapsedQueueService, + private readonly internalEventService: InternalEventService, apLoggerService: ApLoggerService, ) { @@ -181,7 +184,6 @@ export class ApPersonService implements OnModuleInit { @bindThis onModuleInit(): void { this.driveFileEntityService = this.moduleRef.get('DriveFileEntityService'); - this.globalEventService = this.moduleRef.get('GlobalEventService'); this.federatedInstanceService = this.moduleRef.get('FederatedInstanceService'); this.fetchInstanceMetadataService = this.moduleRef.get('FetchInstanceMetadataService'); this.cacheService = this.moduleRef.get('CacheService'); @@ -194,6 +196,7 @@ export class ApPersonService implements OnModuleInit { this.usersChart = this.moduleRef.get('UsersChart'); this.instanceChart = this.moduleRef.get('InstanceChart'); this.accountMoveService = this.moduleRef.get('AccountMoveService'); + this.idService = this.moduleRef.get('IdService'); } /** @@ -301,14 +304,14 @@ export class ApPersonService implements OnModuleInit { withSuspended: opts?.withSuspended ?? true, }; - let userId; + let userId: string | null | undefined; // Resolve URI -> User ID const parsed = this.utilityService.parseUri(uri); if (parsed.local) { userId = parsed.type === 'users' ? parsed.id : null; } else { - userId = await this.uriPersonCache.fetch(uri).catch(() => null); + userId = await this.uriPersonCache.fetchMaybe(uri); } // No match @@ -316,8 +319,7 @@ export class ApPersonService implements OnModuleInit { return null; } - const user = await this.cacheService.findUserById(userId) - .catch(() => null) as MiLocalUser | MiRemoteUser | null; + const user = await this.cacheService.findOptionalUserById(userId) as MiLocalUser | MiRemoteUser | null; if (user?.isDeleted && !_opts.withDeleted) { return null; @@ -329,8 +331,9 @@ export class ApPersonService implements OnModuleInit { return user; } + // TODO fix these "any" types private async resolveAvatarAndBanner(user: MiRemoteUser, icon: any, image: any, bgimg: any): Promise>> { - const [avatar, banner, background] = await Promise.all([icon, image, bgimg].map(img => { + const [avatar, banner, background] = await Promise.all([icon, image, bgimg].map(async img => { // icon and image may be arrays // see https://www.w3.org/TR/activitystreams-vocabulary/#dfn-icon if (Array.isArray(img)) { @@ -343,7 +346,7 @@ export class ApPersonService implements OnModuleInit { return { id: null, url: null, blurhash: null }; } - return this.apImageService.resolveImage(user, img).catch(() => null); + return await this.apImageService.resolveImage(user, img).catch(() => null); })); if (((avatar != null && avatar.id != null) || (banner != null && banner.id != null)) @@ -574,28 +577,23 @@ export class ApPersonService implements OnModuleInit { // Register host if (this.meta.enableStatsForFederatedInstances) { - this.federatedInstanceService.fetchOrRegister(host).then(i => { - this.instancesRepository.increment({ id: i.id }, 'usersCount', 1); + this.federatedInstanceService.fetchOrRegister(host).then(async i => { + await this.collapsedQueueService.updateInstanceQueue.enqueue(i.id, { usersCountDelta: 1 }); if (this.meta.enableChartsForFederatedInstances) { this.instanceChart.newUser(i.host); } - this.fetchInstanceMetadataService.fetchInstanceMetadata(i); + await this.fetchInstanceMetadataService.fetchInstanceMetadataLazy(i); }); } this.usersChart.update(user, true); - // ハッシュタグ更新 - this.hashtagService.updateUsertags(user, tags); - //#region アバターとヘッダー画像をフェッチ try { const updates = await this.resolveAvatarAndBanner(user, person.icon, person.image, person.backgroundUrl); await this.usersRepository.update(user.id, updates); + await this.internalEventService.emit('remoteUserUpdated', { id: user.id }); user = { ...user, ...updates }; - - // Register to the cache - await this.uriPersonCache.set(user.uri, user.id); } catch (err) { // Permanent error implies hidden or inaccessible, which is a normal thing. if (isRetryableError(err)) { @@ -604,16 +602,29 @@ export class ApPersonService implements OnModuleInit { } //#endregion - await this.updateFeatured(user.id, resolver).catch(err => { - // Permanent error implies hidden or inaccessible, which is a normal thing. - if (isRetryableError(err)) { - this.logger.error(`Error updating featured notes: ${renderInlineError(err)}`); - } - }); + // ハッシュタグ更新 + await this.queueService.createUpdateUserTagsJob(user.id); + + await this.updateFeaturedLazy(user); return user; } + /** + * Schedules a deferred update on the background task worker. + * Duplicate updates are automatically skipped. + */ + @bindThis + public async updatePersonLazy(uriOrUser: string | MiUser): Promise { + const user = typeof(uriOrUser) === 'string' + ? await this.fetchPerson(uriOrUser) + : uriOrUser; + + if (user && user.host != null) { + await this.queueService.createUpdateUserJob(user.id); + } + } + /** * Personの情報を更新します。 * Misskeyに対象のPersonが登録されていなければ無視します。 @@ -688,13 +699,16 @@ export class ApPersonService implements OnModuleInit { const profileUrls = url ? [url, person.id] : [person.id]; const verifiedLinks = await verifyFieldLinks(fields, profileUrls, this.httpRequestService); + const featuredUri = person.featured ? getApId(person.featured) : undefined; const updates = { lastFetchedAt: this.timeService.date, inbox: person.inbox, sharedInbox: person.sharedInbox ?? person.endpoints?.sharedInbox ?? null, followersUri: person.followers ? getApId(person.followers) : undefined, - featured: person.featured ? getApId(person.featured) : undefined, + // If the featured collection changes, then reset the fetch timeout. + lastFetchedFeaturedAt: featuredUri !== exist.featured ? null : undefined, + featured: featuredUri, emojis: emojiNames, name: truncate(person.name, nameLength), tags, @@ -751,9 +765,15 @@ export class ApPersonService implements OnModuleInit { return `skip: user ${exist.id} is deleted`; } + // Notify event ASAP + await this.internalEventService.emit('remoteUserUpdated', { id: exist.id }); + + // Do not use "exist" after this point!! + const updated = { ...exist, ...updates }; + if (person.publicKey) { const publicKey = new MiUserPublickey({ - userId: exist.id, + userId: updated.id, keyId: person.publicKey.id, keyPem: person.publicKey.publicKeyPem, }); @@ -767,7 +787,7 @@ export class ApPersonService implements OnModuleInit { this.publicKeyByUserIdCache.set(publicKey.userId, publicKey), ]); } else { - const existingPublicKey = await this.userPublickeysRepository.findOneBy({ userId: exist.id }); + const existingPublicKey = await this.userPublickeysRepository.findOneBy({ userId: updated.id }); if (existingPublicKey) { // Delete key await Promise.all([ @@ -786,7 +806,7 @@ export class ApPersonService implements OnModuleInit { _description = this.apMfmService.htmlToMfm(truncate(person.summary, this.config.maxRemoteBioLength), person.tag); } - await this.userProfilesRepository.update({ userId: exist.id }, { + await this.userProfilesRepository.update({ userId: updated.id }, { url, fields, verifiedLinks, @@ -798,33 +818,25 @@ export class ApPersonService implements OnModuleInit { location: person['vcard:Address'] ?? null, listenbrainz: person.listenbrainz ?? null, }); - - this.globalEventService.publishInternalEvent('remoteUserUpdated', { id: exist.id }); - - // ハッシュタグ更新 - this.hashtagService.updateUsertags(exist, tags); + await this.cacheService.userProfileCache.delete(updated.id); // 該当ユーザーが既にフォロワーになっていた場合はFollowingもアップデートする - if (exist.inbox !== person.inbox || exist.sharedInbox !== (person.sharedInbox ?? person.endpoints?.sharedInbox)) { + if (updated.inbox !== person.inbox || updated.sharedInbox !== (person.sharedInbox ?? person.endpoints?.sharedInbox)) { await this.followingsRepository.update( - { followerId: exist.id }, + { followerId: updated.id }, { followerInbox: person.inbox, followerSharedInbox: person.sharedInbox ?? person.endpoints?.sharedInbox ?? null, }, ); - await this.cacheService.refreshFollowRelationsFor(exist.id); + await this.cacheService.refreshFollowRelationsFor(updated.id); } - await this.updateFeatured(exist.id, resolver).catch(err => { - // Permanent error implies hidden or inaccessible, which is a normal thing. - if (isRetryableError(err)) { - this.logger.error(`Error updating featured notes: ${renderInlineError(err)}`); - } - }); + // ハッシュタグ更新 + await this.queueService.createUpdateUserTagsJob(updated.id); - const updated = { ...exist, ...updates }; + await this.updateFeaturedLazy(updated); // 移行処理を行う if (updated.movedAt && ( @@ -902,43 +914,71 @@ export class ApPersonService implements OnModuleInit { return fields; } + /** + * Schedules a deferred update on the background task worker. + * Duplicate updates are automatically skipped. + */ @bindThis - public async updateFeatured(userId: MiUser['id'], resolver?: Resolver): Promise { - const user = await this.usersRepository.findOneByOrFail({ id: userId, isDeleted: false }); - if (!isRemoteUser(user)) return; - if (!user.featured) return; + public async updateFeaturedLazy(userOrId: MiRemoteUser | MiUser['id']): Promise { + const userId = typeof(userOrId) === 'object' ? userOrId.id : userOrId; + const user = typeof(userOrId) === 'object' ? userOrId : await this.cacheService.findRemoteUserById(userId); - this.logger.info(`Updating the featured: ${user.uri}`); + if (user.isDeleted || user.isSuspended) { + this.logger.debug(`Not updating featured for ${userId}: user is deleted`); + return; + } - const _resolver = resolver ?? this.apResolverService.createResolver(); + if (!user.featured) { + this.logger.debug(`Not updating featured for ${userId}: no featured collection`); + return; + } - // Resolve to (Ordered)Collection Object - const collection = user.featured ? await _resolver.resolveCollection(user.featured, true, user.uri).catch(err => { - // Permanent error implies hidden or inaccessible, which is a normal thing. - if (isRetryableError(err)) { - this.logger.warn(`Failed to update featured notes: ${renderInlineError(err)}`); - } + await this.queueService.createUpdateFeaturedJob(userId); + } - return null; - }) : null; - if (!collection) return; + @bindThis + public async updateFeatured(userOrId: MiRemoteUser | MiUser['id'], resolver?: Resolver): Promise { + const userId = typeof(userOrId) === 'object' ? userOrId.id : userOrId; + const user = typeof(userOrId) === 'object' ? userOrId : await this.cacheService.findRemoteUserById(userId); - if (!isCollectionOrOrderedCollection(collection)) throw new UnrecoverableError(`failed to update user ${user.uri}: featured ${user.featured} is not Collection or OrderedCollection`); + if (user.isDeleted) throw new IdentifiableError(errorCodes.userIsDeleted, `Can't update featured for ${userId}: user is deleted`); + if (user.isSuspended) throw new IdentifiableError(errorCodes.userIsSuspended, `Can't update featured for ${userId}: user is suspended`); + if (!user.featured) throw new IdentifiableError(errorCodes.noFeaturedCollection, `Can't update featured for ${userId}: no featured collection`); - // Resolve to Object(may be Note) arrays - const unresolvedItems = isCollection(collection) ? collection.items : collection.orderedItems; - const items = await Promise.all(toArray(unresolvedItems).map(x => _resolver.resolve(x))); + this.logger.info(`Updating featured notes for: ${user.uri}`); + + resolver ??= this.apResolverService.createResolver(); + + // Mark as updated + await this.usersRepository.update({ id: userId }, { lastFetchedFeaturedAt: this.timeService.date }); + await this.internalEventService.emit('remoteUserUpdated', { id: userId }); // Resolve and regist Notes - const limit = promiseLimit(2); const maxPinned = (await this.roleService.getUserPolicies(user.id)).pinLimit; - const featuredNotes = await Promise.all(items - .filter(item => getApType(item) === 'Note') // TODO: Noteでなくてもいいかも - .slice(0, maxPinned) - .map(item => limit(() => this.apNoteService.resolveNote(item, { - resolver: _resolver, - sentFrom: user.uri, - })))); + const items = await resolver.resolveCollectionItems(user.featured, true, user.uri, maxPinned, 2); + const featuredNotes = await promiseMap(items, async item => { + const itemId = getNullableApId(item); + if (itemId && isPost(item)) { + try { + const note = await this.apNoteService.resolveNote(item, { + resolver: resolver, + sentFrom: itemId, // resolveCollectionItems has already verified this, so we can re-use it to avoid double fetch + }); + + if (note && note.userId !== user.id) { + this.logger.warn(`Ignoring cross-note pin: user ${user.id} tried to pin note ${note.id} belonging to other user ${note.userId}`); + return null; + } + + return note; + } catch (err) { + this.logger.warn(`Couldn't fetch pinned note ${itemId} for user ${user.id} (@${user.username}@${user.host}): ${renderInlineError(err)}`); + } + } + return null; + }, { + limit: 2, + }); await this.db.transaction(async transactionalEntityManager => { await transactionalEntityManager.delete(MiUserNotePining, { userId: user.id }); @@ -947,7 +987,7 @@ export class ApPersonService implements OnModuleInit { let td = 0; for (const note of featuredNotes.filter(x => x != null)) { td -= 1000; - transactionalEntityManager.insert(MiUserNotePining, { + await transactionalEntityManager.insert(MiUserNotePining, { id: this.idService.gen(this.timeService.now + td), userId: user.id, noteId: note.id, @@ -971,6 +1011,7 @@ export class ApPersonService implements OnModuleInit { let dst = await this.fetchPerson(src.movedToUri); if (dst && isLocalUser(dst)) { + // TODO this branch should not be possible // targetがローカルユーザーだった場合データベースから引っ張ってくる dst = await this.usersRepository.findOneByOrFail({ uri: src.movedToUri }) as MiLocalUser; } else if (dst) { diff --git a/packages/backend/src/core/chart/charts/active-users.ts b/packages/backend/src/core/chart/charts/active-users.ts index 20432fb293..4858633973 100644 --- a/packages/backend/src/core/chart/charts/active-users.ts +++ b/packages/backend/src/core/chart/charts/active-users.ts @@ -50,9 +50,9 @@ export default class ActiveUsersChart extends Chart { // eslint-d } @bindThis - public async read(user: { id: MiUser['id'], host: null }): Promise { + public read(user: { id: MiUser['id'], host: null }): void { const createdAt = this.idService.parse(user.id).date; - await this.commit({ + this.commit({ 'read': [user.id], 'registeredWithinWeek': (this.timeService.now - createdAt.getTime() < week) ? [user.id] : [], 'registeredWithinMonth': (this.timeService.now - createdAt.getTime() < month) ? [user.id] : [], @@ -64,8 +64,8 @@ export default class ActiveUsersChart extends Chart { // eslint-d } @bindThis - public async write(user: { id: MiUser['id'], host: null }): Promise { - await this.commit({ + public write(user: { id: MiUser['id'], host: null }): void { + this.commit({ 'write': [user.id], }); } diff --git a/packages/backend/src/core/chart/charts/ap-request.ts b/packages/backend/src/core/chart/charts/ap-request.ts index 8cae5753c7..f4177955b7 100644 --- a/packages/backend/src/core/chart/charts/ap-request.ts +++ b/packages/backend/src/core/chart/charts/ap-request.ts @@ -43,22 +43,22 @@ export default class ApRequestChart extends Chart { // eslint-dis } @bindThis - public async deliverSucc(): Promise { - await this.commit({ + public deliverSucc(): void { + this.commit({ 'deliverSucceeded': 1, }); } @bindThis - public async deliverFail(): Promise { - await this.commit({ + public deliverFail(): void { + this.commit({ 'deliverFailed': 1, }); } @bindThis - public async inbox(): Promise { - await this.commit({ + public inbox(): void { + this.commit({ 'inboxReceived': 1, }); } diff --git a/packages/backend/src/core/chart/charts/drive.ts b/packages/backend/src/core/chart/charts/drive.ts index cce07f3b5b..2818faf1cb 100644 --- a/packages/backend/src/core/chart/charts/drive.ts +++ b/packages/backend/src/core/chart/charts/drive.ts @@ -44,9 +44,9 @@ export default class DriveChart extends Chart { // eslint-disable } @bindThis - public async update(file: MiDriveFile, isAdditional: boolean): Promise { + public update(file: MiDriveFile, isAdditional: boolean): void { const fileSizeKb = file.size / 1000; - await this.commit(file.userHost === null ? { + this.commit(file.userHost === null ? { 'local.incCount': isAdditional ? 1 : 0, 'local.incSize': isAdditional ? fileSizeKb : 0, 'local.decCount': isAdditional ? 0 : 1, diff --git a/packages/backend/src/core/chart/charts/federation.ts b/packages/backend/src/core/chart/charts/federation.ts index 199c263cce..ab160128f5 100644 --- a/packages/backend/src/core/chart/charts/federation.ts +++ b/packages/backend/src/core/chart/charts/federation.ts @@ -118,8 +118,8 @@ export default class FederationChart extends Chart { // eslint-di } @bindThis - public async deliverd(host: string, succeeded: boolean): Promise { - await this.commit(succeeded ? { + public deliverd(host: string, succeeded: boolean): void { + this.commit(succeeded ? { 'deliveredInstances': [host], } : { 'stalled': [host], @@ -127,8 +127,8 @@ export default class FederationChart extends Chart { // eslint-di } @bindThis - public async inbox(host: string): Promise { - await this.commit({ + public inbox(host: string): void { + this.commit({ 'inboxInstances': [host], }); } diff --git a/packages/backend/src/core/chart/charts/instance.ts b/packages/backend/src/core/chart/charts/instance.ts index ca6c1c5026..d3690820e8 100644 --- a/packages/backend/src/core/chart/charts/instance.ts +++ b/packages/backend/src/core/chart/charts/instance.ts @@ -80,31 +80,31 @@ export default class InstanceChart extends Chart { // eslint-disa } @bindThis - public async requestReceived(host: string): Promise { - await this.commit({ + public requestReceived(host: string): void { + this.commit({ 'requests.received': 1, }, this.utilityService.toPuny(host)); } @bindThis - public async requestSent(host: string, isSucceeded: boolean): Promise { - await this.commit({ + public requestSent(host: string, isSucceeded: boolean): void { + this.commit({ 'requests.succeeded': isSucceeded ? 1 : 0, 'requests.failed': isSucceeded ? 0 : 1, }, this.utilityService.toPuny(host)); } @bindThis - public async newUser(host: string): Promise { - await this.commit({ + public newUser(host: string): void { + this.commit({ 'users.total': 1, 'users.inc': 1, }, this.utilityService.toPuny(host)); } @bindThis - public async updateNote(host: string, note: MiNote, isAdditional: boolean): Promise { - await this.commit({ + public updateNote(host: string, note: MiNote, isAdditional: boolean): void { + this.commit({ 'notes.total': isAdditional ? 1 : -1, 'notes.inc': isAdditional ? 1 : 0, 'notes.dec': isAdditional ? 0 : 1, @@ -116,8 +116,8 @@ export default class InstanceChart extends Chart { // eslint-disa } @bindThis - public async updateFollowing(host: string, isAdditional: boolean): Promise { - await this.commit({ + public updateFollowing(host: string, isAdditional: boolean): void { + this.commit({ 'following.total': isAdditional ? 1 : -1, 'following.inc': isAdditional ? 1 : 0, 'following.dec': isAdditional ? 0 : 1, @@ -125,8 +125,8 @@ export default class InstanceChart extends Chart { // eslint-disa } @bindThis - public async updateFollowers(host: string, isAdditional: boolean): Promise { - await this.commit({ + public updateFollowers(host: string, isAdditional: boolean): void { + this.commit({ 'followers.total': isAdditional ? 1 : -1, 'followers.inc': isAdditional ? 1 : 0, 'followers.dec': isAdditional ? 0 : 1, @@ -134,9 +134,9 @@ export default class InstanceChart extends Chart { // eslint-disa } @bindThis - public async updateDrive(file: MiDriveFile, isAdditional: boolean): Promise { + public updateDrive(file: MiDriveFile, isAdditional: boolean): void { const fileSizeKb = file.size / 1000; - await this.commit({ + this.commit({ 'drive.totalFiles': isAdditional ? 1 : -1, 'drive.incFiles': isAdditional ? 1 : 0, 'drive.incUsage': isAdditional ? fileSizeKb : 0, diff --git a/packages/backend/src/core/chart/charts/notes.ts b/packages/backend/src/core/chart/charts/notes.ts index 43cabd0b98..f8c3676009 100644 --- a/packages/backend/src/core/chart/charts/notes.ts +++ b/packages/backend/src/core/chart/charts/notes.ts @@ -56,10 +56,10 @@ export default class NotesChart extends Chart { // eslint-disable } @bindThis - public async update(note: MiNote, isAdditional: boolean): Promise { + public update(note: MiNote, isAdditional: boolean): void { const prefix = note.userHost === null ? 'local' : 'remote'; - await this.commit({ + this.commit({ [`${prefix}.total`]: isAdditional ? 1 : -1, [`${prefix}.inc`]: isAdditional ? 1 : 0, [`${prefix}.dec`]: isAdditional ? 0 : 1, diff --git a/packages/backend/src/core/chart/charts/per-user-drive.ts b/packages/backend/src/core/chart/charts/per-user-drive.ts index 663abc5f00..b1bd7c6173 100644 --- a/packages/backend/src/core/chart/charts/per-user-drive.ts +++ b/packages/backend/src/core/chart/charts/per-user-drive.ts @@ -58,9 +58,9 @@ export default class PerUserDriveChart extends Chart { // eslint- } @bindThis - public async update(file: MiDriveFile, isAdditional: boolean): Promise { + public update(file: MiDriveFile, isAdditional: boolean): void { const fileSizeKb = file.size / 1000; - await this.commit({ + this.commit({ 'totalCount': isAdditional ? 1 : -1, 'totalSize': isAdditional ? fileSizeKb : -fileSizeKb, 'incCount': isAdditional ? 1 : 0, diff --git a/packages/backend/src/core/chart/charts/per-user-following.ts b/packages/backend/src/core/chart/charts/per-user-following.ts index 71678b0573..05e971616a 100644 --- a/packages/backend/src/core/chart/charts/per-user-following.ts +++ b/packages/backend/src/core/chart/charts/per-user-following.ts @@ -70,7 +70,7 @@ export default class PerUserFollowingChart extends Chart { // esl } @bindThis - public async update(follower: { id: MiUser['id']; host: MiUser['host']; }, followee: { id: MiUser['id']; host: MiUser['host']; }, isFollow: boolean): Promise { + public update(follower: { id: MiUser['id']; host: MiUser['host']; }, followee: { id: MiUser['id']; host: MiUser['host']; }, isFollow: boolean): void { const prefixFollower = this.userEntityService.isLocalUser(follower) ? 'local' : 'remote'; const prefixFollowee = this.userEntityService.isLocalUser(followee) ? 'local' : 'remote'; diff --git a/packages/backend/src/core/chart/charts/per-user-pv.ts b/packages/backend/src/core/chart/charts/per-user-pv.ts index 75a61aae07..272bc9180b 100644 --- a/packages/backend/src/core/chart/charts/per-user-pv.ts +++ b/packages/backend/src/core/chart/charts/per-user-pv.ts @@ -44,16 +44,16 @@ export default class PerUserPvChart extends Chart { // eslint-dis } @bindThis - public async commitByUser(user: { id: MiUser['id'] }, key: string): Promise { - await this.commit({ + public commitByUser(user: { id: MiUser['id'] }, key: string): void { + this.commit({ 'upv.user': [key], 'pv.user': 1, }, user.id); } @bindThis - public async commitByVisitor(user: { id: MiUser['id'] }, key: string): Promise { - await this.commit({ + public commitByVisitor(user: { id: MiUser['id'] }, key: string): void { + this.commit({ 'upv.visitor': [key], 'pv.visitor': 1, }, user.id); diff --git a/packages/backend/src/core/chart/charts/per-user-reactions.ts b/packages/backend/src/core/chart/charts/per-user-reactions.ts index 9fb78a28e9..89f6567f50 100644 --- a/packages/backend/src/core/chart/charts/per-user-reactions.ts +++ b/packages/backend/src/core/chart/charts/per-user-reactions.ts @@ -47,7 +47,7 @@ export default class PerUserReactionsChart extends Chart { // esl } @bindThis - public async update(user: { id: MiUser['id'], host: MiUser['host'] }, note: MiNote): Promise { + public update(user: { id: MiUser['id'], host: MiUser['host'] }, note: MiNote): void { const prefix = this.userEntityService.isLocalUser(user) ? 'local' : 'remote'; this.commit({ [`${prefix}.count`]: 1, diff --git a/packages/backend/src/core/chart/charts/test-grouped.ts b/packages/backend/src/core/chart/charts/test-grouped.ts index 6cc48d483d..186e056efc 100644 --- a/packages/backend/src/core/chart/charts/test-grouped.ts +++ b/packages/backend/src/core/chart/charts/test-grouped.ts @@ -48,12 +48,12 @@ export default class TestGroupedChart extends Chart { // eslint-d } @bindThis - public async increment(group: string): Promise { + public increment(group: string): void { if (this.total[group] == null) this.total[group] = 0; this.total[group]++; - await this.commit({ + this.commit({ 'foo.total': 1, 'foo.inc': 1, }, group); diff --git a/packages/backend/src/core/chart/charts/test-intersection.ts b/packages/backend/src/core/chart/charts/test-intersection.ts index d0ae1dab24..f84d498d52 100644 --- a/packages/backend/src/core/chart/charts/test-intersection.ts +++ b/packages/backend/src/core/chart/charts/test-intersection.ts @@ -44,15 +44,15 @@ export default class TestIntersectionChart extends Chart { // esl } @bindThis - public async addA(key: string): Promise { - await this.commit({ + public addA(key: string): void { + this.commit({ a: [key], }); } @bindThis - public async addB(key: string): Promise { - await this.commit({ + public addB(key: string): void { + this.commit({ b: [key], }); } diff --git a/packages/backend/src/core/chart/charts/test-unique.ts b/packages/backend/src/core/chart/charts/test-unique.ts index 54a081fe2a..10a7e0dad2 100644 --- a/packages/backend/src/core/chart/charts/test-unique.ts +++ b/packages/backend/src/core/chart/charts/test-unique.ts @@ -44,8 +44,8 @@ export default class TestUniqueChart extends Chart { // eslint-di } @bindThis - public async uniqueIncrement(key: string): Promise { - await this.commit({ + public uniqueIncrement(key: string): void { + this.commit({ foo: [key], }); } diff --git a/packages/backend/src/core/chart/charts/test.ts b/packages/backend/src/core/chart/charts/test.ts index e95259f3b2..b281d50e67 100644 --- a/packages/backend/src/core/chart/charts/test.ts +++ b/packages/backend/src/core/chart/charts/test.ts @@ -48,20 +48,20 @@ export default class TestChart extends Chart { // eslint-disable- } @bindThis - public async increment(): Promise { + public increment(): void { this.total++; - await this.commit({ + this.commit({ 'foo.total': 1, 'foo.inc': 1, }); } @bindThis - public async decrement(): Promise { + public decrement(): void { this.total--; - await this.commit({ + this.commit({ 'foo.total': -1, 'foo.dec': 1, }); diff --git a/packages/backend/src/core/chart/charts/users.ts b/packages/backend/src/core/chart/charts/users.ts index 91bf972371..ca5e58f07b 100644 --- a/packages/backend/src/core/chart/charts/users.ts +++ b/packages/backend/src/core/chart/charts/users.ts @@ -61,10 +61,10 @@ export default class UsersChart extends Chart { // eslint-disable } @bindThis - public async update(user: { id: MiUser['id'], host: MiUser['host'] }, isAdditional: boolean): Promise { + public update(user: { id: MiUser['id'], host: MiUser['host'] }, isAdditional: boolean): void { const prefix = this.userEntityService.isLocalUser(user) ? 'local' : 'remote'; - await this.commit({ + this.commit({ [`${prefix}.total`]: isAdditional ? 1 : -1, [`${prefix}.inc`]: isAdditional ? 1 : 0, [`${prefix}.dec`]: isAdditional ? 0 : 1, diff --git a/packages/backend/src/core/chart/core.ts b/packages/backend/src/core/chart/core.ts index 2f1ed27132..883a72139f 100644 --- a/packages/backend/src/core/chart/core.ts +++ b/packages/backend/src/core/chart/core.ts @@ -15,6 +15,7 @@ import { dateUTC, isTimeSame, isTimeBefore, subtractTime, addTime } from '@/misc import type Logger from '@/logger.js'; import { bindThis } from '@/decorators.js'; import { MiRepository, miRepository } from '@/models/_.js'; +import { promiseMap } from '@/misc/promise-map.js'; import type { DataSource, Repository } from 'typeorm'; import type { Lock } from 'redis-lock'; @@ -526,13 +527,13 @@ export default abstract class Chart { const groups = removeDuplicates(this.buffer.map(log => log.group)); - await Promise.all( - groups.map(group => - Promise.all([ - this.claimCurrentLog(group, 'hour'), - this.claimCurrentLog(group, 'day'), - ]).then(([logHour, logDay]) => - update(logHour, logDay)))); + await promiseMap(groups, async group => { + const logHour = await this.claimCurrentLog(group, 'hour'); + const logDay = await this.claimCurrentLog(group, 'day'); + await update(logHour, logDay); + }, { + limit: 2, + }); } @bindThis @@ -564,7 +565,7 @@ export default abstract class Chart { ]); }; - return Promise.all([ + return await Promise.all([ this.claimCurrentLog(group, 'hour'), this.claimCurrentLog(group, 'day'), ]).then(([logHour, logDay]) => diff --git a/packages/backend/src/core/entities/BlockingEntityService.ts b/packages/backend/src/core/entities/BlockingEntityService.ts index 1e699032e2..0ba7a0f2dd 100644 --- a/packages/backend/src/core/entities/BlockingEntityService.ts +++ b/packages/backend/src/core/entities/BlockingEntityService.ts @@ -35,6 +35,7 @@ export class BlockingEntityService { ): Promise> { const blocking = typeof src === 'object' ? src : await this.blockingsRepository.findOneByOrFail({ id: src }); + // noinspection ES6MissingAwait return await awaitAll({ id: blocking.id, createdAt: this.idService.parse(blocking.id).date.toISOString(), @@ -53,6 +54,6 @@ export class BlockingEntityService { const _blockees = blockings.map(({ blockee, blockeeId }) => blockee ?? blockeeId); const _userMap = await this.userEntityService.packMany(_blockees, me, { schema: 'UserDetailedNotMe' }) .then(users => new Map(users.map(u => [u.id, u]))); - return Promise.all(blockings.map(blocking => this.pack(blocking, me, { blockee: _userMap.get(blocking.blockeeId) }))); + return await Promise.all(blockings.map(blocking => this.pack(blocking, me, { blockee: _userMap.get(blocking.blockeeId) }))); } } diff --git a/packages/backend/src/core/entities/ChatEntityService.ts b/packages/backend/src/core/entities/ChatEntityService.ts index da112d5444..171a49ebcc 100644 --- a/packages/backend/src/core/entities/ChatEntityService.ts +++ b/packages/backend/src/core/entities/ChatEntityService.ts @@ -117,7 +117,7 @@ export class ChatEntityService { .then(rooms => new Map(rooms.map(r => [r.id, r]))), ]); - return Promise.all(messages.map(message => this.packMessageDetailed(message, me, { _hint_: { packedUsers, packedFiles, packedRooms } }))); + return await Promise.all(messages.map(message => this.packMessageDetailed(message, me, { _hint_: { packedUsers, packedFiles, packedRooms } }))); } @bindThis @@ -165,7 +165,7 @@ export class ChatEntityService { .then(files => new Map(files.map(f => [f.id, f]))), ]); - return Promise.all(messages.map(message => this.packMessageLiteFor1on1(message, { _hint_: { packedFiles } }))); + return await Promise.all(messages.map(message => this.packMessageLiteFor1on1(message, { _hint_: { packedFiles } }))); } @bindThis @@ -228,7 +228,7 @@ export class ChatEntityService { .then(files => new Map(files.map(f => [f.id, f]))), ]); - return Promise.all(messages.map(message => this.packMessageLiteForRoom(message, { _hint_: { packedFiles, packedUsers } }))); + return await Promise.all(messages.map(message => this.packMessageLiteForRoom(message, { _hint_: { packedFiles, packedUsers } }))); } @bindThis @@ -289,7 +289,7 @@ export class ChatEntityService { }).then(memberships => new Map(_rooms.map(r => [r.id, memberships.find(m => m.roomId === r.id)]))), ]); - return Promise.all(_rooms.map(room => this.packRoom(room, me, { _hint_: { packedOwners, memberships } }))); + return await Promise.all(_rooms.map(room => this.packRoom(room, me, { _hint_: { packedOwners, memberships } }))); } @bindThis @@ -322,7 +322,7 @@ export class ChatEntityService { ) { if (invitations.length === 0) return []; - return Promise.all(invitations.map(invitation => this.packRoomInvitation(invitation, me))); + return await Promise.all(invitations.map(invitation => this.packRoomInvitation(invitation, me))); } @bindThis @@ -371,6 +371,6 @@ export class ChatEntityService { .then(rooms => new Map(rooms.map(r => [r.id, r]))), ]); - return Promise.all(memberships.map(membership => this.packRoomMembership(membership, me, { ...options, _hint_: { packedUsers, packedRooms } }))); + return await Promise.all(memberships.map(membership => this.packRoomMembership(membership, me, { ...options, _hint_: { packedUsers, packedRooms } }))); } } diff --git a/packages/backend/src/core/entities/ClipEntityService.ts b/packages/backend/src/core/entities/ClipEntityService.ts index b700fe2efb..fba085767f 100644 --- a/packages/backend/src/core/entities/ClipEntityService.ts +++ b/packages/backend/src/core/entities/ClipEntityService.ts @@ -42,6 +42,7 @@ export class ClipEntityService { const meId = me ? me.id : null; const clip = typeof src === 'object' ? src : await this.clipsRepository.findOneByOrFail({ id: src }); + // noinspection ES6MissingAwait return await awaitAll({ id: clip.id, createdAt: this.idService.parse(clip.id).date.toISOString(), @@ -65,7 +66,7 @@ export class ClipEntityService { const _users = clips.map(({ user, userId }) => user ?? userId); const _userMap = await this.userEntityService.packMany(_users, me) .then(users => new Map(users.map(u => [u.id, u]))); - return Promise.all(clips.map(clip => this.pack(clip, me, { packedUser: _userMap.get(clip.userId) }))); + return await Promise.all(clips.map(clip => this.pack(clip, me, { packedUser: _userMap.get(clip.userId) }))); } } diff --git a/packages/backend/src/core/entities/DriveFileEntityService.ts b/packages/backend/src/core/entities/DriveFileEntityService.ts index a172f81eed..06197ff839 100644 --- a/packages/backend/src/core/entities/DriveFileEntityService.ts +++ b/packages/backend/src/core/entities/DriveFileEntityService.ts @@ -201,6 +201,7 @@ export class DriveFileEntityService implements OnModuleInit { const file = typeof src === 'object' ? src : await this.driveFilesRepository.findOneByOrFail({ id: src }); + // noinspection ES6MissingAwait return await awaitAll>({ id: file.id, createdAt: this.idService.parse(file.id).date.toISOString(), @@ -239,6 +240,7 @@ export class DriveFileEntityService implements OnModuleInit { const file = typeof src === 'object' ? src : await this.driveFilesRepository.findOneBy({ id: src }); if (file == null) return null; + // noinspection ES6MissingAwait return await awaitAll>({ id: file.id, createdAt: this.idService.parse(file.id).date.toISOString(), diff --git a/packages/backend/src/core/entities/EmojiEntityService.ts b/packages/backend/src/core/entities/EmojiEntityService.ts index 5f03df554c..9c3f0adfff 100644 --- a/packages/backend/src/core/entities/EmojiEntityService.ts +++ b/packages/backend/src/core/entities/EmojiEntityService.ts @@ -179,7 +179,7 @@ export class EmojiEntityService implements OnModuleInit { hintRoles = new Map(roles.map(x => [x.id, x])); } - return Promise.all(emojis.map(x => this.packDetailedAdmin(x, { roles: hintRoles }))); + return await Promise.all(emojis.map(x => this.packDetailedAdmin(x, { roles: hintRoles }))); } } diff --git a/packages/backend/src/core/entities/FlashEntityService.ts b/packages/backend/src/core/entities/FlashEntityService.ts index c2575e69aa..47617c5be7 100644 --- a/packages/backend/src/core/entities/FlashEntityService.ts +++ b/packages/backend/src/core/entities/FlashEntityService.ts @@ -77,7 +77,7 @@ export class FlashEntityService { .getRawMany<{ flashLike_flashId: string }>() .then(likes => [...new Set(likes.map(like => like.flashLike_flashId))]) : []; - return Promise.all( + return await Promise.all( flashes.map(flash => this.pack(flash, me, { packedUser: _userMap.get(flash.userId), likedFlashIds: _likedFlashIds, diff --git a/packages/backend/src/core/entities/FollowRequestEntityService.ts b/packages/backend/src/core/entities/FollowRequestEntityService.ts index 0101ec8aa7..05c69b4d56 100644 --- a/packages/backend/src/core/entities/FollowRequestEntityService.ts +++ b/packages/backend/src/core/entities/FollowRequestEntityService.ts @@ -50,7 +50,7 @@ export class FollowRequestEntityService { const _followees = requests.map(({ followee, followeeId }) => followee ?? followeeId); const _userMap = await this.userEntityService.packMany([..._followers, ..._followees], me) .then(users => new Map(users.map(u => [u.id, u]))); - return Promise.all( + return await Promise.all( requests.map(req => { const packedFollower = _userMap.get(req.followerId); const packedFollowee = _userMap.get(req.followeeId); diff --git a/packages/backend/src/core/entities/FollowingEntityService.ts b/packages/backend/src/core/entities/FollowingEntityService.ts index d54c954bf2..5645217ebc 100644 --- a/packages/backend/src/core/entities/FollowingEntityService.ts +++ b/packages/backend/src/core/entities/FollowingEntityService.ts @@ -139,6 +139,7 @@ export class FollowingEntityService { if (opts == null) opts = {}; + // noinspection ES6MissingAwait return await awaitAll({ id: following.id, createdAt: this.idService.parse(following.id).date.toISOString(), @@ -166,7 +167,7 @@ export class FollowingEntityService { const _followers = opts?.populateFollower ? followings.map(({ follower, followerId }) => follower ?? followerId) : []; const _userMap = await this.userEntityService.packMany([..._followees, ..._followers], me, { schema: 'UserDetailedNotMe' }) .then(users => new Map(users.map(u => [u.id, u]))); - return Promise.all( + return await Promise.all( followings.map(following => { const packedFollowee = opts?.populateFollowee ? _userMap.get(following.followeeId) : undefined; const packedFollower = opts?.populateFollower ? _userMap.get(following.followerId) : undefined; diff --git a/packages/backend/src/core/entities/GalleryPostEntityService.ts b/packages/backend/src/core/entities/GalleryPostEntityService.ts index 9746a4c1af..a276f1508c 100644 --- a/packages/backend/src/core/entities/GalleryPostEntityService.ts +++ b/packages/backend/src/core/entities/GalleryPostEntityService.ts @@ -42,6 +42,7 @@ export class GalleryPostEntityService { const meId = me ? me.id : null; const post = typeof src === 'object' ? src : await this.galleryPostsRepository.findOneByOrFail({ id: src }); + // noinspection ES6MissingAwait return await awaitAll({ id: post.id, createdAt: this.idService.parse(post.id).date.toISOString(), @@ -68,7 +69,7 @@ export class GalleryPostEntityService { const _users = posts.map(({ user, userId }) => user ?? userId); const _userMap = await this.userEntityService.packMany(_users, me) .then(users => new Map(users.map(u => [u.id, u]))); - return Promise.all(posts.map(post => this.pack(post, me, { packedUser: _userMap.get(post.userId) }))); + return await Promise.all(posts.map(post => this.pack(post, me, { packedUser: _userMap.get(post.userId) }))); } } diff --git a/packages/backend/src/core/entities/InviteCodeEntityService.ts b/packages/backend/src/core/entities/InviteCodeEntityService.ts index 5d3e823a2a..ceb666ac90 100644 --- a/packages/backend/src/core/entities/InviteCodeEntityService.ts +++ b/packages/backend/src/core/entities/InviteCodeEntityService.ts @@ -62,7 +62,7 @@ export class InviteCodeEntityService { const _usedBys = tickets.map(({ usedBy, usedById }) => usedBy ?? usedById).filter(x => x != null); const _userMap = await this.userEntityService.packMany([..._createdBys, ..._usedBys], me) .then(users => new Map(users.map(u => [u.id, u]))); - return Promise.all( + return await Promise.all( tickets.map(ticket => { const packedCreatedBy = ticket.createdById != null ? _userMap.get(ticket.createdById) : undefined; const packedUsedBy = ticket.usedById != null ? _userMap.get(ticket.usedById) : undefined; diff --git a/packages/backend/src/core/entities/ModerationLogEntityService.ts b/packages/backend/src/core/entities/ModerationLogEntityService.ts index bf1b2a002c..2d1de5cd46 100644 --- a/packages/backend/src/core/entities/ModerationLogEntityService.ts +++ b/packages/backend/src/core/entities/ModerationLogEntityService.ts @@ -34,6 +34,7 @@ export class ModerationLogEntityService { ) { const log = typeof src === 'object' ? src : await this.moderationLogsRepository.findOneByOrFail({ id: src }); + // noinspection ES6MissingAwait return await awaitAll({ id: log.id, createdAt: this.idService.parse(log.id).date.toISOString(), @@ -53,7 +54,7 @@ export class ModerationLogEntityService { const _users = reports.map(({ user, userId }) => user ?? userId); const _userMap = await this.userEntityService.packMany(_users, null, { schema: 'UserDetailedNotMe' }) .then(users => new Map(users.map(u => [u.id, u]))); - return Promise.all(reports.map(report => this.pack(report, { packedUser: _userMap.get(report.userId) }))); + return await Promise.all(reports.map(report => this.pack(report, { packedUser: _userMap.get(report.userId) }))); } } diff --git a/packages/backend/src/core/entities/MutingEntityService.ts b/packages/backend/src/core/entities/MutingEntityService.ts index d361a20271..7d8c3dcde2 100644 --- a/packages/backend/src/core/entities/MutingEntityService.ts +++ b/packages/backend/src/core/entities/MutingEntityService.ts @@ -36,6 +36,7 @@ export class MutingEntityService { ): Promise> { const muting = typeof src === 'object' ? src : await this.mutingsRepository.findOneByOrFail({ id: src }); + // noinspection ES6MissingAwait return await awaitAll({ id: muting.id, createdAt: this.idService.parse(muting.id).date.toISOString(), @@ -55,7 +56,7 @@ export class MutingEntityService { const _mutees = mutings.map(({ mutee, muteeId }) => mutee ?? muteeId); const _userMap = await this.userEntityService.packMany(_mutees, me, { schema: 'UserDetailedNotMe' }) .then(users => new Map(users.map(u => [u.id, u]))); - return Promise.all(mutings.map(muting => this.pack(muting, me, { packedMutee: _userMap.get(muting.muteeId) }))); + return await Promise.all(mutings.map(muting => this.pack(muting, me, { packedMutee: _userMap.get(muting.muteeId) }))); } } diff --git a/packages/backend/src/core/entities/NoteEntityService.ts b/packages/backend/src/core/entities/NoteEntityService.ts index 373f05332f..310e361736 100644 --- a/packages/backend/src/core/entities/NoteEntityService.ts +++ b/packages/backend/src/core/entities/NoteEntityService.ts @@ -592,6 +592,7 @@ export class NoteEntityService implements OnModuleInit { const bypassSilence = opts.bypassSilence || note.userId === meId; + // noinspection ES6MissingAwait const packed: Packed<'Note'> = await awaitAll({ id: note.id, threadId, diff --git a/packages/backend/src/core/entities/NoteFavoriteEntityService.ts b/packages/backend/src/core/entities/NoteFavoriteEntityService.ts index 3cdafe48ad..3d879b0dff 100644 --- a/packages/backend/src/core/entities/NoteFavoriteEntityService.ts +++ b/packages/backend/src/core/entities/NoteFavoriteEntityService.ts @@ -5,12 +5,13 @@ import { Inject, Injectable } from '@nestjs/common'; import { DI } from '@/di-symbols.js'; -import type { NoteFavoritesRepository } from '@/models/_.js'; +import type { MiNote, NoteFavoritesRepository } from '@/models/_.js'; import type { } from '@/models/Blocking.js'; import type { MiUser } from '@/models/User.js'; import type { MiNoteFavorite } from '@/models/NoteFavorite.js'; import { bindThis } from '@/decorators.js'; import { IdService } from '@/core/IdService.js'; +import type { Packed } from '@/misc/json-schema.js'; import { NoteEntityService } from './NoteEntityService.js'; @Injectable() @@ -28,6 +29,7 @@ export class NoteFavoriteEntityService { public async pack( src: MiNoteFavorite['id'] | MiNoteFavorite, me?: { id: MiUser['id'] } | null | undefined, + notes?: Map>, ) { const favorite = typeof src === 'object' ? src : await this.noteFavoritesRepository.findOneByOrFail({ id: src }); @@ -35,15 +37,18 @@ export class NoteFavoriteEntityService { id: favorite.id, createdAt: this.idService.parse(favorite.id).date.toISOString(), noteId: favorite.noteId, - note: await this.noteEntityService.pack(favorite.note ?? favorite.noteId, me), + note: notes?.get(favorite.noteId) ?? await this.noteEntityService.pack(favorite.note ?? favorite.noteId, me), }; } @bindThis - public packMany( - favorites: any[], + public async packMany( + favorites: (MiNoteFavorite & { note: MiNote })[], me: { id: MiUser['id'] }, ) { - return Promise.all(favorites.map(x => this.pack(x, me))); + const packedNotes = await this.noteEntityService.packMany(favorites.map(f => f.note), me); + const packedNotesMap = new Map(packedNotes.map(n => [n.id, n])); + + return Promise.all(favorites.map(x => this.pack(x, me, packedNotesMap))); } } diff --git a/packages/backend/src/core/entities/NoteReactionEntityService.ts b/packages/backend/src/core/entities/NoteReactionEntityService.ts index 2b0d69b261..1f6d52d20c 100644 --- a/packages/backend/src/core/entities/NoteReactionEntityService.ts +++ b/packages/backend/src/core/entities/NoteReactionEntityService.ts @@ -88,6 +88,6 @@ export class NoteReactionEntityService implements OnModuleInit { const _users = reactions.map(({ user, userId }) => user ?? userId); const _userMap = await this.userEntityService.packMany(_users, me) .then(users => new Map(users.map(u => [u.id, u]))); - return Promise.all(reactions.map(reaction => this.pack(reaction, me, opts, { packedUser: _userMap.get(reaction.userId) }))); + return await Promise.all(reactions.map(reaction => this.pack(reaction, me, opts, { packedUser: _userMap.get(reaction.userId) }))); } } diff --git a/packages/backend/src/core/entities/PageEntityService.ts b/packages/backend/src/core/entities/PageEntityService.ts index 46bf51bb6d..5a56b9cec5 100644 --- a/packages/backend/src/core/entities/PageEntityService.ts +++ b/packages/backend/src/core/entities/PageEntityService.ts @@ -83,11 +83,12 @@ export class PageEntityService { }; migrate(page.content); if (migrated) { - this.pagesRepository.update(page.id, { + await this.pagesRepository.update(page.id, { content: page.content, }); } + // noinspection ES6MissingAwait return await awaitAll({ id: page.id, createdAt: this.idService.parse(page.id).date.toISOString(), @@ -104,10 +105,13 @@ export class PageEntityService { font: page.font, script: page.script, eyeCatchingImageId: page.eyeCatchingImageId, - eyeCatchingImage: page.eyeCatchingImageId ? await this.driveFileEntityService.pack(page.eyeCatchingImageId) : null, - attachedFiles: this.driveFileEntityService.packMany((await Promise.all(attachedFiles)).filter(x => x != null)), + eyeCatchingImage: page.eyeCatchingImageId ? this.driveFileEntityService.pack(page.eyeCatchingImageId) : null, + attachedFiles: Promise + .all(attachedFiles) + .then(fs => fs.filter(x => x != null)) + .then(fs => this.driveFileEntityService.packMany(fs)), likedCount: page.likedCount, - isLiked: meId ? await this.pageLikesRepository.exists({ where: { pageId: page.id, userId: meId } }) : undefined, + isLiked: meId ? this.pageLikesRepository.exists({ where: { pageId: page.id, userId: meId } }) : undefined, }); } @@ -119,7 +123,7 @@ export class PageEntityService { const _users = pages.map(({ user, userId }) => user ?? userId); const _userMap = await this.userEntityService.packMany(_users, me) .then(users => new Map(users.map(u => [u.id, u]))); - return Promise.all(pages.map(page => this.pack(page, me, { packedUser: _userMap.get(page.userId) }))); + return await Promise.all(pages.map(page => this.pack(page, me, { packedUser: _userMap.get(page.userId) }))); } } diff --git a/packages/backend/src/core/entities/RenoteMutingEntityService.ts b/packages/backend/src/core/entities/RenoteMutingEntityService.ts index e4e154109a..199fc2366e 100644 --- a/packages/backend/src/core/entities/RenoteMutingEntityService.ts +++ b/packages/backend/src/core/entities/RenoteMutingEntityService.ts @@ -36,6 +36,7 @@ export class RenoteMutingEntityService { ): Promise> { const muting = typeof src === 'object' ? src : await this.renoteMutingsRepository.findOneByOrFail({ id: src }); + // noinspection ES6MissingAwait return await awaitAll({ id: muting.id, createdAt: this.idService.parse(muting.id).date.toISOString(), @@ -54,7 +55,7 @@ export class RenoteMutingEntityService { const _users = mutings.map(({ mutee, muteeId }) => mutee ?? muteeId); const _userMap = await this.userEntityService.packMany(_users, me, { schema: 'UserDetailedNotMe' }) .then(users => new Map(users.map(u => [u.id, u]))); - return Promise.all(mutings.map(muting => this.pack(muting, me, { packedMutee: _userMap.get(muting.muteeId) }))); + return await Promise.all(mutings.map(muting => this.pack(muting, me, { packedMutee: _userMap.get(muting.muteeId) }))); } } diff --git a/packages/backend/src/core/entities/UserEntityService.ts b/packages/backend/src/core/entities/UserEntityService.ts index 6e7f15610f..5ea53c32ba 100644 --- a/packages/backend/src/core/entities/UserEntityService.ts +++ b/packages/backend/src/core/entities/UserEntityService.ts @@ -485,7 +485,7 @@ export class UserEntityService implements OnModuleInit { if (user.avatarId != null && user.avatarUrl === null) { const avatar = await this.driveFilesRepository.findOneByOrFail({ id: user.avatarId }); user.avatarUrl = this.driveFileEntityService.getPublicUrl(avatar, 'avatar'); - this.usersRepository.update(user.id, { + await this.usersRepository.update(user.id, { avatarUrl: user.avatarUrl, avatarBlurhash: avatar.blurhash, }); @@ -493,7 +493,7 @@ export class UserEntityService implements OnModuleInit { if (user.bannerId != null && user.bannerUrl === null) { const banner = await this.driveFilesRepository.findOneByOrFail({ id: user.bannerId }); user.bannerUrl = this.driveFileEntityService.getPublicUrl(banner); - this.usersRepository.update(user.id, { + await this.usersRepository.update(user.id, { bannerUrl: user.bannerUrl, bannerBlurhash: banner.blurhash, }); @@ -501,7 +501,7 @@ export class UserEntityService implements OnModuleInit { if (user.backgroundId != null && user.backgroundUrl === null) { const background = await this.driveFilesRepository.findOneByOrFail({ id: user.backgroundId }); user.backgroundUrl = this.driveFileEntityService.getPublicUrl(background); - this.usersRepository.update(user.id, { + await this.usersRepository.update(user.id, { backgroundUrl: user.backgroundUrl, backgroundBlurhash: background.blurhash, }); @@ -581,6 +581,7 @@ export class UserEntityService implements OnModuleInit { const bypassSilence = isMe || (myFollowings ? myFollowings.has(user.id) : false); + // noinspection ES6MissingAwait const packed = { id: user.id, name: user.name, @@ -644,6 +645,7 @@ export class UserEntityService implements OnModuleInit { ...(isDetailed ? { url: profile!.url, uri: user.uri, + // TODO hints for all of this movedTo: user.movedToUri ? Promise.resolve(opts.userIdsByUri?.get(user.movedToUri) ?? this.apPersonService.resolvePerson(user.movedToUri).then(user => user.id).catch(() => null)) : null, movedToUri: user.movedToUri, // alsoKnownAs moved from packedUserDetailedNotMeOnly for privacy @@ -894,7 +896,7 @@ export class UserEntityService implements OnModuleInit { myFollowingsPromise, ]); - return Promise.all( + return await Promise.all( _users.map(u => this.pack( u, me, diff --git a/packages/backend/src/core/entities/UserListEntityService.ts b/packages/backend/src/core/entities/UserListEntityService.ts index 2722d52195..55a15dd07b 100644 --- a/packages/backend/src/core/entities/UserListEntityService.ts +++ b/packages/backend/src/core/entities/UserListEntityService.ts @@ -67,7 +67,7 @@ export class UserListEntityService { const _users = memberships.map(({ user, userId }) => user ?? userId); const _userMap = await this.userEntityService.packMany(_users) .then(users => new Map(users.map(u => [u.id, u]))); - return Promise.all(memberships.map(async x => ({ + return await Promise.all(memberships.map(async x => ({ id: x.id, createdAt: this.idService.parse(x.id).date.toISOString(), userId: x.userId, diff --git a/packages/backend/src/daemons/QueueStatsService.ts b/packages/backend/src/daemons/QueueStatsService.ts index 3779172517..767e80bc81 100644 --- a/packages/backend/src/daemons/QueueStatsService.ts +++ b/packages/backend/src/daemons/QueueStatsService.ts @@ -24,6 +24,7 @@ export interface StatsEntry { export interface Stats { deliver: StatsEntry, inbox: StatsEntry, + background: StatsEntry, } const ev = new Xev(); @@ -35,9 +36,11 @@ export class QueueStatsService implements OnApplicationShutdown { private intervalId?: TimerHandle; private activeDeliverJobs = 0; private activeInboxJobs = 0; + private activeBackgroundJobs = 0; private deliverQueueEvents?: Bull.QueueEvents; private inboxQueueEvents?: Bull.QueueEvents; + private backgroundQueueEvents?: Bull.QueueEvents; private log?: Stats[]; @@ -60,6 +63,11 @@ export class QueueStatsService implements OnApplicationShutdown { this.activeInboxJobs++; } + @bindThis + private onBackgroundActive() { + this.activeBackgroundJobs++; + } + @bindThis private onRequestQueueStatsLog(x: { id: string, length?: number }) { if (this.log) { @@ -80,13 +88,16 @@ export class QueueStatsService implements OnApplicationShutdown { this.deliverQueueEvents = new Bull.QueueEvents(QUEUE.DELIVER, baseQueueOptions(this.config, QUEUE.DELIVER)); this.inboxQueueEvents = new Bull.QueueEvents(QUEUE.INBOX, baseQueueOptions(this.config, QUEUE.INBOX)); + this.backgroundQueueEvents = new Bull.QueueEvents(QUEUE.BACKGROUND_TASK, baseQueueOptions(this.config, QUEUE.BACKGROUND_TASK)); this.deliverQueueEvents.on('active', this.onDeliverActive); this.inboxQueueEvents.on('active', this.onInboxActive); + this.backgroundQueueEvents.on('active', this.onBackgroundActive); const tick = async () => { const deliverJobCounts = await this.queueService.deliverQueue.getJobCounts(); const inboxJobCounts = await this.queueService.inboxQueue.getJobCounts(); + const backgroundJobCounts = await this.queueService.backgroundTaskQueue.getJobCounts(); const stats = { deliver: { @@ -101,6 +112,12 @@ export class QueueStatsService implements OnApplicationShutdown { waiting: inboxJobCounts.waiting, delayed: inboxJobCounts.delayed, }, + background: { + activeSincePrevTick: this.activeBackgroundJobs, + active: backgroundJobCounts.active, + waiting: backgroundJobCounts.waiting, + delayed: backgroundJobCounts.delayed, + }, }; ev.emit('queueStats', stats); @@ -112,6 +129,7 @@ export class QueueStatsService implements OnApplicationShutdown { this.activeDeliverJobs = 0; this.activeInboxJobs = 0; + this.activeBackgroundJobs = 0; }; tick(); @@ -120,7 +138,7 @@ export class QueueStatsService implements OnApplicationShutdown { } @bindThis - public async stop() { + public async stop(): Promise { if (this.intervalId) { this.timeService.stopTimer(this.intervalId); } @@ -130,12 +148,15 @@ export class QueueStatsService implements OnApplicationShutdown { this.deliverQueueEvents?.off('active', this.onDeliverActive); this.inboxQueueEvents?.off('active', this.onInboxActive); + this.backgroundQueueEvents?.off('active', this.onBackgroundActive); await this.deliverQueueEvents?.close(); await this.inboxQueueEvents?.close(); + await this.backgroundQueueEvents?.close(); this.activeDeliverJobs = 0; this.activeInboxJobs = 0; + this.activeBackgroundJobs = 0; } @bindThis diff --git a/packages/backend/src/di-symbols.ts b/packages/backend/src/di-symbols.ts index e2c73562c8..d34431cdea 100644 --- a/packages/backend/src/di-symbols.ts +++ b/packages/backend/src/di-symbols.ts @@ -93,7 +93,7 @@ export const DI = { chatRoomsRepository: Symbol('chatRoomsRepository'), chatRoomMembershipsRepository: Symbol('chatRoomMembershipsRepository'), chatRoomInvitationsRepository: Symbol('chatRoomInvitationsRepository'), - noteEditRepository: Symbol('noteEditRepository'), + noteEditsRepository: Symbol('noteEditsRepository'), bubbleGameRecordsRepository: Symbol('bubbleGameRecordsRepository'), reversiGamesRepository: Symbol('reversiGamesRepository'), noteScheduleRepository: Symbol('noteScheduleRepository'), diff --git a/packages/backend/src/misc/collapsed-queue.ts b/packages/backend/src/misc/collapsed-queue.ts index 168a4d7680..d9e86dd4f5 100644 --- a/packages/backend/src/misc/collapsed-queue.ts +++ b/packages/backend/src/misc/collapsed-queue.ts @@ -3,45 +3,169 @@ * SPDX-License-Identifier: AGPL-3.0-only */ +import promiseLimit from 'promise-limit'; import type { TimeService, TimerHandle } from '@/global/TimeService.js'; +import { InternalEventService } from '@/global/InternalEventService.js'; +import { bindThis } from '@/decorators.js'; +import { Serialized } from '@/types.js'; type Job = { value: V; timer: TimerHandle; }; -// TODO: redis使えるようにする -export class CollapsedQueue { - private jobs: Map> = new Map(); +// TODO document IPC sync process + +// sync cross-process: +// 1. Emit internal events when scheduling timer, performing queue, and enqueuing data +// 2. On enqueue, mark ID as deferred. +// 3. On perform, clear mark. +// 4. On performAll, skip deferred IDs. +// 5. On enqueue when ID is deferred, send data as event instead. +// 6. On delete, clear mark. +// 7. On delete when ID is deferred, do nothing. + +export class CollapsedQueue { + private readonly limiter?: ReturnType>; + private readonly jobs: Map> = new Map(); + private readonly deferredKeys = new Set(); constructor( - protected readonly timeService: TimeService, - private timeout: number, - private collapse: (oldValue: V, newValue: V) => V, - private perform: (key: K, value: V) => Promise, - ) {} - - enqueue(key: K, value: V) { - if (this.jobs.has(key)) { - const old = this.jobs.get(key)!; - const merged = this.collapse(old.value, value); - this.jobs.set(key, { ...old, value: merged }); - } else { - const timer = this.timeService.startTimer(() => { - const job = this.jobs.get(key)!; - this.jobs.delete(key); - this.perform(key, job.value); - }, this.timeout); - this.jobs.set(key, { value, timer }); + private readonly internalEventService: InternalEventService, + private readonly timeService: TimeService, + public readonly name: string, + private readonly timeout: number, + private readonly collapse: (oldValue: V, newValue: V) => V, + private readonly perform: (key: string, value: V) => Promise, + private readonly opts?: { + onError?: (queue: CollapsedQueue, error: unknown) => void | Promise, + concurrency?: number, + redisParser?: (data: Serialized) => V, + }, + ) { + if (opts?.concurrency) { + this.limiter = promiseLimit(opts.concurrency); } + + this.internalEventService.on('collapsedQueueDefer', this.onDefer, { ignoreLocal: true }); + this.internalEventService.on('collapsedQueueEnqueue', this.onEnqueue, { ignoreLocal: true }); } + @bindThis + async enqueue(key: string, value: V) { + // If deferred, then send it out to the owning process + if (this.deferredKeys.has(key)) { + await this.internalEventService.emit('collapsedQueueEnqueue', { name: this.name, key, value }); + return; + } + + // If already queued, then merge + const job = this.jobs.get(key); + if (job) { + job.value = this.collapse(job.value, value); + return; + } + + // Otherwise, create a new job + const timer = this.timeService.startTimer(async () => { + const job = this.jobs.get(key); + if (!job) return; + + this.jobs.delete(key); + await this._perform(key, job.value); + }, this.timeout); + this.jobs.set(key, { value, timer }); + + // Mark as deferred so other processes will forward their state to us + await this.internalEventService.emit('collapsedQueueDefer', { name: this.name, key, deferred: true }); + } + + @bindThis + async delete(key: string) { + const job = this.jobs.get(key); + if (!job) return; + + this.timeService.stopTimer(job.timer); + this.jobs.delete(key); + await this.internalEventService.emit('collapsedQueueDefer', { name: this.name, key, deferred: false }); + } + + @bindThis async performAllNow() { - const entries = [...this.jobs.entries()]; - this.jobs.clear(); - for (const [_key, job] of entries) { + for (const job of this.jobs.values()) { this.timeService.stopTimer(job.timer); } - await Promise.allSettled(entries.map(([key, job]) => this.perform(key, job.value))); + + const entries = Array.from(this.jobs.entries()); + this.jobs.clear(); + + return await Promise.all(entries.map(([key, job]) => this._perform(key, job.value))); + } + + private async _perform(key: string, value: V) { + try { + await this.internalEventService.emit('collapsedQueueDefer', { name: this.name, key, deferred: false }); + + if (this.limiter) { + await this.limiter(async () => { + await this.perform(key, value); + }); + } else { + await this.perform(key, value); + } + + return true; + } catch (err) { + await this.opts?.onError?.(this, err); + return false; + } + } + + //#region Events from other processes + @bindThis + private async onDefer(data: { name: string, key: string, deferred: boolean }) { + if (data.name !== this.name) return; + + // Check for and recover from de-sync conditions where multiple processes try to "own" the same job. + const job = this.jobs.get(data.key); + if (job) { + if (data.deferred) { + // If another process tries to claim our job, then give it to them and queue our latest state. + this.timeService.stopTimer(job.timer); + this.jobs.delete(data.key); + await this.internalEventService.emit('collapsedQueueEnqueue', { name: this.name, key: data.key, value: job.value }); + } else { + // If another process tries to release our job, then just continue. + return; + } + } + + if (data.deferred) { + this.deferredKeys.add(data.key); + } else { + this.deferredKeys.delete(data.key); + } + } + + @bindThis + private async onEnqueue(data: { name: string, key: string, value: unknown }) { + if (data.name !== this.name) return; + + // Only enqueue if not deferred + if (!this.deferredKeys.has(data.key)) { + const value = this.opts?.redisParser + ? this.opts.redisParser(data.value as Serialized) + : data.value as V; + + await this.enqueue(data.key, value); + } + } + //#endregion + + async dispose() { + this.internalEventService.off('collapsedQueueDefer', this.onDefer); + this.internalEventService.off('collapsedQueueEnqueue', this.onEnqueue); + + return await this.performAllNow(); } } diff --git a/packages/backend/src/misc/identifiable-error.ts b/packages/backend/src/misc/identifiable-error.ts index 9497791ea1..782c2efd67 100644 --- a/packages/backend/src/misc/identifiable-error.ts +++ b/packages/backend/src/misc/identifiable-error.ts @@ -25,3 +25,15 @@ export class IdentifiableError extends Error { this.isRetryable = isRetryable; } } + +/** + * Standard error codes to reference throughout the app + */ +export const errorCodes = { + // User has been deleted (hard or soft deleted) + userIsDeleted: '4cac9436-baa3-4955-a368-7628aea676cf', + // User is suspended (directly or by instance) + userIsSuspended: '1e56d624-737f-48e4-beb6-0bdddb9fa809', + // User has no valid featured collection (not defined, invalid, etc) + noFeaturedCollection: '2aa4766e-b7d8-4291-a671-56800498b085', +} as const; diff --git a/packages/backend/src/misc/is-renote.ts b/packages/backend/src/misc/is-renote.ts index fcaafaf95a..cb88b64b8c 100644 --- a/packages/backend/src/misc/is-renote.ts +++ b/packages/backend/src/misc/is-renote.ts @@ -5,42 +5,51 @@ import type { MiNote } from '@/models/Note.js'; import type { Packed } from '@/misc/json-schema.js'; +import type { NoteEdit } from '@/models/NoteEdit.js'; // NoteEntityService.isPureRenote とよしなにリンク -type Renote = +export type Renote = MiNote & { renoteId: NonNullable }; -type Quote = +export type Quote = Renote & ({ text: NonNullable } | { cw: NonNullable } | { replyId: NonNullable - reply: NonNullable + reply: NonNullable // TODO this is wrong } | { hasPoll: true + } | { + fileIds: [string, ...string[]] }); -type PureRenote = +export type PureRenote = Renote & { text: null, cw: null, replyId: null, hasPoll: false, - fileIds: { - length: 0, - }, + fileIds: [], }; -export function isRenote(note: MiNote): note is Renote { +export function isRenote(note: MiNote): note is Renote; +export function isRenote(note: NoteEdit): note is RenoteEdit; +export function isRenote(note: MinimalNote): note is MinimalRenote; +export function isRenote(note: MiNote | NoteEdit | MinimalNote): note is Renote | RenoteEdit | MinimalRenote; +export function isRenote(note: MiNote | NoteEdit | MinimalNote): note is Renote | RenoteEdit | MinimalRenote { return note.renoteId != null; } -export function isQuote(note: Renote): note is Quote { +export function isQuote(note: Renote): note is Quote; +export function isQuote(note: RenoteEdit): note is QuoteEdit; +export function isQuote(note: MinimalNote): note is MinimalQuote; +export function isQuote(note: Renote | RenoteEdit | MinimalNote): note is Quote | QuoteEdit | MinimalQuote; +export function isQuote(note: Renote | RenoteEdit | MinimalNote): note is Quote | QuoteEdit | MinimalQuote { // NOTE: SYNC WITH NoteCreateService.isQuote return note.text != null || note.cw != null || @@ -49,7 +58,11 @@ export function isQuote(note: Renote): note is Quote { note.fileIds.length > 0; } -export function isPureRenote(note: MiNote): note is PureRenote { +export function isPureRenote(note: MiNote): note is PureRenote; +export function isPureRenote(note: NoteEdit): note is PureRenoteEdit; +export function isPureRenote(note: MinimalNote): note is MinimalPureRenote; +export function isPureRenote(note: MiNote | NoteEdit | MinimalNote): note is PureRenote | PureRenoteEdit | MinimalPureRenote; +export function isPureRenote(note: MiNote | NoteEdit | MinimalNote): note is PureRenote | PureRenoteEdit | MinimalPureRenote { return isRenote(note) && !isQuote(note); } @@ -68,15 +81,16 @@ type PackedQuote = } | { poll: NonNullable['poll']> } | { - fileIds: NonNullable['fileIds']> + fileIds: [string, ...string[]] }); type PackedPureRenote = PackedRenote & { - text: NonNullable['text']>; - cw: NonNullable['cw']>; - replyId: NonNullable['replyId']>; - poll: NonNullable['poll']>; - fileIds: NonNullable['fileIds']>; + text: null; + cw: null; + replyId: null; + reply: null; + poll: null; + fileIds: []; }; export function isRenotePacked(note: Packed<'Note'>): note is PackedRenote { @@ -94,3 +108,58 @@ export function isQuotePacked(note: PackedRenote): note is PackedQuote { export function isPackedPureRenote(note: Packed<'Note'>): note is PackedPureRenote { return isRenotePacked(note) && !isQuotePacked(note); } + +export type RenoteEdit = + NoteEdit & { + renoteId: NonNullable + }; + +export type QuoteEdit = + RenoteEdit & ({ + text: NonNullable + } | { + cw: NonNullable + } | { + replyId: NonNullable + } | { + hasPoll: true + } | { + fileIds: [string, ...string[]], + }); + +export type PureRenoteEdit = + RenoteEdit & { + text: null, + cw: null, + replyId: null, + reply: null, + hasPoll: false, + fileIds: [], + }; + +export type MinimalNote = Pick; + +export type MinimalRenote = MinimalNote & { + renoteId: string; +}; + +export type MinimalQuote = MinimalRenote & ({ + text: NonNullable +} | { + cw: NonNullable +} | { + replyId: NonNullable +} | { + hasPoll: true +} | { + fileIds: [string, ...string[]], +}); + +export type MinimalPureRenote = MinimalRenote & { + text: null, + cw: null, + replyId: null, + reply: null, + hasPoll: false, + fileIds: [], +}; diff --git a/packages/backend/src/misc/promise-map.ts b/packages/backend/src/misc/promise-map.ts new file mode 100644 index 0000000000..e3fc5f1d65 --- /dev/null +++ b/packages/backend/src/misc/promise-map.ts @@ -0,0 +1,107 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import promiseLimit from 'promise-limit'; + +/** + * Pipes a stream of values through an async mapping callback to produce a new stream of results. + * Avoids extra work by bailing out if any promise rejects or the caller stops iterating the stream. + * + * Can optionally accept a concurrency limit and/or abort signal to further customize behavior. + * If a limit is provided, then no more than that many promises will execute at once. + * If a signal is provided, then all promises will terminate when the signal aborts. + * A signal cannot be provided without a limit, as that would be a no-op. + */ +export async function promiseMap( + values: Iterable | AsyncIterable, + callback: (value: Input, index: number) => Promise, + opts?: { + limit: number | ReturnType>; + signal?: AbortSignal; + }, +): Promise { + // Parse the configured limit or create no-op + const limiter = createLimiter(opts?.limit); + + // Internal state + const outputs: Output[] = []; + const errors: unknown[] = []; + const queue: Promise[] = []; + + let count = 0; + for await (const input of values) { + // Capture the destination index to make sure items are returned in the same order + const index = count; + count++; + + // Stop when any promise fails + if (errors.length > 0) { + break; + } + + // Kick off the next item + const promise = limiter(async () => { + // Check for rejection without throwing any new errors + if (errors.length > 0) return; + + try { + // Checking the abort signal here covers all locations. + // 1. It bails the callback directly. + // 2. The error is written to errors, which breaks out of the loop + opts?.signal?.throwIfAborted(); + + // Populate the next value + outputs[index] = await callback(input, index); + } catch (err) { + errors.push(err); + } + }); + + // But don't forget about it! + queue.push(promise); + } + + // Wait for everything to complete + await Promise.allSettled(queue); + + // Failed - consolidate and throw errors + if (errors.length > 0) { + throwResults(errors); + } + + // Success - return results + return outputs; +} + +type Limiter = (cb: () => Promise) => Promise; + +function createLimiter(limit: undefined | number | ReturnType>): Limiter { + if (!limit) { + return cb => cb(); + } + + if (typeof limit === 'number') { + return promiseLimit(limit); + } + + return limit; +} + +function throwResults(errors: unknown[]): never { + if (errors.length === 0) { + // Shouldn't happen + throw new Error('Mapping promise rejected'); + } + + if (errors.length === 1) { + if (errors[0] instanceof Error) { + throw errors[0]; + } else { + throw new Error('Mapping promise rejected', { cause: errors[0] }); + } + } + + throw new AggregateError(errors); +} diff --git a/packages/backend/src/models/LatestNote.ts b/packages/backend/src/models/LatestNote.ts index 37efb0d4b6..f33c84cc27 100644 --- a/packages/backend/src/models/LatestNote.ts +++ b/packages/backend/src/models/LatestNote.ts @@ -6,7 +6,7 @@ import { PrimaryColumn, Entity, JoinColumn, Column, ManyToOne } from 'typeorm'; import { MiUser } from '@/models/User.js'; import { MiNote } from '@/models/Note.js'; -import { isQuote, isRenote } from '@/misc/is-renote.js'; +import { isQuote, isRenote, MinimalNote } from '@/misc/is-renote.js'; /** * Maps a user to the most recent post by that user. @@ -76,7 +76,7 @@ export class SkLatestNote { /** * Generates a compound key matching a provided note. */ - static keyFor(note: MiNote) { + static keyFor(note: MinimalNote) { return { userId: note.userId, isPublic: note.visibility === 'public', @@ -88,7 +88,7 @@ export class SkLatestNote { /** * Checks if two notes would produce equivalent compound keys. */ - static areEquivalent(first: MiNote, second: MiNote): boolean { + static areEquivalent(first: MinimalNote, second: MinimalNote): boolean { const firstKey = SkLatestNote.keyFor(first); const secondKey = SkLatestNote.keyFor(second); diff --git a/packages/backend/src/models/NoteEdit.ts b/packages/backend/src/models/NoteEdit.ts index 449c974d52..5b7132771d 100644 --- a/packages/backend/src/models/NoteEdit.ts +++ b/packages/backend/src/models/NoteEdit.ts @@ -7,6 +7,8 @@ import { Entity, JoinColumn, Column, ManyToOne, PrimaryColumn, Index } from 'typ import { id } from './util/id.js'; import { MiNote } from './Note.js'; import type { MiDriveFile } from './DriveFile.js'; +import { MiUser } from '@/models/User.js'; +import { noteVisibilities } from '@/types.js'; @Entity() export class NoteEdit { @@ -26,17 +28,63 @@ export class NoteEdit { @JoinColumn() public note: MiNote | null; + @Column({ + ...id(), + comment: 'The ID of author.', + }) + public userId: MiUser['id']; + + @ManyToOne(type => MiUser, { + onDelete: 'CASCADE', + }) + @JoinColumn() + public user: MiUser | null; + + @Column({ + ...id(), + nullable: true, + comment: 'The ID of renote target. Will always be null for older edits', + }) + public renoteId: MiNote['id'] | null; + + @ManyToOne(() => MiNote, { + onDelete: 'CASCADE', + }) + @JoinColumn() + public renote: MiNote | null; + + @Column({ + ...id(), + nullable: true, + comment: 'The ID of reply target. Will always be null for older edits', + }) + public replyId: MiNote['id'] | null; + + @ManyToOne(() => MiNote, { + onDelete: 'CASCADE', + }) + @JoinColumn() + public reply: MiNote | null; + + @Column('enum', { enum: noteVisibilities }) + public visibility: typeof noteVisibilities[number]; + @Column('text', { nullable: true, }) public newText: string | null; - @Column('varchar', { - length: 512, + @Column('text', { nullable: true, + comment: 'Will always be null for older edits', }) public cw: string | null; + @Column('text', { + nullable: true, + }) + public newCw: string | null; + @Column({ ...id(), array: true, @@ -52,11 +100,17 @@ export class NoteEdit { @Column('text', { nullable: true, }) - public oldText: string | null; + public text: string | null; @Column('timestamp with time zone', { comment: 'The old date from before the edit', nullable: true, }) public oldDate: Date | null; + + @Column('boolean', { + default: false, + comment: 'Whether this revision had a poll. Will always be false for older edits', + }) + public hasPoll: boolean; } diff --git a/packages/backend/src/models/RepositoryModule.ts b/packages/backend/src/models/RepositoryModule.ts index 5e0154fe50..476765cc37 100644 --- a/packages/backend/src/models/RepositoryModule.ts +++ b/packages/backend/src/models/RepositoryModule.ts @@ -519,7 +519,7 @@ const $userMemosRepository: Provider = { }; const $noteEditRepository: Provider = { - provide: DI.noteEditRepository, + provide: DI.noteEditsRepository, useFactory: (db: DataSource) => db.getRepository(NoteEdit), inject: [DI.db], }; diff --git a/packages/backend/src/models/User.ts b/packages/backend/src/models/User.ts index b2b13e1724..10c142c53b 100644 --- a/packages/backend/src/models/User.ts +++ b/packages/backend/src/models/User.ts @@ -28,6 +28,11 @@ export class MiUser { }) public lastFetchedAt: Date | null; + @Column('timestamp with time zone', { + nullable: true, + }) + public lastFetchedFeaturedAt?: Date | null; + @Index() @Column('timestamp with time zone', { nullable: true, diff --git a/packages/backend/src/models/_.ts b/packages/backend/src/models/_.ts index e362230d7e..1e742b0bf8 100644 --- a/packages/backend/src/models/_.ts +++ b/packages/backend/src/models/_.ts @@ -118,12 +118,12 @@ export const miRepository = { if (opt.replication) { const queryRunner = this.manager.connection.createQueryRunner('master'); try { - return this.insertOneImpl(entity, findOptions, queryRunner); + return await this.insertOneImpl(entity, findOptions, queryRunner); } finally { await queryRunner.release(); } } else { - return this.insertOneImpl(entity, findOptions); + return await this.insertOneImpl(entity, findOptions); } }, async insertOneImpl(entity, findOptions?, queryRunner?) { @@ -326,5 +326,5 @@ export type ChatRoomInvitationsRepository = Repository & M export type ChatApprovalsRepository = Repository & MiRepository; export type BubbleGameRecordsRepository = Repository & MiRepository; export type ReversiGamesRepository = Repository & MiRepository; -export type NoteEditRepository = Repository & MiRepository; +export type NoteEditsRepository = Repository & MiRepository; export type NoteScheduleRepository = Repository & MiRepository; diff --git a/packages/backend/src/queue/QueueProcessorModule.ts b/packages/backend/src/queue/QueueProcessorModule.ts index b6469229d2..d76b29340d 100644 --- a/packages/backend/src/queue/QueueProcessorModule.ts +++ b/packages/backend/src/queue/QueueProcessorModule.ts @@ -46,7 +46,7 @@ import { RelationshipProcessorService } from './processors/RelationshipProcessor import { ScheduleNotePostProcessorService } from './processors/ScheduleNotePostProcessorService.js'; import { CleanupApLogsProcessorService } from './processors/CleanupApLogsProcessorService.js'; import { HibernateUsersProcessorService } from './processors/HibernateUsersProcessorService.js'; - +import { BackgroundTaskProcessorService } from './processors/BackgroundTaskProcessorService.js'; @Module({ imports: [ CoreModule, @@ -93,6 +93,7 @@ import { HibernateUsersProcessorService } from './processors/HibernateUsersProce ScheduleNotePostProcessorService, CleanupApLogsProcessorService, HibernateUsersProcessorService, + BackgroundTaskProcessorService, ], exports: [ QueueProcessorService, diff --git a/packages/backend/src/queue/QueueProcessorService.ts b/packages/backend/src/queue/QueueProcessorService.ts index cd85db3122..0124660227 100644 --- a/packages/backend/src/queue/QueueProcessorService.ts +++ b/packages/backend/src/queue/QueueProcessorService.ts @@ -14,6 +14,7 @@ import { CheckModeratorsActivityProcessorService } from '@/queue/processors/Chec import { TimeService } from '@/global/TimeService.js'; import { renderFullError } from '@/misc/render-full-error.js'; import { renderInlineError } from '@/misc/render-inline-error.js'; +import { isRetryableError } from '@/misc/is-retryable-error.js'; import { UserWebhookDeliverProcessorService } from './processors/UserWebhookDeliverProcessorService.js'; import { SystemWebhookDeliverProcessorService } from './processors/SystemWebhookDeliverProcessorService.js'; import { EndedPollNotificationProcessorService } from './processors/EndedPollNotificationProcessorService.js'; @@ -53,9 +54,16 @@ import { QUEUE, baseWorkerOptions } from './const.js'; import { ImportNotesProcessorService } from './processors/ImportNotesProcessorService.js'; import { CleanupApLogsProcessorService } from './processors/CleanupApLogsProcessorService.js'; import { HibernateUsersProcessorService } from './processors/HibernateUsersProcessorService.js'; +import { BackgroundTaskProcessorService } from './processors/BackgroundTaskProcessorService.js'; // ref. https://github.com/misskey-dev/misskey/pull/7635#issue-971097019 -function httpRelatedBackoff(attemptsMade: number) { +function httpRelatedBackoff(attemptsMade: number, type?: string, error?: Error) { + // Don't retry permanent errors + // https://docs.bullmq.io/guide/retrying-failing-jobs#custom-back-off-strategies + if (error && !isRetryableError(error)) { + return -1; + } + const baseDelay = 60 * 1000; // 1min const maxBackoff = 8 * 60 * 60 * 1000; // 8hours let backoff = (Math.pow(2, attemptsMade) - 1) * baseDelay; @@ -95,6 +103,7 @@ export class QueueProcessorService implements OnApplicationShutdown { private objectStorageQueueWorker: Bull.Worker; private endedPollNotificationQueueWorker: Bull.Worker; private schedulerNotePostQueueWorker: Bull.Worker; + private readonly backgroundTaskWorker: Bull.Worker; constructor( @Inject(DI.config) @@ -140,6 +149,7 @@ export class QueueProcessorService implements OnApplicationShutdown { private readonly timeService: TimeService, private readonly cleanupApLogsProcessorService: CleanupApLogsProcessorService, private readonly hibernateUsersProcessorService: HibernateUsersProcessorService, + private readonly backgroundTaskProcessorService: BackgroundTaskProcessorService, ) { this.logger = this.queueLoggerService.logger; @@ -565,6 +575,49 @@ export class QueueProcessorService implements OnApplicationShutdown { .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); } //#endregion + + //#region background tasks + { + const logger = this.logger.createSubLogger('backgroundTask'); + + this.backgroundTaskWorker = new Bull.Worker(QUEUE.BACKGROUND_TASK, (job) => this.backgroundTaskProcessorService.process(job), { + ...baseWorkerOptions(this.config, QUEUE.BACKGROUND_TASK), + autorun: false, + concurrency: this.config.backgroundJobConcurrency ?? 32, + limiter: { + max: this.config.backgroundJobPerSec ?? 256, + duration: 1000, + }, + settings: { + backoffStrategy: httpRelatedBackoff, + }, + // Keep a lot of jobs, because this queue moves *fast*! + // https://docs.bullmq.io/guide/workers/auto-removal-of-jobs + removeOnComplete: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 1000, + }, + removeOnFail: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 1000, + }, + }); + this.backgroundTaskWorker + .on('active', (job) => logger.debug(`active id=${job.id}`)) + .on('completed', (job, result) => logger.debug(`completed(${result}) id=${job.id}`)) + .on('failed', (job, err) => { + this.logError(logger, err, job); + if (config.sentryForBackend) { + Sentry.captureMessage(`Queue: ${QUEUE.BACKGROUND_TASK}: ${job?.name ?? '?'}: ${err.name}: ${err.message}`, { + level: 'error', + extra: { job, err }, + }); + } + }) + .on('error', (err: Error) => this.logError(logger, err)) + .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); + } + //#endregion } private logError(logger: Logger, err: unknown, job?: Bull.Job | null): void { @@ -606,6 +659,7 @@ export class QueueProcessorService implements OnApplicationShutdown { this.objectStorageQueueWorker.run(), this.endedPollNotificationQueueWorker.run(), this.schedulerNotePostQueueWorker.run(), + this.backgroundTaskWorker.run(), ]); } @@ -622,6 +676,7 @@ export class QueueProcessorService implements OnApplicationShutdown { this.objectStorageQueueWorker.close(), this.endedPollNotificationQueueWorker.close(), this.schedulerNotePostQueueWorker.close(), + this.backgroundTaskWorker.close(), ]).then(res => { for (const result of res) { if (result.status === 'rejected') { diff --git a/packages/backend/src/queue/const.ts b/packages/backend/src/queue/const.ts index 17c6b81736..44192c280e 100644 --- a/packages/backend/src/queue/const.ts +++ b/packages/backend/src/queue/const.ts @@ -18,6 +18,7 @@ export const QUEUE = { USER_WEBHOOK_DELIVER: 'userWebhookDeliver', SYSTEM_WEBHOOK_DELIVER: 'systemWebhookDeliver', SCHEDULE_NOTE_POST: 'scheduleNotePost', + BACKGROUND_TASK: 'backgroundTask', }; export function baseQueueOptions(config: Config, queueName: typeof QUEUE[keyof typeof QUEUE]): Bull.QueueOptions { diff --git a/packages/backend/src/queue/processors/BackgroundTaskProcessorService.ts b/packages/backend/src/queue/processors/BackgroundTaskProcessorService.ts new file mode 100644 index 0000000000..d25424b02f --- /dev/null +++ b/packages/backend/src/queue/processors/BackgroundTaskProcessorService.ts @@ -0,0 +1,348 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Inject, Injectable } from '@nestjs/common'; +import * as Bull from 'bullmq'; +import { BackgroundTaskJobData, CheckHibernationBackgroundTask, PostDeliverBackgroundTask, PostInboxBackgroundTask, PostNoteBackgroundTask, UpdateFeaturedBackgroundTask, UpdateInstanceBackgroundTask, UpdateUserTagsBackgroundTask, UpdateUserBackgroundTask, UpdateNoteTagsBackgroundTask, DeleteFileBackgroundTask, UpdateLatestNoteBackgroundTask, PostSuspendBackgroundTask, PostUnsuspendBackgroundTask, DeleteApLogsBackgroundTask } from '@/queue/types.js'; +import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js'; +import { QueueLoggerService } from '@/queue/QueueLoggerService.js'; +import Logger from '@/logger.js'; +import { DI } from '@/di-symbols.js'; +import { CacheService } from '@/core/CacheService.js'; +import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; +import { FetchInstanceMetadataService } from '@/core/FetchInstanceMetadataService.js'; +import { MiMeta } from '@/models/Meta.js'; +import InstanceChart from '@/core/chart/charts/instance.js'; +import ApRequestChart from '@/core/chart/charts/ap-request.js'; +import FederationChart from '@/core/chart/charts/federation.js'; +import { NoteCreateService } from '@/core/NoteCreateService.js'; +import type { DriveFilesRepository, NoteEditsRepository, NotesRepository, PollsRepository } from '@/models/_.js'; +import { MiUser } from '@/models/_.js'; +import { NoteEditService } from '@/core/NoteEditService.js'; +import { HashtagService } from '@/core/HashtagService.js'; +import { DriveService } from '@/core/DriveService.js'; +import { LatestNoteService } from '@/core/LatestNoteService.js'; +import { trackTask } from '@/misc/promise-tracker.js'; +import { UserSuspendService } from '@/core/UserSuspendService.js'; +import { ApLogService } from '@/core/ApLogService.js'; +import { CollapsedQueueService } from '@/core/CollapsedQueueService.js'; +import { isRemoteUser } from '@/models/User.js'; +import { errorCodes, IdentifiableError } from '@/misc/identifiable-error.js'; +import { TimeService } from '@/global/TimeService.js'; + +@Injectable() +export class BackgroundTaskProcessorService { + private readonly logger: Logger; + + constructor( + @Inject(DI.meta) + private readonly meta: MiMeta, + + @Inject(DI.notesRepository) + private readonly notesRepository: NotesRepository, + + @Inject(DI.driveFilesRepository) + private readonly driveFilesRepository: DriveFilesRepository, + + @Inject(DI.noteEditsRepository) + private readonly noteEditsRepository: NoteEditsRepository, + + @Inject(DI.pollsRepository) + private readonly pollsRepository: PollsRepository, + + private readonly apPersonService: ApPersonService, + private readonly cacheService: CacheService, + private readonly federatedInstanceService: FederatedInstanceService, + private readonly fetchInstanceMetadataService: FetchInstanceMetadataService, + private readonly instanceChart: InstanceChart, + private readonly apRequestChart: ApRequestChart, + private readonly federationChart: FederationChart, + private readonly collapsedQueueService: CollapsedQueueService, + private readonly noteCreateService: NoteCreateService, + private readonly noteEditService: NoteEditService, + private readonly hashtagService: HashtagService, + private readonly driveService: DriveService, + private readonly latestNoteService: LatestNoteService, + private readonly userSuspendService: UserSuspendService, + private readonly apLogService: ApLogService, + private readonly timeService: TimeService, + + queueLoggerService: QueueLoggerService, + ) { + this.logger = queueLoggerService.logger.createSubLogger('background-task'); + } + + public async process(job: Bull.Job): Promise { + if (job.data.type === 'update-user') { + return await this.processUpdateUser(job.data); + } else if (job.data.type === 'update-featured') { + return await this.processUpdateFeatured(job.data); + } else if (job.data.type === 'update-user-tags') { + return await this.processUpdateUserTags(job.data); + } else if (job.data.type === 'update-note-tags') { + return await this.processUpdateNoteTags(job.data); + } else if (job.data.type === 'update-instance') { + return await this.processUpdateInstance(job.data); + } else if (job.data.type === 'post-deliver') { + return await this.processPostDeliver(job.data); + } else if (job.data.type === 'post-inbox') { + return await this.processPostInbox(job.data); + } else if (job.data.type === 'post-note') { + return await this.processPostNote(job.data); + } else if (job.data.type === 'delete-file') { + return await this.processDeleteFile(job.data); + } else if (job.data.type === 'update-latest-note') { + return await this.processUpdateLatestNote(job.data); + } else if (job.data.type === 'post-suspend') { + return await this.processPostSuspend(job.data); + } else if (job.data.type === 'post-unsuspend') { + return await this.processPostUnsuspend(job.data); + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + } else if (job.data.type === 'delete-ap-logs') { + return await this.processDeleteApLogs(job.data); + } else { + this.logger.warn(`Can't process unknown job type "${job.data}"; this is likely a bug. Full job data:`, job.data); + throw new Error(`Unknown job type ${job.data}, see system logs for details`); + } + } + + private async processUpdateUser(task: UpdateUserBackgroundTask): Promise { + const user = await this.cacheService.findOptionalUserById(task.userId); + if (!user || user.isDeleted) return `Skipping update-user task: user ${task.userId} has been deleted`; + if (user.isSuspended) return `Skipping update-user task: user ${task.userId} is suspended`; + if (!isRemoteUser(user)) return `Skipping update-user task: user ${task.userId} is local`; + + if (user.lastFetchedAt && this.timeService.now - user.lastFetchedAt.getTime() < 1000 * 60 * 60 * 24) { + return `Skipping update-user task: user ${task.userId} was recently updated`; + } + + await this.apPersonService.updatePerson(user.uri); + return 'ok'; + } + + private async processUpdateFeatured(task: UpdateFeaturedBackgroundTask): Promise { + const user = await this.cacheService.findOptionalUserById(task.userId); + if (!user || user.isDeleted) return `Skipping update-featured task: user ${task.userId} has been deleted`; + if (user.isSuspended) return `Skipping update-featured task: user ${task.userId} is suspended`; + if (!isRemoteUser(user)) return `Skipping update-featured task: user ${task.userId} is local`; + if (!user.featured) return `Skipping update-featured task: user ${task.userId} has no featured collection`; + + if (user.lastFetchedFeaturedAt && this.timeService.now - user.lastFetchedFeaturedAt.getTime() < 1000 * 60 * 60 * 24) { + return `Skipping update-featured task: user ${task.userId} was recently updated`; + } + + try { + await this.apPersonService.updateFeatured(user); + } catch (err) { + if (err instanceof IdentifiableError) { + if (err.id === errorCodes.userIsSuspended) return err.message; + if (err.id === errorCodes.userIsDeleted) return err.message; + if (err.id === errorCodes.noFeaturedCollection) return err.message; + } + throw err; + } + return 'ok'; + } + + private async processUpdateUserTags(task: UpdateUserTagsBackgroundTask): Promise { + const user = await this.cacheService.findOptionalUserById(task.userId); + if (!user || user.isDeleted) return `Skipping update-user-tags task: user ${task.userId} has been deleted`; + if (user.isSuspended) return `Skipping update-user-tags task: user ${task.userId} is suspended`; + if (!isRemoteUser(user)) return `Skipping update-user-tags task: user ${task.userId} is local`; + + await this.hashtagService.updateUsertags(user, user.tags); + return 'ok'; + } + + private async processUpdateNoteTags(task: UpdateNoteTagsBackgroundTask): Promise { + const note = await this.notesRepository.findOneBy({ id: task.noteId }); + if (!note) return `Skipping update-note-tags task: note ${task.noteId} has been deleted`; + const user = await this.cacheService.findUserById(note.userId); + if (user.isSuspended) return `Skipping update-note-tags task: note ${task.noteId}'s user ${note.userId} is suspended`; + + await this.hashtagService.updateHashtags(user, note.tags); + return 'ok'; + } + + private async processUpdateInstance(task: UpdateInstanceBackgroundTask): Promise { + const instance = await this.federatedInstanceService.fetch(task.host); + if (instance.isBlocked) return `Skipping update-instance task: instance ${task.host} is blocked`; + if (instance.suspensionState === 'goneSuspended') return `Skipping update-instance task: instance ${task.host} is gone`; + + if (instance.infoUpdatedAt && this.timeService.now - instance.infoUpdatedAt.getTime() < 1000 * 60 * 60 * 24) { + return `Skipping update-instance task: instance ${task.host} was recently updated`; + } + + await this.fetchInstanceMetadataService.fetchInstanceMetadata(instance); + return 'ok'; + } + + private async processPostDeliver(task: PostDeliverBackgroundTask): Promise { + const instance = await this.federatedInstanceService.fetchOrRegister(task.host); + if (instance.isBlocked) return `Skipping post-deliver task: instance ${task.host} is blocked`; + + const success = task.result === 'success'; + + // isNotResponding should be the inverse of success, because: + // 1. We expect success (success=true) from a responding instance (isNotResponding=false). + // 2. We expect failure (success=false) from a non-responding instance (isNotResponding=true). + // If they are equal, then we need to update the cached state. + const updateNotResponding = success === instance.isNotResponding; + + // If we get a permanent failure, then we need to immediately suspend the instance + const updateGoneSuspended = task.result === 'perm-fail' && instance.suspensionState !== 'goneSuspended'; + + // Check if we need to auto-suspend the instance + const updateAutoSuspended = instance.isNotResponding && instance.notRespondingSince && instance.suspensionState === 'none' && instance.notRespondingSince.getTime() <= this.timeService.now - 1000 * 60 * 60 * 24 * 7; + + // This is messy, but we need to minimize updates to space in Postgres blocks. + if (updateNotResponding || updateGoneSuspended || updateAutoSuspended) { + await this.collapsedQueueService.updateInstanceQueue.enqueue(instance.id, { + notRespondingSince: updateNotResponding ? (success ? null : this.timeService.date) : undefined, + shouldSuspendGone: updateGoneSuspended || undefined, + shouldSuspendNotResponding: updateAutoSuspended || undefined, + }); + } + + // Update instance metadata (deferred) + if (success && this.meta.enableStatsForFederatedInstances) { + await this.fetchInstanceMetadataService.fetchInstanceMetadataLazy(instance); + } + + // Update charts + if (this.meta.enableChartsForFederatedInstances) { + this.instanceChart.requestSent(task.host, success); + } + if (success) { + this.apRequestChart.deliverSucc(); + } else { + this.apRequestChart.deliverFail(); + } + this.federationChart.deliverd(task.host, success); + + return 'ok'; + } + + private async processPostInbox(task: PostInboxBackgroundTask): Promise { + const instance = await this.federatedInstanceService.fetchOrRegister(task.host); + if (instance.isBlocked) return `Skipping post-inbox task: instance ${task.host} is blocked`; + + // Update charts + if (this.meta.enableChartsForFederatedInstances) { + this.instanceChart.requestReceived(task.host); + } + this.apRequestChart.inbox(); + this.federationChart.inbox(task.host); + + // Update instance metadata (deferred) + await this.fetchInstanceMetadataService.fetchInstanceMetadataLazy(instance); + + // Unsuspend instance (deferred) + await this.collapsedQueueService.updateInstanceQueue.enqueue(instance.id, { + latestRequestReceivedAt: this.timeService.date, + shouldUnsuspend: instance.suspensionState === 'autoSuspendedForNotResponding', + }); + + return 'ok'; + } + + private async processPostNote(task: PostNoteBackgroundTask): Promise { + const note = await this.notesRepository.findOne({ + where: { id: task.noteId }, + relations: { renote: true, reply: true, channel: true }, + }); + if (!note) return `Skipping post-note task: note ${task.noteId} has been deleted`; + const user = await this.cacheService.findUserById(note.userId); + if (user.isSuspended) return `Skipping post-note task: note ${task.noteId}'s user ${note.userId} is suspended`; + note.user = user; + + const mentionedUsers = await this.cacheService.findUsersById(note.mentions); + const poll = await this.pollsRepository.findOneBy({ noteId: note.id }); + + if (task.edit) { + await this.noteEditService.postNoteEdited(note, user, { ...note, poll }, task.silent, Array.from(mentionedUsers.values())); + } else { + await this.noteCreateService.postNoteCreated(note, user, { ...note, poll }, task.silent, Array.from(mentionedUsers.values())); + } + + return 'ok'; + } + + private async processDeleteFile(task: DeleteFileBackgroundTask): Promise { + const file = await this.driveFilesRepository.findOneBy({ id: task.fileId }); + if (!file) return `Skipping delete-file task: file ${task.fileId} has been deleted`; + + let deleter: MiUser | undefined = undefined; + if (task.deleterId) { + deleter = await this.cacheService.findOptionalUserById(task.deleterId); + if (!deleter) { + this.logger.warn(`[delete-file] Deleting user ${task.deleterId} has been deleted; proceeding with null deleter`); + } + } + + await this.driveService.deleteFileSync(file, task.isExpired, deleter); + return 'ok'; + } + + private async processUpdateLatestNote(task: UpdateLatestNoteBackgroundTask): Promise { + const note = await this.notesRepository.findOneBy({ id: task.note.id }); + + if (note) { + const lastEdit = await this.noteEditsRepository.findOne({ + where: { noteId: task.note.id }, + order: { id: 'desc' }, + }); + + if (lastEdit) { + // Update + await this.latestNoteService.handleUpdatedNote(lastEdit, note); + } else { + // Create + await this.latestNoteService.handleCreatedNote(note); + } + } else { + // Delete + await this.latestNoteService.handleDeletedNote(task.note); + } + + return 'ok'; + } + + private async processPostSuspend(task: PostSuspendBackgroundTask): Promise { + const user = await this.cacheService.findOptionalUserById(task.userId); + if (!user || user.isDeleted) return `Skipping post-suspend task: user ${task.userId} has been deleted`; + + await trackTask(async () => { + await this.userSuspendService.postSuspend(user); + }); + + return 'ok'; + } + + private async processPostUnsuspend(task: PostUnsuspendBackgroundTask): Promise { + const user = await this.cacheService.findOptionalUserById(task.userId); + if (!user || user.isDeleted) return `Skipping post-unsuspend task: user ${task.userId} has been deleted`; + + await trackTask(async () => { + await this.userSuspendService.postUnsuspend(user); + }); + + return 'ok'; + } + + private async processDeleteApLogs(task: DeleteApLogsBackgroundTask): Promise { + if (task.dataType === 'object') { + await this.apLogService.deleteObjectLogs(task.data); + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + } else if (task.dataType === 'inbox') { + await this.apLogService.deleteInboxLogs(task.data); + } else { + this.logger.warn(`Can't process unknown data type "${task.dataType}"; this is likely a bug. Full task data:`, task); + throw new Error(`Unknown task type ${task.dataType}, see system logs for details`); + } + + return 'ok'; + } +} diff --git a/packages/backend/src/queue/processors/CleanProcessorService.ts b/packages/backend/src/queue/processors/CleanProcessorService.ts index f678801e08..eec468e3e4 100644 --- a/packages/backend/src/queue/processors/CleanProcessorService.ts +++ b/packages/backend/src/queue/processors/CleanProcessorService.ts @@ -13,6 +13,7 @@ import { IdService } from '@/core/IdService.js'; import type { Config } from '@/config.js'; import { ReversiService } from '@/core/ReversiService.js'; import { TimeService } from '@/global/TimeService.js'; +import { CollapsedQueueService } from '@/core/CollapsedQueueService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; @@ -37,6 +38,7 @@ export class CleanProcessorService { private reversiService: ReversiService, private idService: IdService, private readonly timeService: TimeService, + private readonly collapsedQueueService: CollapsedQueueService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('clean'); } @@ -51,6 +53,7 @@ export class CleanProcessorService { // 使われてないアンテナを停止 if (this.config.deactivateAntennaThreshold > 0) { + await this.collapsedQueueService.updateAntennaQueue.performAllNow(); await this.antennasRepository.update({ lastUsedAt: LessThan(new Date(this.timeService.now - this.config.deactivateAntennaThreshold)), }, { diff --git a/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts b/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts index 06a4b7ab7f..f5190e67fe 100644 --- a/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts +++ b/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts @@ -4,9 +4,9 @@ */ import { Inject, Injectable } from '@nestjs/common'; -import { In, MoreThan } from 'typeorm'; +import { In, IsNull, MoreThan } from 'typeorm'; import { DI } from '@/di-symbols.js'; -import type { DriveFilesRepository, NoteReactionsRepository, NotesRepository, UserProfilesRepository, UsersRepository, NoteScheduleRepository, MiNoteSchedule, FollowingsRepository, FollowRequestsRepository, BlockingsRepository, MutingsRepository, ClipsRepository, ClipNotesRepository, LatestNotesRepository, NoteEditRepository, NoteFavoritesRepository, PollVotesRepository, PollsRepository, SigninsRepository, UserIpsRepository, RegistryItemsRepository } from '@/models/_.js'; +import type { DriveFilesRepository, NoteReactionsRepository, NotesRepository, UserProfilesRepository, UsersRepository, NoteScheduleRepository, MiNoteSchedule, FollowingsRepository, FollowRequestsRepository, BlockingsRepository, MutingsRepository, ClipsRepository, ClipNotesRepository, LatestNotesRepository, NoteEditsRepository, NoteFavoritesRepository, PollVotesRepository, PollsRepository, SigninsRepository, UserIpsRepository, RegistryItemsRepository, MiUser } from '@/models/_.js'; import type Logger from '@/logger.js'; import { DriveService } from '@/core/DriveService.js'; import type { MiDriveFile } from '@/models/DriveFile.js'; @@ -19,6 +19,7 @@ import { ApLogService } from '@/core/ApLogService.js'; import { ReactionService } from '@/core/ReactionService.js'; import { QueueService } from '@/core/QueueService.js'; import { CacheService } from '@/core/CacheService.js'; +import { NoteDeleteService } from '@/core/NoteDeleteService.js'; import { QueueLoggerService } from '@/queue/QueueLoggerService.js'; import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js'; import * as Acct from '@/misc/acct.js'; @@ -69,8 +70,8 @@ export class DeleteAccountProcessorService { @Inject(DI.latestNotesRepository) private readonly latestNotesRepository: LatestNotesRepository, - @Inject(DI.noteEditRepository) - private readonly noteEditRepository: NoteEditRepository, + @Inject(DI.noteEditsRepository) + private readonly noteEditsRepository: NoteEditsRepository, @Inject(DI.noteFavoritesRepository) private readonly noteFavoritesRepository: NoteFavoritesRepository, @@ -99,6 +100,7 @@ export class DeleteAccountProcessorService { private readonly apLogService: ApLogService, private readonly cacheService: CacheService, private readonly apPersonService: ApPersonService, + private readonly noteDeleteService: NoteDeleteService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('delete-account'); } @@ -293,11 +295,12 @@ export class DeleteAccountProcessorService { const notes = await this.notesRepository.find({ where: { userId: user.id, + replyId: IsNull(), ...(cursor ? { id: MoreThan(cursor) } : {}), }, take: 100, order: { - id: 1, + id: 'desc', }, }) as MiNote[]; @@ -318,7 +321,17 @@ export class DeleteAccountProcessorService { const ids = notes.map(note => note.id); - await this.noteEditRepository.delete({ + const replies = await this.notesRepository.find({ + where: { replyId: In(ids) }, + relations: { user: true }, + }); + + // Delete replies through the usual service to ensure we get all "cascading notes" logic. + for (const reply of replies) { + await this.noteDeleteService.delete(reply.user as MiUser, reply, undefined, true); + } + + await this.noteEditsRepository.delete({ noteId: In(ids), }); await this.notesRepository.delete({ @@ -332,8 +345,7 @@ export class DeleteAccountProcessorService { // Delete note AP logs const noteUris = notes.map(n => n.uri).filter(u => !!u) as string[]; if (noteUris.length > 0) { - await this.apLogService.deleteObjectLogs(noteUris) - .catch(err => this.logger.error(err, `Failed to delete AP logs for notes of user '${user.uri ?? user.id}'`)); + await this.apLogService.deleteObjectLogs(noteUris); } } @@ -371,12 +383,10 @@ export class DeleteAccountProcessorService { { // Delete actor logs if (user.uri) { - await this.apLogService.deleteObjectLogs(user.uri) - .catch(err => this.logger.error(err, `Failed to delete AP logs for user '${user.uri}'`)); + await this.apLogService.deleteObjectLogs(user.uri); } - await this.apLogService.deleteInboxLogs(user.id) - .catch(err => this.logger.error(err, `Failed to delete AP logs for user '${user.uri}'`)); + await this.apLogService.deleteInboxLogs(user.id); this.logger.info('All AP logs deleted'); } diff --git a/packages/backend/src/queue/processors/DeliverProcessorService.ts b/packages/backend/src/queue/processors/DeliverProcessorService.ts index 0b1ef03a7a..7e75a2105c 100644 --- a/packages/backend/src/queue/processors/DeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/DeliverProcessorService.ts @@ -21,6 +21,7 @@ import { StatusError } from '@/misc/status-error.js'; import { UtilityService } from '@/core/UtilityService.js'; import { TimeService } from '@/global/TimeService.js'; import { bindThis } from '@/decorators.js'; +import { QueueService } from '@/core/QueueService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type { DeliverJobData } from '../types.js'; @@ -44,13 +45,14 @@ export class DeliverProcessorService { private federationChart: FederationChart, private queueLoggerService: QueueLoggerService, private readonly timeService: TimeService, + private readonly queueService: QueueService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('deliver'); } @bindThis public async process(job: Bull.Job): Promise { - const { host } = new URL(job.data.to); + const host = this.utilityService.extractDbHost(job.data.to); if (!this.utilityService.isFederationAllowedUri(job.data.to)) { return 'skip (blocked)'; @@ -72,66 +74,19 @@ export class DeliverProcessorService { try { await this.apRequestService.signedPost(job.data.user, job.data.to, job.data.content, job.data.digest); - this.apRequestChart.deliverSucc(); - this.federationChart.deliverd(host, true); - // Update instance stats - process.nextTick(async () => { - if (i == null) return; - - if (i.isNotResponding) { - this.federatedInstanceService.update(i.id, { - isNotResponding: false, - notRespondingSince: null, - }); - } - - if (this.meta.enableChartsForFederatedInstances) { - this.instanceChart.requestSent(i.host, true); - } - }); + await this.queueService.createPostDeliverJob(host, 'success'); return 'Success'; } catch (res) { - this.apRequestChart.deliverFail(); - this.federationChart.deliverd(host, false); - // Update instance stats - this.federatedInstanceService.fetchOrRegister(host).then(i => { - if (!i.isNotResponding) { - this.federatedInstanceService.update(i.id, { - isNotResponding: true, - notRespondingSince: this.timeService.date, - }); - } else if (i.notRespondingSince) { - // 1週間以上不通ならサスペンド - if (i.suspensionState === 'none' && i.notRespondingSince.getTime() <= this.timeService.now - 1000 * 60 * 60 * 24 * 7) { - this.federatedInstanceService.update(i.id, { - suspensionState: 'autoSuspendedForNotResponding', - }); - } - } else { - // isNotRespondingがtrueでnotRespondingSinceがnullの場合はnotRespondingSinceをセット - // notRespondingSinceは新たな機能なので、それ以前のデータにはnotRespondingSinceがない場合がある - this.federatedInstanceService.update(i.id, { - notRespondingSince: this.timeService.date, - }); - } - - if (this.meta.enableChartsForFederatedInstances) { - this.instanceChart.requestSent(i.host, false); - } - }); + const isPerm = job.data.isSharedInbox && res instanceof StatusError && res.statusCode === 410; + await this.queueService.createPostDeliverJob(host, isPerm ? 'perm-fail' : 'temp-fail'); if (res instanceof StatusError && !res.isRetryable) { // 4xx // 相手が閉鎖していることを明示しているため、配送停止する if (job.data.isSharedInbox && res.statusCode === 410) { - this.federatedInstanceService.fetchOrRegister(host).then(i => { - this.federatedInstanceService.update(i.id, { - suspensionState: 'goneSuspended', - }); - }); throw new Bull.UnrecoverableError(`${host} is gone`); } throw new Bull.UnrecoverableError(`${res.statusCode} ${res.statusMessage}`); diff --git a/packages/backend/src/queue/processors/ImportNotesProcessorService.ts b/packages/backend/src/queue/processors/ImportNotesProcessorService.ts index 6f5e929673..a9d179d90e 100644 --- a/packages/backend/src/queue/processors/ImportNotesProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportNotesProcessorService.ts @@ -417,7 +417,7 @@ export class ImportNotesProcessorService { const hashtags = extractApHashtagObjects(toot.object.tag).map((x) => x.name).filter((x): x is string => x != null); try { - text = await this.mfmService.fromHtml(toot.object.content, hashtags); + text = this.mfmService.fromHtml(toot.object.content, hashtags); } catch (error) { text = undefined; } @@ -487,7 +487,7 @@ export class ImportNotesProcessorService { const hashtags = extractApHashtagObjects(post.object.tag).map((x) => x.name).filter((x): x is string => x != null); try { - text = await this.mfmService.fromHtml(post.object.content, hashtags); + text = this.mfmService.fromHtml(post.object.content, hashtags); } catch (error) { text = undefined; } diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index 13b2885263..f8542945df 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -30,21 +30,26 @@ import { DI } from '@/di-symbols.js'; import { SkApInboxLog } from '@/models/_.js'; import type { Config } from '@/config.js'; import { ApLogService, calculateDurationSince } from '@/core/ApLogService.js'; -import { UpdateInstanceQueue } from '@/core/UpdateInstanceQueue.js'; import { TimeService } from '@/global/TimeService.js'; import { isRetryableError } from '@/misc/is-retryable-error.js'; import { renderInlineError } from '@/misc/render-inline-error.js'; +import { QueueService } from '@/core/QueueService.js'; +import { trackPromise } from '@/misc/promise-tracker.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type { InboxJobData } from '../types.js'; +// Moved to CollapsedQueueService +/* type UpdateInstanceJob = { latestRequestReceivedAt: Date, shouldUnsuspend: boolean, }; + */ @Injectable() export class InboxProcessorService implements OnApplicationShutdown { private logger: Logger; + // Moved to CollapsedQueueService //private updateInstanceQueue: CollapsedQueue; constructor( @@ -66,8 +71,8 @@ export class InboxProcessorService implements OnApplicationShutdown { private federationChart: FederationChart, private queueLoggerService: QueueLoggerService, private readonly apLogService: ApLogService, - private readonly updateInstanceQueue: UpdateInstanceQueue, private readonly timeService: TimeService, + private readonly queueService: QueueService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('inbox'); } @@ -103,8 +108,8 @@ export class InboxProcessorService implements OnApplicationShutdown { log.duration = calculateDurationSince(startTime); // Save or finalize asynchronously - this.apLogService.saveInboxLog(log) - .catch(err => this.logger.error('Failed to record AP activity:', err)); + trackPromise(this.apLogService.saveInboxLog(log) + .catch(err => this.logger.error('Failed to record AP activity:', err))); } } @@ -258,28 +263,8 @@ export class InboxProcessorService implements OnApplicationShutdown { log.authUserId = authUser.user.id; } - this.apRequestChart.inbox(); - this.federationChart.inbox(authUser.user.host); - // Update instance stats - process.nextTick(async () => { - const i = await (this.meta.enableStatsForFederatedInstances - ? this.federatedInstanceService.fetchOrRegister(authUser.user.host) - : this.federatedInstanceService.fetch(authUser.user.host)); - - if (i == null) return; - - this.updateInstanceQueue.enqueue(i.id, { - latestRequestReceivedAt: this.timeService.date, - shouldUnsuspend: i.suspensionState === 'autoSuspendedForNotResponding', - }); - - if (this.meta.enableChartsForFederatedInstances) { - this.instanceChart.requestReceived(i.host); - } - - this.fetchInstanceMetadataService.fetchInstanceMetadata(i); - }); + await this.queueService.createPostInboxJob(authUser.user.host); // アクティビティを処理 try { @@ -314,27 +299,8 @@ export class InboxProcessorService implements OnApplicationShutdown { return 'ok'; } - @bindThis - public collapseUpdateInstanceJobs(oldJob: UpdateInstanceJob, newJob: UpdateInstanceJob) { - const latestRequestReceivedAt = oldJob.latestRequestReceivedAt < newJob.latestRequestReceivedAt - ? newJob.latestRequestReceivedAt - : oldJob.latestRequestReceivedAt; - const shouldUnsuspend = oldJob.shouldUnsuspend || newJob.shouldUnsuspend; - return { - latestRequestReceivedAt, - shouldUnsuspend, - }; - } - - @bindThis - public async performUpdateInstance(id: string, job: UpdateInstanceJob) { - await this.federatedInstanceService.update(id, { - latestRequestReceivedAt: this.timeService.date, - isNotResponding: false, - // もしサーバーが死んでるために配信が止まっていた場合には自動的に復活させてあげる - suspensionState: job.shouldUnsuspend ? 'none' : undefined, - }); - } + // collapseUpdateInstanceJobs moved to CollapsedQueueService + // performUpdateInstance moved to CollapsedQueueService @bindThis public async dispose(): Promise {} diff --git a/packages/backend/src/queue/types.ts b/packages/backend/src/queue/types.ts index 6dc9f88034..f37ff581a0 100644 --- a/packages/backend/src/queue/types.ts +++ b/packages/backend/src/queue/types.ts @@ -12,6 +12,7 @@ import type { MiWebhook, WebhookEventTypes } from '@/models/Webhook.js'; import type { IActivity } from '@/core/activitypub/type.js'; import type { SystemWebhookPayload } from '@/core/SystemWebhookService.js'; import type { UserWebhookPayload } from '@/core/UserWebhookService.js'; +import type { MinimalNote } from '@/misc/is-renote.js'; import type httpSignature from '@peertube/http-signature'; export type DeliverJobData = { @@ -168,3 +169,94 @@ export type ThinUser = { export type ScheduleNotePostJobData = { scheduleNoteId: MiNote['id']; }; + +export type BackgroundTaskJobData = + UpdateUserBackgroundTask | + UpdateFeaturedBackgroundTask | + UpdateUserTagsBackgroundTask | + UpdateNoteTagsBackgroundTask | + UpdateInstanceBackgroundTask | + PostDeliverBackgroundTask | + PostInboxBackgroundTask | + PostNoteBackgroundTask | + DeleteFileBackgroundTask | + UpdateLatestNoteBackgroundTask | + PostSuspendBackgroundTask | + PostUnsuspendBackgroundTask | + DeleteApLogsBackgroundTask; + +export type UpdateUserBackgroundTask = { + type: 'update-user'; + userId: string; +}; + +export type UpdateFeaturedBackgroundTask = { + type: 'update-featured'; + userId: string; +}; + +export type UpdateUserTagsBackgroundTask = { + type: 'update-user-tags'; + userId: string; +}; + +export type UpdateNoteTagsBackgroundTask = { + type: 'update-note-tags'; + noteId: string; +}; + +export type UpdateInstanceBackgroundTask = { + type: 'update-instance'; + host: string; +}; + +export type PostDeliverBackgroundTask = { + type: 'post-deliver'; + host: string; + result: 'success' | 'temp-fail' | 'perm-fail'; +}; + +export type PostInboxBackgroundTask = { + type: 'post-inbox'; + host: string; +}; + +export type PostNoteBackgroundTask = { + type: 'post-note'; + noteId: string; + silent: boolean; + edit: boolean; +}; + +export type CheckHibernationBackgroundTask = { + type: 'check-hibernation'; + userId: string; +}; + +export type DeleteFileBackgroundTask = { + type: 'delete-file'; + fileId: string; + isExpired?: boolean; + deleterId?: string; +}; + +export type UpdateLatestNoteBackgroundTask = { + type: 'update-latest-note'; + note: MinimalNote; +}; + +export type PostSuspendBackgroundTask = { + type: 'post-suspend'; + userId: string; +}; + +export type PostUnsuspendBackgroundTask = { + type: 'post-unsuspend'; + userId: string; +}; + +export type DeleteApLogsBackgroundTask = { + type: 'delete-ap-logs'; + dataType: 'inbox' | 'object'; + data: string | string[]; +}; diff --git a/packages/backend/src/server/ActivityPubServerService.ts b/packages/backend/src/server/ActivityPubServerService.ts index e73de15241..de9d588bfa 100644 --- a/packages/backend/src/server/ActivityPubServerService.ts +++ b/packages/backend/src/server/ActivityPubServerService.ts @@ -14,7 +14,7 @@ import accepts from 'accepts'; import vary from 'vary'; import secureJson from 'secure-json-parse'; import { DI } from '@/di-symbols.js'; -import type { FollowingsRepository, NotesRepository, EmojisRepository, NoteReactionsRepository, UserProfilesRepository, UserNotePiningsRepository, UsersRepository, FollowRequestsRepository, MiMeta } from '@/models/_.js'; +import type { FollowingsRepository, NotesRepository, EmojisRepository, NoteReactionsRepository, UserProfilesRepository, UserNotePiningsRepository, UsersRepository, FollowRequestsRepository, MiMeta, MiUserNotePining } from '@/models/_.js'; import * as url from '@/misc/prelude/url.js'; import type { Config } from '@/config.js'; import { ApRendererService } from '@/core/activitypub/ApRendererService.js'; @@ -40,6 +40,7 @@ import { CustomEmojiService, encodeEmojiKey } from '@/core/CustomEmojiService.js import type { FastifyInstance, FastifyRequest, FastifyReply, FastifyPluginOptions, FastifyBodyParser } from 'fastify'; import type { FindOptionsWhere } from 'typeorm'; import { FanoutTimelineEndpointService } from '@/core/FanoutTimelineEndpointService.js'; +import { promiseMap } from '@/misc/promise-map.js'; const ACTIVITY_JSON = 'application/activity+json; charset=utf-8'; const LD_JSON = 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"; charset=utf-8'; @@ -418,7 +419,7 @@ export class ActivityPubServerService { const inStock = followings.length === limit + 1; if (inStock) followings.pop(); - const renderedFollowers = await Promise.all(followings.map(following => this.apRendererService.renderFollowUser(following.followerId))); + const renderedFollowers = await promiseMap(followings, async following => this.apRendererService.renderFollowUser(following.followerId), { limit: 4 }); const rendered = this.apRendererService.renderOrderedCollectionPage( `${partOf}?${url.query({ page: 'true', @@ -515,7 +516,7 @@ export class ActivityPubServerService { const inStock = followings.length === limit + 1; if (inStock) followings.pop(); - const renderedFollowees = await Promise.all(followings.map(following => this.apRendererService.renderFollowUser(following.followeeId))); + const renderedFollowees = await promiseMap(followings, async following => this.apRendererService.renderFollowUser(following.followeeId), { limit: 4 }); const rendered = this.apRendererService.renderOrderedCollectionPage( `${partOf}?${url.query({ page: 'true', @@ -555,10 +556,7 @@ export class ActivityPubServerService { const userId = request.params.user; - const user = await this.usersRepository.findOneBy({ - id: userId, - host: IsNull(), - }); + const user = await this.cacheService.findLocalUserById(userId); if (user == null) { reply.code(404); @@ -568,13 +566,14 @@ export class ActivityPubServerService { const pinings = await this.userNotePiningsRepository.find({ where: { userId: user.id }, order: { id: 'DESC' }, - }); + relations: { note: true }, + }) as (MiUserNotePining & { note: MiNote })[]; - const pinnedNotes = (await Promise.all(pinings.map(pining => - this.notesRepository.findOneByOrFail({ id: pining.noteId })))) + const pinnedNotes = pinings + .map(pin => pin.note) .filter(note => !note.localOnly && ['public', 'home'].includes(note.visibility) && !isPureRenote(note)); - const renderedNotes = await Promise.all(pinnedNotes.map(note => this.apRendererService.renderNote(note, user))); + const renderedNotes = await promiseMap(pinnedNotes, async note => await this.apRendererService.renderNote(note, user), { limit: 4 }); const rendered = this.apRendererService.renderOrderedCollection( `${this.config.url}/users/${userId}/collections/featured`, @@ -664,7 +663,7 @@ export class ActivityPubServerService { if (sinceId) notes.reverse(); - const activities = await Promise.all(notes.map(note => this.packActivity(note, user))); + const activities = await promiseMap(notes, async note => await this.packActivity(note, user)); const rendered = this.apRendererService.renderOrderedCollectionPage( `${partOf}?${url.query({ page: 'true', @@ -1092,14 +1091,8 @@ export class ActivityPubServerService { // check if the following exists. const [follower, followee] = await Promise.all([ - this.usersRepository.findOneBy({ - id: request.params.follower, - host: IsNull(), - }), - this.usersRepository.findOneBy({ - id: request.params.followee, - host: Not(IsNull()), - }), + this.cacheService.findLocalUserById(request.params.follower), + this.cacheService.findRemoteUserById(request.params.followee), ]) as [MiLocalUser | MiRemoteUser | null, MiLocalUser | MiRemoteUser | null]; if (follower == null || followee == null) { @@ -1134,14 +1127,8 @@ export class ActivityPubServerService { } const [follower, followee] = await Promise.all([ - this.usersRepository.findOneBy({ - id: followRequest.followerId, - host: IsNull(), - }), - this.usersRepository.findOneBy({ - id: followRequest.followeeId, - host: Not(IsNull()), - }), + this.cacheService.findLocalUserById(followRequest.followerId), + this.cacheService.findRemoteUserById(followRequest.followeeId), ]) as [MiLocalUser | MiRemoteUser | null, MiLocalUser | MiRemoteUser | null]; if (follower == null || followee == null) { diff --git a/packages/backend/src/server/api/ApiCallService.ts b/packages/backend/src/server/api/ApiCallService.ts index 6605783ff1..be78a63ebb 100644 --- a/packages/backend/src/server/api/ApiCallService.ts +++ b/packages/backend/src/server/api/ApiCallService.ts @@ -145,11 +145,11 @@ export class ApiCallService implements OnApplicationShutdown { } @bindThis - public handleRequest( + public async handleRequest( endpoint: IEndpoint & { exec: any }, request: FastifyRequest<{ Body: Record | undefined, Querystring: Record }>, reply: FastifyReply, - ): void { + ): Promise { // Tell crawlers not to index API endpoints. // https://developers.google.com/search/docs/crawling-indexing/block-indexing reply.header('X-Robots-Tag', 'noindex'); @@ -166,8 +166,8 @@ export class ApiCallService implements OnApplicationShutdown { reply.code(400); return; } - this.authenticateService.authenticate(token).then(([user, app]) => { - this.call(endpoint, user, app, body, null, request, reply).then((res) => { + await this.authenticateService.authenticate(token).then(async ([user, app]) => { + await this.call(endpoint, user, app, body, null, request, reply).then((res) => { if (request.method === 'GET' && endpoint.meta.cacheSec && !token && !user) { reply.header('Cache-Control', `public, max-age=${endpoint.meta.cacheSec}`); } @@ -177,7 +177,8 @@ export class ApiCallService implements OnApplicationShutdown { }); if (user) { - this.logIp(request, user); + // logIp records errors directly + this.logIp(request, user).catch(() => null); } }).catch(err => { this.#sendAuthenticationError(reply, err); @@ -225,8 +226,8 @@ export class ApiCallService implements OnApplicationShutdown { reply.code(400); return; } - this.authenticateService.authenticate(token).then(([user, app]) => { - this.call(endpoint, user, app, fields, { + await this.authenticateService.authenticate(token).then(async ([user, app]) => { + await this.call(endpoint, user, app, fields, { name: multipartData.filename, path: path, }, request, reply).then((res) => { @@ -237,7 +238,8 @@ export class ApiCallService implements OnApplicationShutdown { }); if (user) { - this.logIp(request, user); + // logIp records errors directly + this.logIp(request, user).catch(() => null); } }).catch(err => { cleanup(); @@ -268,7 +270,7 @@ export class ApiCallService implements OnApplicationShutdown { } @bindThis - private logIp(request: FastifyRequest, user: MiLocalUser) { + private async logIp(request: FastifyRequest, user: MiLocalUser) { if (!this.meta.enableIpLogging) return; const ip = request.ip; if (!ip) { @@ -285,12 +287,13 @@ export class ApiCallService implements OnApplicationShutdown { } try { - this.userIpsRepository.createQueryBuilder().insert().values({ + await this.userIpsRepository.createQueryBuilder().insert().values({ createdAt: this.timeService.date, userId: user.id, ip: ip, }).orIgnore(true).execute(); - } catch { + } catch (err) { + this.logger.warn(`Failed to save IP address ${ip} for user ${user.id}: ${renderInlineError(err)}`); } } } diff --git a/packages/backend/src/server/api/ApiServerService.ts b/packages/backend/src/server/api/ApiServerService.ts index 6ed139ad77..8900048de0 100644 --- a/packages/backend/src/server/api/ApiServerService.ts +++ b/packages/backend/src/server/api/ApiServerService.ts @@ -194,7 +194,7 @@ export class ApiServerService { }); if (token && token.session != null && !token.fetched) { - this.accessTokensRepository.update(token.id, { + await this.accessTokensRepository.update(token.id, { fetched: true, }); diff --git a/packages/backend/src/server/api/AuthenticateService.ts b/packages/backend/src/server/api/AuthenticateService.ts index 2c5615c084..23ba7a98b9 100644 --- a/packages/backend/src/server/api/AuthenticateService.ts +++ b/packages/backend/src/server/api/AuthenticateService.ts @@ -16,6 +16,7 @@ import { bindThis } from '@/decorators.js'; import { attachCallerId } from '@/misc/attach-caller-id.js'; import { CacheManagementService, type ManagedMemoryKVCache } from '@/global/CacheManagementService.js'; import { TimeService } from '@/global/TimeService.js'; +import { CollapsedQueueService } from '@/core/CollapsedQueueService.js'; export class AuthenticationError extends Error { // Fix the error name in stack traces - https://stackoverflow.com/a/71573071 @@ -43,6 +44,7 @@ export class AuthenticateService { private cacheService: CacheService, private readonly timeService: TimeService, + private readonly collapsedQueueService: CollapsedQueueService, cacheManagementService: CacheManagementService, ) { @@ -79,7 +81,7 @@ export class AuthenticateService { throw new AuthenticationError('invalid signature'); } - this.accessTokensRepository.update(accessToken.id, { + await this.collapsedQueueService.updateAccessTokenQueue.enqueue(accessToken.id, { lastUsedAt: this.timeService.date, }); diff --git a/packages/backend/src/server/api/GetterService.ts b/packages/backend/src/server/api/GetterService.ts index edf70c0185..47b4e2f1bc 100644 --- a/packages/backend/src/server/api/GetterService.ts +++ b/packages/backend/src/server/api/GetterService.ts @@ -5,7 +5,7 @@ import { Inject, Injectable } from '@nestjs/common'; import { DI } from '@/di-symbols.js'; -import type { NotesRepository, UsersRepository, NoteEditRepository } from '@/models/_.js'; +import type { NotesRepository, UsersRepository, NoteEditsRepository } from '@/models/_.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; import type { MiLocalUser, MiRemoteUser, MiUser } from '@/models/User.js'; import { isRemoteUser, isLocalUser } from '@/models/User.js'; @@ -22,8 +22,8 @@ export class GetterService { @Inject(DI.notesRepository) private notesRepository: NotesRepository, - @Inject(DI.noteEditRepository) - private noteEditRepository: NoteEditRepository, + @Inject(DI.noteEditsRepository) + private noteEditsRepository: NoteEditsRepository, private readonly cacheService: CacheService, ) { @@ -59,7 +59,7 @@ export class GetterService { */ @bindThis public async getEdits(noteId: MiNote['id']) { - const edits = await this.noteEditRepository.findBy({ noteId: noteId }).catch(() => { + const edits = await this.noteEditsRepository.findBy({ noteId: noteId }).catch(() => { throw new IdentifiableError('9725d0ce-ba28-4dde-95a7-2cbb2c15de24', `Note ${noteId} does not exist`); }); diff --git a/packages/backend/src/server/api/SigninApiService.ts b/packages/backend/src/server/api/SigninApiService.ts index a53fec88d0..6ff5ea13f1 100644 --- a/packages/backend/src/server/api/SigninApiService.ts +++ b/packages/backend/src/server/api/SigninApiService.ts @@ -243,7 +243,7 @@ export class SigninApiService { if (same) { if (profile.password!.startsWith('$2')) { const newHash = await argon2.hash(password); - this.userProfilesRepository.update(user.id, { + await this.userProfilesRepository.update(user.id, { password: newHash, }); } @@ -267,7 +267,7 @@ export class SigninApiService { try { if (profile.password!.startsWith('$2')) { const newHash = await argon2.hash(password); - this.userProfilesRepository.update(user.id, { + await this.userProfilesRepository.update(user.id, { password: newHash, }); } diff --git a/packages/backend/src/server/api/StreamingApiServerService.ts b/packages/backend/src/server/api/StreamingApiServerService.ts index 20d8fc4ca2..88e6574263 100644 --- a/packages/backend/src/server/api/StreamingApiServerService.ts +++ b/packages/backend/src/server/api/StreamingApiServerService.ts @@ -271,11 +271,11 @@ export class StreamingApiServerService implements OnApplicationShutdown { this.#connections.set(connection, this.timeService.now); // TODO use collapsed queue - const userUpdateIntervalId = user ? this.timeService.startTimer(() => { - this.usersService.updateLastActiveDate(user); + const userUpdateIntervalId = user ? this.timeService.startTimer(async () => { + await this.usersService.updateLastActiveDate(user); }, 1000 * 60 * 5, { repeated: true }) : null; if (user) { - this.usersService.updateLastActiveDate(user); + await this.usersService.updateLastActiveDate(user); } const pong = () => { this.#connections.set(connection, this.timeService.now); diff --git a/packages/backend/src/server/api/endpoints/admin/abuse-report/notification-recipient/create.ts b/packages/backend/src/server/api/endpoints/admin/abuse-report/notification-recipient/create.ts index bdfbcba518..35c1eb225a 100644 --- a/packages/backend/src/server/api/endpoints/admin/abuse-report/notification-recipient/create.ts +++ b/packages/backend/src/server/api/endpoints/admin/abuse-report/notification-recipient/create.ts @@ -116,7 +116,7 @@ export default class extends Endpoint { // eslint- me, ); - return this.abuseReportNotificationRecipientEntityService.pack(result); + return await this.abuseReportNotificationRecipientEntityService.pack(result); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/abuse-report/notification-recipient/list.ts b/packages/backend/src/server/api/endpoints/admin/abuse-report/notification-recipient/list.ts index dad9161a8a..f36d356205 100644 --- a/packages/backend/src/server/api/endpoints/admin/abuse-report/notification-recipient/list.ts +++ b/packages/backend/src/server/api/endpoints/admin/abuse-report/notification-recipient/list.ts @@ -49,7 +49,7 @@ export default class extends Endpoint { // eslint- ) { super(meta, paramDef, async (ps) => { const recipients = await this.abuseReportNotificationService.fetchRecipients({ method: ps.method }); - return this.abuseReportNotificationRecipientEntityService.packMany(recipients); + return await this.abuseReportNotificationRecipientEntityService.packMany(recipients); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/abuse-report/notification-recipient/show.ts b/packages/backend/src/server/api/endpoints/admin/abuse-report/notification-recipient/show.ts index 557798f946..b2e5e197e3 100644 --- a/packages/backend/src/server/api/endpoints/admin/abuse-report/notification-recipient/show.ts +++ b/packages/backend/src/server/api/endpoints/admin/abuse-report/notification-recipient/show.ts @@ -58,7 +58,7 @@ export default class extends Endpoint { // eslint- throw new ApiError(meta.errors.noSuchRecipient); } - return this.abuseReportNotificationRecipientEntityService.pack(recipients[0]); + return await this.abuseReportNotificationRecipientEntityService.pack(recipients[0]); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/abuse-report/notification-recipient/update.ts b/packages/backend/src/server/api/endpoints/admin/abuse-report/notification-recipient/update.ts index bd4b485217..c3d88412b3 100644 --- a/packages/backend/src/server/api/endpoints/admin/abuse-report/notification-recipient/update.ts +++ b/packages/backend/src/server/api/endpoints/admin/abuse-report/notification-recipient/update.ts @@ -122,7 +122,7 @@ export default class extends Endpoint { // eslint- me, ); - return this.abuseReportNotificationRecipientEntityService.pack(result); + return await this.abuseReportNotificationRecipientEntityService.pack(result); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/captcha/current.ts b/packages/backend/src/server/api/endpoints/admin/captcha/current.ts index 7cc1bc675a..d620f937b1 100644 --- a/packages/backend/src/server/api/endpoints/admin/captcha/current.ts +++ b/packages/backend/src/server/api/endpoints/admin/captcha/current.ts @@ -75,7 +75,7 @@ export default class extends Endpoint { // eslint- private captchaService: CaptchaService, ) { super(meta, paramDef, async () => { - return this.captchaService.get(); + return await this.captchaService.get(); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/delete-all-files-of-a-user.ts b/packages/backend/src/server/api/endpoints/admin/delete-all-files-of-a-user.ts index 2ace85062a..f2c8ac7d42 100644 --- a/packages/backend/src/server/api/endpoints/admin/delete-all-files-of-a-user.ts +++ b/packages/backend/src/server/api/endpoints/admin/delete-all-files-of-a-user.ts @@ -50,7 +50,7 @@ export default class extends Endpoint { // eslint- }); for (const file of files) { - this.driveService.deleteFile(file); + await this.driveService.deleteFile(file); } }); } diff --git a/packages/backend/src/server/api/endpoints/admin/drive/cleanup.ts b/packages/backend/src/server/api/endpoints/admin/drive/cleanup.ts index f5d20366cf..3ca2c348a4 100644 --- a/packages/backend/src/server/api/endpoints/admin/drive/cleanup.ts +++ b/packages/backend/src/server/api/endpoints/admin/drive/cleanup.ts @@ -43,7 +43,7 @@ export default class extends Endpoint { // eslint- }); for (const file of files) { - this.driveService.deleteFile(file); + await this.driveService.deleteFile(file); } }); } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/add.ts b/packages/backend/src/server/api/endpoints/admin/emoji/add.ts index 5ef8307df0..130d2dd2cc 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/add.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/add.ts @@ -106,7 +106,7 @@ export default class extends Endpoint { // eslint- roleIdsThatCanBeUsedThisEmojiAsReaction: ps.roleIdsThatCanBeUsedThisEmojiAsReaction ?? [], }, me); - return this.emojiEntityService.packDetailed(emoji); + return await this.emojiEntityService.packDetailed(emoji); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts b/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts index a7d88954d9..7765a762d9 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts @@ -100,7 +100,7 @@ export default class extends Endpoint { // eslint- roleIdsThatCanBeUsedThisEmojiAsReaction: emoji.roleIdsThatCanBeUsedThisEmojiAsReaction, }, me); - return this.emojiEntityService.packDetailed(addedEmoji); + return await this.emojiEntityService.packDetailed(addedEmoji); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts b/packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts index 7f4ba083cf..0af5295ecb 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts @@ -108,7 +108,7 @@ export default class extends Endpoint { // eslint- .limit(ps.limit) .getMany(); - return this.emojiEntityService.packDetailedMany(emojis); + return await this.emojiEntityService.packDetailedMany(emojis); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/list.ts b/packages/backend/src/server/api/endpoints/admin/emoji/list.ts index b1b8e63d2f..9a5826d13d 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/list.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/list.ts @@ -111,7 +111,7 @@ export default class extends Endpoint { // eslint- emojis = await q.take(ps.limit).skip(ps.offset ?? 0).getMany(); } - return this.emojiEntityService.packDetailedMany(emojis); + return await this.emojiEntityService.packDetailedMany(emojis); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/federation/delete-all-files.ts b/packages/backend/src/server/api/endpoints/admin/federation/delete-all-files.ts index 89fd4be99c..b98d0a7163 100644 --- a/packages/backend/src/server/api/endpoints/admin/federation/delete-all-files.ts +++ b/packages/backend/src/server/api/endpoints/admin/federation/delete-all-files.ts @@ -45,7 +45,7 @@ export default class extends Endpoint { // eslint- }); for (const file of files) { - this.driveService.deleteFile(file); + await this.driveService.deleteFile(file); } }); } diff --git a/packages/backend/src/server/api/endpoints/admin/federation/refresh-remote-instance-metadata.ts b/packages/backend/src/server/api/endpoints/admin/federation/refresh-remote-instance-metadata.ts index 556e291025..d6f38601ea 100644 --- a/packages/backend/src/server/api/endpoints/admin/federation/refresh-remote-instance-metadata.ts +++ b/packages/backend/src/server/api/endpoints/admin/federation/refresh-remote-instance-metadata.ts @@ -42,7 +42,7 @@ export default class extends Endpoint { // eslint- throw new Error('instance not found'); } - this.fetchInstanceMetadataService.fetchInstanceMetadata(instance, true); + await this.fetchInstanceMetadataService.fetchInstanceMetadata(instance, true); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/federation/remove-all-following.ts b/packages/backend/src/server/api/endpoints/admin/federation/remove-all-following.ts index e5d85e1d57..19df5b1355 100644 --- a/packages/backend/src/server/api/endpoints/admin/federation/remove-all-following.ts +++ b/packages/backend/src/server/api/endpoints/admin/federation/remove-all-following.ts @@ -9,6 +9,7 @@ import type { FollowingsRepository, UsersRepository } from '@/models/_.js'; import { DI } from '@/di-symbols.js'; import { QueueService } from '@/core/QueueService.js'; import { ModerationLogService } from '@/core/ModerationLogService.js'; +import { promiseMap } from '@/misc/promise-map.js'; export const meta = { tags: ['admin'], @@ -48,16 +49,20 @@ export default class extends Endpoint { // eslint- }, ]); - const pairs = await Promise.all(followings.map(f => Promise.all([ - this.usersRepository.findOneByOrFail({ id: f.followerId }), - this.usersRepository.findOneByOrFail({ id: f.followeeId }), - ]).then(([from, to]) => [{ id: from.id }, { id: to.id }]))); + const pairs = await promiseMap(followings, async f => { + const [from, to] = await Promise.all([ + this.usersRepository.findOneByOrFail({ id: f.followerId }), + this.usersRepository.findOneByOrFail({ id: f.followeeId }), + ]); + + return [{ id: from.id }, { id: to.id }]; + }); await this.moderationLogService.log(me, 'severFollowRelations', { host: ps.host, }); - this.queueService.createUnfollowJob(pairs.map(p => ({ from: p[0], to: p[1], silent: true }))); + await this.queueService.createUnfollowJob(pairs.map(p => ({ from: p[0], to: p[1], silent: true }))); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/queue/clear.ts b/packages/backend/src/server/api/endpoints/admin/queue/clear.ts index 81cb4b8119..c10e200f86 100644 --- a/packages/backend/src/server/api/endpoints/admin/queue/clear.ts +++ b/packages/backend/src/server/api/endpoints/admin/queue/clear.ts @@ -32,9 +32,9 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - this.queueService.queueClear(ps.queue, ps.state); + await this.queueService.queueClear(ps.queue, ps.state); - this.moderationLogService.log(me, 'clearQueue'); + await this.moderationLogService.log(me, 'clearQueue'); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/queue/jobs.ts b/packages/backend/src/server/api/endpoints/admin/queue/jobs.ts index aba68376ad..be85a50990 100644 --- a/packages/backend/src/server/api/endpoints/admin/queue/jobs.ts +++ b/packages/backend/src/server/api/endpoints/admin/queue/jobs.ts @@ -14,6 +14,76 @@ export const meta = { requireCredential: true, requireModerator: true, kind: 'read:admin:queue', + + res: { + type: 'array', + nullable: false, optional: false, + items: { + type: 'object', + nullable: false, optional: false, + properties: { + id: { + type: 'string', + nullable: false, optional: true, + }, + name: { + type: 'string', + nullable: false, optional: false, + }, + data: { + type: 'object', + nullable: true, optional: true, + additionalProperties: true, + }, + opts: { + type: 'object', + nullable: false, optional: false, + additionalProperties: true, + }, + timestamp: { + type: 'number', + nullable: false, optional: false, + }, + processedOn: { + type: 'number', + nullable: false, optional: true, + }, + processedBy: { + type: 'string', + nullable: false, optional: true, + }, + finishedOn: { + type: 'number', + nullable: false, optional: true, + }, + progress: {}, + attempts: { + type: 'number', + nullable: false, optional: false, + }, + delay: { + type: 'number', + nullable: false, optional: false, + }, + failedReason: { + type: 'string', + nullable: false, optional: true, + }, + stackTrace: { + type: 'array', + nullable: false, optional: true, + items: { + type: 'string', + }, + }, + returnValue: {}, + isFailed: { + type: 'boolean', + nullable: false, optional: true, + }, + }, + }, + }, } as const; export const paramDef = { @@ -32,7 +102,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - return this.queueService.queueGetJobs(ps.queue, ps.state, ps.search); + return await this.queueService.queueGetJobs(ps.queue, ps.state, ps.search); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/queue/promote-jobs.ts b/packages/backend/src/server/api/endpoints/admin/queue/promote-jobs.ts index d22385e261..6ea49bdb7b 100644 --- a/packages/backend/src/server/api/endpoints/admin/queue/promote-jobs.ts +++ b/packages/backend/src/server/api/endpoints/admin/queue/promote-jobs.ts @@ -31,9 +31,9 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - this.queueService.queuePromoteJobs(ps.queue); + await this.queueService.queuePromoteJobs(ps.queue); - this.moderationLogService.log(me, 'promoteQueue'); + await this.moderationLogService.log(me, 'promoteQueue'); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/queue/queue-stats.ts b/packages/backend/src/server/api/endpoints/admin/queue/queue-stats.ts index 10ce48332a..eafc1998db 100644 --- a/packages/backend/src/server/api/endpoints/admin/queue/queue-stats.ts +++ b/packages/backend/src/server/api/endpoints/admin/queue/queue-stats.ts @@ -30,7 +30,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - return this.queueService.queueGetQueue(ps.queue); + return await this.queueService.queueGetQueue(ps.queue); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/queue/queues.ts b/packages/backend/src/server/api/endpoints/admin/queue/queues.ts index 3a38275f60..83def8bf27 100644 --- a/packages/backend/src/server/api/endpoints/admin/queue/queues.ts +++ b/packages/backend/src/server/api/endpoints/admin/queue/queues.ts @@ -29,7 +29,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - return this.queueService.queueGetQueues(); + return await this.queueService.queueGetQueues(); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/queue/remove-job.ts b/packages/backend/src/server/api/endpoints/admin/queue/remove-job.ts index 2c73f689d0..e2ca37c75d 100644 --- a/packages/backend/src/server/api/endpoints/admin/queue/remove-job.ts +++ b/packages/backend/src/server/api/endpoints/admin/queue/remove-job.ts @@ -32,7 +32,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - this.queueService.queueRemoveJob(ps.queue, ps.jobId); + await this.queueService.queueRemoveJob(ps.queue, ps.jobId); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/queue/retry-job.ts b/packages/backend/src/server/api/endpoints/admin/queue/retry-job.ts index b2603128f8..517f99e67f 100644 --- a/packages/backend/src/server/api/endpoints/admin/queue/retry-job.ts +++ b/packages/backend/src/server/api/endpoints/admin/queue/retry-job.ts @@ -32,7 +32,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - this.queueService.queueRetryJob(ps.queue, ps.jobId); + await this.queueService.queueRetryJob(ps.queue, ps.jobId); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/queue/show-job.ts b/packages/backend/src/server/api/endpoints/admin/queue/show-job.ts index 63747b5540..c931d57cb2 100644 --- a/packages/backend/src/server/api/endpoints/admin/queue/show-job.ts +++ b/packages/backend/src/server/api/endpoints/admin/queue/show-job.ts @@ -32,7 +32,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - return this.queueService.queueGetJob(ps.queue, ps.jobId); + return await this.queueService.queueGetJob(ps.queue, ps.jobId); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/show-user.ts b/packages/backend/src/server/api/endpoints/admin/show-user.ts index 49ce0f8bd4..00f4413c74 100644 --- a/packages/backend/src/server/api/endpoints/admin/show-user.ts +++ b/packages/backend/src/server/api/endpoints/admin/show-user.ts @@ -300,14 +300,10 @@ export default class extends Endpoint { // eslint- ) { super(meta, paramDef, async (ps, me) => { const [user, profile] = await Promise.all([ - this.usersRepository.findOneBy({ id: ps.userId }), - this.userProfilesRepository.findOneBy({ userId: ps.userId }), + this.cacheService.findUserById(ps.userId), + this.cacheService.userProfileCache.fetch(ps.userId), ]); - if (user == null || profile == null) { - throw new Error('user not found'); - } - const isModerator = await this.roleService.isModerator(user); const isAdministrator = await this.roleService.isAdministrator(user); const isSilenced = user.isSilenced || !(await this.roleService.getUserPolicies(user.id)).canPublicNote; diff --git a/packages/backend/src/server/api/endpoints/admin/system-webhook/create.ts b/packages/backend/src/server/api/endpoints/admin/system-webhook/create.ts index 28071e7a33..7dc5f1fd39 100644 --- a/packages/backend/src/server/api/endpoints/admin/system-webhook/create.ts +++ b/packages/backend/src/server/api/endpoints/admin/system-webhook/create.ts @@ -79,7 +79,7 @@ export default class extends Endpoint { // eslint- me, ); - return this.systemWebhookEntityService.pack(result); + return await this.systemWebhookEntityService.pack(result); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/system-webhook/list.ts b/packages/backend/src/server/api/endpoints/admin/system-webhook/list.ts index 7a440a774e..c776459c5e 100644 --- a/packages/backend/src/server/api/endpoints/admin/system-webhook/list.ts +++ b/packages/backend/src/server/api/endpoints/admin/system-webhook/list.ts @@ -54,7 +54,7 @@ export default class extends Endpoint { // eslint- isActive: ps.isActive, on: ps.on, }); - return this.systemWebhookEntityService.packMany(webhooks); + return await this.systemWebhookEntityService.packMany(webhooks); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/system-webhook/show.ts b/packages/backend/src/server/api/endpoints/admin/system-webhook/show.ts index 75862c96a7..024c00e59e 100644 --- a/packages/backend/src/server/api/endpoints/admin/system-webhook/show.ts +++ b/packages/backend/src/server/api/endpoints/admin/system-webhook/show.ts @@ -56,7 +56,7 @@ export default class extends Endpoint { // eslint- throw new ApiError(meta.errors.noSuchSystemWebhook); } - return this.systemWebhookEntityService.pack(webhooks[0]); + return await this.systemWebhookEntityService.pack(webhooks[0]); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/system-webhook/update.ts b/packages/backend/src/server/api/endpoints/admin/system-webhook/update.ts index 8d68bb8f87..72f7df2ad0 100644 --- a/packages/backend/src/server/api/endpoints/admin/system-webhook/update.ts +++ b/packages/backend/src/server/api/endpoints/admin/system-webhook/update.ts @@ -85,7 +85,7 @@ export default class extends Endpoint { // eslint- me, ); - return this.systemWebhookEntityService.pack(result); + return await this.systemWebhookEntityService.pack(result); }); } } diff --git a/packages/backend/src/server/api/endpoints/announcements.ts b/packages/backend/src/server/api/endpoints/announcements.ts index 08528ce826..c21d358ead 100644 --- a/packages/backend/src/server/api/endpoints/announcements.ts +++ b/packages/backend/src/server/api/endpoints/announcements.ts @@ -70,7 +70,7 @@ export default class extends Endpoint { // eslint- const announcements = await query.limit(ps.limit).getMany(); - return this.announcementEntityService.packMany(announcements, me); + return await this.announcementEntityService.packMany(announcements, me); }); } } diff --git a/packages/backend/src/server/api/endpoints/antennas/list.ts b/packages/backend/src/server/api/endpoints/antennas/list.ts index ed7d1c9daa..c370143d4f 100644 --- a/packages/backend/src/server/api/endpoints/antennas/list.ts +++ b/packages/backend/src/server/api/endpoints/antennas/list.ts @@ -8,6 +8,7 @@ import { Endpoint } from '@/server/api/endpoint-base.js'; import type { AntennasRepository } from '@/models/_.js'; import { AntennaEntityService } from '@/core/entities/AntennaEntityService.js'; import { DI } from '@/di-symbols.js'; +import { promiseMap } from '@/misc/promise-map.js'; export const meta = { tags: ['antennas', 'account'], @@ -52,7 +53,7 @@ export default class extends Endpoint { // eslint- userId: me.id, }); - return await Promise.all(antennas.map(x => this.antennaEntityService.pack(x))); + return await promiseMap(antennas, async x => await this.antennaEntityService.pack(x), { limit: 4 }); }); } } diff --git a/packages/backend/src/server/api/endpoints/antennas/notes.ts b/packages/backend/src/server/api/endpoints/antennas/notes.ts index 683d5b9e91..5eddfe96fa 100644 --- a/packages/backend/src/server/api/endpoints/antennas/notes.ts +++ b/packages/backend/src/server/api/endpoints/antennas/notes.ts @@ -16,6 +16,7 @@ import { FanoutTimelineService } from '@/core/FanoutTimelineService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import { trackPromise } from '@/misc/promise-tracker.js'; import ActiveUsersChart from '@/core/chart/charts/active-users.js'; +import { CollapsedQueueService } from '@/core/CollapsedQueueService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -79,6 +80,7 @@ export default class extends Endpoint { // eslint- private globalEventService: GlobalEventService, private readonly activeUsersChart: ActiveUsersChart, private readonly timeService: TimeService, + private readonly collapsedQueueService: CollapsedQueueService, ) { super(meta, paramDef, async (ps, me) => { const untilId = ps.untilId ?? (ps.untilDate ? this.idService.gen(ps.untilDate!) : null); @@ -96,9 +98,10 @@ export default class extends Endpoint { // eslint- // falseだった場合はアンテナの配信先が増えたことを通知したい const needPublishEvent = !antenna.isActive; - antenna.isActive = true; - antenna.lastUsedAt = this.timeService.date; - trackPromise(this.antennasRepository.update(antenna.id, antenna)); + await this.collapsedQueueService.updateAntennaQueue.enqueue(antenna.id, { + isActive: true, + lastUsedAt: this.timeService.date, + }); if (needPublishEvent) { this.globalEventService.publishInternalEvent('antennaUpdated', antenna); diff --git a/packages/backend/src/server/api/endpoints/ap/get.ts b/packages/backend/src/server/api/endpoints/ap/get.ts index e3e68b50af..bdd7ea4d01 100644 --- a/packages/backend/src/server/api/endpoints/ap/get.ts +++ b/packages/backend/src/server/api/endpoints/ap/get.ts @@ -91,7 +91,7 @@ export default class extends Endpoint { // eslint- const object = await resolver.resolve(uri, ps.allowAnonymous ?? false); if (ps.expandCollectionItems && isCollectionOrOrderedCollection(object)) { - const items = await resolver.resolveCollectionItems(object, ps.expandCollectionLimit, ps.allowAnonymous ?? false); + const items = await resolver.resolveCollectionItems(object, ps.allowAnonymous ?? false, uri, ps.expandCollectionLimit); if (isOrderedCollection(object) || isOrderedCollectionPage(object)) { object.orderedItems = items; diff --git a/packages/backend/src/server/api/endpoints/auth/session/userkey.ts b/packages/backend/src/server/api/endpoints/auth/session/userkey.ts index 1027eeb4d4..a80a0705d3 100644 --- a/packages/backend/src/server/api/endpoints/auth/session/userkey.ts +++ b/packages/backend/src/server/api/endpoints/auth/session/userkey.ts @@ -113,7 +113,7 @@ export default class extends Endpoint { // eslint- }); // Delete session - this.authSessionsRepository.delete(session.id); + await this.authSessionsRepository.delete(session.id); return { accessToken: accessToken.token, diff --git a/packages/backend/src/server/api/endpoints/channels/followed.ts b/packages/backend/src/server/api/endpoints/channels/followed.ts index 415f7ee29a..d25ba82593 100644 --- a/packages/backend/src/server/api/endpoints/channels/followed.ts +++ b/packages/backend/src/server/api/endpoints/channels/followed.ts @@ -9,6 +9,7 @@ import type { ChannelFollowingsRepository } from '@/models/_.js'; import { QueryService } from '@/core/QueryService.js'; import { ChannelEntityService } from '@/core/entities/ChannelEntityService.js'; import { DI } from '@/di-symbols.js'; +import { promiseMap } from '@/misc/promise-map.js'; export const meta = { tags: ['channels', 'account'], @@ -69,7 +70,7 @@ export default class extends Endpoint { // eslint- .limit(ps.limit) .getMany(); - return await Promise.all(followings.map(x => this.channelEntityService.pack(x.followeeId, me))); + return await promiseMap(followings, async x => await this.channelEntityService.pack(x.followeeId, me), { limit: 4 }); }); } } diff --git a/packages/backend/src/server/api/endpoints/channels/my-favorites.ts b/packages/backend/src/server/api/endpoints/channels/my-favorites.ts index 72a1cc0cf9..8cfe7a65fb 100644 --- a/packages/backend/src/server/api/endpoints/channels/my-favorites.ts +++ b/packages/backend/src/server/api/endpoints/channels/my-favorites.ts @@ -8,6 +8,7 @@ import { Endpoint } from '@/server/api/endpoint-base.js'; import type { ChannelFavoritesRepository } from '@/models/_.js'; import { ChannelEntityService } from '@/core/entities/ChannelEntityService.js'; import { DI } from '@/di-symbols.js'; +import { promiseMap } from '@/misc/promise-map.js'; export const meta = { tags: ['channels', 'account'], @@ -56,7 +57,7 @@ export default class extends Endpoint { // eslint- const favorites = await query .getMany(); - return await Promise.all(favorites.map(x => this.channelEntityService.pack(x.channel!, me))); + return await promiseMap(favorites, async x => await this.channelEntityService.pack(x.channel!, me), { limit: 4 }); }); } } diff --git a/packages/backend/src/server/api/endpoints/channels/search.ts b/packages/backend/src/server/api/endpoints/channels/search.ts index 9476c494a3..6483a08116 100644 --- a/packages/backend/src/server/api/endpoints/channels/search.ts +++ b/packages/backend/src/server/api/endpoints/channels/search.ts @@ -11,6 +11,7 @@ import type { ChannelsRepository } from '@/models/_.js'; import { ChannelEntityService } from '@/core/entities/ChannelEntityService.js'; import { DI } from '@/di-symbols.js'; import { sqlLikeEscape } from '@/misc/sql-like-escape.js'; +import { promiseMap } from '@/misc/promise-map.js'; export const meta = { tags: ['channels'], @@ -75,7 +76,7 @@ export default class extends Endpoint { // eslint- .limit(ps.limit) .getMany(); - return await Promise.all(channels.map(x => this.channelEntityService.pack(x, me))); + return await promiseMap(channels, async x => await this.channelEntityService.pack(x, me), { limit: 4 }); }); } } diff --git a/packages/backend/src/server/api/endpoints/chat/messages/show.ts b/packages/backend/src/server/api/endpoints/chat/messages/show.ts index 9a2bbb8742..950666deb9 100644 --- a/packages/backend/src/server/api/endpoints/chat/messages/show.ts +++ b/packages/backend/src/server/api/endpoints/chat/messages/show.ts @@ -59,7 +59,7 @@ export default class extends Endpoint { // eslint- if (message.fromUserId !== me.id && message.toUserId !== me.id && !(await this.roleService.isModerator(me))) { throw new ApiError(meta.errors.noSuchMessage); } - return this.chatEntityService.packMessageDetailed(message, me); + return await this.chatEntityService.packMessageDetailed(message, me); }); } } diff --git a/packages/backend/src/server/api/endpoints/chat/rooms/invitations/inbox.ts b/packages/backend/src/server/api/endpoints/chat/rooms/invitations/inbox.ts index 8a02d1c704..40688b026d 100644 --- a/packages/backend/src/server/api/endpoints/chat/rooms/invitations/inbox.ts +++ b/packages/backend/src/server/api/endpoints/chat/rooms/invitations/inbox.ts @@ -50,7 +50,7 @@ export default class extends Endpoint { // eslint- await this.chatService.checkChatAvailability(me.id, 'read'); const invitations = await this.chatService.getReceivedRoomInvitationsWithPagination(me.id, ps.limit, ps.sinceId, ps.untilId); - return this.chatEntityService.packRoomInvitations(invitations, me); + return await this.chatEntityService.packRoomInvitations(invitations, me); }); } } diff --git a/packages/backend/src/server/api/endpoints/chat/rooms/invitations/outbox.ts b/packages/backend/src/server/api/endpoints/chat/rooms/invitations/outbox.ts index 0702ba086c..61e6402cab 100644 --- a/packages/backend/src/server/api/endpoints/chat/rooms/invitations/outbox.ts +++ b/packages/backend/src/server/api/endpoints/chat/rooms/invitations/outbox.ts @@ -63,7 +63,7 @@ export default class extends Endpoint { // eslint- } const invitations = await this.chatService.getSentRoomInvitationsWithPagination(ps.roomId, ps.limit, ps.sinceId, ps.untilId); - return this.chatEntityService.packRoomInvitations(invitations, me); + return await this.chatEntityService.packRoomInvitations(invitations, me); }); } } diff --git a/packages/backend/src/server/api/endpoints/chat/rooms/joining.ts b/packages/backend/src/server/api/endpoints/chat/rooms/joining.ts index ba9242c762..c9a5288527 100644 --- a/packages/backend/src/server/api/endpoints/chat/rooms/joining.ts +++ b/packages/backend/src/server/api/endpoints/chat/rooms/joining.ts @@ -51,7 +51,7 @@ export default class extends Endpoint { // eslint- const memberships = await this.chatService.getMyMemberships(me.id, ps.limit, ps.sinceId, ps.untilId); - return this.chatEntityService.packRoomMemberships(memberships, me, { + return await this.chatEntityService.packRoomMemberships(memberships, me, { populateUser: false, populateRoom: true, }); diff --git a/packages/backend/src/server/api/endpoints/chat/rooms/members.ts b/packages/backend/src/server/api/endpoints/chat/rooms/members.ts index f5ffa21d32..388513e939 100644 --- a/packages/backend/src/server/api/endpoints/chat/rooms/members.ts +++ b/packages/backend/src/server/api/endpoints/chat/rooms/members.ts @@ -67,7 +67,7 @@ export default class extends Endpoint { // eslint- const memberships = await this.chatService.getRoomMembershipsWithPagination(room.id, ps.limit, ps.sinceId, ps.untilId); - return this.chatEntityService.packRoomMemberships(memberships, me, { + return await this.chatEntityService.packRoomMemberships(memberships, me, { populateUser: true, populateRoom: false, }); diff --git a/packages/backend/src/server/api/endpoints/chat/rooms/owned.ts b/packages/backend/src/server/api/endpoints/chat/rooms/owned.ts index accf7e1bee..d9cd60a39e 100644 --- a/packages/backend/src/server/api/endpoints/chat/rooms/owned.ts +++ b/packages/backend/src/server/api/endpoints/chat/rooms/owned.ts @@ -50,7 +50,7 @@ export default class extends Endpoint { // eslint- await this.chatService.checkChatAvailability(me.id, 'read'); const rooms = await this.chatService.getOwnedRoomsWithPagination(me.id, ps.limit, ps.sinceId, ps.untilId); - return this.chatEntityService.packRooms(rooms, me); + return await this.chatEntityService.packRooms(rooms, me); }); } } diff --git a/packages/backend/src/server/api/endpoints/chat/rooms/show.ts b/packages/backend/src/server/api/endpoints/chat/rooms/show.ts index 50da210d81..e4bc07cc0c 100644 --- a/packages/backend/src/server/api/endpoints/chat/rooms/show.ts +++ b/packages/backend/src/server/api/endpoints/chat/rooms/show.ts @@ -54,7 +54,7 @@ export default class extends Endpoint { // eslint- throw new ApiError(meta.errors.noSuchRoom); } - return this.chatEntityService.packRoom(room, me); + return await this.chatEntityService.packRoom(room, me); }); } } diff --git a/packages/backend/src/server/api/endpoints/chat/rooms/update.ts b/packages/backend/src/server/api/endpoints/chat/rooms/update.ts index 0cd62cb040..423cff1227 100644 --- a/packages/backend/src/server/api/endpoints/chat/rooms/update.ts +++ b/packages/backend/src/server/api/endpoints/chat/rooms/update.ts @@ -61,7 +61,7 @@ export default class extends Endpoint { // eslint- description: ps.description, }); - return this.chatEntityService.packRoom(updated, me); + return await this.chatEntityService.packRoom(updated, me); }); } } diff --git a/packages/backend/src/server/api/endpoints/clips/my-favorites.ts b/packages/backend/src/server/api/endpoints/clips/my-favorites.ts index 1f9b24e6c9..9278e0578d 100644 --- a/packages/backend/src/server/api/endpoints/clips/my-favorites.ts +++ b/packages/backend/src/server/api/endpoints/clips/my-favorites.ts @@ -56,7 +56,7 @@ export default class extends Endpoint { // eslint- const favorites = await query .getMany(); - return this.clipEntityService.packMany(favorites.map(x => x.clip!), me); + return await this.clipEntityService.packMany(favorites.map(x => x.clip!), me); }); } } diff --git a/packages/backend/src/server/api/endpoints/drive/folders.ts b/packages/backend/src/server/api/endpoints/drive/folders.ts index 525cb8c5d6..bd2ae8bd18 100644 --- a/packages/backend/src/server/api/endpoints/drive/folders.ts +++ b/packages/backend/src/server/api/endpoints/drive/folders.ts @@ -10,6 +10,7 @@ import { QueryService } from '@/core/QueryService.js'; import { DriveFolderEntityService } from '@/core/entities/DriveFolderEntityService.js'; import { DI } from '@/di-symbols.js'; import { sqlLikeEscape } from '@/misc/sql-like-escape.js'; +import { promiseMap } from '@/misc/promise-map.js'; export const meta = { tags: ['drive'], @@ -71,7 +72,7 @@ export default class extends Endpoint { // eslint- } const folders = await query.limit(ps.limit).getMany(); - return await Promise.all(folders.map(folder => this.driveFolderEntityService.pack(folder))); + return await promiseMap(folders, async folder => await this.driveFolderEntityService.pack(folder), { limit: 4 }); }); } } diff --git a/packages/backend/src/server/api/endpoints/drive/folders/find.ts b/packages/backend/src/server/api/endpoints/drive/folders/find.ts index 950aeacea0..2c4cf857c4 100644 --- a/packages/backend/src/server/api/endpoints/drive/folders/find.ts +++ b/packages/backend/src/server/api/endpoints/drive/folders/find.ts @@ -9,6 +9,7 @@ import { Endpoint } from '@/server/api/endpoint-base.js'; import type { DriveFoldersRepository } from '@/models/_.js'; import { DriveFolderEntityService } from '@/core/entities/DriveFolderEntityService.js'; import { DI } from '@/di-symbols.js'; +import { promiseMap } from '@/misc/promise-map.js'; export const meta = { tags: ['drive'], @@ -58,7 +59,7 @@ export default class extends Endpoint { // eslint- parentId: ps.parentId ?? IsNull(), }); - return await Promise.all(folders.map(folder => this.driveFolderEntityService.pack(folder))); + return await promiseMap(folders, async folder => await this.driveFolderEntityService.pack(folder), { limit: 4 }); }); } } diff --git a/packages/backend/src/server/api/endpoints/drive/folders/update.ts b/packages/backend/src/server/api/endpoints/drive/folders/update.ts index 8d51d09ea6..06beb82c7a 100644 --- a/packages/backend/src/server/api/endpoints/drive/folders/update.ts +++ b/packages/backend/src/server/api/endpoints/drive/folders/update.ts @@ -125,7 +125,7 @@ export default class extends Endpoint { // eslint- } // Update - this.driveFoldersRepository.update(folder.id, { + await this.driveFoldersRepository.update(folder.id, { name: folder.name, parentId: folder.parentId, }); diff --git a/packages/backend/src/server/api/endpoints/emoji.ts b/packages/backend/src/server/api/endpoints/emoji.ts index caef5d1528..7b1be89453 100644 --- a/packages/backend/src/server/api/endpoints/emoji.ts +++ b/packages/backend/src/server/api/endpoints/emoji.ts @@ -53,7 +53,7 @@ export default class extends Endpoint { // eslint- super(meta, paramDef, async (ps, me) => { const emoji = await this.customEmojiService.emojisByKeyCache.fetch(ps.name); - return this.emojiEntityService.packDetailed(emoji); + return await this.emojiEntityService.packDetailed(emoji); }); } } diff --git a/packages/backend/src/server/api/endpoints/export-custom-emojis.ts b/packages/backend/src/server/api/endpoints/export-custom-emojis.ts index 5ff099524d..53c157c5cd 100644 --- a/packages/backend/src/server/api/endpoints/export-custom-emojis.ts +++ b/packages/backend/src/server/api/endpoints/export-custom-emojis.ts @@ -29,7 +29,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - this.queueService.createExportCustomEmojisJob(me); + await this.queueService.createExportCustomEmojisJob(me); }); } } diff --git a/packages/backend/src/server/api/endpoints/federation/followers.ts b/packages/backend/src/server/api/endpoints/federation/followers.ts index 9add00ccde..ddf9cf0cbd 100644 --- a/packages/backend/src/server/api/endpoints/federation/followers.ts +++ b/packages/backend/src/server/api/endpoints/federation/followers.ts @@ -50,7 +50,7 @@ export default class extends Endpoint { // eslint- private followingEntityService: FollowingEntityService, ) { super(meta, paramDef, async (ps, me) => { - return this.followingEntityService.getFollowers(me, ps); + return await this.followingEntityService.getFollowers(me, ps); }); } } diff --git a/packages/backend/src/server/api/endpoints/federation/following.ts b/packages/backend/src/server/api/endpoints/federation/following.ts index 849bb61fb4..2ae3a0d485 100644 --- a/packages/backend/src/server/api/endpoints/federation/following.ts +++ b/packages/backend/src/server/api/endpoints/federation/following.ts @@ -50,7 +50,7 @@ export default class extends Endpoint { // eslint- private followingEntityService: FollowingEntityService, ) { super(meta, paramDef, async (ps, me) => { - return this.followingEntityService.getFollowing(me, ps); + return await this.followingEntityService.getFollowing(me, ps); }); } } diff --git a/packages/backend/src/server/api/endpoints/federation/update-remote-user.ts b/packages/backend/src/server/api/endpoints/federation/update-remote-user.ts index 27c3e63ade..c93ac79036 100644 --- a/packages/backend/src/server/api/endpoints/federation/update-remote-user.ts +++ b/packages/backend/src/server/api/endpoints/federation/update-remote-user.ts @@ -48,7 +48,7 @@ export default class extends Endpoint { // eslint- private readonly cacheService: CacheService, ) { super(meta, paramDef, async (ps) => { - const user = await this.cacheService.findRemoteUserById(ps.userId); + const user = await this.cacheService.findOptionalRemoteUserById(ps.userId); if (!user) { throw new ApiError(meta.errors.noSuchUser); diff --git a/packages/backend/src/server/api/endpoints/flash/my-likes.ts b/packages/backend/src/server/api/endpoints/flash/my-likes.ts index 22eae381da..baaf5ba845 100644 --- a/packages/backend/src/server/api/endpoints/flash/my-likes.ts +++ b/packages/backend/src/server/api/endpoints/flash/my-likes.ts @@ -72,7 +72,7 @@ export default class extends Endpoint { // eslint- .limit(ps.limit) .getMany(); - return this.flashLikeEntityService.packMany(likes, me); + return await this.flashLikeEntityService.packMany(likes, me); }); } } diff --git a/packages/backend/src/server/api/endpoints/gallery/posts/create.ts b/packages/backend/src/server/api/endpoints/gallery/posts/create.ts index f280dd8986..7193cf993f 100644 --- a/packages/backend/src/server/api/endpoints/gallery/posts/create.ts +++ b/packages/backend/src/server/api/endpoints/gallery/posts/create.ts @@ -13,6 +13,7 @@ import { IdService } from '@/core/IdService.js'; import { GalleryPostEntityService } from '@/core/entities/GalleryPostEntityService.js'; import { DI } from '@/di-symbols.js'; import { TimeService } from '@/global/TimeService.js'; +import { In } from 'typeorm'; export const meta = { tags: ['gallery'], @@ -66,12 +67,10 @@ export default class extends Endpoint { // eslint- private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { - const files = (await Promise.all(ps.fileIds.map(fileId => - this.driveFilesRepository.findOneBy({ - id: fileId, - userId: me.id, - }), - ))).filter(x => x != null); + const files = await this.driveFilesRepository.findBy({ + id: In(ps.fileIds), + userId: me.id, + }); if (files.length === 0) { throw new Error('no files specified'); diff --git a/packages/backend/src/server/api/endpoints/get-online-users-count.ts b/packages/backend/src/server/api/endpoints/get-online-users-count.ts index 08a037f7a3..0d5663b367 100644 --- a/packages/backend/src/server/api/endpoints/get-online-users-count.ts +++ b/packages/backend/src/server/api/endpoints/get-online-users-count.ts @@ -3,13 +3,14 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { MoreThan } from 'typeorm'; +import { IsNull, MoreThan } from 'typeorm'; import { Inject, Injectable } from '@nestjs/common'; import { USER_ONLINE_THRESHOLD } from '@/const.js'; import type { UsersRepository } from '@/models/_.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { DI } from '@/di-symbols.js'; import { TimeService } from '@/global/TimeService.js'; +import { CacheManagementService, type ManagedMemorySingleCache } from '@/global/CacheManagementService.js'; export const meta = { tags: ['meta'], @@ -23,7 +24,11 @@ export const meta = { properties: { count: { type: 'number', - nullable: false, + nullable: false, optional: false, + }, + countAcrossNetwork: { + type: 'number', + nullable: false, optional: false, }, }, }, @@ -44,19 +49,32 @@ export const paramDef = { @Injectable() export default class extends Endpoint { // eslint-disable-line import/no-default-export + private readonly cache: ManagedMemorySingleCache<{ count: number, countAcrossNetwork: number }>; + constructor( @Inject(DI.usersRepository) private usersRepository: UsersRepository, private readonly timeService: TimeService, + + cacheManagementService: CacheManagementService, ) { super(meta, paramDef, async () => { - const count = await this.usersRepository.countBy({ - lastActiveDate: MoreThan(new Date(this.timeService.now - USER_ONLINE_THRESHOLD)), - }); + return this.cache.fetch(async () => { + const countAcrossNetwork = await this.usersRepository.countBy({ + lastActiveDate: MoreThan(new Date(this.timeService.now - USER_ONLINE_THRESHOLD)), + }); + const count = await this.usersRepository.countBy({ + lastActiveDate: MoreThan(new Date(this.timeService.now - USER_ONLINE_THRESHOLD)), + host: IsNull(), + }); - return { - count, - }; + return { + count, + countAcrossNetwork, + }; + }); }); + + this.cache = cacheManagementService.createMemorySingleCache<{ count: number, countAcrossNetwork: number }>('onlineUsers', { lifetime: 1000 * 60 }); // 1 minute } } diff --git a/packages/backend/src/server/api/endpoints/hashtags/list.ts b/packages/backend/src/server/api/endpoints/hashtags/list.ts index b49c907432..b59d25b18b 100644 --- a/packages/backend/src/server/api/endpoints/hashtags/list.ts +++ b/packages/backend/src/server/api/endpoints/hashtags/list.ts @@ -91,7 +91,7 @@ export default class extends Endpoint { // eslint- const tags = await query.limit(ps.limit).getMany(); - return this.hashtagEntityService.packMany(tags); + return await this.hashtagEntityService.packMany(tags); }); } } diff --git a/packages/backend/src/server/api/endpoints/hashtags/users.ts b/packages/backend/src/server/api/endpoints/hashtags/users.ts index 1be9f6a553..160d3c1cc3 100644 --- a/packages/backend/src/server/api/endpoints/hashtags/users.ts +++ b/packages/backend/src/server/api/endpoints/hashtags/users.ts @@ -12,6 +12,7 @@ import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { DI } from '@/di-symbols.js'; import { RoleService } from '@/core/RoleService.js'; import { TimeService } from '@/global/TimeService.js'; +import { promiseMap } from '@/misc/promise-map.js'; export const meta = { requireCredential: false, @@ -97,7 +98,7 @@ export default class extends Endpoint { // eslint- // 2. A span of more than "limit" consecutive non-trendable users may cause the pagination to stop early. // Unfortunately, there's no better solution unless we refactor role policies to be persisted to the DB. if (ps.trending) { - const usersWithRoles = await Promise.all(users.map(async u => [u, await this.roleService.getUserPolicies(u)] as const)); + const usersWithRoles = await promiseMap(users, async u => [u, await this.roleService.getUserPolicies(u)] as const, { limit: 4 }); users = usersWithRoles .filter(([,p]) => p.canTrend) .map(([u]) => u); diff --git a/packages/backend/src/server/api/endpoints/i/authorized-apps.ts b/packages/backend/src/server/api/endpoints/i/authorized-apps.ts index 447fd18dcf..ad9adac304 100644 --- a/packages/backend/src/server/api/endpoints/i/authorized-apps.ts +++ b/packages/backend/src/server/api/endpoints/i/authorized-apps.ts @@ -9,6 +9,7 @@ import { Endpoint } from '@/server/api/endpoint-base.js'; import type { AccessTokensRepository } from '@/models/_.js'; import { AppEntityService } from '@/core/entities/AppEntityService.js'; import { DI } from '@/di-symbols.js'; +import { promiseMap } from '@/misc/promise-map.js'; export const meta = { requireCredential: true, @@ -88,9 +89,11 @@ export default class extends Endpoint { // eslint- }, }); - return await Promise.all(tokens.map(token => this.appEntityService.pack(token.appId!, me, { + return await promiseMap(tokens, async token => await this.appEntityService.pack(token.appId!, me, { detail: true, - }))); + }), { + limit: 4, + }); }); } } diff --git a/packages/backend/src/server/api/endpoints/i/export-antennas.ts b/packages/backend/src/server/api/endpoints/i/export-antennas.ts index 77fb4a895f..c7ea520fb2 100644 --- a/packages/backend/src/server/api/endpoints/i/export-antennas.ts +++ b/packages/backend/src/server/api/endpoints/i/export-antennas.ts @@ -29,7 +29,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - this.queueService.createExportAntennasJob(me); + await this.queueService.createExportAntennasJob(me); }); } } diff --git a/packages/backend/src/server/api/endpoints/i/export-blocking.ts b/packages/backend/src/server/api/endpoints/i/export-blocking.ts index 7573018bec..84ad687cd6 100644 --- a/packages/backend/src/server/api/endpoints/i/export-blocking.ts +++ b/packages/backend/src/server/api/endpoints/i/export-blocking.ts @@ -29,7 +29,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - this.queueService.createExportBlockingJob(me); + await this.queueService.createExportBlockingJob(me); }); } } diff --git a/packages/backend/src/server/api/endpoints/i/export-clips.ts b/packages/backend/src/server/api/endpoints/i/export-clips.ts index 10d1fdac73..9611a32a29 100644 --- a/packages/backend/src/server/api/endpoints/i/export-clips.ts +++ b/packages/backend/src/server/api/endpoints/i/export-clips.ts @@ -29,7 +29,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - this.queueService.createExportClipsJob(me); + await this.queueService.createExportClipsJob(me); }); } } diff --git a/packages/backend/src/server/api/endpoints/i/export-data.ts b/packages/backend/src/server/api/endpoints/i/export-data.ts index d9a1e087b9..9465ac30ce 100644 --- a/packages/backend/src/server/api/endpoints/i/export-data.ts +++ b/packages/backend/src/server/api/endpoints/i/export-data.ts @@ -29,7 +29,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - this.queueService.createExportAccountDataJob(me); + await this.queueService.createExportAccountDataJob(me); }); } } diff --git a/packages/backend/src/server/api/endpoints/i/export-favorites.ts b/packages/backend/src/server/api/endpoints/i/export-favorites.ts index 5e03f70170..d482618769 100644 --- a/packages/backend/src/server/api/endpoints/i/export-favorites.ts +++ b/packages/backend/src/server/api/endpoints/i/export-favorites.ts @@ -29,7 +29,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - this.queueService.createExportFavoritesJob(me); + await this.queueService.createExportFavoritesJob(me); }); } } diff --git a/packages/backend/src/server/api/endpoints/i/export-following.ts b/packages/backend/src/server/api/endpoints/i/export-following.ts index 2e5ba14737..774704ba2a 100644 --- a/packages/backend/src/server/api/endpoints/i/export-following.ts +++ b/packages/backend/src/server/api/endpoints/i/export-following.ts @@ -32,7 +32,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - this.queueService.createExportFollowingJob(me, ps.excludeMuting, ps.excludeInactive); + await this.queueService.createExportFollowingJob(me, ps.excludeMuting, ps.excludeInactive); }); } } diff --git a/packages/backend/src/server/api/endpoints/i/export-mute.ts b/packages/backend/src/server/api/endpoints/i/export-mute.ts index 0384cf142b..902cea1ac2 100644 --- a/packages/backend/src/server/api/endpoints/i/export-mute.ts +++ b/packages/backend/src/server/api/endpoints/i/export-mute.ts @@ -29,7 +29,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - this.queueService.createExportMuteJob(me); + await this.queueService.createExportMuteJob(me); }); } } diff --git a/packages/backend/src/server/api/endpoints/i/export-notes.ts b/packages/backend/src/server/api/endpoints/i/export-notes.ts index db4e78f667..a3d7d2c0d9 100644 --- a/packages/backend/src/server/api/endpoints/i/export-notes.ts +++ b/packages/backend/src/server/api/endpoints/i/export-notes.ts @@ -29,7 +29,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - this.queueService.createExportNotesJob(me); + await this.queueService.createExportNotesJob(me); }); } } diff --git a/packages/backend/src/server/api/endpoints/i/export-user-lists.ts b/packages/backend/src/server/api/endpoints/i/export-user-lists.ts index 6cd662102c..72ebb4f16e 100644 --- a/packages/backend/src/server/api/endpoints/i/export-user-lists.ts +++ b/packages/backend/src/server/api/endpoints/i/export-user-lists.ts @@ -29,7 +29,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, ) { super(meta, paramDef, async (ps, me) => { - this.queueService.createExportUserListsJob(me); + await this.queueService.createExportUserListsJob(me); }); } } diff --git a/packages/backend/src/server/api/endpoints/i/favorites.ts b/packages/backend/src/server/api/endpoints/i/favorites.ts index 49d47e1624..e1b1be3f73 100644 --- a/packages/backend/src/server/api/endpoints/i/favorites.ts +++ b/packages/backend/src/server/api/endpoints/i/favorites.ts @@ -5,7 +5,7 @@ import { Inject, Injectable } from '@nestjs/common'; import { Endpoint } from '@/server/api/endpoint-base.js'; -import type { NoteFavoritesRepository } from '@/models/_.js'; +import type { MiNote, MiNoteFavorite, NoteFavoritesRepository } from '@/models/_.js'; import { QueryService } from '@/core/QueryService.js'; import { NoteFavoriteEntityService } from '@/core/entities/NoteFavoriteEntityService.js'; import { DI } from '@/di-symbols.js'; @@ -56,11 +56,11 @@ export default class extends Endpoint { // eslint- super(meta, paramDef, async (ps, me) => { const query = this.queryService.makePaginationQuery(this.noteFavoritesRepository.createQueryBuilder('favorite'), ps.sinceId, ps.untilId) .andWhere('favorite.userId = :meId', { meId: me.id }) - .leftJoinAndSelect('favorite.note', 'note'); + .innerJoinAndSelect('favorite.note', 'note'); const favorites = await query .limit(ps.limit) - .getMany(); + .getMany() as (MiNoteFavorite & { note: MiNote })[]; return await this.noteFavoriteEntityService.packMany(favorites, me); }); diff --git a/packages/backend/src/server/api/endpoints/i/page-likes.ts b/packages/backend/src/server/api/endpoints/i/page-likes.ts index 19baa9726d..53a9b0f3b9 100644 --- a/packages/backend/src/server/api/endpoints/i/page-likes.ts +++ b/packages/backend/src/server/api/endpoints/i/page-likes.ts @@ -72,7 +72,7 @@ export default class extends Endpoint { // eslint- .limit(ps.limit) .getMany(); - return this.pageLikeEntityService.packMany(likes, me); + return await this.pageLikeEntityService.packMany(likes, me); }); } } diff --git a/packages/backend/src/server/api/endpoints/i/signin-history.ts b/packages/backend/src/server/api/endpoints/i/signin-history.ts index 8b39b87a7f..3094747a74 100644 --- a/packages/backend/src/server/api/endpoints/i/signin-history.ts +++ b/packages/backend/src/server/api/endpoints/i/signin-history.ts @@ -9,6 +9,7 @@ import type { SigninsRepository } from '@/models/_.js'; import { QueryService } from '@/core/QueryService.js'; import { SigninEntityService } from '@/core/entities/SigninEntityService.js'; import { DI } from '@/di-symbols.js'; +import { promiseMap } from '@/misc/promise-map.js'; export const meta = { requireCredential: true, @@ -56,7 +57,7 @@ export default class extends Endpoint { // eslint- const history = await query.limit(ps.limit).getMany(); - return await Promise.all(history.map(record => this.signinEntityService.pack(record))); + return await promiseMap(history, async record => await this.signinEntityService.pack(record), { limit: 4 }); }); } } diff --git a/packages/backend/src/server/api/endpoints/i/update.ts b/packages/backend/src/server/api/endpoints/i/update.ts index b2396d93eb..9804e5052b 100644 --- a/packages/backend/src/server/api/endpoints/i/update.ts +++ b/packages/backend/src/server/api/endpoints/i/update.ts @@ -36,6 +36,7 @@ import { notificationRecieveConfig } from '@/models/json-schema/user.js'; import { userUnsignedFetchOptions } from '@/const.js'; import { renderInlineError } from '@/misc/render-inline-error.js'; import { trackPromise } from '@/misc/promise-tracker.js'; +import { QueueService } from '@/core/QueueService.js'; import { ApiLoggerService } from '../../ApiLoggerService.js'; import { ApiError } from '../../error.js'; @@ -318,6 +319,7 @@ export default class extends Endpoint { // eslint- private httpRequestService: HttpRequestService, private avatarDecorationService: AvatarDecorationService, private utilityService: UtilityService, + private readonly queueService: QueueService, ) { super(meta, paramDef, async (ps, _user, token) => { const user = await this.usersRepository.findOneByOrFail({ id: _user.id }) as MiLocalUser; @@ -606,9 +608,6 @@ export default class extends Endpoint { // eslint- updates.emojis = emojis; updates.tags = tags; - - // ハッシュタグ更新 - this.hashtagService.updateUsertags(user, tags); //#endregion if (Object.keys(updates).length > 0) { @@ -639,9 +638,12 @@ export default class extends Endpoint { // eslint- // Publish meUpdated event this.globalEventService.publishMainStream(user.id, 'meUpdated', iObj); + // ハッシュタグ更新 + await this.queueService.createUpdateUserTagsJob(user.id); + // 鍵垢を解除したとき、溜まっていたフォローリクエストがあるならすべて承認 if (user.isLocked && ps.isLocked === false) { - await this.userFollowingService.acceptAllFollowRequests(user); + trackPromise(this.userFollowingService.acceptAllFollowRequests(user)); } // フォロワーにUpdateを配信 diff --git a/packages/backend/src/server/api/endpoints/my/apps.ts b/packages/backend/src/server/api/endpoints/my/apps.ts index 3fb0d1b3b7..58996f60e7 100644 --- a/packages/backend/src/server/api/endpoints/my/apps.ts +++ b/packages/backend/src/server/api/endpoints/my/apps.ts @@ -8,6 +8,7 @@ import { Endpoint } from '@/server/api/endpoint-base.js'; import type { AppsRepository } from '@/models/_.js'; import { AppEntityService } from '@/core/entities/AppEntityService.js'; import { DI } from '@/di-symbols.js'; +import { promiseMap } from '@/misc/promise-map.js'; export const meta = { tags: ['account', 'app'], @@ -60,9 +61,11 @@ export default class extends Endpoint { // eslint- skip: ps.offset, }); - return await Promise.all(apps.map(app => this.appEntityService.pack(app, me, { + return await promiseMap(apps, async app => await this.appEntityService.pack(app, me, { detail: true, - }))); + }), { + limit: 4, + }); }); } } diff --git a/packages/backend/src/server/api/endpoints/notes/delete.ts b/packages/backend/src/server/api/endpoints/notes/delete.ts index 9d7c9a9081..e37e8869e9 100644 --- a/packages/backend/src/server/api/endpoints/notes/delete.ts +++ b/packages/backend/src/server/api/endpoints/notes/delete.ts @@ -70,7 +70,7 @@ export default class extends Endpoint { // eslint- } // この操作を行うのが投稿者とは限らない(例えばモデレーター)ため - await this.noteDeleteService.delete(await this.usersRepository.findOneByOrFail({ id: note.userId }), note, false, me); + await this.noteDeleteService.delete(await this.usersRepository.findOneByOrFail({ id: note.userId }), note, me); }); } } diff --git a/packages/backend/src/server/api/endpoints/notes/polls/vote.ts b/packages/backend/src/server/api/endpoints/notes/polls/vote.ts index f7a5db8739..5bc104aad3 100644 --- a/packages/backend/src/server/api/endpoints/notes/polls/vote.ts +++ b/packages/backend/src/server/api/endpoints/notes/polls/vote.ts @@ -16,6 +16,7 @@ import { GlobalEventService } from '@/core/GlobalEventService.js'; import { DI } from '@/di-symbols.js'; import { UserBlockingService } from '@/core/UserBlockingService.js'; import { TimeService } from '@/global/TimeService.js'; +import { trackPromise } from '@/misc/promise-tracker.js'; import { ApiError } from '../../../error.js'; export const meta = { @@ -176,7 +177,7 @@ export default class extends Endpoint { // eslint- } // リモートフォロワーにUpdate配信 - await this.pollService.deliverQuestionUpdate(note); + trackPromise(this.pollService.deliverQuestionUpdate(note)); }); } } diff --git a/packages/backend/src/server/api/endpoints/notes/schedule/list.ts b/packages/backend/src/server/api/endpoints/notes/schedule/list.ts index cbf3a961c0..bfc0010014 100644 --- a/packages/backend/src/server/api/endpoints/notes/schedule/list.ts +++ b/packages/backend/src/server/api/endpoints/notes/schedule/list.ts @@ -15,6 +15,8 @@ import { QueryService } from '@/core/QueryService.js'; import { Packed } from '@/misc/json-schema.js'; import { noteVisibilities } from '@/types.js'; import { bindThis } from '@/decorators.js'; +import { promiseMap } from '@/misc/promise-map.js'; +import { In } from 'typeorm'; export const meta = { tags: ['notes'], @@ -95,6 +97,9 @@ export default class extends Endpoint { // eslint- const query = this.queryService.makePaginationQuery(this.noteScheduleRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId) .andWhere('note.userId = :userId', { userId: me.id }); const scheduleNotes = await query.limit(ps.limit).getMany(); + const refNoteIds = scheduleNotes.flatMap(s => [s.note.reply, s.note.renote]).filter(id => id != null); + const refNotesList = await this.notesRepository.findBy({ id: In(refNoteIds) }); + const refNotesMap = new Map(refNotesList.map(n => [n.id, n])); const user = await this.userEntityService.pack(me, me); const scheduleNotesPack: { id: string; @@ -111,9 +116,9 @@ export default class extends Endpoint { // eslint- }; userId: string; scheduledAt: string; - }[] = await Promise.all(scheduleNotes.map(async (item: MiNoteSchedule) => { - const renote = await this.fetchNote(item.note.renote, me); - const reply = await this.fetchNote(item.note.reply, me); + }[] = await promiseMap(scheduleNotes, async (item: MiNoteSchedule) => { + const renote = await this.fetchNote(item.note.renote, me, refNotesMap); + const reply = await this.fetchNote(item.note.reply, me, refNotesMap); return { ...item, @@ -136,7 +141,9 @@ export default class extends Endpoint { // eslint- poll: item.note.poll ? await this.fillPoll(item.note.poll) : undefined, }, }; - })); + }, { + limit: 4, + }); return scheduleNotesPack; }); @@ -146,12 +153,13 @@ export default class extends Endpoint { // eslint- private async fetchNote( id: MiNote['id'] | null | undefined, me: MiUser, + hint?: Map, ): Promise | null> { if (id) { - const note = await this.notesRepository.findOneBy({ id }); + const note = hint?.get(id) ?? await this.notesRepository.findOneBy({ id }); if (note) { note.reactionAndUserPairCache ??= []; - return this.noteEntityService.pack(note, me); + return await this.noteEntityService.pack(note, me); } } return null; diff --git a/packages/backend/src/server/api/endpoints/notes/unrenote.ts b/packages/backend/src/server/api/endpoints/notes/unrenote.ts index 4a3ca8b656..7443b1d26e 100644 --- a/packages/backend/src/server/api/endpoints/notes/unrenote.ts +++ b/packages/backend/src/server/api/endpoints/notes/unrenote.ts @@ -10,6 +10,7 @@ import { Endpoint } from '@/server/api/endpoint-base.js'; import { NoteDeleteService } from '@/core/NoteDeleteService.js'; import { DI } from '@/di-symbols.js'; import { GetterService } from '@/server/api/GetterService.js'; +import { isQuote, Renote } from '@/misc/is-renote.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -64,14 +65,14 @@ export default class extends Endpoint { // eslint- const renotes = await this.notesRepository.findBy({ userId: me.id, renoteId: note.id, - }); + }) as Renote[]; // TODO inline this into the above query for (const note of renotes) { if (ps.quote) { - if (note.text) this.noteDeleteService.delete(me, note, false); + if (isQuote(note)) await this.noteDeleteService.delete(me, note); } else { - if (!note.text) this.noteDeleteService.delete(me, note, false); + if (!isQuote(note)) await this.noteDeleteService.delete(me, note); } } }); diff --git a/packages/backend/src/server/api/endpoints/notes/versions.ts b/packages/backend/src/server/api/endpoints/notes/versions.ts index 1c6f9838f5..92a1f14dce 100644 --- a/packages/backend/src/server/api/endpoints/notes/versions.ts +++ b/packages/backend/src/server/api/endpoints/notes/versions.ts @@ -108,7 +108,7 @@ export default class extends Endpoint { // eslint- editArray.push({ oldDate: (edit.oldDate ?? edit.updatedAt).toISOString(), updatedAt: edit.updatedAt.toISOString(), - text: edit.oldText ?? edit.newText ?? null, + text: edit.text ?? null, }); } diff --git a/packages/backend/src/server/api/endpoints/notifications/flush.ts b/packages/backend/src/server/api/endpoints/notifications/flush.ts index ab78435b89..ed619d5dd2 100644 --- a/packages/backend/src/server/api/endpoints/notifications/flush.ts +++ b/packages/backend/src/server/api/endpoints/notifications/flush.ts @@ -33,7 +33,7 @@ export default class extends Endpoint { // eslint- private notificationService: NotificationService, ) { super(meta, paramDef, async (ps, me) => { - this.notificationService.flushAllNotifications(me.id); + await this.notificationService.flushAllNotifications(me.id); }); } } diff --git a/packages/backend/src/server/api/endpoints/notifications/mark-all-as-read.ts b/packages/backend/src/server/api/endpoints/notifications/mark-all-as-read.ts index bc83f8d794..4dfe0b92b7 100644 --- a/packages/backend/src/server/api/endpoints/notifications/mark-all-as-read.ts +++ b/packages/backend/src/server/api/endpoints/notifications/mark-all-as-read.ts @@ -33,7 +33,7 @@ export default class extends Endpoint { // eslint- private notificationService: NotificationService, ) { super(meta, paramDef, async (ps, me) => { - this.notificationService.readAllNotification(me.id, true); + await this.notificationService.readAllNotification(me.id, true); }); } } diff --git a/packages/backend/src/server/api/endpoints/reset-password.ts b/packages/backend/src/server/api/endpoints/reset-password.ts index 813550bbcd..dd4e387a20 100644 --- a/packages/backend/src/server/api/endpoints/reset-password.ts +++ b/packages/backend/src/server/api/endpoints/reset-password.ts @@ -67,7 +67,7 @@ export default class extends Endpoint { // eslint- password: hash, }); - this.passwordResetRequestsRepository.delete(req.id); + await this.passwordResetRequestsRepository.delete(req.id); }); } } diff --git a/packages/backend/src/server/api/endpoints/users.ts b/packages/backend/src/server/api/endpoints/users.ts index 1efab2afca..ab4a8bf991 100644 --- a/packages/backend/src/server/api/endpoints/users.ts +++ b/packages/backend/src/server/api/endpoints/users.ts @@ -12,6 +12,7 @@ import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { DI } from '@/di-symbols.js'; import { RoleService } from '@/core/RoleService.js'; import { TimeService } from '@/global/TimeService.js'; +import { promiseMap } from '@/misc/promise-map.js'; import type { SelectQueryBuilder } from 'typeorm'; export const meta = { @@ -113,7 +114,7 @@ export default class extends Endpoint { // eslint- // 1. It may return less than "limit" results. // 2. A span of more than "limit" consecutive non-trendable users may cause the pagination to stop early. // Unfortunately, there's no better solution unless we refactor role policies to be persisted to the DB. - const usersWithRoles = await Promise.all(allUsers.map(async u => [u, await this.roleService.getUserPolicies(u)] as const)); + const usersWithRoles = await promiseMap(allUsers, async u => [u, await this.roleService.getUserPolicies(u)] as const, { limit: 4 }); const users = usersWithRoles .filter(([,p]) => p.canTrend) .map(([u]) => u); diff --git a/packages/backend/src/server/api/endpoints/users/lists/get-memberships.ts b/packages/backend/src/server/api/endpoints/users/lists/get-memberships.ts index 559f08b654..142176deb2 100644 --- a/packages/backend/src/server/api/endpoints/users/lists/get-memberships.ts +++ b/packages/backend/src/server/api/endpoints/users/lists/get-memberships.ts @@ -108,7 +108,7 @@ export default class extends Endpoint { .limit(ps.limit) .getMany(); - return this.userListEntityService.packMembershipsMany(memberships); + return await this.userListEntityService.packMembershipsMany(memberships); }); } } diff --git a/packages/backend/src/server/api/endpoints/users/lists/list.ts b/packages/backend/src/server/api/endpoints/users/lists/list.ts index 976da9512d..2599ea0e34 100644 --- a/packages/backend/src/server/api/endpoints/users/lists/list.ts +++ b/packages/backend/src/server/api/endpoints/users/lists/list.ts @@ -9,6 +9,7 @@ import { Endpoint } from '@/server/api/endpoint-base.js'; import { UserListEntityService } from '@/core/entities/UserListEntityService.js'; import { ApiError } from '@/server/api/error.js'; import { DI } from '@/di-symbols.js'; +import { promiseMap } from '@/misc/promise-map.js'; export const meta = { tags: ['lists', 'account'], @@ -88,7 +89,7 @@ export default class extends Endpoint { isPublic: true, }); - return await Promise.all(userLists.map(x => this.userListEntityService.pack(x, me?.id))); + return await promiseMap(userLists, async x => await this.userListEntityService.pack(x, me?.id), { limit: 4 }); }); } } diff --git a/packages/backend/src/server/api/endpoints/users/show.ts b/packages/backend/src/server/api/endpoints/users/show.ts index 1733c5e460..57ed7a1482 100644 --- a/packages/backend/src/server/api/endpoints/users/show.ts +++ b/packages/backend/src/server/api/endpoints/users/show.ts @@ -138,7 +138,7 @@ export default class extends Endpoint { // eslint- if (ps.username) { user = await this.remoteUserResolveService.resolveUser(ps.username, ps.host ?? null).catch(() => null); } else if (ps.userId != null) { - user = await this.cacheService.findUserById(ps.userId).catch(() => null); + user = await this.cacheService.findOptionalUserById(ps.userId); } if (user == null && ps.host != null) { diff --git a/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts b/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts index 1d2a1db625..287bb9036e 100644 --- a/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts +++ b/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts @@ -23,6 +23,7 @@ import { parseTimelineArgs, TimelineArgs, toBoolean } from './argsUtils.js'; import { convertAnnouncement, convertAttachment, MastodonConverters, convertRelationship } from './MastodonConverters.js'; import type { Entity } from 'megalodon'; import type { FastifyInstance, FastifyPluginOptions } from 'fastify'; +import { promiseMap } from '@/misc/promise-map.js'; @Injectable() export class MastodonApiServerService { @@ -178,7 +179,7 @@ export class MastodonApiServerService { const { client, me } = await this.clientService.getAuthClient(_request); const data = await client.getBookmarks(parseTimelineArgs(_request.query)); - const response = await Promise.all(data.data.map((status) => this.mastoConverters.convertStatus(status, me))); + const response = await promiseMap(data.data, async (status) => await this.mastoConverters.convertStatus(status, me), { limit: 4 }); return reply.send(response); }); @@ -200,7 +201,7 @@ export class MastodonApiServerService { userId: me.id, }; const data = await client.getFavourites(args); - const response = await Promise.all(data.data.map((status) => this.mastoConverters.convertStatus(status, me))); + const response = await promiseMap(data.data, async (status) => await this.mastoConverters.convertStatus(status, me), { limit: 4 }); return reply.send(response); }); @@ -209,7 +210,7 @@ export class MastodonApiServerService { const client = this.clientService.getClient(_request); const data = await client.getMutes(parseTimelineArgs(_request.query)); - const response = await Promise.all(data.data.map((account) => this.mastoConverters.convertAccount(account))); + const response = await promiseMap(data.data, async (account) => await this.mastoConverters.convertAccount(account), { limit: 4 }); return reply.send(response); }); @@ -218,7 +219,7 @@ export class MastodonApiServerService { const client = this.clientService.getClient(_request); const data = await client.getBlocks(parseTimelineArgs(_request.query)); - const response = await Promise.all(data.data.map((account) => this.mastoConverters.convertAccount(account))); + const response = await promiseMap(data.data, async (account) => await this.mastoConverters.convertAccount(account), { limit: 4 }); return reply.send(response); }); @@ -228,7 +229,7 @@ export class MastodonApiServerService { const limit = _request.query.limit ? parseInt(_request.query.limit) : 20; const data = await client.getFollowRequests(limit); - const response = await Promise.all(data.data.map((account) => this.mastoConverters.convertAccount(account as Entity.Account))); + const response = await promiseMap(data.data, async (account) => await this.mastoConverters.convertAccount(account), { limit: 4 }); return reply.send(response); }); diff --git a/packages/backend/src/server/api/mastodon/MastodonConverters.ts b/packages/backend/src/server/api/mastodon/MastodonConverters.ts index 6b0283bf55..6abd918bc6 100644 --- a/packages/backend/src/server/api/mastodon/MastodonConverters.ts +++ b/packages/backend/src/server/api/mastodon/MastodonConverters.ts @@ -13,7 +13,7 @@ import { MfmService } from '@/core/MfmService.js'; import type { Config } from '@/config.js'; import { IMentionedRemoteUsers, MiNote } from '@/models/Note.js'; import type { MiLocalUser, MiUser } from '@/models/User.js'; -import type { NoteEditRepository, UserProfilesRepository } from '@/models/_.js'; +import type { NoteEditsRepository, UserProfilesRepository } from '@/models/_.js'; import { awaitAll } from '@/misc/prelude/await-all.js'; import { CustomEmojiService } from '@/core/CustomEmojiService.js'; import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js'; @@ -24,6 +24,7 @@ import { GetterService } from '@/server/api/GetterService.js'; import { appendContentWarning } from '@/misc/append-content-warning.js'; import { isRenote } from '@/misc/is-renote.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; +import { promiseMap } from '@/misc/promise-map.js'; // Missing from Megalodon apparently // https://docs.joinmastodon.org/entities/StatusEdit/ @@ -60,8 +61,8 @@ export class MastodonConverters { @Inject(DI.userProfilesRepository) private readonly userProfilesRepository: UserProfilesRepository, - @Inject(DI.noteEditRepository) - private readonly noteEditRepository: NoteEditRepository, + @Inject(DI.noteEditsRepository) + private readonly noteEditsRepository: NoteEditsRepository, private readonly mfmService: MfmService, private readonly getterService: GetterService, @@ -175,7 +176,7 @@ export class MastodonConverters { const bioText = profile?.description && this.mfmService.toMastoApiHtml(mfm.parse(profile.description)); - return awaitAll({ + return await awaitAll({ id: account.id, username: user.username, acct: acct, @@ -214,7 +215,7 @@ export class MastodonConverters { const noteUser = await this.getUser(note.userId); const noteInstance = noteUser.instance ?? (noteUser.host ? await this.federatedInstanceService.fetch(noteUser.host) : null); const account = await this.convertAccount(noteUser); - const edits = await this.noteEditRepository.find({ where: { noteId: note.id }, order: { id: 'ASC' } }); + const edits = await this.noteEditsRepository.find({ where: { noteId: note.id }, order: { id: 'ASC' } }); const history: StatusEdit[] = []; const mentionedRemoteUsers = JSON.parse(note.mentionedRemoteUsers); @@ -283,11 +284,15 @@ export class MastodonConverters { }); }); - const mentions = Promise.all(note.mentions.map(p => - this.getUser(p) - .then(u => this.encode(u, mentionedRemoteUsers)) - .catch(() => null))) - .then((p: (Entity.Mention | null)[]) => p.filter(m => m != null)); + const mentions = promiseMap(note.mentions, async p => { + try { + const u = await this.getUser(p); + return this.encode(u, mentionedRemoteUsers); + } catch { + return null; + } + }, { limit: 4 }) + .then((p: Entity.Mention[]) => p.filter(m => m)); const tags = note.tags.map(tag => { return { @@ -363,7 +368,7 @@ export class MastodonConverters { public async convertConversation(conversation: Entity.Conversation, me: MiLocalUser | null): Promise { return { id: conversation.id, - accounts: await Promise.all(conversation.accounts.map((a: Entity.Account) => this.convertAccount(a))), + accounts: await promiseMap(conversation.accounts, async (a: Entity.Account) => await this.convertAccount(a), { limit: 4 }), last_status: conversation.last_status ? await this.convertStatus(conversation.last_status, me) : null, unread: conversation.unread, }; diff --git a/packages/backend/src/server/api/mastodon/endpoints/account.ts b/packages/backend/src/server/api/mastodon/endpoints/account.ts index 0d22ac8ab5..260ab58ece 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/account.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/account.ts @@ -12,6 +12,7 @@ import type { AccessTokensRepository, UserProfilesRepository } from '@/models/_. import { attachMinMaxPagination } from '@/server/api/mastodon/pagination.js'; import { MastodonConverters, convertRelationship, convertFeaturedTag, convertList } from '../MastodonConverters.js'; import type { FastifyInstance } from 'fastify'; +import { promiseMap } from '@/misc/promise-map.js'; interface ApiAccountMastodonRoute { Params: { id?: string }, @@ -188,7 +189,7 @@ export class ApiAccountMastodon { const { client, me } = await this.clientService.getAuthClient(request); const args = parseTimelineArgs(request.query); const data = await client.getAccountStatuses(request.params.id, args); - const response = await Promise.all(data.data.map(async (status) => await this.mastoConverters.convertStatus(status, me))); + const response = await promiseMap(data.data, async status => await this.mastoConverters.convertStatus(status, me), { limit: 2 }); attachMinMaxPagination(request, reply, response); return reply.send(response); @@ -212,7 +213,7 @@ export class ApiAccountMastodon { request.params.id, parseTimelineArgs(request.query), ); - const response = await Promise.all(data.data.map(async (account) => await this.mastoConverters.convertAccount(account))); + const response = await promiseMap(data.data, async account => await this.mastoConverters.convertAccount(account), { limit: 2 }); attachMinMaxPagination(request, reply, response); return reply.send(response); @@ -226,7 +227,7 @@ export class ApiAccountMastodon { request.params.id, parseTimelineArgs(request.query), ); - const response = await Promise.all(data.data.map(async (account) => await this.mastoConverters.convertAccount(account))); + const response = await promiseMap(data.data, async account => await this.mastoConverters.convertAccount(account), { limit: 2 }); attachMinMaxPagination(request, reply, response); return reply.send(response); diff --git a/packages/backend/src/server/api/mastodon/endpoints/notifications.ts b/packages/backend/src/server/api/mastodon/endpoints/notifications.ts index f6cc59e782..dc2c0fefdc 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/notifications.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/notifications.ts @@ -10,6 +10,7 @@ import { MastodonConverters } from '@/server/api/mastodon/MastodonConverters.js' import { attachMinMaxPagination } from '@/server/api/mastodon/pagination.js'; import { MastodonClientService } from '../MastodonClientService.js'; import type { FastifyInstance } from 'fastify'; +import { promiseMap } from '@/misc/promise-map.js'; interface ApiNotifyMastodonRoute { Params: { @@ -29,7 +30,7 @@ export class ApiNotificationsMastodon { fastify.get('/v1/notifications', async (request, reply) => { const { client, me } = await this.clientService.getAuthClient(request); const data = await client.getNotifications(parseTimelineArgs(request.query)); - const notifications = await Promise.all(data.data.map(n => this.mastoConverters.convertNotification(n, me))); + const notifications = await promiseMap(data.data, async n => await this.mastoConverters.convertNotification(n, me), { limit: 4 }); const response: MastodonEntity.Notification[] = []; for (const notification of notifications) { // Notifications for inaccessible notes will be null and should be ignored diff --git a/packages/backend/src/server/api/mastodon/endpoints/search.ts b/packages/backend/src/server/api/mastodon/endpoints/search.ts index 0d1df8c06b..89da9daef9 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/search.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/search.ts @@ -6,6 +6,7 @@ import { Injectable } from '@nestjs/common'; import { MastodonClientService } from '@/server/api/mastodon/MastodonClientService.js'; import { attachMinMaxPagination, attachOffsetPagination } from '@/server/api/mastodon/pagination.js'; +import { promiseMap } from '@/misc/promise-map.js'; import { MastodonConverters } from '../MastodonConverters.js'; import { parseTimelineArgs, TimelineArgs, toBoolean, toInt } from '../argsUtils.js'; import { ApiError } from '../../error.js'; @@ -52,8 +53,8 @@ export class ApiSearchMastodon { const { data } = await client.search(request.query.q, { type, ...query }); const response = { ...data, - accounts: await Promise.all(data.accounts.map((account: Entity.Account) => this.mastoConverters.convertAccount(account))), - statuses: await Promise.all(data.statuses.map((status: Entity.Status) => this.mastoConverters.convertStatus(status, me))), + accounts: await promiseMap(data.accounts, (account: Entity.Account) => this.mastoConverters.convertAccount(account), { limit: 3 }), + statuses: await promiseMap(data.statuses, (status: Entity.Status) => this.mastoConverters.convertStatus(status, me), { limit: 3 }), }; if (type === 'hashtags') { @@ -89,8 +90,8 @@ export class ApiSearchMastodon { const stat = !type || type === 'statuses' ? await client.search(request.query.q, { type: 'statuses', ...query }) : null; const tags = !type || type === 'hashtags' ? await client.search(request.query.q, { type: 'hashtags', ...query }) : null; const response = { - accounts: await Promise.all(acct?.data.accounts.map((account: Entity.Account) => this.mastoConverters.convertAccount(account)) ?? []), - statuses: await Promise.all(stat?.data.statuses.map((status: Entity.Status) => this.mastoConverters.convertStatus(status, me)) ?? []), + accounts: acct ? await promiseMap(acct.data.accounts, async (account: Entity.Account) => await this.mastoConverters.convertAccount(account), { limit: 3 }) : [], + statuses: acct ? await promiseMap(acct.data.statuses, async (status: Entity.Status) => this.mastoConverters.convertStatus(status, me), { limit: 3 }) : [], hashtags: tags?.data.hashtags ?? [], }; @@ -123,7 +124,7 @@ export class ApiSearchMastodon { const data = await res.json() as Entity.Status[]; const me = await this.clientService.getAuth(request); - const response = await Promise.all(data.map(status => this.mastoConverters.convertStatus(status, me))); + const response = await promiseMap(data, async status => await this.mastoConverters.convertStatus(status, me), { limit: 4 }); attachMinMaxPagination(request, reply, response); return reply.send(response); @@ -150,12 +151,12 @@ export class ApiSearchMastodon { await verifyResponse(res); const data = await res.json() as Entity.Account[]; - const response = await Promise.all(data.map(async entry => { - return { - source: 'global', - account: await this.mastoConverters.convertAccount(entry), - }; - })); + const response = await promiseMap(data, async entry => ({ + source: 'global', + account: await this.mastoConverters.convertAccount(entry), + }), { + limit: 4, + }); attachOffsetPagination(request, reply, response); return reply.send(response); diff --git a/packages/backend/src/server/api/mastodon/endpoints/status.ts b/packages/backend/src/server/api/mastodon/endpoints/status.ts index f5942a5267..e126a22f58 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/status.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/status.ts @@ -11,6 +11,7 @@ import { MastodonClientService } from '@/server/api/mastodon/MastodonClientServi import { MastodonDataService } from '@/server/api/mastodon/MastodonDataService.js'; import { getNoteSummary } from '@/misc/get-note-summary.js'; import { isPureRenote } from '@/misc/is-renote.js'; +import { promiseMap } from '@/misc/promise-map.js'; import { convertAttachment, convertPoll, MastodonConverters } from '../MastodonConverters.js'; import type { Entity } from 'megalodon'; import type { FastifyInstance } from 'fastify'; @@ -97,8 +98,8 @@ export class ApiStatusMastodon { const { client, me } = await this.clientService.getAuthClient(_request); const { data } = await client.getStatusContext(_request.params.id, parseTimelineArgs(_request.query)); - const ancestors = await Promise.all(data.ancestors.map(async (status: Entity.Status) => await this.mastoConverters.convertStatus(status, me))); - const descendants = await Promise.all(data.descendants.map(async (status: Entity.Status) => await this.mastoConverters.convertStatus(status, me))); + const ancestors = await promiseMap(data.ancestors, async (status: Entity.Status) => await this.mastoConverters.convertStatus(status, me), { limit: 4 }); + const descendants = await promiseMap(data.descendants, async (status: Entity.Status) => await this.mastoConverters.convertStatus(status, me), { limit: 4 }); const response = { ancestors, descendants }; return reply.send(response); @@ -118,7 +119,7 @@ export class ApiStatusMastodon { const client = this.clientService.getClient(_request); const data = await client.getStatusRebloggedBy(_request.params.id); - const response = await Promise.all(data.data.map((account: Entity.Account) => this.mastoConverters.convertAccount(account))); + const response = await promiseMap(data.data, async (account: Entity.Account) => await this.mastoConverters.convertAccount(account), { limit: 4 }); return reply.send(response); }); @@ -128,7 +129,7 @@ export class ApiStatusMastodon { const client = this.clientService.getClient(_request); const data = await client.getStatusFavouritedBy(_request.params.id); - const response = await Promise.all(data.data.map((account: Entity.Account) => this.mastoConverters.convertAccount(account))); + const response = await promiseMap(data.data, async (account: Entity.Account) => await this.mastoConverters.convertAccount(account), { limit: 4 }); return reply.send(response); }); diff --git a/packages/backend/src/server/api/mastodon/endpoints/timeline.ts b/packages/backend/src/server/api/mastodon/endpoints/timeline.ts index b2f7b18dc9..df56c34c82 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/timeline.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/timeline.ts @@ -6,6 +6,7 @@ import { Injectable } from '@nestjs/common'; import { MastodonClientService } from '@/server/api/mastodon/MastodonClientService.js'; import { attachMinMaxPagination } from '@/server/api/mastodon/pagination.js'; +import { promiseMap } from '@/misc/promise-map.js'; import { convertList, MastodonConverters } from '../MastodonConverters.js'; import { parseTimelineArgs, TimelineArgs, toBoolean } from '../argsUtils.js'; import type { Entity } from 'megalodon'; @@ -25,7 +26,7 @@ export class ApiTimelineMastodon { const data = toBoolean(request.query.local) ? await client.getLocalTimeline(query) : await client.getPublicTimeline(query); - const response = await Promise.all(data.data.map((status: Entity.Status) => this.mastoConverters.convertStatus(status, me))); + const response = await promiseMap(data.data, async (status: Entity.Status) => await this.mastoConverters.convertStatus(status, me), { limit: 4 }); attachMinMaxPagination(request, reply, response); return reply.send(response); @@ -35,7 +36,7 @@ export class ApiTimelineMastodon { const { client, me } = await this.clientService.getAuthClient(request); const query = parseTimelineArgs(request.query); const data = await client.getHomeTimeline(query); - const response = await Promise.all(data.data.map((status: Entity.Status) => this.mastoConverters.convertStatus(status, me))); + const response = await promiseMap(data.data, async (status: Entity.Status) => await this.mastoConverters.convertStatus(status, me), { limit: 4 }); attachMinMaxPagination(request, reply, response); return reply.send(response); @@ -47,7 +48,7 @@ export class ApiTimelineMastodon { const { client, me } = await this.clientService.getAuthClient(request); const query = parseTimelineArgs(request.query); const data = await client.getTagTimeline(request.params.hashtag, query); - const response = await Promise.all(data.data.map((status: Entity.Status) => this.mastoConverters.convertStatus(status, me))); + const response = await promiseMap(data.data, async (status: Entity.Status) => await this.mastoConverters.convertStatus(status, me), { limit: 4 }); attachMinMaxPagination(request, reply, response); return reply.send(response); @@ -59,7 +60,7 @@ export class ApiTimelineMastodon { const { client, me } = await this.clientService.getAuthClient(request); const query = parseTimelineArgs(request.query); const data = await client.getListTimeline(request.params.id, query); - const response = await Promise.all(data.data.map(async (status: Entity.Status) => await this.mastoConverters.convertStatus(status, me))); + const response = await promiseMap(data.data, async (status: Entity.Status) => await this.mastoConverters.convertStatus(status, me), { limit: 4 }); attachMinMaxPagination(request, reply, response); return reply.send(response); @@ -69,7 +70,7 @@ export class ApiTimelineMastodon { const { client, me } = await this.clientService.getAuthClient(request); const query = parseTimelineArgs(request.query); const data = await client.getConversationTimeline(query); - const response = await Promise.all(data.data.map((conversation: Entity.Conversation) => this.mastoConverters.convertConversation(conversation, me))); + const response = await promiseMap(data.data, async (conversation: Entity.Conversation) => await this.mastoConverters.convertConversation(conversation, me), { limit: 4 }); attachMinMaxPagination(request, reply, response); return reply.send(response); @@ -99,7 +100,7 @@ export class ApiTimelineMastodon { const client = this.clientService.getClient(request); const data = await client.getAccountsInList(request.params.id, parseTimelineArgs(request.query)); - const response = await Promise.all(data.data.map((account: Entity.Account) => this.mastoConverters.convertAccount(account))); + const response = await promiseMap(data.data, async (account: Entity.Account) => await this.mastoConverters.convertAccount(account), { limit: 4 }); attachMinMaxPagination(request, reply, response); return reply.send(response); diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index 850f47ad52..e5468d2134 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -551,13 +551,17 @@ export type Serialized = { ? string : T[K] extends (Date | null) ? (string | null) - : T[K] extends Record - ? Serialized - : T[K] extends (Record | null) - ? (Serialized | null) - : T[K] extends (Record | undefined) - ? (Serialized | undefined) - : T[K]; + : T[K] extends (Date | undefined) + ? (string | undefined) + : T[K] extends (Date | null | undefined) + ? (string | null | undefined) + : T[K] extends Record + ? Serialized + : T[K] extends (Record | null) + ? (Serialized | null) + : T[K] extends (Record | undefined) + ? (Serialized | undefined) + : T[K]; }; export type FilterUnionByProperty< diff --git a/packages/backend/test/misc/immediateBackgroundTasks.ts b/packages/backend/test/misc/immediateBackgroundTasks.ts new file mode 100644 index 0000000000..a60a2cd7f8 --- /dev/null +++ b/packages/backend/test/misc/immediateBackgroundTasks.ts @@ -0,0 +1,41 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import type { MiRemoteUser } from '@/models/User.js'; +import type { MiInstance } from '@/models/Instance.js'; +import type { Resolver } from '@/core/activitypub/ApResolverService.js'; +import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js'; +import { FetchInstanceMetadataService } from '@/core/FetchInstanceMetadataService.js'; +import { bindThis } from '@/decorators.js'; +import { errorCodes, IdentifiableError } from '@/misc/identifiable-error.js'; + +export class ImmediateApPersonService extends ApPersonService { + public resolver?: Resolver; + + @bindThis + async updatePersonLazy(uriOrUser: string | MiRemoteUser): Promise { + const userId = typeof(uriOrUser) === 'object' ? uriOrUser.id : uriOrUser; + await this.updatePerson(userId, this.resolver); + } + + @bindThis + async updateFeaturedLazy(userOrId: string | MiRemoteUser): Promise { + await this.updateFeatured(userOrId, this.resolver).catch(err => { + if (err instanceof IdentifiableError) { + if (err.id === errorCodes.userIsSuspended) return; + if (err.id === errorCodes.userIsDeleted) return; + if (err.id === errorCodes.noFeaturedCollection) return; + } + throw err; + }); + } +} + +export class ImmediateFetchInstanceMetadataService extends FetchInstanceMetadataService { + @bindThis + async fetchInstanceMetadataLazy(instance: MiInstance): Promise { + return await this.fetchInstanceMetadata(instance); + } +} diff --git a/packages/backend/test/misc/mock-resolver.ts b/packages/backend/test/misc/mock-resolver.ts index 3bb71a1471..04233232d9 100644 --- a/packages/backend/test/misc/mock-resolver.ts +++ b/packages/backend/test/misc/mock-resolver.ts @@ -22,6 +22,7 @@ import type { UsersRepository, } from '@/models/_.js'; import type { CacheService } from '@/core/CacheService.js'; +import type { MiLocalUser } from '@/models/User.js'; import { ApLogService } from '@/core/ApLogService.js'; import { LoggerService } from '@/core/LoggerService.js'; import { ApUtilityService } from '@/core/activitypub/ApUtilityService.js'; @@ -115,8 +116,9 @@ export class MockResolver extends Resolver { return this.#remoteGetTrials; } - public async resolve(value: string | [string]): Promise; - public async resolve(value: string | IObject | [string | IObject]): Promise; + public async resolve(value: string | [string], allowAnonymous?: boolean, fetchUser?: MiLocalUser): Promise; + public async resolve(value: string | IObjectWithId | [string | IObjectWithId], allowAnonymous?: boolean, fetchUser?: MiLocalUser): Promise; + public async resolve(value: string | IObject | [string | IObject], allowAnonymous?: boolean, fetchUser?: MiLocalUser): Promise; @bindThis public async resolve(value: string | IObject | [string | IObject]): Promise { value = fromTuple(value); diff --git a/packages/backend/test/unit/activitypub.ts b/packages/backend/test/unit/activitypub.ts index eccad42633..e80beedeed 100644 --- a/packages/backend/test/unit/activitypub.ts +++ b/packages/backend/test/unit/activitypub.ts @@ -11,6 +11,7 @@ import { Test, TestingModule } from '@nestjs/testing'; import { jest } from '@jest/globals'; import { MockApResolverService } from '../misc/MockApResolverService.js'; import { MockConsole } from '../misc/MockConsole.js'; +import { ImmediateApPersonService, ImmediateFetchInstanceMetadataService } from '../misc/immediateBackgroundTasks.js'; import type { Config } from '@/config.js'; import type { MiLocalUser, MiRemoteUser } from '@/models/User.js'; import { ApImageService } from '@/core/activitypub/models/ApImageService.js'; @@ -25,8 +26,9 @@ import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; import { LoggerService } from '@/core/LoggerService.js'; import { CacheManagementService } from '@/global/CacheManagementService.js'; import { ApResolverService } from '@/core/activitypub/ApResolverService.js'; +import { FetchInstanceMetadataService } from '@/core/FetchInstanceMetadataService.js'; import type { IActor, IApDocument, ICollection, IObject, IPost } from '@/core/activitypub/type.js'; -import { MiMeta, MiNote, MiUser, MiUserKeypair, UserProfilesRepository, UserPublickeysRepository, UserKeypairsRepository, UsersRepository, NotesRepository } from '@/models/_.js'; +import { MiMeta, MiNote, MiUser, MiUserKeypair, UserProfilesRepository, UserPublickeysRepository, UserKeypairsRepository, UsersRepository, NotesRepository, UserNotePiningsRepository } from '@/models/_.js'; import { DI } from '@/di-symbols.js'; import { secureRndstr } from '@/misc/secure-rndstr.js'; import { DownloadService } from '@/core/DownloadService.js'; @@ -75,7 +77,7 @@ function createRandomFeaturedCollection(actor: NonTransientIActor, length: numbe return { '@context': 'https://www.w3.org/ns/activitystreams', type: 'Collection', - id: actor.outbox as string, + id: actor.featured as string | null ?? `${actor.id}/featured`, totalItems: items.length, items, }; @@ -108,6 +110,7 @@ describe('ActivityPub', () => { let cacheManagementService: CacheManagementService; let mockConsole: MockConsole; let notesRepository: NotesRepository; + let userNotePiningsRepository: UserNotePiningsRepository; const metaInitial = { id: 'x', @@ -163,6 +166,8 @@ describe('ActivityPub', () => { .overrideProvider(DI.meta).useValue(meta) .overrideProvider(ApResolverService).useClass(MockApResolverService) .overrideProvider(DI.console).useClass(MockConsole) + .overrideProvider(FetchInstanceMetadataService).useClass(ImmediateFetchInstanceMetadataService) + .overrideProvider(ApPersonService).useClass(ImmediateApPersonService) .compile(); await app.init(); @@ -184,6 +189,7 @@ describe('ActivityPub', () => { cacheManagementService = app.get(CacheManagementService); mockConsole = app.get(DI.console); notesRepository = app.get(DI.notesRepository); + userNotePiningsRepository = app.get(DI.userNotePiningsRepository); }); afterAll(async () => { @@ -380,7 +386,7 @@ describe('ActivityPub', () => { resolver.register(actor2.id, actor2); resolver.register(actor2Note.id, actor2Note); - await personService.createPerson(actor1.id, resolver); + const created = await personService.createPerson(actor1.id, resolver); // actor2Note is from a different server and needs to be fetched again assert.deepStrictEqual( @@ -394,6 +400,10 @@ describe('ActivityPub', () => { // Reflects the original content instead of the fraud assert.strictEqual(note.text, 'test test foo'); assert.strictEqual(note.uri, actor2Note.id); + + // Cross-user pin should be rejected + const pinExists = await userNotePiningsRepository.existsBy({ userId: created.id, noteId: note.id }); + expect(pinExists).toBe(false); }); test('Fetch a note that is a featured note of the attributed actor', async () => { diff --git a/packages/backend/test/unit/misc/promise-map.ts b/packages/backend/test/unit/misc/promise-map.ts new file mode 100644 index 0000000000..81d21c8e0e --- /dev/null +++ b/packages/backend/test/unit/misc/promise-map.ts @@ -0,0 +1,170 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { setTimeout } from 'node:timers/promises'; +import promiseLimit from 'promise-limit'; +import { promiseMap } from '@/misc/promise-map.js'; + +async function randomDelay() { + await setTimeout(10 * Math.abs(Math.random())); +} + +describe(promiseMap, () => { + it('should return empty array with no input', async () => { + const result = await promiseMap([] as string[], async () => 'wrong'); + expect(result).toHaveLength(0); + }); + + it('should map items in correct order', async () => { + const items = [1, 2, 3, 4, 5]; + + const results = await promiseMap(items, async i => { + await randomDelay(); + return String(i); + }); + + expect(results).toEqual(['1', '2', '3', '4', '5']); + }); + + it('should accept async iterable input', async () => { + async function *generator() { + yield 1; + yield 2; + yield 3; + } + + const results = await promiseMap(generator(), async i => String(i)); + + expect(results).toEqual(['1', '2', '3']); + }); + + it('should accept limit input', async () => { + const items = [1, 2, 3, 4, 5]; + + let inProgress = 0; + let maxProgress = 0; + + const results = await promiseMap(items, async i => { + inProgress++; + maxProgress = Math.max(maxProgress, inProgress); + + await randomDelay(); + + inProgress--; + return String(i); + }, { + limit: 2, + }); + + expect(results).toEqual(['1', '2', '3', '4', '5']); + expect(maxProgress).toEqual(2); + }); + + it('should accept limit as instance', async () => { + const items = [1, 2, 3, 4, 5]; + const limit = promiseLimit(2); + + let inProgress = 0; + let maxProgress = 0; + + const results = await promiseMap(items, async i => { + inProgress++; + maxProgress = Math.max(maxProgress, inProgress); + + await randomDelay(); + + inProgress--; + return String(i); + }, { + limit, + }); + + expect(results).toEqual(['1', '2', '3', '4', '5']); + expect(maxProgress).toEqual(2); + }); + + it('should reject when signal aborts', async () => { + const items = [1, 2, 3, 4, 5]; + const controller = new AbortController(); + + const promise = promiseMap(items, async i => { + if (i === 3) { + controller.abort(new Error('test abort')); + } + + return String(i); + }, { + limit: 1, + signal: controller.signal, + }); + + await expect(promise).rejects.toThrow('abort'); + }); + + it('should abort when signal aborts', async () => { + const items = [1, 2, 3, 4, 5]; + const controller = new AbortController(); + + const processed: number[] = []; + + await promiseMap(items, async i => { + if (i === 3) { + controller.abort('test abort'); + } + + processed.push(i); + return String(i); + }, { + limit: 1, + signal: controller.signal, + }).catch(() => null); + + expect(processed).toEqual([1, 2, 3]); + }); + + it('should reject when promise rejects', async () => { + const items = [1, 2, 3, 4, 5]; + + const promise = promiseMap(items, async i => { + if (i === 3) { + throw new Error('test error'); + } + + return String(i); + }); + + await expect(promise).rejects.toThrow('test'); + }); + + it('should abort when promise rejects', async () => { + const items = [1, 2, 3, 4, 5]; + + const processed: number[] = []; + + await promiseMap(items, async i => { + if (i === 3) { + throw new Error('test error'); + } + + processed.push(i); + + return String(i); + }).catch(() => null); + + expect(processed).toEqual([1, 2]); + }); + + it('should aggregate all errors', async () => { + const items = [1, 2, 3, 4, 5]; + + const promise = promiseMap(items, async i => { + await setTimeout(10); + + throw new Error(`test error: ${i}`); + }); + + await expect(promise).rejects.toThrow(AggregateError); + }); +}); diff --git a/packages/frontend/src/pages/admin/job-queue.vue b/packages/frontend/src/pages/admin/job-queue.vue index 155277c976..a37c8f872b 100644 --- a/packages/frontend/src/pages/admin/job-queue.vue +++ b/packages/frontend/src/pages/admin/job-queue.vue @@ -204,6 +204,7 @@ const QUEUE_TYPES = [ 'userWebhookDeliver', 'systemWebhookDeliver', 'scheduleNotePost', + 'backgroundTask', ] as const; const tab: Ref = ref('-'); diff --git a/packages/frontend/src/pages/admin/overview.queue.vue b/packages/frontend/src/pages/admin/overview.queue.vue index cf07cddced..9fafbd4f24 100644 --- a/packages/frontend/src/pages/admin/overview.queue.vue +++ b/packages/frontend/src/pages/admin/overview.queue.vue @@ -38,7 +38,6 @@ SPDX-License-Identifier: AGPL-3.0-only import { markRaw, onMounted, onUnmounted, ref, useTemplateRef } from 'vue'; import * as Misskey from 'misskey-js'; import XChart from './overview.queue.chart.vue'; -import type { ApQueueDomain } from '@/pages/admin/queue.vue'; import number from '@/filters/number.js'; import { useStream } from '@/stream.js'; @@ -54,7 +53,7 @@ const chartDelayed = useTemplateRef('chartDelayed'); const chartWaiting = useTemplateRef('chartWaiting'); const props = defineProps<{ - domain: ApQueueDomain; + domain: 'inbox' | 'deliver' | 'background'; }>(); function onStats(stats: Misskey.entities.QueueStats) { @@ -63,17 +62,17 @@ function onStats(stats: Misskey.entities.QueueStats) { delayed.value = stats[props.domain].delayed; waiting.value = stats[props.domain].waiting; - chartProcess.value.pushData(stats[props.domain].activeSincePrevTick); - chartActive.value.pushData(stats[props.domain].active); - chartDelayed.value.pushData(stats[props.domain].delayed); - chartWaiting.value.pushData(stats[props.domain].waiting); + chartProcess.value?.pushData(stats[props.domain].activeSincePrevTick); + chartActive.value?.pushData(stats[props.domain].active); + chartDelayed.value?.pushData(stats[props.domain].delayed); + chartWaiting.value?.pushData(stats[props.domain].waiting); } function onStatsLog(statsLog: Misskey.entities.QueueStatsLog) { - const dataProcess: Misskey.entities.QueueStats[ApQueueDomain]['activeSincePrevTick'][] = []; - const dataActive: Misskey.entities.QueueStats[ApQueueDomain]['active'][] = []; - const dataDelayed: Misskey.entities.QueueStats[ApQueueDomain]['delayed'][] = []; - const dataWaiting: Misskey.entities.QueueStats[ApQueueDomain]['waiting'][] = []; + const dataProcess: number[] = []; + const dataActive: number[] = []; + const dataDelayed: number[] = []; + const dataWaiting: number[] = []; for (const stats of [...statsLog].reverse()) { dataProcess.push(stats[props.domain].activeSincePrevTick); @@ -82,10 +81,10 @@ function onStatsLog(statsLog: Misskey.entities.QueueStatsLog) { dataWaiting.push(stats[props.domain].waiting); } - chartProcess.value.setData(dataProcess); - chartActive.value.setData(dataActive); - chartDelayed.value.setData(dataDelayed); - chartWaiting.value.setData(dataWaiting); + chartProcess.value?.setData(dataProcess); + chartActive.value?.setData(dataActive); + chartDelayed.value?.setData(dataDelayed); + chartWaiting.value?.setData(dataWaiting); } onMounted(() => { diff --git a/packages/frontend/src/pages/admin/overview.vue b/packages/frontend/src/pages/admin/overview.vue index caa888b51d..1c299ea6fb 100644 --- a/packages/frontend/src/pages/admin/overview.vue +++ b/packages/frontend/src/pages/admin/overview.vue @@ -60,13 +60,17 @@ SPDX-License-Identifier: AGPL-3.0-only + + + + +