diff --git a/packages/backend/src/core/AntennaService.ts b/packages/backend/src/core/AntennaService.ts index 8404da2a08..ca451256db 100644 --- a/packages/backend/src/core/AntennaService.ts +++ b/packages/backend/src/core/AntennaService.ts @@ -19,6 +19,7 @@ import type { MiAntenna } from '@/models/Antenna.js'; import type { MiNote } from '@/models/Note.js'; import type { MiUser } from '@/models/User.js'; import { InternalEventService } from '@/core/InternalEventService.js'; +import { promiseMap } from '@/misc/promise-map.js'; import { CacheService } from './CacheService.js'; import type { OnApplicationShutdown } from '@nestjs/common'; @@ -95,7 +96,10 @@ export class AntennaService implements OnApplicationShutdown { @bindThis public async addNoteToAntennas(note: MiNote, noteUser: { id: MiUser['id']; username: string; host: string | null; isBot: boolean; }): Promise { const antennas = await this.getAntennas(); - const antennasWithMatchResult = await Promise.all(antennas.map(antenna => this.checkHitAntenna(antenna, note, noteUser).then(hit => [antenna, hit] as const))); + const antennasWithMatchResult = await promiseMap(antennas, async antenna => { + const hit = await this.checkHitAntenna(antenna, note, noteUser); + return [antenna, hit] as const; + }); const matchedAntennas = antennasWithMatchResult.filter(([, hit]) => hit).map(([antenna]) => antenna); const redisPipeline = this.redisForTimelines.pipeline(); diff --git a/packages/backend/src/core/CustomEmojiService.ts b/packages/backend/src/core/CustomEmojiService.ts index dff6560378..15adf0faae 100644 --- a/packages/backend/src/core/CustomEmojiService.ts +++ b/packages/backend/src/core/CustomEmojiService.ts @@ -23,6 +23,7 @@ import { DriveService } from '@/core/DriveService.js'; import { CacheManagementService, type ManagedQuantumKVCache } from '@/global/CacheManagementService.js'; import { TimeService } from '@/global/TimeService.js'; import { LoggerService } from '@/core/LoggerService.js'; +import { promiseMap } from '@/misc/promise-map.js'; import { isRetryableSymbol } from '@/misc/is-retryable-error.js'; import type Logger from '@/logger.js'; import { KeyNotFoundError } from '@/misc/errors/KeyNotFoundError.js'; @@ -577,7 +578,7 @@ export class CustomEmojiService { */ @bindThis public async populateEmojis(emojiNames: string[], noteUserHost: string | null): Promise> { - const emojis = await Promise.all(emojiNames.map(x => this.populateEmoji(x, noteUserHost))); + const emojis = await promiseMap(emojiNames, async x => await this.populateEmoji(x, noteUserHost), { limit: 4 }); const res = {} as Record; for (let i = 0; i < emojiNames.length; i++) { const resolvedEmoji = emojis[i]; diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index c1c48d1708..489edcf5dd 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -60,6 +60,7 @@ import { CacheService } from '@/core/CacheService.js'; import { TimeService } from '@/global/TimeService.js'; import { NoteVisibilityService } from '@/core/NoteVisibilityService.js'; import { CollapsedQueueService } from '@/core/CollapsedQueueService.js'; +import { promiseMap } from '@/misc/promise-map.js'; type NotificationType = 'reply' | 'renote' | 'quote' | 'mention'; @@ -889,20 +890,16 @@ export class NoteCreateService implements OnApplicationShutdown { } @bindThis - private async extractMentionedUsers(user: { host: MiUser['host']; }, tokens: mfm.MfmNode[]): Promise { - if (tokens == null) return []; + public async extractMentionedUsers(user: { host: MiUser['host']; }, tokens: mfm.MfmNode[]): Promise { + if (tokens == null || tokens.length === 0) return []; - const mentions = extractMentions(tokens); - let mentionedUsers = (await Promise.all(mentions.map(m => - this.remoteUserResolveService.resolveUser(m.username, m.host ?? user.host).catch(() => null), - ))).filter(x => x != null); + const allMentions = extractMentions(tokens); + const mentions = new Map(allMentions.map(m => [`${m.username.toLowerCase()}@${m.host?.toLowerCase()}`, m])); - // Drop duplicate users - mentionedUsers = mentionedUsers.filter((u, i, self) => - i === self.findIndex(u2 => u.id === u2.id), - ); + const allMentionedUsers = await promiseMap(mentions.values(), async m => await this.remoteUserResolveService.resolveUser(m.username, m.host ?? user.host).catch(() => null), { limit: 2 }); + const mentionedUsers = new Map(allMentionedUsers.filter(u => u != null).map(u => [u.id, u])); - return mentionedUsers; + return Array.from(mentionedUsers.values()); } @bindThis diff --git a/packages/backend/src/core/NoteEditService.ts b/packages/backend/src/core/NoteEditService.ts index df98283930..0e36cc1c63 100644 --- a/packages/backend/src/core/NoteEditService.ts +++ b/packages/backend/src/core/NoteEditService.ts @@ -3,18 +3,16 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { setImmediate } from 'node:timers/promises'; import * as mfm from 'mfm-js'; -import { DataSource, In, IsNull, LessThan } from 'typeorm'; +import { DataSource, In } from 'typeorm'; import * as Redis from 'ioredis'; import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import { UnrecoverableError } from 'bullmq'; -import { extractMentions } from '@/misc/extract-mentions.js'; import { extractCustomEmojisFromMfm } from '@/misc/extract-custom-emojis-from-mfm.js'; import { extractHashtags } from '@/misc/extract-hashtags.js'; import type { IMentionedRemoteUsers } from '@/models/Note.js'; import { MiNote } from '@/models/Note.js'; -import type { NoteEditsRepository, ChannelFollowingsRepository, ChannelsRepository, FollowingsRepository, InstancesRepository, MiFollowing, MiMeta, MutingsRepository, NotesRepository, NoteThreadMutingsRepository, UserListMembershipsRepository, UserProfilesRepository, UsersRepository, PollsRepository } from '@/models/_.js'; +import type { NoteEditsRepository, ChannelFollowingsRepository, ChannelsRepository, FollowingsRepository, InstancesRepository, MiMeta, MutingsRepository, NotesRepository, NoteThreadMutingsRepository, UserListMembershipsRepository, UserProfilesRepository, UsersRepository, PollsRepository } from '@/models/_.js'; import type { MiDriveFile } from '@/models/DriveFile.js'; import type { MiApp } from '@/models/App.js'; import { concat } from '@/misc/prelude/array.js'; @@ -430,7 +428,7 @@ export class NoteEditService implements OnApplicationShutdown { emojis = data.apEmojis ?? extractCustomEmojisFromMfm(combinedTokens); - mentionedUsers = data.apMentions ?? await this.extractMentionedUsers(user, combinedTokens); + mentionedUsers = data.apMentions ?? await this.noteCreateService.extractMentionedUsers(user, combinedTokens); } // if the host is media-silenced, custom emojis are not allowed @@ -789,23 +787,6 @@ export class NoteEditService implements OnApplicationShutdown { await this.searchService.indexNote(note); } - @bindThis - private async extractMentionedUsers(user: { host: MiUser['host']; }, tokens: mfm.MfmNode[]): Promise { - if (tokens == null) return []; - - const mentions = extractMentions(tokens); - let mentionedUsers = (await Promise.all(mentions.map(m => - this.remoteUserResolveService.resolveUser(m.username, m.host ?? user.host).catch(() => null), - ))).filter(x => x !== null) as MiUser[]; - - // Drop duplicate users - mentionedUsers = mentionedUsers.filter((u, i, self) => - i === self.findIndex(u2 => u.id === u2.id), - ); - - return mentionedUsers; - } - @bindThis private async pushToTl(note: MiNote, user: { id: MiUser['id']; host: MiUser['host']; }) { if (!this.meta.enableFanoutTimeline) return; diff --git a/packages/backend/src/core/ReactionService.ts b/packages/backend/src/core/ReactionService.ts index 070083784f..6af04dcba5 100644 --- a/packages/backend/src/core/ReactionService.ts +++ b/packages/backend/src/core/ReactionService.ts @@ -34,6 +34,7 @@ import { CacheService } from '@/core/CacheService.js'; import { NoteVisibilityService } from '@/core/NoteVisibilityService.js'; import { TimeService } from '@/global/TimeService.js'; import { CollapsedQueueService } from '@/core/CollapsedQueueService.js'; +import { promiseMap } from '@/misc/promise-map.js'; import type { DataSource } from 'typeorm'; const FALLBACK = '\u2764'; @@ -298,7 +299,7 @@ export class ReactionService implements OnModuleInit { if (['public', 'home', 'followers'].includes(note.visibility)) { dm.addFollowersRecipe(); } else if (note.visibility === 'specified') { - const visibleUsers = await Promise.all(note.visibleUserIds.map(id => this.usersRepository.findOneBy({ id }))); + const visibleUsers = await promiseMap(note.visibleUserIds, async id => await this.cacheService.findOptionalUserById(id), { limit: 2 }); for (const u of visibleUsers.filter(u => u && isRemoteUser(u))) { dm.addDirectRecipe(u as MiRemoteUser); } diff --git a/packages/backend/src/core/activitypub/ApAudienceService.ts b/packages/backend/src/core/activitypub/ApAudienceService.ts index 15c4546063..e6c00aa5e7 100644 --- a/packages/backend/src/core/activitypub/ApAudienceService.ts +++ b/packages/backend/src/core/activitypub/ApAudienceService.ts @@ -8,6 +8,7 @@ import promiseLimit from 'promise-limit'; import type { MiRemoteUser, MiUser } from '@/models/User.js'; import { concat, unique } from '@/misc/prelude/array.js'; import { bindThis } from '@/decorators.js'; +import { promiseMap } from '@/misc/promise-map.js'; import { getApIds } from './type.js'; import { ApPersonService } from './models/ApPersonService.js'; import type { ApObject } from './type.js'; @@ -37,10 +38,12 @@ export class ApAudienceService { const others = unique(concat([toGroups.other, ccGroups.other])); - const limit = promiseLimit(2); - const mentionedUsers = (await Promise.all( - others.map(id => limit(() => this.apPersonService.resolvePerson(id, resolver).catch(() => null))), - )).filter(x => x != null); + const resolved = await promiseMap(others, async x => { + return await this.apPersonService.resolvePerson(x, resolver).catch(() => null) as MiUser | null; + }, { + limit: 2, + }); + const mentionedUsers = resolved.filter(x => x != null); // If no audience is specified, then assume public if ( diff --git a/packages/backend/src/core/activitypub/ApResolverService.ts b/packages/backend/src/core/activitypub/ApResolverService.ts index 506686cb46..a1510db1b3 100644 --- a/packages/backend/src/core/activitypub/ApResolverService.ts +++ b/packages/backend/src/core/activitypub/ApResolverService.ts @@ -5,7 +5,6 @@ import { Inject, Injectable } from '@nestjs/common'; import { IsNull, Not } from 'typeorm'; -import promiseLimit from 'promise-limit'; import type { MiLocalUser, MiRemoteUser } from '@/models/User.js'; import type { NotesRepository, PollsRepository, NoteReactionsRepository, UsersRepository, FollowRequestsRepository, MiMeta, SkApFetchLog } from '@/models/_.js'; import type { Config } from '@/config.js'; @@ -24,6 +23,7 @@ import { toArray } from '@/misc/prelude/array.js'; import { isPureRenote } from '@/misc/is-renote.js'; import { CacheService } from '@/core/CacheService.js'; import { trackPromise } from '@/misc/promise-tracker.js'; +import { promiseMap } from '@/misc/promise-map.js'; import { AnyCollection, getApId, getNullableApId, IObjectWithId, isCollection, isCollectionOrOrderedCollection, isCollectionPage, isOrderedCollection, isOrderedCollectionPage } from './type.js'; import { ApDbResolverService } from './ApDbResolverService.js'; import { ApRendererService } from './ApRendererService.js'; @@ -151,21 +151,20 @@ export class Resolver { const recursionLimit = this.recursionLimit - this.history.size; const batchLimit = Math.min(source.length, recursionLimit, itemLimit); - const limiter = promiseLimit(concurrency); - const batch = await Promise.all(source - .slice(0, batchLimit) - .map(item => limiter(async () => { - if (sentFrom) { - // Use secureResolve to avoid re-fetching items that were included inline. - return await this.secureResolve(item, sentFrom, allowAnonymousItems); - } else if (allowAnonymousItems) { - return await this.resolveAnonymous(item); - } else { - // ID is required if we have neither sentFrom not allowAnonymousItems - const id = getApId(item); - return await this.resolve(id); - } - }))); + const batch = await promiseMap(source.slice(0, batchLimit), async item => { + if (sentFrom) { + // Use secureResolve to avoid re-fetching items that were included inline. + return await this.secureResolve(item, sentFrom, allowAnonymousItems); + } else if (allowAnonymousItems) { + return await this.resolveAnonymous(item); + } else { + // ID is required if we have neither sentFrom not allowAnonymousItems + const id = getApId(item); + return await this.resolve(id); + } + }, { + limit: concurrency, + }); destination.push(...batch); }; diff --git a/packages/backend/src/core/activitypub/models/ApMentionService.ts b/packages/backend/src/core/activitypub/models/ApMentionService.ts index 2cd151fa04..1aab5894d3 100644 --- a/packages/backend/src/core/activitypub/models/ApMentionService.ts +++ b/packages/backend/src/core/activitypub/models/ApMentionService.ts @@ -12,6 +12,7 @@ import { isMention } from '../type.js'; import { Resolver } from '../ApResolverService.js'; import { ApPersonService } from './ApPersonService.js'; import type { IObject, IApMention } from '../type.js'; +import { promiseMap } from '@/misc/promise-map.js'; @Injectable() export class ApMentionService { @@ -24,12 +25,13 @@ export class ApMentionService { public async extractApMentions(tags: IObject | IObject[] | null | undefined, resolver: Resolver): Promise { const hrefs = unique(this.extractApMentionObjects(tags).map(x => x.href)); - const limit = promiseLimit(2); - const mentionedUsers = (await Promise.all( - hrefs.map(x => limit(() => this.apPersonService.resolvePerson(x, resolver).catch(() => null))), - )).filter(x => x != null); + const mentionedUsers = await promiseMap(hrefs, async x => { + return await this.apPersonService.resolvePerson(x, resolver).catch(() => null) as MiUser | null; + }, { + limit: 2, + }); - return mentionedUsers; + return mentionedUsers.filter(resolved => resolved != null); } @bindThis diff --git a/packages/backend/src/core/activitypub/models/ApNoteService.ts b/packages/backend/src/core/activitypub/models/ApNoteService.ts index 9b717af32d..690110a7f7 100644 --- a/packages/backend/src/core/activitypub/models/ApNoteService.ts +++ b/packages/backend/src/core/activitypub/models/ApNoteService.ts @@ -6,7 +6,6 @@ import { Inject, Injectable, OnModuleInit } from '@nestjs/common'; import { In } from 'typeorm'; import { UnrecoverableError } from 'bullmq'; -import promiseLimit from 'promise-limit'; import { ModuleRef } from '@nestjs/core'; import { DI } from '@/di-symbols.js'; import type { UsersRepository, PollsRepository, EmojisRepository, NotesRepository, MiMeta } from '@/models/_.js'; @@ -32,6 +31,7 @@ import { renderInlineError } from '@/misc/render-inline-error.js'; import { extractMediaFromHtml } from '@/core/activitypub/misc/extract-media-from-html.js'; import { extractMediaFromMfm } from '@/core/activitypub/misc/extract-media-from-mfm.js'; import { getContentByType } from '@/core/activitypub/misc/get-content-by-type.js'; +import { promiseMap } from '@/misc/promise-map.js'; import { trackPromise } from '@/misc/promise-tracker.js'; import { CustomEmojiService, encodeEmojiKey, isValidEmojiName } from '@/core/CustomEmojiService.js'; import { TimeService } from '@/global/TimeService.js'; @@ -583,8 +583,8 @@ export class ApNoteService implements OnModuleInit { const emojiKeys = eomjiTags.map(tag => encodeEmojiKey({ name: tag.name, host })); const existingEmojis = await this.customEmojiService.emojisByKeyCache.fetchMany(emojiKeys); - return await Promise.all(eomjiTags.map(async tag => { - const name = tag.name; + return await promiseMap(eomjiTags, async tag => { + const name = tag.name.replaceAll(':', ''); tag.icon = toSingle(tag.icon); const exists = existingEmojis.values.find(x => x.name === name); @@ -627,7 +627,9 @@ export class ApNoteService implements OnModuleInit { // _misskey_license が存在しなければ `null` license: (tag._misskey_license?.freeText ?? null), }); - })); + }, { + limit: 4, + }); } /** @@ -691,7 +693,7 @@ export class ApNoteService implements OnModuleInit { } }; - const results = await Promise.all(Array.from(quoteUris).map(u => resolveQuote(u))); + const results = await promiseMap(quoteUris, async u => resolveQuote(u), { limit: 2 }); // Success - return the quote const quote = results.find(r => typeof(r) === 'object'); @@ -753,14 +755,10 @@ export class ApNoteService implements OnModuleInit { // Resolve all files w/ concurrency 2. // This prevents one big file from blocking the others. - const limiter = promiseLimit(2); - const results = await Promise - .all(Array - .from(attachments.values()) - .map(attach => limiter(async () => { - attach.sensitive ??= note.sensitive; - return await this.resolveImage(actor, attach); - }))); + const results = await promiseMap(attachments.values(), async attach => { + attach.sensitive ??= note.sensitive; + return await this.resolveImage(actor, attach); + }); // Process results let hasFileError = false; diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index 692b04f1c8..5f31d529ad 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -48,7 +48,8 @@ import { renderInlineError } from '@/misc/render-inline-error.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; import { QueueService } from '@/core/QueueService.js'; import { CollapsedQueueService } from '@/core/CollapsedQueueService.js'; -import { getApId, getApType, isActor, isCollection, isCollectionOrOrderedCollection, isPropertyValue } from '../type.js'; +import { promiseMap } from '@/misc/promise-map.js'; +import { getApId, getApType, getNullableApId, isActor, isPost, isPropertyValue } from '../type.js'; import { ApLoggerService } from '../ApLoggerService.js'; import { extractApHashtags } from './tag.js'; import type { OnModuleInit } from '@nestjs/common'; @@ -933,35 +934,27 @@ export class ApPersonService implements OnModuleInit { this.logger.info(`Updating the featured: ${user.uri}`); - const _resolver = resolver ?? this.apResolverService.createResolver(); - - // Resolve to (Ordered)Collection Object - const collection = user.featured ? await _resolver.resolveCollection(user.featured, true, user.uri).catch(err => { - // Permanent error implies hidden or inaccessible, which is a normal thing. - if (isRetryableError(err)) { - this.logger.warn(`Failed to update featured notes: ${renderInlineError(err)}`); - } - - return null; - }) : null; - if (!collection) return; - - if (!isCollectionOrOrderedCollection(collection)) throw new UnrecoverableError(`failed to update user ${user.uri}: featured ${user.featured} is not Collection or OrderedCollection`); - - // Resolve to Object(may be Note) arrays - const unresolvedItems = isCollection(collection) ? collection.items : collection.orderedItems; - const items = await Promise.all(toArray(unresolvedItems).map(x => _resolver.resolve(x))); + resolver ??= this.apResolverService.createResolver(); // Resolve and regist Notes - const limit = promiseLimit(2); const maxPinned = (await this.roleService.getUserPolicies(user.id)).pinLimit; - const featuredNotes = await Promise.all(items - .filter(item => getApType(item) === 'Note') // TODO: Noteでなくてもいいかも - .slice(0, maxPinned) - .map(item => limit(() => this.apNoteService.resolveNote(item, { - resolver: _resolver, - sentFrom: user.uri, - })))); + const items = await resolver.resolveCollectionItems(user.featured, true, user.uri, maxPinned, 2); + const featuredNotes = await promiseMap(items, async item => { + const itemId = getNullableApId(item); + if (itemId && isPost(item)) { + try { + return await this.apNoteService.resolveNote(item, { + resolver: resolver, + sentFrom: user.uri, + }); + } catch (err) { + this.logger.warn(`Couldn't fetch pinned note ${itemId} for user ${user.id} (@${user.username}@${user.host}): ${renderInlineError(err)}`); + } + } + return null; + }, { + limit: 2, + }); await this.db.transaction(async transactionalEntityManager => { await transactionalEntityManager.delete(MiUserNotePining, { userId: user.id }); diff --git a/packages/backend/src/core/chart/core.ts b/packages/backend/src/core/chart/core.ts index 2f1ed27132..883a72139f 100644 --- a/packages/backend/src/core/chart/core.ts +++ b/packages/backend/src/core/chart/core.ts @@ -15,6 +15,7 @@ import { dateUTC, isTimeSame, isTimeBefore, subtractTime, addTime } from '@/misc import type Logger from '@/logger.js'; import { bindThis } from '@/decorators.js'; import { MiRepository, miRepository } from '@/models/_.js'; +import { promiseMap } from '@/misc/promise-map.js'; import type { DataSource, Repository } from 'typeorm'; import type { Lock } from 'redis-lock'; @@ -526,13 +527,13 @@ export default abstract class Chart { const groups = removeDuplicates(this.buffer.map(log => log.group)); - await Promise.all( - groups.map(group => - Promise.all([ - this.claimCurrentLog(group, 'hour'), - this.claimCurrentLog(group, 'day'), - ]).then(([logHour, logDay]) => - update(logHour, logDay)))); + await promiseMap(groups, async group => { + const logHour = await this.claimCurrentLog(group, 'hour'); + const logDay = await this.claimCurrentLog(group, 'day'); + await update(logHour, logDay); + }, { + limit: 2, + }); } @bindThis @@ -564,7 +565,7 @@ export default abstract class Chart { ]); }; - return Promise.all([ + return await Promise.all([ this.claimCurrentLog(group, 'hour'), this.claimCurrentLog(group, 'day'), ]).then(([logHour, logDay]) => diff --git a/packages/backend/src/core/entities/NoteFavoriteEntityService.ts b/packages/backend/src/core/entities/NoteFavoriteEntityService.ts index 3cdafe48ad..3d879b0dff 100644 --- a/packages/backend/src/core/entities/NoteFavoriteEntityService.ts +++ b/packages/backend/src/core/entities/NoteFavoriteEntityService.ts @@ -5,12 +5,13 @@ import { Inject, Injectable } from '@nestjs/common'; import { DI } from '@/di-symbols.js'; -import type { NoteFavoritesRepository } from '@/models/_.js'; +import type { MiNote, NoteFavoritesRepository } from '@/models/_.js'; import type { } from '@/models/Blocking.js'; import type { MiUser } from '@/models/User.js'; import type { MiNoteFavorite } from '@/models/NoteFavorite.js'; import { bindThis } from '@/decorators.js'; import { IdService } from '@/core/IdService.js'; +import type { Packed } from '@/misc/json-schema.js'; import { NoteEntityService } from './NoteEntityService.js'; @Injectable() @@ -28,6 +29,7 @@ export class NoteFavoriteEntityService { public async pack( src: MiNoteFavorite['id'] | MiNoteFavorite, me?: { id: MiUser['id'] } | null | undefined, + notes?: Map>, ) { const favorite = typeof src === 'object' ? src : await this.noteFavoritesRepository.findOneByOrFail({ id: src }); @@ -35,15 +37,18 @@ export class NoteFavoriteEntityService { id: favorite.id, createdAt: this.idService.parse(favorite.id).date.toISOString(), noteId: favorite.noteId, - note: await this.noteEntityService.pack(favorite.note ?? favorite.noteId, me), + note: notes?.get(favorite.noteId) ?? await this.noteEntityService.pack(favorite.note ?? favorite.noteId, me), }; } @bindThis - public packMany( - favorites: any[], + public async packMany( + favorites: (MiNoteFavorite & { note: MiNote })[], me: { id: MiUser['id'] }, ) { - return Promise.all(favorites.map(x => this.pack(x, me))); + const packedNotes = await this.noteEntityService.packMany(favorites.map(f => f.note), me); + const packedNotesMap = new Map(packedNotes.map(n => [n.id, n])); + + return Promise.all(favorites.map(x => this.pack(x, me, packedNotesMap))); } } diff --git a/packages/backend/src/core/entities/PageEntityService.ts b/packages/backend/src/core/entities/PageEntityService.ts index 3106001676..5a56b9cec5 100644 --- a/packages/backend/src/core/entities/PageEntityService.ts +++ b/packages/backend/src/core/entities/PageEntityService.ts @@ -105,10 +105,13 @@ export class PageEntityService { font: page.font, script: page.script, eyeCatchingImageId: page.eyeCatchingImageId, - eyeCatchingImage: page.eyeCatchingImageId ? await this.driveFileEntityService.pack(page.eyeCatchingImageId) : null, - attachedFiles: this.driveFileEntityService.packMany((await Promise.all(attachedFiles)).filter(x => x != null)), + eyeCatchingImage: page.eyeCatchingImageId ? this.driveFileEntityService.pack(page.eyeCatchingImageId) : null, + attachedFiles: Promise + .all(attachedFiles) + .then(fs => fs.filter(x => x != null)) + .then(fs => this.driveFileEntityService.packMany(fs)), likedCount: page.likedCount, - isLiked: meId ? await this.pageLikesRepository.exists({ where: { pageId: page.id, userId: meId } }) : undefined, + isLiked: meId ? this.pageLikesRepository.exists({ where: { pageId: page.id, userId: meId } }) : undefined, }); } diff --git a/packages/backend/src/core/entities/UserEntityService.ts b/packages/backend/src/core/entities/UserEntityService.ts index bacc3c3fde..5ea53c32ba 100644 --- a/packages/backend/src/core/entities/UserEntityService.ts +++ b/packages/backend/src/core/entities/UserEntityService.ts @@ -645,6 +645,7 @@ export class UserEntityService implements OnModuleInit { ...(isDetailed ? { url: profile!.url, uri: user.uri, + // TODO hints for all of this movedTo: user.movedToUri ? Promise.resolve(opts.userIdsByUri?.get(user.movedToUri) ?? this.apPersonService.resolvePerson(user.movedToUri).then(user => user.id).catch(() => null)) : null, movedToUri: user.movedToUri, // alsoKnownAs moved from packedUserDetailedNotMeOnly for privacy diff --git a/packages/backend/src/misc/promise-map.ts b/packages/backend/src/misc/promise-map.ts new file mode 100644 index 0000000000..e3fc5f1d65 --- /dev/null +++ b/packages/backend/src/misc/promise-map.ts @@ -0,0 +1,107 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import promiseLimit from 'promise-limit'; + +/** + * Pipes a stream of values through an async mapping callback to produce a new stream of results. + * Avoids extra work by bailing out if any promise rejects or the caller stops iterating the stream. + * + * Can optionally accept a concurrency limit and/or abort signal to further customize behavior. + * If a limit is provided, then no more than that many promises will execute at once. + * If a signal is provided, then all promises will terminate when the signal aborts. + * A signal cannot be provided without a limit, as that would be a no-op. + */ +export async function promiseMap( + values: Iterable | AsyncIterable, + callback: (value: Input, index: number) => Promise, + opts?: { + limit: number | ReturnType>; + signal?: AbortSignal; + }, +): Promise { + // Parse the configured limit or create no-op + const limiter = createLimiter(opts?.limit); + + // Internal state + const outputs: Output[] = []; + const errors: unknown[] = []; + const queue: Promise[] = []; + + let count = 0; + for await (const input of values) { + // Capture the destination index to make sure items are returned in the same order + const index = count; + count++; + + // Stop when any promise fails + if (errors.length > 0) { + break; + } + + // Kick off the next item + const promise = limiter(async () => { + // Check for rejection without throwing any new errors + if (errors.length > 0) return; + + try { + // Checking the abort signal here covers all locations. + // 1. It bails the callback directly. + // 2. The error is written to errors, which breaks out of the loop + opts?.signal?.throwIfAborted(); + + // Populate the next value + outputs[index] = await callback(input, index); + } catch (err) { + errors.push(err); + } + }); + + // But don't forget about it! + queue.push(promise); + } + + // Wait for everything to complete + await Promise.allSettled(queue); + + // Failed - consolidate and throw errors + if (errors.length > 0) { + throwResults(errors); + } + + // Success - return results + return outputs; +} + +type Limiter = (cb: () => Promise) => Promise; + +function createLimiter(limit: undefined | number | ReturnType>): Limiter { + if (!limit) { + return cb => cb(); + } + + if (typeof limit === 'number') { + return promiseLimit(limit); + } + + return limit; +} + +function throwResults(errors: unknown[]): never { + if (errors.length === 0) { + // Shouldn't happen + throw new Error('Mapping promise rejected'); + } + + if (errors.length === 1) { + if (errors[0] instanceof Error) { + throw errors[0]; + } else { + throw new Error('Mapping promise rejected', { cause: errors[0] }); + } + } + + throw new AggregateError(errors); +} diff --git a/packages/backend/src/server/ActivityPubServerService.ts b/packages/backend/src/server/ActivityPubServerService.ts index e73de15241..de9d588bfa 100644 --- a/packages/backend/src/server/ActivityPubServerService.ts +++ b/packages/backend/src/server/ActivityPubServerService.ts @@ -14,7 +14,7 @@ import accepts from 'accepts'; import vary from 'vary'; import secureJson from 'secure-json-parse'; import { DI } from '@/di-symbols.js'; -import type { FollowingsRepository, NotesRepository, EmojisRepository, NoteReactionsRepository, UserProfilesRepository, UserNotePiningsRepository, UsersRepository, FollowRequestsRepository, MiMeta } from '@/models/_.js'; +import type { FollowingsRepository, NotesRepository, EmojisRepository, NoteReactionsRepository, UserProfilesRepository, UserNotePiningsRepository, UsersRepository, FollowRequestsRepository, MiMeta, MiUserNotePining } from '@/models/_.js'; import * as url from '@/misc/prelude/url.js'; import type { Config } from '@/config.js'; import { ApRendererService } from '@/core/activitypub/ApRendererService.js'; @@ -40,6 +40,7 @@ import { CustomEmojiService, encodeEmojiKey } from '@/core/CustomEmojiService.js import type { FastifyInstance, FastifyRequest, FastifyReply, FastifyPluginOptions, FastifyBodyParser } from 'fastify'; import type { FindOptionsWhere } from 'typeorm'; import { FanoutTimelineEndpointService } from '@/core/FanoutTimelineEndpointService.js'; +import { promiseMap } from '@/misc/promise-map.js'; const ACTIVITY_JSON = 'application/activity+json; charset=utf-8'; const LD_JSON = 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"; charset=utf-8'; @@ -418,7 +419,7 @@ export class ActivityPubServerService { const inStock = followings.length === limit + 1; if (inStock) followings.pop(); - const renderedFollowers = await Promise.all(followings.map(following => this.apRendererService.renderFollowUser(following.followerId))); + const renderedFollowers = await promiseMap(followings, async following => this.apRendererService.renderFollowUser(following.followerId), { limit: 4 }); const rendered = this.apRendererService.renderOrderedCollectionPage( `${partOf}?${url.query({ page: 'true', @@ -515,7 +516,7 @@ export class ActivityPubServerService { const inStock = followings.length === limit + 1; if (inStock) followings.pop(); - const renderedFollowees = await Promise.all(followings.map(following => this.apRendererService.renderFollowUser(following.followeeId))); + const renderedFollowees = await promiseMap(followings, async following => this.apRendererService.renderFollowUser(following.followeeId), { limit: 4 }); const rendered = this.apRendererService.renderOrderedCollectionPage( `${partOf}?${url.query({ page: 'true', @@ -555,10 +556,7 @@ export class ActivityPubServerService { const userId = request.params.user; - const user = await this.usersRepository.findOneBy({ - id: userId, - host: IsNull(), - }); + const user = await this.cacheService.findLocalUserById(userId); if (user == null) { reply.code(404); @@ -568,13 +566,14 @@ export class ActivityPubServerService { const pinings = await this.userNotePiningsRepository.find({ where: { userId: user.id }, order: { id: 'DESC' }, - }); + relations: { note: true }, + }) as (MiUserNotePining & { note: MiNote })[]; - const pinnedNotes = (await Promise.all(pinings.map(pining => - this.notesRepository.findOneByOrFail({ id: pining.noteId })))) + const pinnedNotes = pinings + .map(pin => pin.note) .filter(note => !note.localOnly && ['public', 'home'].includes(note.visibility) && !isPureRenote(note)); - const renderedNotes = await Promise.all(pinnedNotes.map(note => this.apRendererService.renderNote(note, user))); + const renderedNotes = await promiseMap(pinnedNotes, async note => await this.apRendererService.renderNote(note, user), { limit: 4 }); const rendered = this.apRendererService.renderOrderedCollection( `${this.config.url}/users/${userId}/collections/featured`, @@ -664,7 +663,7 @@ export class ActivityPubServerService { if (sinceId) notes.reverse(); - const activities = await Promise.all(notes.map(note => this.packActivity(note, user))); + const activities = await promiseMap(notes, async note => await this.packActivity(note, user)); const rendered = this.apRendererService.renderOrderedCollectionPage( `${partOf}?${url.query({ page: 'true', @@ -1092,14 +1091,8 @@ export class ActivityPubServerService { // check if the following exists. const [follower, followee] = await Promise.all([ - this.usersRepository.findOneBy({ - id: request.params.follower, - host: IsNull(), - }), - this.usersRepository.findOneBy({ - id: request.params.followee, - host: Not(IsNull()), - }), + this.cacheService.findLocalUserById(request.params.follower), + this.cacheService.findRemoteUserById(request.params.followee), ]) as [MiLocalUser | MiRemoteUser | null, MiLocalUser | MiRemoteUser | null]; if (follower == null || followee == null) { @@ -1134,14 +1127,8 @@ export class ActivityPubServerService { } const [follower, followee] = await Promise.all([ - this.usersRepository.findOneBy({ - id: followRequest.followerId, - host: IsNull(), - }), - this.usersRepository.findOneBy({ - id: followRequest.followeeId, - host: Not(IsNull()), - }), + this.cacheService.findLocalUserById(followRequest.followerId), + this.cacheService.findRemoteUserById(followRequest.followeeId), ]) as [MiLocalUser | MiRemoteUser | null, MiLocalUser | MiRemoteUser | null]; if (follower == null || followee == null) { diff --git a/packages/backend/src/server/api/GetterService.ts b/packages/backend/src/server/api/GetterService.ts index 47b4e2f1bc..ee8018dfab 100644 --- a/packages/backend/src/server/api/GetterService.ts +++ b/packages/backend/src/server/api/GetterService.ts @@ -12,6 +12,7 @@ import { isRemoteUser, isLocalUser } from '@/models/User.js'; import type { MiNote } from '@/models/Note.js'; import { CacheService } from '@/core/CacheService.js'; import { bindThis } from '@/decorators.js'; +import { CacheService } from '@/core/CacheService.js'; @Injectable() export class GetterService { diff --git a/packages/backend/src/server/api/endpoints/admin/federation/remove-all-following.ts b/packages/backend/src/server/api/endpoints/admin/federation/remove-all-following.ts index 8a66e004d7..19df5b1355 100644 --- a/packages/backend/src/server/api/endpoints/admin/federation/remove-all-following.ts +++ b/packages/backend/src/server/api/endpoints/admin/federation/remove-all-following.ts @@ -9,6 +9,7 @@ import type { FollowingsRepository, UsersRepository } from '@/models/_.js'; import { DI } from '@/di-symbols.js'; import { QueueService } from '@/core/QueueService.js'; import { ModerationLogService } from '@/core/ModerationLogService.js'; +import { promiseMap } from '@/misc/promise-map.js'; export const meta = { tags: ['admin'], @@ -48,10 +49,14 @@ export default class extends Endpoint { // eslint- }, ]); - const pairs = await Promise.all(followings.map(f => Promise.all([ - this.usersRepository.findOneByOrFail({ id: f.followerId }), - this.usersRepository.findOneByOrFail({ id: f.followeeId }), - ]).then(([from, to]) => [{ id: from.id }, { id: to.id }]))); + const pairs = await promiseMap(followings, async f => { + const [from, to] = await Promise.all([ + this.usersRepository.findOneByOrFail({ id: f.followerId }), + this.usersRepository.findOneByOrFail({ id: f.followeeId }), + ]); + + return [{ id: from.id }, { id: to.id }]; + }); await this.moderationLogService.log(me, 'severFollowRelations', { host: ps.host, diff --git a/packages/backend/src/server/api/endpoints/admin/show-user.ts b/packages/backend/src/server/api/endpoints/admin/show-user.ts index 49ce0f8bd4..00f4413c74 100644 --- a/packages/backend/src/server/api/endpoints/admin/show-user.ts +++ b/packages/backend/src/server/api/endpoints/admin/show-user.ts @@ -300,14 +300,10 @@ export default class extends Endpoint { // eslint- ) { super(meta, paramDef, async (ps, me) => { const [user, profile] = await Promise.all([ - this.usersRepository.findOneBy({ id: ps.userId }), - this.userProfilesRepository.findOneBy({ userId: ps.userId }), + this.cacheService.findUserById(ps.userId), + this.cacheService.userProfileCache.fetch(ps.userId), ]); - if (user == null || profile == null) { - throw new Error('user not found'); - } - const isModerator = await this.roleService.isModerator(user); const isAdministrator = await this.roleService.isAdministrator(user); const isSilenced = user.isSilenced || !(await this.roleService.getUserPolicies(user.id)).canPublicNote; diff --git a/packages/backend/src/server/api/endpoints/antennas/list.ts b/packages/backend/src/server/api/endpoints/antennas/list.ts index ed7d1c9daa..c370143d4f 100644 --- a/packages/backend/src/server/api/endpoints/antennas/list.ts +++ b/packages/backend/src/server/api/endpoints/antennas/list.ts @@ -8,6 +8,7 @@ import { Endpoint } from '@/server/api/endpoint-base.js'; import type { AntennasRepository } from '@/models/_.js'; import { AntennaEntityService } from '@/core/entities/AntennaEntityService.js'; import { DI } from '@/di-symbols.js'; +import { promiseMap } from '@/misc/promise-map.js'; export const meta = { tags: ['antennas', 'account'], @@ -52,7 +53,7 @@ export default class extends Endpoint { // eslint- userId: me.id, }); - return await Promise.all(antennas.map(x => this.antennaEntityService.pack(x))); + return await promiseMap(antennas, async x => await this.antennaEntityService.pack(x), { limit: 4 }); }); } } diff --git a/packages/backend/src/server/api/endpoints/channels/followed.ts b/packages/backend/src/server/api/endpoints/channels/followed.ts index 415f7ee29a..d25ba82593 100644 --- a/packages/backend/src/server/api/endpoints/channels/followed.ts +++ b/packages/backend/src/server/api/endpoints/channels/followed.ts @@ -9,6 +9,7 @@ import type { ChannelFollowingsRepository } from '@/models/_.js'; import { QueryService } from '@/core/QueryService.js'; import { ChannelEntityService } from '@/core/entities/ChannelEntityService.js'; import { DI } from '@/di-symbols.js'; +import { promiseMap } from '@/misc/promise-map.js'; export const meta = { tags: ['channels', 'account'], @@ -69,7 +70,7 @@ export default class extends Endpoint { // eslint- .limit(ps.limit) .getMany(); - return await Promise.all(followings.map(x => this.channelEntityService.pack(x.followeeId, me))); + return await promiseMap(followings, async x => await this.channelEntityService.pack(x.followeeId, me), { limit: 4 }); }); } } diff --git a/packages/backend/src/server/api/endpoints/channels/my-favorites.ts b/packages/backend/src/server/api/endpoints/channels/my-favorites.ts index 72a1cc0cf9..8cfe7a65fb 100644 --- a/packages/backend/src/server/api/endpoints/channels/my-favorites.ts +++ b/packages/backend/src/server/api/endpoints/channels/my-favorites.ts @@ -8,6 +8,7 @@ import { Endpoint } from '@/server/api/endpoint-base.js'; import type { ChannelFavoritesRepository } from '@/models/_.js'; import { ChannelEntityService } from '@/core/entities/ChannelEntityService.js'; import { DI } from '@/di-symbols.js'; +import { promiseMap } from '@/misc/promise-map.js'; export const meta = { tags: ['channels', 'account'], @@ -56,7 +57,7 @@ export default class extends Endpoint { // eslint- const favorites = await query .getMany(); - return await Promise.all(favorites.map(x => this.channelEntityService.pack(x.channel!, me))); + return await promiseMap(favorites, async x => await this.channelEntityService.pack(x.channel!, me), { limit: 4 }); }); } } diff --git a/packages/backend/src/server/api/endpoints/channels/search.ts b/packages/backend/src/server/api/endpoints/channels/search.ts index 9476c494a3..6483a08116 100644 --- a/packages/backend/src/server/api/endpoints/channels/search.ts +++ b/packages/backend/src/server/api/endpoints/channels/search.ts @@ -11,6 +11,7 @@ import type { ChannelsRepository } from '@/models/_.js'; import { ChannelEntityService } from '@/core/entities/ChannelEntityService.js'; import { DI } from '@/di-symbols.js'; import { sqlLikeEscape } from '@/misc/sql-like-escape.js'; +import { promiseMap } from '@/misc/promise-map.js'; export const meta = { tags: ['channels'], @@ -75,7 +76,7 @@ export default class extends Endpoint { // eslint- .limit(ps.limit) .getMany(); - return await Promise.all(channels.map(x => this.channelEntityService.pack(x, me))); + return await promiseMap(channels, async x => await this.channelEntityService.pack(x, me), { limit: 4 }); }); } } diff --git a/packages/backend/src/server/api/endpoints/drive/folders.ts b/packages/backend/src/server/api/endpoints/drive/folders.ts index 525cb8c5d6..bd2ae8bd18 100644 --- a/packages/backend/src/server/api/endpoints/drive/folders.ts +++ b/packages/backend/src/server/api/endpoints/drive/folders.ts @@ -10,6 +10,7 @@ import { QueryService } from '@/core/QueryService.js'; import { DriveFolderEntityService } from '@/core/entities/DriveFolderEntityService.js'; import { DI } from '@/di-symbols.js'; import { sqlLikeEscape } from '@/misc/sql-like-escape.js'; +import { promiseMap } from '@/misc/promise-map.js'; export const meta = { tags: ['drive'], @@ -71,7 +72,7 @@ export default class extends Endpoint { // eslint- } const folders = await query.limit(ps.limit).getMany(); - return await Promise.all(folders.map(folder => this.driveFolderEntityService.pack(folder))); + return await promiseMap(folders, async folder => await this.driveFolderEntityService.pack(folder), { limit: 4 }); }); } } diff --git a/packages/backend/src/server/api/endpoints/drive/folders/find.ts b/packages/backend/src/server/api/endpoints/drive/folders/find.ts index 950aeacea0..2c4cf857c4 100644 --- a/packages/backend/src/server/api/endpoints/drive/folders/find.ts +++ b/packages/backend/src/server/api/endpoints/drive/folders/find.ts @@ -9,6 +9,7 @@ import { Endpoint } from '@/server/api/endpoint-base.js'; import type { DriveFoldersRepository } from '@/models/_.js'; import { DriveFolderEntityService } from '@/core/entities/DriveFolderEntityService.js'; import { DI } from '@/di-symbols.js'; +import { promiseMap } from '@/misc/promise-map.js'; export const meta = { tags: ['drive'], @@ -58,7 +59,7 @@ export default class extends Endpoint { // eslint- parentId: ps.parentId ?? IsNull(), }); - return await Promise.all(folders.map(folder => this.driveFolderEntityService.pack(folder))); + return await promiseMap(folders, async folder => await this.driveFolderEntityService.pack(folder), { limit: 4 }); }); } } diff --git a/packages/backend/src/server/api/endpoints/gallery/posts/create.ts b/packages/backend/src/server/api/endpoints/gallery/posts/create.ts index f280dd8986..7193cf993f 100644 --- a/packages/backend/src/server/api/endpoints/gallery/posts/create.ts +++ b/packages/backend/src/server/api/endpoints/gallery/posts/create.ts @@ -13,6 +13,7 @@ import { IdService } from '@/core/IdService.js'; import { GalleryPostEntityService } from '@/core/entities/GalleryPostEntityService.js'; import { DI } from '@/di-symbols.js'; import { TimeService } from '@/global/TimeService.js'; +import { In } from 'typeorm'; export const meta = { tags: ['gallery'], @@ -66,12 +67,10 @@ export default class extends Endpoint { // eslint- private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { - const files = (await Promise.all(ps.fileIds.map(fileId => - this.driveFilesRepository.findOneBy({ - id: fileId, - userId: me.id, - }), - ))).filter(x => x != null); + const files = await this.driveFilesRepository.findBy({ + id: In(ps.fileIds), + userId: me.id, + }); if (files.length === 0) { throw new Error('no files specified'); diff --git a/packages/backend/src/server/api/endpoints/hashtags/users.ts b/packages/backend/src/server/api/endpoints/hashtags/users.ts index 1be9f6a553..160d3c1cc3 100644 --- a/packages/backend/src/server/api/endpoints/hashtags/users.ts +++ b/packages/backend/src/server/api/endpoints/hashtags/users.ts @@ -12,6 +12,7 @@ import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { DI } from '@/di-symbols.js'; import { RoleService } from '@/core/RoleService.js'; import { TimeService } from '@/global/TimeService.js'; +import { promiseMap } from '@/misc/promise-map.js'; export const meta = { requireCredential: false, @@ -97,7 +98,7 @@ export default class extends Endpoint { // eslint- // 2. A span of more than "limit" consecutive non-trendable users may cause the pagination to stop early. // Unfortunately, there's no better solution unless we refactor role policies to be persisted to the DB. if (ps.trending) { - const usersWithRoles = await Promise.all(users.map(async u => [u, await this.roleService.getUserPolicies(u)] as const)); + const usersWithRoles = await promiseMap(users, async u => [u, await this.roleService.getUserPolicies(u)] as const, { limit: 4 }); users = usersWithRoles .filter(([,p]) => p.canTrend) .map(([u]) => u); diff --git a/packages/backend/src/server/api/endpoints/i/authorized-apps.ts b/packages/backend/src/server/api/endpoints/i/authorized-apps.ts index 447fd18dcf..ad9adac304 100644 --- a/packages/backend/src/server/api/endpoints/i/authorized-apps.ts +++ b/packages/backend/src/server/api/endpoints/i/authorized-apps.ts @@ -9,6 +9,7 @@ import { Endpoint } from '@/server/api/endpoint-base.js'; import type { AccessTokensRepository } from '@/models/_.js'; import { AppEntityService } from '@/core/entities/AppEntityService.js'; import { DI } from '@/di-symbols.js'; +import { promiseMap } from '@/misc/promise-map.js'; export const meta = { requireCredential: true, @@ -88,9 +89,11 @@ export default class extends Endpoint { // eslint- }, }); - return await Promise.all(tokens.map(token => this.appEntityService.pack(token.appId!, me, { + return await promiseMap(tokens, async token => await this.appEntityService.pack(token.appId!, me, { detail: true, - }))); + }), { + limit: 4, + }); }); } } diff --git a/packages/backend/src/server/api/endpoints/i/favorites.ts b/packages/backend/src/server/api/endpoints/i/favorites.ts index 49d47e1624..e1b1be3f73 100644 --- a/packages/backend/src/server/api/endpoints/i/favorites.ts +++ b/packages/backend/src/server/api/endpoints/i/favorites.ts @@ -5,7 +5,7 @@ import { Inject, Injectable } from '@nestjs/common'; import { Endpoint } from '@/server/api/endpoint-base.js'; -import type { NoteFavoritesRepository } from '@/models/_.js'; +import type { MiNote, MiNoteFavorite, NoteFavoritesRepository } from '@/models/_.js'; import { QueryService } from '@/core/QueryService.js'; import { NoteFavoriteEntityService } from '@/core/entities/NoteFavoriteEntityService.js'; import { DI } from '@/di-symbols.js'; @@ -56,11 +56,11 @@ export default class extends Endpoint { // eslint- super(meta, paramDef, async (ps, me) => { const query = this.queryService.makePaginationQuery(this.noteFavoritesRepository.createQueryBuilder('favorite'), ps.sinceId, ps.untilId) .andWhere('favorite.userId = :meId', { meId: me.id }) - .leftJoinAndSelect('favorite.note', 'note'); + .innerJoinAndSelect('favorite.note', 'note'); const favorites = await query .limit(ps.limit) - .getMany(); + .getMany() as (MiNoteFavorite & { note: MiNote })[]; return await this.noteFavoriteEntityService.packMany(favorites, me); }); diff --git a/packages/backend/src/server/api/endpoints/i/signin-history.ts b/packages/backend/src/server/api/endpoints/i/signin-history.ts index 8b39b87a7f..3094747a74 100644 --- a/packages/backend/src/server/api/endpoints/i/signin-history.ts +++ b/packages/backend/src/server/api/endpoints/i/signin-history.ts @@ -9,6 +9,7 @@ import type { SigninsRepository } from '@/models/_.js'; import { QueryService } from '@/core/QueryService.js'; import { SigninEntityService } from '@/core/entities/SigninEntityService.js'; import { DI } from '@/di-symbols.js'; +import { promiseMap } from '@/misc/promise-map.js'; export const meta = { requireCredential: true, @@ -56,7 +57,7 @@ export default class extends Endpoint { // eslint- const history = await query.limit(ps.limit).getMany(); - return await Promise.all(history.map(record => this.signinEntityService.pack(record))); + return await promiseMap(history, async record => await this.signinEntityService.pack(record), { limit: 4 }); }); } } diff --git a/packages/backend/src/server/api/endpoints/my/apps.ts b/packages/backend/src/server/api/endpoints/my/apps.ts index 3fb0d1b3b7..58996f60e7 100644 --- a/packages/backend/src/server/api/endpoints/my/apps.ts +++ b/packages/backend/src/server/api/endpoints/my/apps.ts @@ -8,6 +8,7 @@ import { Endpoint } from '@/server/api/endpoint-base.js'; import type { AppsRepository } from '@/models/_.js'; import { AppEntityService } from '@/core/entities/AppEntityService.js'; import { DI } from '@/di-symbols.js'; +import { promiseMap } from '@/misc/promise-map.js'; export const meta = { tags: ['account', 'app'], @@ -60,9 +61,11 @@ export default class extends Endpoint { // eslint- skip: ps.offset, }); - return await Promise.all(apps.map(app => this.appEntityService.pack(app, me, { + return await promiseMap(apps, async app => await this.appEntityService.pack(app, me, { detail: true, - }))); + }), { + limit: 4, + }); }); } } diff --git a/packages/backend/src/server/api/endpoints/notes/schedule/list.ts b/packages/backend/src/server/api/endpoints/notes/schedule/list.ts index 374a4248d1..bfc0010014 100644 --- a/packages/backend/src/server/api/endpoints/notes/schedule/list.ts +++ b/packages/backend/src/server/api/endpoints/notes/schedule/list.ts @@ -15,6 +15,8 @@ import { QueryService } from '@/core/QueryService.js'; import { Packed } from '@/misc/json-schema.js'; import { noteVisibilities } from '@/types.js'; import { bindThis } from '@/decorators.js'; +import { promiseMap } from '@/misc/promise-map.js'; +import { In } from 'typeorm'; export const meta = { tags: ['notes'], @@ -95,6 +97,9 @@ export default class extends Endpoint { // eslint- const query = this.queryService.makePaginationQuery(this.noteScheduleRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId) .andWhere('note.userId = :userId', { userId: me.id }); const scheduleNotes = await query.limit(ps.limit).getMany(); + const refNoteIds = scheduleNotes.flatMap(s => [s.note.reply, s.note.renote]).filter(id => id != null); + const refNotesList = await this.notesRepository.findBy({ id: In(refNoteIds) }); + const refNotesMap = new Map(refNotesList.map(n => [n.id, n])); const user = await this.userEntityService.pack(me, me); const scheduleNotesPack: { id: string; @@ -111,9 +116,9 @@ export default class extends Endpoint { // eslint- }; userId: string; scheduledAt: string; - }[] = await Promise.all(scheduleNotes.map(async (item: MiNoteSchedule) => { - const renote = await this.fetchNote(item.note.renote, me); - const reply = await this.fetchNote(item.note.reply, me); + }[] = await promiseMap(scheduleNotes, async (item: MiNoteSchedule) => { + const renote = await this.fetchNote(item.note.renote, me, refNotesMap); + const reply = await this.fetchNote(item.note.reply, me, refNotesMap); return { ...item, @@ -136,7 +141,9 @@ export default class extends Endpoint { // eslint- poll: item.note.poll ? await this.fillPoll(item.note.poll) : undefined, }, }; - })); + }, { + limit: 4, + }); return scheduleNotesPack; }); @@ -146,9 +153,10 @@ export default class extends Endpoint { // eslint- private async fetchNote( id: MiNote['id'] | null | undefined, me: MiUser, + hint?: Map, ): Promise | null> { if (id) { - const note = await this.notesRepository.findOneBy({ id }); + const note = hint?.get(id) ?? await this.notesRepository.findOneBy({ id }); if (note) { note.reactionAndUserPairCache ??= []; return await this.noteEntityService.pack(note, me); diff --git a/packages/backend/src/server/api/endpoints/users.ts b/packages/backend/src/server/api/endpoints/users.ts index 1efab2afca..ab4a8bf991 100644 --- a/packages/backend/src/server/api/endpoints/users.ts +++ b/packages/backend/src/server/api/endpoints/users.ts @@ -12,6 +12,7 @@ import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { DI } from '@/di-symbols.js'; import { RoleService } from '@/core/RoleService.js'; import { TimeService } from '@/global/TimeService.js'; +import { promiseMap } from '@/misc/promise-map.js'; import type { SelectQueryBuilder } from 'typeorm'; export const meta = { @@ -113,7 +114,7 @@ export default class extends Endpoint { // eslint- // 1. It may return less than "limit" results. // 2. A span of more than "limit" consecutive non-trendable users may cause the pagination to stop early. // Unfortunately, there's no better solution unless we refactor role policies to be persisted to the DB. - const usersWithRoles = await Promise.all(allUsers.map(async u => [u, await this.roleService.getUserPolicies(u)] as const)); + const usersWithRoles = await promiseMap(allUsers, async u => [u, await this.roleService.getUserPolicies(u)] as const, { limit: 4 }); const users = usersWithRoles .filter(([,p]) => p.canTrend) .map(([u]) => u); diff --git a/packages/backend/src/server/api/endpoints/users/lists/list.ts b/packages/backend/src/server/api/endpoints/users/lists/list.ts index 976da9512d..2599ea0e34 100644 --- a/packages/backend/src/server/api/endpoints/users/lists/list.ts +++ b/packages/backend/src/server/api/endpoints/users/lists/list.ts @@ -9,6 +9,7 @@ import { Endpoint } from '@/server/api/endpoint-base.js'; import { UserListEntityService } from '@/core/entities/UserListEntityService.js'; import { ApiError } from '@/server/api/error.js'; import { DI } from '@/di-symbols.js'; +import { promiseMap } from '@/misc/promise-map.js'; export const meta = { tags: ['lists', 'account'], @@ -88,7 +89,7 @@ export default class extends Endpoint { isPublic: true, }); - return await Promise.all(userLists.map(x => this.userListEntityService.pack(x, me?.id))); + return await promiseMap(userLists, async x => await this.userListEntityService.pack(x, me?.id), { limit: 4 }); }); } } diff --git a/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts b/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts index 1d2a1db625..287bb9036e 100644 --- a/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts +++ b/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts @@ -23,6 +23,7 @@ import { parseTimelineArgs, TimelineArgs, toBoolean } from './argsUtils.js'; import { convertAnnouncement, convertAttachment, MastodonConverters, convertRelationship } from './MastodonConverters.js'; import type { Entity } from 'megalodon'; import type { FastifyInstance, FastifyPluginOptions } from 'fastify'; +import { promiseMap } from '@/misc/promise-map.js'; @Injectable() export class MastodonApiServerService { @@ -178,7 +179,7 @@ export class MastodonApiServerService { const { client, me } = await this.clientService.getAuthClient(_request); const data = await client.getBookmarks(parseTimelineArgs(_request.query)); - const response = await Promise.all(data.data.map((status) => this.mastoConverters.convertStatus(status, me))); + const response = await promiseMap(data.data, async (status) => await this.mastoConverters.convertStatus(status, me), { limit: 4 }); return reply.send(response); }); @@ -200,7 +201,7 @@ export class MastodonApiServerService { userId: me.id, }; const data = await client.getFavourites(args); - const response = await Promise.all(data.data.map((status) => this.mastoConverters.convertStatus(status, me))); + const response = await promiseMap(data.data, async (status) => await this.mastoConverters.convertStatus(status, me), { limit: 4 }); return reply.send(response); }); @@ -209,7 +210,7 @@ export class MastodonApiServerService { const client = this.clientService.getClient(_request); const data = await client.getMutes(parseTimelineArgs(_request.query)); - const response = await Promise.all(data.data.map((account) => this.mastoConverters.convertAccount(account))); + const response = await promiseMap(data.data, async (account) => await this.mastoConverters.convertAccount(account), { limit: 4 }); return reply.send(response); }); @@ -218,7 +219,7 @@ export class MastodonApiServerService { const client = this.clientService.getClient(_request); const data = await client.getBlocks(parseTimelineArgs(_request.query)); - const response = await Promise.all(data.data.map((account) => this.mastoConverters.convertAccount(account))); + const response = await promiseMap(data.data, async (account) => await this.mastoConverters.convertAccount(account), { limit: 4 }); return reply.send(response); }); @@ -228,7 +229,7 @@ export class MastodonApiServerService { const limit = _request.query.limit ? parseInt(_request.query.limit) : 20; const data = await client.getFollowRequests(limit); - const response = await Promise.all(data.data.map((account) => this.mastoConverters.convertAccount(account as Entity.Account))); + const response = await promiseMap(data.data, async (account) => await this.mastoConverters.convertAccount(account), { limit: 4 }); return reply.send(response); }); diff --git a/packages/backend/src/server/api/mastodon/MastodonConverters.ts b/packages/backend/src/server/api/mastodon/MastodonConverters.ts index 963bde3726..6abd918bc6 100644 --- a/packages/backend/src/server/api/mastodon/MastodonConverters.ts +++ b/packages/backend/src/server/api/mastodon/MastodonConverters.ts @@ -24,6 +24,7 @@ import { GetterService } from '@/server/api/GetterService.js'; import { appendContentWarning } from '@/misc/append-content-warning.js'; import { isRenote } from '@/misc/is-renote.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; +import { promiseMap } from '@/misc/promise-map.js'; // Missing from Megalodon apparently // https://docs.joinmastodon.org/entities/StatusEdit/ @@ -283,11 +284,15 @@ export class MastodonConverters { }); }); - const mentions = Promise.all(note.mentions.map(p => - this.getUser(p) - .then(u => this.encode(u, mentionedRemoteUsers)) - .catch(() => null))) - .then((p: (Entity.Mention | null)[]) => p.filter(m => m != null)); + const mentions = promiseMap(note.mentions, async p => { + try { + const u = await this.getUser(p); + return this.encode(u, mentionedRemoteUsers); + } catch { + return null; + } + }, { limit: 4 }) + .then((p: Entity.Mention[]) => p.filter(m => m)); const tags = note.tags.map(tag => { return { @@ -363,7 +368,7 @@ export class MastodonConverters { public async convertConversation(conversation: Entity.Conversation, me: MiLocalUser | null): Promise { return { id: conversation.id, - accounts: await Promise.all(conversation.accounts.map((a: Entity.Account) => this.convertAccount(a))), + accounts: await promiseMap(conversation.accounts, async (a: Entity.Account) => await this.convertAccount(a), { limit: 4 }), last_status: conversation.last_status ? await this.convertStatus(conversation.last_status, me) : null, unread: conversation.unread, }; diff --git a/packages/backend/src/server/api/mastodon/endpoints/account.ts b/packages/backend/src/server/api/mastodon/endpoints/account.ts index 0d22ac8ab5..260ab58ece 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/account.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/account.ts @@ -12,6 +12,7 @@ import type { AccessTokensRepository, UserProfilesRepository } from '@/models/_. import { attachMinMaxPagination } from '@/server/api/mastodon/pagination.js'; import { MastodonConverters, convertRelationship, convertFeaturedTag, convertList } from '../MastodonConverters.js'; import type { FastifyInstance } from 'fastify'; +import { promiseMap } from '@/misc/promise-map.js'; interface ApiAccountMastodonRoute { Params: { id?: string }, @@ -188,7 +189,7 @@ export class ApiAccountMastodon { const { client, me } = await this.clientService.getAuthClient(request); const args = parseTimelineArgs(request.query); const data = await client.getAccountStatuses(request.params.id, args); - const response = await Promise.all(data.data.map(async (status) => await this.mastoConverters.convertStatus(status, me))); + const response = await promiseMap(data.data, async status => await this.mastoConverters.convertStatus(status, me), { limit: 2 }); attachMinMaxPagination(request, reply, response); return reply.send(response); @@ -212,7 +213,7 @@ export class ApiAccountMastodon { request.params.id, parseTimelineArgs(request.query), ); - const response = await Promise.all(data.data.map(async (account) => await this.mastoConverters.convertAccount(account))); + const response = await promiseMap(data.data, async account => await this.mastoConverters.convertAccount(account), { limit: 2 }); attachMinMaxPagination(request, reply, response); return reply.send(response); @@ -226,7 +227,7 @@ export class ApiAccountMastodon { request.params.id, parseTimelineArgs(request.query), ); - const response = await Promise.all(data.data.map(async (account) => await this.mastoConverters.convertAccount(account))); + const response = await promiseMap(data.data, async account => await this.mastoConverters.convertAccount(account), { limit: 2 }); attachMinMaxPagination(request, reply, response); return reply.send(response); diff --git a/packages/backend/src/server/api/mastodon/endpoints/notifications.ts b/packages/backend/src/server/api/mastodon/endpoints/notifications.ts index f6cc59e782..dc2c0fefdc 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/notifications.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/notifications.ts @@ -10,6 +10,7 @@ import { MastodonConverters } from '@/server/api/mastodon/MastodonConverters.js' import { attachMinMaxPagination } from '@/server/api/mastodon/pagination.js'; import { MastodonClientService } from '../MastodonClientService.js'; import type { FastifyInstance } from 'fastify'; +import { promiseMap } from '@/misc/promise-map.js'; interface ApiNotifyMastodonRoute { Params: { @@ -29,7 +30,7 @@ export class ApiNotificationsMastodon { fastify.get('/v1/notifications', async (request, reply) => { const { client, me } = await this.clientService.getAuthClient(request); const data = await client.getNotifications(parseTimelineArgs(request.query)); - const notifications = await Promise.all(data.data.map(n => this.mastoConverters.convertNotification(n, me))); + const notifications = await promiseMap(data.data, async n => await this.mastoConverters.convertNotification(n, me), { limit: 4 }); const response: MastodonEntity.Notification[] = []; for (const notification of notifications) { // Notifications for inaccessible notes will be null and should be ignored diff --git a/packages/backend/src/server/api/mastodon/endpoints/search.ts b/packages/backend/src/server/api/mastodon/endpoints/search.ts index 0d1df8c06b..89da9daef9 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/search.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/search.ts @@ -6,6 +6,7 @@ import { Injectable } from '@nestjs/common'; import { MastodonClientService } from '@/server/api/mastodon/MastodonClientService.js'; import { attachMinMaxPagination, attachOffsetPagination } from '@/server/api/mastodon/pagination.js'; +import { promiseMap } from '@/misc/promise-map.js'; import { MastodonConverters } from '../MastodonConverters.js'; import { parseTimelineArgs, TimelineArgs, toBoolean, toInt } from '../argsUtils.js'; import { ApiError } from '../../error.js'; @@ -52,8 +53,8 @@ export class ApiSearchMastodon { const { data } = await client.search(request.query.q, { type, ...query }); const response = { ...data, - accounts: await Promise.all(data.accounts.map((account: Entity.Account) => this.mastoConverters.convertAccount(account))), - statuses: await Promise.all(data.statuses.map((status: Entity.Status) => this.mastoConverters.convertStatus(status, me))), + accounts: await promiseMap(data.accounts, (account: Entity.Account) => this.mastoConverters.convertAccount(account), { limit: 3 }), + statuses: await promiseMap(data.statuses, (status: Entity.Status) => this.mastoConverters.convertStatus(status, me), { limit: 3 }), }; if (type === 'hashtags') { @@ -89,8 +90,8 @@ export class ApiSearchMastodon { const stat = !type || type === 'statuses' ? await client.search(request.query.q, { type: 'statuses', ...query }) : null; const tags = !type || type === 'hashtags' ? await client.search(request.query.q, { type: 'hashtags', ...query }) : null; const response = { - accounts: await Promise.all(acct?.data.accounts.map((account: Entity.Account) => this.mastoConverters.convertAccount(account)) ?? []), - statuses: await Promise.all(stat?.data.statuses.map((status: Entity.Status) => this.mastoConverters.convertStatus(status, me)) ?? []), + accounts: acct ? await promiseMap(acct.data.accounts, async (account: Entity.Account) => await this.mastoConverters.convertAccount(account), { limit: 3 }) : [], + statuses: acct ? await promiseMap(acct.data.statuses, async (status: Entity.Status) => this.mastoConverters.convertStatus(status, me), { limit: 3 }) : [], hashtags: tags?.data.hashtags ?? [], }; @@ -123,7 +124,7 @@ export class ApiSearchMastodon { const data = await res.json() as Entity.Status[]; const me = await this.clientService.getAuth(request); - const response = await Promise.all(data.map(status => this.mastoConverters.convertStatus(status, me))); + const response = await promiseMap(data, async status => await this.mastoConverters.convertStatus(status, me), { limit: 4 }); attachMinMaxPagination(request, reply, response); return reply.send(response); @@ -150,12 +151,12 @@ export class ApiSearchMastodon { await verifyResponse(res); const data = await res.json() as Entity.Account[]; - const response = await Promise.all(data.map(async entry => { - return { - source: 'global', - account: await this.mastoConverters.convertAccount(entry), - }; - })); + const response = await promiseMap(data, async entry => ({ + source: 'global', + account: await this.mastoConverters.convertAccount(entry), + }), { + limit: 4, + }); attachOffsetPagination(request, reply, response); return reply.send(response); diff --git a/packages/backend/src/server/api/mastodon/endpoints/status.ts b/packages/backend/src/server/api/mastodon/endpoints/status.ts index f5942a5267..e126a22f58 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/status.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/status.ts @@ -11,6 +11,7 @@ import { MastodonClientService } from '@/server/api/mastodon/MastodonClientServi import { MastodonDataService } from '@/server/api/mastodon/MastodonDataService.js'; import { getNoteSummary } from '@/misc/get-note-summary.js'; import { isPureRenote } from '@/misc/is-renote.js'; +import { promiseMap } from '@/misc/promise-map.js'; import { convertAttachment, convertPoll, MastodonConverters } from '../MastodonConverters.js'; import type { Entity } from 'megalodon'; import type { FastifyInstance } from 'fastify'; @@ -97,8 +98,8 @@ export class ApiStatusMastodon { const { client, me } = await this.clientService.getAuthClient(_request); const { data } = await client.getStatusContext(_request.params.id, parseTimelineArgs(_request.query)); - const ancestors = await Promise.all(data.ancestors.map(async (status: Entity.Status) => await this.mastoConverters.convertStatus(status, me))); - const descendants = await Promise.all(data.descendants.map(async (status: Entity.Status) => await this.mastoConverters.convertStatus(status, me))); + const ancestors = await promiseMap(data.ancestors, async (status: Entity.Status) => await this.mastoConverters.convertStatus(status, me), { limit: 4 }); + const descendants = await promiseMap(data.descendants, async (status: Entity.Status) => await this.mastoConverters.convertStatus(status, me), { limit: 4 }); const response = { ancestors, descendants }; return reply.send(response); @@ -118,7 +119,7 @@ export class ApiStatusMastodon { const client = this.clientService.getClient(_request); const data = await client.getStatusRebloggedBy(_request.params.id); - const response = await Promise.all(data.data.map((account: Entity.Account) => this.mastoConverters.convertAccount(account))); + const response = await promiseMap(data.data, async (account: Entity.Account) => await this.mastoConverters.convertAccount(account), { limit: 4 }); return reply.send(response); }); @@ -128,7 +129,7 @@ export class ApiStatusMastodon { const client = this.clientService.getClient(_request); const data = await client.getStatusFavouritedBy(_request.params.id); - const response = await Promise.all(data.data.map((account: Entity.Account) => this.mastoConverters.convertAccount(account))); + const response = await promiseMap(data.data, async (account: Entity.Account) => await this.mastoConverters.convertAccount(account), { limit: 4 }); return reply.send(response); }); diff --git a/packages/backend/src/server/api/mastodon/endpoints/timeline.ts b/packages/backend/src/server/api/mastodon/endpoints/timeline.ts index b2f7b18dc9..df56c34c82 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/timeline.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/timeline.ts @@ -6,6 +6,7 @@ import { Injectable } from '@nestjs/common'; import { MastodonClientService } from '@/server/api/mastodon/MastodonClientService.js'; import { attachMinMaxPagination } from '@/server/api/mastodon/pagination.js'; +import { promiseMap } from '@/misc/promise-map.js'; import { convertList, MastodonConverters } from '../MastodonConverters.js'; import { parseTimelineArgs, TimelineArgs, toBoolean } from '../argsUtils.js'; import type { Entity } from 'megalodon'; @@ -25,7 +26,7 @@ export class ApiTimelineMastodon { const data = toBoolean(request.query.local) ? await client.getLocalTimeline(query) : await client.getPublicTimeline(query); - const response = await Promise.all(data.data.map((status: Entity.Status) => this.mastoConverters.convertStatus(status, me))); + const response = await promiseMap(data.data, async (status: Entity.Status) => await this.mastoConverters.convertStatus(status, me), { limit: 4 }); attachMinMaxPagination(request, reply, response); return reply.send(response); @@ -35,7 +36,7 @@ export class ApiTimelineMastodon { const { client, me } = await this.clientService.getAuthClient(request); const query = parseTimelineArgs(request.query); const data = await client.getHomeTimeline(query); - const response = await Promise.all(data.data.map((status: Entity.Status) => this.mastoConverters.convertStatus(status, me))); + const response = await promiseMap(data.data, async (status: Entity.Status) => await this.mastoConverters.convertStatus(status, me), { limit: 4 }); attachMinMaxPagination(request, reply, response); return reply.send(response); @@ -47,7 +48,7 @@ export class ApiTimelineMastodon { const { client, me } = await this.clientService.getAuthClient(request); const query = parseTimelineArgs(request.query); const data = await client.getTagTimeline(request.params.hashtag, query); - const response = await Promise.all(data.data.map((status: Entity.Status) => this.mastoConverters.convertStatus(status, me))); + const response = await promiseMap(data.data, async (status: Entity.Status) => await this.mastoConverters.convertStatus(status, me), { limit: 4 }); attachMinMaxPagination(request, reply, response); return reply.send(response); @@ -59,7 +60,7 @@ export class ApiTimelineMastodon { const { client, me } = await this.clientService.getAuthClient(request); const query = parseTimelineArgs(request.query); const data = await client.getListTimeline(request.params.id, query); - const response = await Promise.all(data.data.map(async (status: Entity.Status) => await this.mastoConverters.convertStatus(status, me))); + const response = await promiseMap(data.data, async (status: Entity.Status) => await this.mastoConverters.convertStatus(status, me), { limit: 4 }); attachMinMaxPagination(request, reply, response); return reply.send(response); @@ -69,7 +70,7 @@ export class ApiTimelineMastodon { const { client, me } = await this.clientService.getAuthClient(request); const query = parseTimelineArgs(request.query); const data = await client.getConversationTimeline(query); - const response = await Promise.all(data.data.map((conversation: Entity.Conversation) => this.mastoConverters.convertConversation(conversation, me))); + const response = await promiseMap(data.data, async (conversation: Entity.Conversation) => await this.mastoConverters.convertConversation(conversation, me), { limit: 4 }); attachMinMaxPagination(request, reply, response); return reply.send(response); @@ -99,7 +100,7 @@ export class ApiTimelineMastodon { const client = this.clientService.getClient(request); const data = await client.getAccountsInList(request.params.id, parseTimelineArgs(request.query)); - const response = await Promise.all(data.data.map((account: Entity.Account) => this.mastoConverters.convertAccount(account))); + const response = await promiseMap(data.data, async (account: Entity.Account) => await this.mastoConverters.convertAccount(account), { limit: 4 }); attachMinMaxPagination(request, reply, response); return reply.send(response); diff --git a/packages/backend/test/unit/misc/promise-map.ts b/packages/backend/test/unit/misc/promise-map.ts new file mode 100644 index 0000000000..81d21c8e0e --- /dev/null +++ b/packages/backend/test/unit/misc/promise-map.ts @@ -0,0 +1,170 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { setTimeout } from 'node:timers/promises'; +import promiseLimit from 'promise-limit'; +import { promiseMap } from '@/misc/promise-map.js'; + +async function randomDelay() { + await setTimeout(10 * Math.abs(Math.random())); +} + +describe(promiseMap, () => { + it('should return empty array with no input', async () => { + const result = await promiseMap([] as string[], async () => 'wrong'); + expect(result).toHaveLength(0); + }); + + it('should map items in correct order', async () => { + const items = [1, 2, 3, 4, 5]; + + const results = await promiseMap(items, async i => { + await randomDelay(); + return String(i); + }); + + expect(results).toEqual(['1', '2', '3', '4', '5']); + }); + + it('should accept async iterable input', async () => { + async function *generator() { + yield 1; + yield 2; + yield 3; + } + + const results = await promiseMap(generator(), async i => String(i)); + + expect(results).toEqual(['1', '2', '3']); + }); + + it('should accept limit input', async () => { + const items = [1, 2, 3, 4, 5]; + + let inProgress = 0; + let maxProgress = 0; + + const results = await promiseMap(items, async i => { + inProgress++; + maxProgress = Math.max(maxProgress, inProgress); + + await randomDelay(); + + inProgress--; + return String(i); + }, { + limit: 2, + }); + + expect(results).toEqual(['1', '2', '3', '4', '5']); + expect(maxProgress).toEqual(2); + }); + + it('should accept limit as instance', async () => { + const items = [1, 2, 3, 4, 5]; + const limit = promiseLimit(2); + + let inProgress = 0; + let maxProgress = 0; + + const results = await promiseMap(items, async i => { + inProgress++; + maxProgress = Math.max(maxProgress, inProgress); + + await randomDelay(); + + inProgress--; + return String(i); + }, { + limit, + }); + + expect(results).toEqual(['1', '2', '3', '4', '5']); + expect(maxProgress).toEqual(2); + }); + + it('should reject when signal aborts', async () => { + const items = [1, 2, 3, 4, 5]; + const controller = new AbortController(); + + const promise = promiseMap(items, async i => { + if (i === 3) { + controller.abort(new Error('test abort')); + } + + return String(i); + }, { + limit: 1, + signal: controller.signal, + }); + + await expect(promise).rejects.toThrow('abort'); + }); + + it('should abort when signal aborts', async () => { + const items = [1, 2, 3, 4, 5]; + const controller = new AbortController(); + + const processed: number[] = []; + + await promiseMap(items, async i => { + if (i === 3) { + controller.abort('test abort'); + } + + processed.push(i); + return String(i); + }, { + limit: 1, + signal: controller.signal, + }).catch(() => null); + + expect(processed).toEqual([1, 2, 3]); + }); + + it('should reject when promise rejects', async () => { + const items = [1, 2, 3, 4, 5]; + + const promise = promiseMap(items, async i => { + if (i === 3) { + throw new Error('test error'); + } + + return String(i); + }); + + await expect(promise).rejects.toThrow('test'); + }); + + it('should abort when promise rejects', async () => { + const items = [1, 2, 3, 4, 5]; + + const processed: number[] = []; + + await promiseMap(items, async i => { + if (i === 3) { + throw new Error('test error'); + } + + processed.push(i); + + return String(i); + }).catch(() => null); + + expect(processed).toEqual([1, 2]); + }); + + it('should aggregate all errors', async () => { + const items = [1, 2, 3, 4, 5]; + + const promise = promiseMap(items, async i => { + await setTimeout(10); + + throw new Error(`test error: ${i}`); + }); + + await expect(promise).rejects.toThrow(AggregateError); + }); +});