misty.cafe/mistykey
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

limit unbounded parallel async maps

Browse source
This commit is contained in:
Hazelnoot 2025-06-27 12:58:11 -04:00
parent 4ceff00dac
commit 8b3ea577da
42 changed files with 508 additions and 219 deletions
Download patch file Download diff file Expand all files Collapse all files

6
packages/backend/src/core/AntennaService.ts
View file

@ -19,6 +19,7 @@ import type { MiAntenna } from '@/models/Antenna.js';
import type { MiNote } from '@/models/Note.js';
import type { MiUser } from '@/models/User.js';
import { InternalEventService } from '@/core/InternalEventService.js';
import { promiseMap } from '@/misc/promise-map.js';
import { CacheService } from './CacheService.js';
import type { OnApplicationShutdown } from '@nestjs/common';
@ -95,7 +96,10 @@ export class AntennaService implements OnApplicationShutdown {
@bindThis
public async addNoteToAntennas(note: MiNote, noteUser: { id: MiUser['id']; username: string; host: string | null; isBot: boolean; }): Promise<void> {
const antennas = await this.getAntennas();
const antennasWithMatchResult = await Promise.all(antennas.map(antenna => this.checkHitAntenna(antenna, note, noteUser).then(hit => [antenna, hit] as const)));
const antennasWithMatchResult = await promiseMap(antennas, async antenna => {
const hit = await this.checkHitAntenna(antenna, note, noteUser);
return [antenna, hit] as const;
});
const matchedAntennas = antennasWithMatchResult.filter(([, hit]) => hit).map(([antenna]) => antenna);
const redisPipeline = this.redisForTimelines.pipeline();

3
packages/backend/src/core/CustomEmojiService.ts
View file

@ -23,6 +23,7 @@ import { DriveService } from '@/core/DriveService.js';
import { CacheManagementService, type ManagedQuantumKVCache } from '@/global/CacheManagementService.js';
import { TimeService } from '@/global/TimeService.js';
import { LoggerService } from '@/core/LoggerService.js';
import { promiseMap } from '@/misc/promise-map.js';
import { isRetryableSymbol } from '@/misc/is-retryable-error.js';
import type Logger from '@/logger.js';
import { KeyNotFoundError } from '@/misc/errors/KeyNotFoundError.js';
@ -577,7 +578,7 @@ export class CustomEmojiService {
*/
@bindThis
public async populateEmojis(emojiNames: string[], noteUserHost: string | null): Promise<Record<string, string>> {
const emojis = await Promise.all(emojiNames.map(x => this.populateEmoji(x, noteUserHost)));
const emojis = await promiseMap(emojiNames, async x => await this.populateEmoji(x, noteUserHost), { limit: 4 });
const res = {} as Record<string, string>;
for (let i = 0; i < emojiNames.length; i++) {
const resolvedEmoji = emojis[i];

19
packages/backend/src/core/NoteCreateService.ts
View file

@ -60,6 +60,7 @@ import { CacheService } from '@/core/CacheService.js';
import { TimeService } from '@/global/TimeService.js';
import { NoteVisibilityService } from '@/core/NoteVisibilityService.js';
import { CollapsedQueueService } from '@/core/CollapsedQueueService.js';
import { promiseMap } from '@/misc/promise-map.js';
type NotificationType = 'reply' | 'renote' | 'quote' | 'mention';
@ -889,20 +890,16 @@ export class NoteCreateService implements OnApplicationShutdown {
}
@bindThis
private async extractMentionedUsers(user: { host: MiUser['host']; }, tokens: mfm.MfmNode[]): Promise<MiUser[]> {
if (tokens == null) return [];
public async extractMentionedUsers(user: { host: MiUser['host']; }, tokens: mfm.MfmNode[]): Promise<MiUser[]> {
if (tokens == null || tokens.length === 0) return [];
const mentions = extractMentions(tokens);
let mentionedUsers = (await Promise.all(mentions.map(m =>
this.remoteUserResolveService.resolveUser(m.username, m.host ?? user.host).catch(() => null),
))).filter(x => x != null);
const allMentions = extractMentions(tokens);
const mentions = new Map(allMentions.map(m => [`${m.username.toLowerCase()}@${m.host?.toLowerCase()}`, m]));
// Drop duplicate users
mentionedUsers = mentionedUsers.filter((u, i, self) =>
i === self.findIndex(u2 => u.id === u2.id),
);
const allMentionedUsers = await promiseMap(mentions.values(), async m => await this.remoteUserResolveService.resolveUser(m.username, m.host ?? user.host).catch(() => null), { limit: 2 });
const mentionedUsers = new Map(allMentionedUsers.filter(u => u != null).map(u => [u.id, u]));
return mentionedUsers;
return Array.from(mentionedUsers.values());
}
@bindThis

25
packages/backend/src/core/NoteEditService.ts
View file

@ -3,18 +3,16 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { setImmediate } from 'node:timers/promises';
import * as mfm from 'mfm-js';
import { DataSource, In, IsNull, LessThan } from 'typeorm';
import { DataSource, In } from 'typeorm';
import * as Redis from 'ioredis';
import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
import { UnrecoverableError } from 'bullmq';
import { extractMentions } from '@/misc/extract-mentions.js';
import { extractCustomEmojisFromMfm } from '@/misc/extract-custom-emojis-from-mfm.js';
import { extractHashtags } from '@/misc/extract-hashtags.js';
import type { IMentionedRemoteUsers } from '@/models/Note.js';
import { MiNote } from '@/models/Note.js';
import type { NoteEditsRepository, ChannelFollowingsRepository, ChannelsRepository, FollowingsRepository, InstancesRepository, MiFollowing, MiMeta, MutingsRepository, NotesRepository, NoteThreadMutingsRepository, UserListMembershipsRepository, UserProfilesRepository, UsersRepository, PollsRepository } from '@/models/_.js';
import type { NoteEditsRepository, ChannelFollowingsRepository, ChannelsRepository, FollowingsRepository, InstancesRepository, MiMeta, MutingsRepository, NotesRepository, NoteThreadMutingsRepository, UserListMembershipsRepository, UserProfilesRepository, UsersRepository, PollsRepository } from '@/models/_.js';
import type { MiDriveFile } from '@/models/DriveFile.js';
import type { MiApp } from '@/models/App.js';
import { concat } from '@/misc/prelude/array.js';
@ -430,7 +428,7 @@ export class NoteEditService implements OnApplicationShutdown {
emojis = data.apEmojis ?? extractCustomEmojisFromMfm(combinedTokens);
mentionedUsers = data.apMentions ?? await this.extractMentionedUsers(user, combinedTokens);
mentionedUsers = data.apMentions ?? await this.noteCreateService.extractMentionedUsers(user, combinedTokens);
}
// if the host is media-silenced, custom emojis are not allowed
@ -789,23 +787,6 @@ export class NoteEditService implements OnApplicationShutdown {
await this.searchService.indexNote(note);
}
@bindThis
private async extractMentionedUsers(user: { host: MiUser['host']; }, tokens: mfm.MfmNode[]): Promise<MiUser[]> {
if (tokens == null) return [];
const mentions = extractMentions(tokens);
let mentionedUsers = (await Promise.all(mentions.map(m =>
this.remoteUserResolveService.resolveUser(m.username, m.host ?? user.host).catch(() => null),
))).filter(x => x !== null) as MiUser[];
// Drop duplicate users
mentionedUsers = mentionedUsers.filter((u, i, self) =>
i === self.findIndex(u2 => u.id === u2.id),
);
return mentionedUsers;
}
@bindThis
private async pushToTl(note: MiNote, user: { id: MiUser['id']; host: MiUser['host']; }) {
if (!this.meta.enableFanoutTimeline) return;

3
packages/backend/src/core/ReactionService.ts
View file

@ -34,6 +34,7 @@ import { CacheService } from '@/core/CacheService.js';
import { NoteVisibilityService } from '@/core/NoteVisibilityService.js';
import { TimeService } from '@/global/TimeService.js';
import { CollapsedQueueService } from '@/core/CollapsedQueueService.js';
import { promiseMap } from '@/misc/promise-map.js';
import type { DataSource } from 'typeorm';
const FALLBACK = '\u2764';
@ -298,7 +299,7 @@ export class ReactionService implements OnModuleInit {
if (['public', 'home', 'followers'].includes(note.visibility)) {
dm.addFollowersRecipe();
} else if (note.visibility === 'specified') {
const visibleUsers = await Promise.all(note.visibleUserIds.map(id => this.usersRepository.findOneBy({ id })));
const visibleUsers = await promiseMap(note.visibleUserIds, async id => await this.cacheService.findOptionalUserById(id), { limit: 2 });
for (const u of visibleUsers.filter(u => u && isRemoteUser(u))) {
dm.addDirectRecipe(u as MiRemoteUser);
}

11
packages/backend/src/core/activitypub/ApAudienceService.ts
View file

@ -8,6 +8,7 @@ import promiseLimit from 'promise-limit';
import type { MiRemoteUser, MiUser } from '@/models/User.js';
import { concat, unique } from '@/misc/prelude/array.js';
import { bindThis } from '@/decorators.js';
import { promiseMap } from '@/misc/promise-map.js';
import { getApIds } from './type.js';
import { ApPersonService } from './models/ApPersonService.js';
import type { ApObject } from './type.js';
@ -37,10 +38,12 @@ export class ApAudienceService {
const others = unique(concat([toGroups.other, ccGroups.other]));
const limit = promiseLimit<MiUser | null>(2);
const mentionedUsers = (await Promise.all(
others.map(id => limit(() => this.apPersonService.resolvePerson(id, resolver).catch(() => null))),
)).filter(x => x != null);
const resolved = await promiseMap(others, async x => {
return await this.apPersonService.resolvePerson(x, resolver).catch(() => null) as MiUser | null;
}, {
limit: 2,
});
const mentionedUsers = resolved.filter(x => x != null);
// If no audience is specified, then assume public
if (

31
packages/backend/src/core/activitypub/ApResolverService.ts
View file

@ -5,7 +5,6 @@
import { Inject, Injectable } from '@nestjs/common';
import { IsNull, Not } from 'typeorm';
import promiseLimit from 'promise-limit';
import type { MiLocalUser, MiRemoteUser } from '@/models/User.js';
import type { NotesRepository, PollsRepository, NoteReactionsRepository, UsersRepository, FollowRequestsRepository, MiMeta, SkApFetchLog } from '@/models/_.js';
import type { Config } from '@/config.js';
@ -24,6 +23,7 @@ import { toArray } from '@/misc/prelude/array.js';
import { isPureRenote } from '@/misc/is-renote.js';
import { CacheService } from '@/core/CacheService.js';
import { trackPromise } from '@/misc/promise-tracker.js';
import { promiseMap } from '@/misc/promise-map.js';
import { AnyCollection, getApId, getNullableApId, IObjectWithId, isCollection, isCollectionOrOrderedCollection, isCollectionPage, isOrderedCollection, isOrderedCollectionPage } from './type.js';
import { ApDbResolverService } from './ApDbResolverService.js';
import { ApRendererService } from './ApRendererService.js';
@ -151,21 +151,20 @@ export class Resolver {
const recursionLimit = this.recursionLimit - this.history.size;
const batchLimit = Math.min(source.length, recursionLimit, itemLimit);
const limiter = promiseLimit<IObject>(concurrency);
const batch = await Promise.all(source
.slice(0, batchLimit)
.map(item => limiter(async () => {
if (sentFrom) {
// Use secureResolve to avoid re-fetching items that were included inline.
return await this.secureResolve(item, sentFrom, allowAnonymousItems);
} else if (allowAnonymousItems) {
return await this.resolveAnonymous(item);
} else {
// ID is required if we have neither sentFrom not allowAnonymousItems
const id = getApId(item);
return await this.resolve(id);
}
})));
const batch = await promiseMap(source.slice(0, batchLimit), async item => {
if (sentFrom) {
// Use secureResolve to avoid re-fetching items that were included inline.
return await this.secureResolve(item, sentFrom, allowAnonymousItems);
} else if (allowAnonymousItems) {
return await this.resolveAnonymous(item);
} else {
// ID is required if we have neither sentFrom not allowAnonymousItems
const id = getApId(item);
return await this.resolve(id);
}
}, {
limit: concurrency,
});
destination.push(...batch);
};

12
packages/backend/src/core/activitypub/models/ApMentionService.ts
View file

@ -12,6 +12,7 @@ import { isMention } from '../type.js';
import { Resolver } from '../ApResolverService.js';
import { ApPersonService } from './ApPersonService.js';
import type { IObject, IApMention } from '../type.js';
import { promiseMap } from '@/misc/promise-map.js';
@Injectable()
export class ApMentionService {
@ -24,12 +25,13 @@ export class ApMentionService {
public async extractApMentions(tags: IObject | IObject[] | null | undefined, resolver: Resolver): Promise<MiUser[]> {
const hrefs = unique(this.extractApMentionObjects(tags).map(x => x.href));
const limit = promiseLimit<MiUser | null>(2);
const mentionedUsers = (await Promise.all(
hrefs.map(x => limit(() => this.apPersonService.resolvePerson(x, resolver).catch(() => null))),
)).filter(x => x != null);
const mentionedUsers = await promiseMap(hrefs, async x => {
return await this.apPersonService.resolvePerson(x, resolver).catch(() => null) as MiUser | null;
}, {
limit: 2,
});
return mentionedUsers;
return mentionedUsers.filter(resolved => resolved != null);
}
@bindThis

24
packages/backend/src/core/activitypub/models/ApNoteService.ts
View file

@ -6,7 +6,6 @@
import { Inject, Injectable, OnModuleInit } from '@nestjs/common';
import { In } from 'typeorm';
import { UnrecoverableError } from 'bullmq';
import promiseLimit from 'promise-limit';
import { ModuleRef } from '@nestjs/core';
import { DI } from '@/di-symbols.js';
import type { UsersRepository, PollsRepository, EmojisRepository, NotesRepository, MiMeta } from '@/models/_.js';
@ -32,6 +31,7 @@ import { renderInlineError } from '@/misc/render-inline-error.js';
import { extractMediaFromHtml } from '@/core/activitypub/misc/extract-media-from-html.js';
import { extractMediaFromMfm } from '@/core/activitypub/misc/extract-media-from-mfm.js';
import { getContentByType } from '@/core/activitypub/misc/get-content-by-type.js';
import { promiseMap } from '@/misc/promise-map.js';
import { trackPromise } from '@/misc/promise-tracker.js';
import { CustomEmojiService, encodeEmojiKey, isValidEmojiName } from '@/core/CustomEmojiService.js';
import { TimeService } from '@/global/TimeService.js';
@ -583,8 +583,8 @@ export class ApNoteService implements OnModuleInit {
const emojiKeys = eomjiTags.map(tag => encodeEmojiKey({ name: tag.name, host }));
const existingEmojis = await this.customEmojiService.emojisByKeyCache.fetchMany(emojiKeys);
return await Promise.all(eomjiTags.map(async tag => {
const name = tag.name;
return await promiseMap(eomjiTags, async tag => {
const name = tag.name.replaceAll(':', '');
tag.icon = toSingle(tag.icon);
const exists = existingEmojis.values.find(x => x.name === name);
@ -627,7 +627,9 @@ export class ApNoteService implements OnModuleInit {
// _misskey_license が存在しなければ `null`
license: (tag._misskey_license?.freeText ?? null),
});
}));
}, {
limit: 4,
});
}
/**
@ -691,7 +693,7 @@ export class ApNoteService implements OnModuleInit {
}
};
const results = await Promise.all(Array.from(quoteUris).map(u => resolveQuote(u)));
const results = await promiseMap(quoteUris, async u => resolveQuote(u), { limit: 2 });
// Success - return the quote
const quote = results.find(r => typeof(r) === 'object');
@ -753,14 +755,10 @@ export class ApNoteService implements OnModuleInit {
// Resolve all files w/ concurrency 2.
// This prevents one big file from blocking the others.
const limiter = promiseLimit<MiDriveFile | null>(2);
const results = await Promise
.all(Array
.from(attachments.values())
.map(attach => limiter(async () => {
attach.sensitive ??= note.sensitive;
return await this.resolveImage(actor, attach);
})));
const results = await promiseMap(attachments.values(), async attach => {
attach.sensitive ??= note.sensitive;
return await this.resolveImage(actor, attach);
});
// Process results
let hasFileError = false;

47
packages/backend/src/core/activitypub/models/ApPersonService.ts
View file

@ -48,7 +48,8 @@ import { renderInlineError } from '@/misc/render-inline-error.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import { QueueService } from '@/core/QueueService.js';
import { CollapsedQueueService } from '@/core/CollapsedQueueService.js';
import { getApId, getApType, isActor, isCollection, isCollectionOrOrderedCollection, isPropertyValue } from '../type.js';
import { promiseMap } from '@/misc/promise-map.js';
import { getApId, getApType, getNullableApId, isActor, isPost, isPropertyValue } from '../type.js';
import { ApLoggerService } from '../ApLoggerService.js';
import { extractApHashtags } from './tag.js';
import type { OnModuleInit } from '@nestjs/common';
@ -933,35 +934,27 @@ export class ApPersonService implements OnModuleInit {
this.logger.info(`Updating the featured: ${user.uri}`);
const _resolver = resolver ?? this.apResolverService.createResolver();
// Resolve to (Ordered)Collection Object
const collection = user.featured ? await _resolver.resolveCollection(user.featured, true, user.uri).catch(err => {
// Permanent error implies hidden or inaccessible, which is a normal thing.
if (isRetryableError(err)) {
this.logger.warn(`Failed to update featured notes: ${renderInlineError(err)}`);
}
return null;
}) : null;
if (!collection) return;
if (!isCollectionOrOrderedCollection(collection)) throw new UnrecoverableError(`failed to update user ${user.uri}: featured ${user.featured} is not Collection or OrderedCollection`);
// Resolve to Object(may be Note) arrays
const unresolvedItems = isCollection(collection) ? collection.items : collection.orderedItems;
const items = await Promise.all(toArray(unresolvedItems).map(x => _resolver.resolve(x)));
resolver ??= this.apResolverService.createResolver();
// Resolve and regist Notes
const limit = promiseLimit<MiNote | null>(2);
const maxPinned = (await this.roleService.getUserPolicies(user.id)).pinLimit;
const featuredNotes = await Promise.all(items
.filter(item => getApType(item) === 'Note') // TODO: Noteでなくてもいいかも
.slice(0, maxPinned)
.map(item => limit(() => this.apNoteService.resolveNote(item, {
resolver: _resolver,
sentFrom: user.uri,
}))));
const items = await resolver.resolveCollectionItems(user.featured, true, user.uri, maxPinned, 2);
const featuredNotes = await promiseMap(items, async item => {
const itemId = getNullableApId(item);
if (itemId && isPost(item)) {
try {
return await this.apNoteService.resolveNote(item, {
resolver: resolver,
sentFrom: user.uri,
});
} catch (err) {
this.logger.warn(`Couldn't fetch pinned note ${itemId} for user ${user.id} (@${user.username}@${user.host}): ${renderInlineError(err)}`);
}
}
return null;
}, {
limit: 2,
});
await this.db.transaction(async transactionalEntityManager => {
await transactionalEntityManager.delete(MiUserNotePining, { userId: user.id });

17
packages/backend/src/core/chart/core.ts
View file

@ -15,6 +15,7 @@ import { dateUTC, isTimeSame, isTimeBefore, subtractTime, addTime } from '@/misc
import type Logger from '@/logger.js';
import { bindThis } from '@/decorators.js';
import { MiRepository, miRepository } from '@/models/_.js';
import { promiseMap } from '@/misc/promise-map.js';
import type { DataSource, Repository } from 'typeorm';
import type { Lock } from 'redis-lock';
@ -526,13 +527,13 @@ export default abstract class Chart<T extends Schema> {
const groups = removeDuplicates(this.buffer.map(log => log.group));
await Promise.all(
groups.map(group =>
Promise.all([
this.claimCurrentLog(group, 'hour'),
this.claimCurrentLog(group, 'day'),
]).then(([logHour, logDay]) =>
update(logHour, logDay))));
await promiseMap(groups, async group => {
const logHour = await this.claimCurrentLog(group, 'hour');
const logDay = await this.claimCurrentLog(group, 'day');
await update(logHour, logDay);
}, {
limit: 2,
});
}
@bindThis
@ -564,7 +565,7 @@ export default abstract class Chart<T extends Schema> {
]);
};
return Promise.all([
return await Promise.all([
this.claimCurrentLog(group, 'hour'),
this.claimCurrentLog(group, 'day'),
]).then(([logHour, logDay]) =>

15
packages/backend/src/core/entities/NoteFavoriteEntityService.ts
View file

@ -5,12 +5,13 @@
import { Inject, Injectable } from '@nestjs/common';
import { DI } from '@/di-symbols.js';
import type { NoteFavoritesRepository } from '@/models/_.js';
import type { MiNote, NoteFavoritesRepository } from '@/models/_.js';
import type { } from '@/models/Blocking.js';
import type { MiUser } from '@/models/User.js';
import type { MiNoteFavorite } from '@/models/NoteFavorite.js';
import { bindThis } from '@/decorators.js';
import { IdService } from '@/core/IdService.js';
import type { Packed } from '@/misc/json-schema.js';
import { NoteEntityService } from './NoteEntityService.js';
@Injectable()
@ -28,6 +29,7 @@ export class NoteFavoriteEntityService {
public async pack(
src: MiNoteFavorite['id'] | MiNoteFavorite,
me?: { id: MiUser['id'] } | null | undefined,
notes?: Map<string, Packed<'Note'>>,
) {
const favorite = typeof src === 'object' ? src : await this.noteFavoritesRepository.findOneByOrFail({ id: src });
@ -35,15 +37,18 @@ export class NoteFavoriteEntityService {
id: favorite.id,
createdAt: this.idService.parse(favorite.id).date.toISOString(),
noteId: favorite.noteId,
note: await this.noteEntityService.pack(favorite.note ?? favorite.noteId, me),
note: notes?.get(favorite.noteId) ?? await this.noteEntityService.pack(favorite.note ?? favorite.noteId, me),
};
}
@bindThis
public packMany(
favorites: any[],
public async packMany(
favorites: (MiNoteFavorite & { note: MiNote })[],
me: { id: MiUser['id'] },
) {
return Promise.all(favorites.map(x => this.pack(x, me)));
const packedNotes = await this.noteEntityService.packMany(favorites.map(f => f.note), me);
const packedNotesMap = new Map(packedNotes.map(n => [n.id, n]));
return Promise.all(favorites.map(x => this.pack(x, me, packedNotesMap)));
}
}

9
packages/backend/src/core/entities/PageEntityService.ts
View file

@ -105,10 +105,13 @@ export class PageEntityService {
font: page.font,
script: page.script,
eyeCatchingImageId: page.eyeCatchingImageId,
eyeCatchingImage: page.eyeCatchingImageId ? await this.driveFileEntityService.pack(page.eyeCatchingImageId) : null,
attachedFiles: this.driveFileEntityService.packMany((await Promise.all(attachedFiles)).filter(x => x != null)),
eyeCatchingImage: page.eyeCatchingImageId ? this.driveFileEntityService.pack(page.eyeCatchingImageId) : null,
attachedFiles: Promise
.all(attachedFiles)
.then(fs => fs.filter(x => x != null))
.then(fs => this.driveFileEntityService.packMany(fs)),
likedCount: page.likedCount,
isLiked: meId ? await this.pageLikesRepository.exists({ where: { pageId: page.id, userId: meId } }) : undefined,
isLiked: meId ? this.pageLikesRepository.exists({ where: { pageId: page.id, userId: meId } }) : undefined,
});
}

1
packages/backend/src/core/entities/UserEntityService.ts
View file

@ -645,6 +645,7 @@ export class UserEntityService implements OnModuleInit {
...(isDetailed ? {
url: profile!.url,
uri: user.uri,
// TODO hints for all of this
movedTo: user.movedToUri ? Promise.resolve(opts.userIdsByUri?.get(user.movedToUri) ?? this.apPersonService.resolvePerson(user.movedToUri).then(user => user.id).catch(() => null)) : null,
movedToUri: user.movedToUri,
// alsoKnownAs moved from packedUserDetailedNotMeOnly for privacy

107
packages/backend/src/misc/promise-map.ts Normal file
View file

@ -0,0 +1,107 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
import promiseLimit from 'promise-limit';
/**
* Pipes a stream of values through an async mapping callback to produce a new stream of results.
* Avoids extra work by bailing out if any promise rejects or the caller stops iterating the stream.
*
* Can optionally accept a concurrency limit and/or abort signal to further customize behavior.
* If a limit is provided, then no more than that many promises will execute at once.
* If a signal is provided, then all promises will terminate when the signal aborts.
* A signal cannot be provided without a limit, as that would be a no-op.
*/
export async function promiseMap<Input, Output>(
values: Iterable<Input> | AsyncIterable<Input>,
callback: (value: Input, index: number) => Promise<Output>,
opts?: {
limit: number | ReturnType<typeof promiseLimit<void>>;
signal?: AbortSignal;
},
): Promise<Output[]> {
// Parse the configured limit or create no-op
const limiter = createLimiter(opts?.limit);
// Internal state
const outputs: Output[] = [];
const errors: unknown[] = [];
const queue: Promise<void>[] = [];
let count = 0;
for await (const input of values) {
// Capture the destination index to make sure items are returned in the same order
const index = count;
count++;
// Stop when any promise fails
if (errors.length > 0) {
break;
}
// Kick off the next item
const promise = limiter(async () => {
// Check for rejection without throwing any new errors
if (errors.length > 0) return;
try {
// Checking the abort signal here covers all locations.
// 1. It bails the callback directly.
// 2. The error is written to errors, which breaks out of the loop
opts?.signal?.throwIfAborted();
// Populate the next value
outputs[index] = await callback(input, index);
} catch (err) {
errors.push(err);
}
});
// But don't forget about it!
queue.push(promise);
}
// Wait for everything to complete
await Promise.allSettled(queue);
// Failed - consolidate and throw errors
if (errors.length > 0) {
throwResults(errors);
}
// Success - return results
return outputs;
}
type Limiter = (cb: () => Promise<void>) => Promise<void>;
function createLimiter(limit: undefined | number | ReturnType<typeof promiseLimit<void>>): Limiter {
if (!limit) {
return cb => cb();
}
if (typeof limit === 'number') {
return promiseLimit<void>(limit);
}
return limit;
}
function throwResults(errors: unknown[]): never {
if (errors.length === 0) {
// Shouldn't happen
throw new Error('Mapping promise rejected');
}
if (errors.length === 1) {
if (errors[0] instanceof Error) {
throw errors[0];
} else {
throw new Error('Mapping promise rejected', { cause: errors[0] });
}
}
throw new AggregateError(errors);
}

43
packages/backend/src/server/ActivityPubServerService.ts
View file

@ -14,7 +14,7 @@ import accepts from 'accepts';
import vary from 'vary';
import secureJson from 'secure-json-parse';
import { DI } from '@/di-symbols.js';
import type { FollowingsRepository, NotesRepository, EmojisRepository, NoteReactionsRepository, UserProfilesRepository, UserNotePiningsRepository, UsersRepository, FollowRequestsRepository, MiMeta } from '@/models/_.js';
import type { FollowingsRepository, NotesRepository, EmojisRepository, NoteReactionsRepository, UserProfilesRepository, UserNotePiningsRepository, UsersRepository, FollowRequestsRepository, MiMeta, MiUserNotePining } from '@/models/_.js';
import * as url from '@/misc/prelude/url.js';
import type { Config } from '@/config.js';
import { ApRendererService } from '@/core/activitypub/ApRendererService.js';
@ -40,6 +40,7 @@ import { CustomEmojiService, encodeEmojiKey } from '@/core/CustomEmojiService.js
import type { FastifyInstance, FastifyRequest, FastifyReply, FastifyPluginOptions, FastifyBodyParser } from 'fastify';
import type { FindOptionsWhere } from 'typeorm';
import { FanoutTimelineEndpointService } from '@/core/FanoutTimelineEndpointService.js';
import { promiseMap } from '@/misc/promise-map.js';
const ACTIVITY_JSON = 'application/activity+json; charset=utf-8';
const LD_JSON = 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"; charset=utf-8';
@ -418,7 +419,7 @@ export class ActivityPubServerService {
const inStock = followings.length === limit + 1;
if (inStock) followings.pop();
const renderedFollowers = await Promise.all(followings.map(following => this.apRendererService.renderFollowUser(following.followerId)));
const renderedFollowers = await promiseMap(followings, async following => this.apRendererService.renderFollowUser(following.followerId), { limit: 4 });
const rendered = this.apRendererService.renderOrderedCollectionPage(
`${partOf}?${url.query({
page: 'true',
@ -515,7 +516,7 @@ export class ActivityPubServerService {
const inStock = followings.length === limit + 1;
if (inStock) followings.pop();
const renderedFollowees = await Promise.all(followings.map(following => this.apRendererService.renderFollowUser(following.followeeId)));
const renderedFollowees = await promiseMap(followings, async following => this.apRendererService.renderFollowUser(following.followeeId), { limit: 4 });
const rendered = this.apRendererService.renderOrderedCollectionPage(
`${partOf}?${url.query({
page: 'true',
@ -555,10 +556,7 @@ export class ActivityPubServerService {
const userId = request.params.user;
const user = await this.usersRepository.findOneBy({
id: userId,
host: IsNull(),
});
const user = await this.cacheService.findLocalUserById(userId);
if (user == null) {
reply.code(404);
@ -568,13 +566,14 @@ export class ActivityPubServerService {
const pinings = await this.userNotePiningsRepository.find({
where: { userId: user.id },
order: { id: 'DESC' },
});
relations: { note: true },
}) as (MiUserNotePining & { note: MiNote })[];
const pinnedNotes = (await Promise.all(pinings.map(pining =>
this.notesRepository.findOneByOrFail({ id: pining.noteId }))))
const pinnedNotes = pinings
.map(pin => pin.note)
.filter(note => !note.localOnly && ['public', 'home'].includes(note.visibility) && !isPureRenote(note));
const renderedNotes = await Promise.all(pinnedNotes.map(note => this.apRendererService.renderNote(note, user)));
const renderedNotes = await promiseMap(pinnedNotes, async note => await this.apRendererService.renderNote(note, user), { limit: 4 });
const rendered = this.apRendererService.renderOrderedCollection(
`${this.config.url}/users/${userId}/collections/featured`,
@ -664,7 +663,7 @@ export class ActivityPubServerService {
if (sinceId) notes.reverse();
const activities = await Promise.all(notes.map(note => this.packActivity(note, user)));
const activities = await promiseMap(notes, async note => await this.packActivity(note, user));
const rendered = this.apRendererService.renderOrderedCollectionPage(
`${partOf}?${url.query({
page: 'true',
@ -1092,14 +1091,8 @@ export class ActivityPubServerService {
// check if the following exists.
const [follower, followee] = await Promise.all([
this.usersRepository.findOneBy({
id: request.params.follower,
host: IsNull(),
}),
this.usersRepository.findOneBy({
id: request.params.followee,
host: Not(IsNull()),
}),
this.cacheService.findLocalUserById(request.params.follower),
this.cacheService.findRemoteUserById(request.params.followee),
]) as [MiLocalUser | MiRemoteUser | null, MiLocalUser | MiRemoteUser | null];
if (follower == null || followee == null) {
@ -1134,14 +1127,8 @@ export class ActivityPubServerService {
}
const [follower, followee] = await Promise.all([
this.usersRepository.findOneBy({
id: followRequest.followerId,
host: IsNull(),
}),
this.usersRepository.findOneBy({
id: followRequest.followeeId,
host: Not(IsNull()),
}),
this.cacheService.findLocalUserById(followRequest.followerId),
this.cacheService.findRemoteUserById(followRequest.followeeId),
]) as [MiLocalUser | MiRemoteUser | null, MiLocalUser | MiRemoteUser | null];
if (follower == null || followee == null) {

1
packages/backend/src/server/api/GetterService.ts
View file

@ -12,6 +12,7 @@ import { isRemoteUser, isLocalUser } from '@/models/User.js';
import type { MiNote } from '@/models/Note.js';
import { CacheService } from '@/core/CacheService.js';
import { bindThis } from '@/decorators.js';
import { CacheService } from '@/core/CacheService.js';
@Injectable()
export class GetterService {

13
packages/backend/src/server/api/endpoints/admin/federation/remove-all-following.ts
View file

@ -9,6 +9,7 @@ import type { FollowingsRepository, UsersRepository } from '@/models/_.js';
import { DI } from '@/di-symbols.js';
import { QueueService } from '@/core/QueueService.js';
import { ModerationLogService } from '@/core/ModerationLogService.js';
import { promiseMap } from '@/misc/promise-map.js';
export const meta = {
tags: ['admin'],
@ -48,10 +49,14 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
},
]);
const pairs = await Promise.all(followings.map(f => Promise.all([
this.usersRepository.findOneByOrFail({ id: f.followerId }),
this.usersRepository.findOneByOrFail({ id: f.followeeId }),
]).then(([from, to]) => [{ id: from.id }, { id: to.id }])));
const pairs = await promiseMap(followings, async f => {
const [from, to] = await Promise.all([
this.usersRepository.findOneByOrFail({ id: f.followerId }),
this.usersRepository.findOneByOrFail({ id: f.followeeId }),
]);
return [{ id: from.id }, { id: to.id }];
});
await this.moderationLogService.log(me, 'severFollowRelations', {
host: ps.host,

8
packages/backend/src/server/api/endpoints/admin/show-user.ts
View file

@ -300,14 +300,10 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
) {
super(meta, paramDef, async (ps, me) => {
const [user, profile] = await Promise.all([
this.usersRepository.findOneBy({ id: ps.userId }),
this.userProfilesRepository.findOneBy({ userId: ps.userId }),
this.cacheService.findUserById(ps.userId),
this.cacheService.userProfileCache.fetch(ps.userId),
]);
if (user == null || profile == null) {
throw new Error('user not found');
}
const isModerator = await this.roleService.isModerator(user);
const isAdministrator = await this.roleService.isAdministrator(user);
const isSilenced = user.isSilenced || !(await this.roleService.getUserPolicies(user.id)).canPublicNote;

3
packages/backend/src/server/api/endpoints/antennas/list.ts
View file

@ -8,6 +8,7 @@ import { Endpoint } from '@/server/api/endpoint-base.js';
import type { AntennasRepository } from '@/models/_.js';
import { AntennaEntityService } from '@/core/entities/AntennaEntityService.js';
import { DI } from '@/di-symbols.js';
import { promiseMap } from '@/misc/promise-map.js';
export const meta = {
tags: ['antennas', 'account'],
@ -52,7 +53,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
userId: me.id,
});
return await Promise.all(antennas.map(x => this.antennaEntityService.pack(x)));
return await promiseMap(antennas, async x => await this.antennaEntityService.pack(x), { limit: 4 });
});
}
}

3
packages/backend/src/server/api/endpoints/channels/followed.ts
View file

@ -9,6 +9,7 @@ import type { ChannelFollowingsRepository } from '@/models/_.js';
import { QueryService } from '@/core/QueryService.js';
import { ChannelEntityService } from '@/core/entities/ChannelEntityService.js';
import { DI } from '@/di-symbols.js';
import { promiseMap } from '@/misc/promise-map.js';
export const meta = {
tags: ['channels', 'account'],
@ -69,7 +70,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
.limit(ps.limit)
.getMany();
return await Promise.all(followings.map(x => this.channelEntityService.pack(x.followeeId, me)));
return await promiseMap(followings, async x => await this.channelEntityService.pack(x.followeeId, me), { limit: 4 });
});
}
}

3
packages/backend/src/server/api/endpoints/channels/my-favorites.ts
View file

@ -8,6 +8,7 @@ import { Endpoint } from '@/server/api/endpoint-base.js';
import type { ChannelFavoritesRepository } from '@/models/_.js';
import { ChannelEntityService } from '@/core/entities/ChannelEntityService.js';
import { DI } from '@/di-symbols.js';
import { promiseMap } from '@/misc/promise-map.js';
export const meta = {
tags: ['channels', 'account'],
@ -56,7 +57,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
const favorites = await query
.getMany();
return await Promise.all(favorites.map(x => this.channelEntityService.pack(x.channel!, me)));
return await promiseMap(favorites, async x => await this.channelEntityService.pack(x.channel!, me), { limit: 4 });
});
}
}

3
packages/backend/src/server/api/endpoints/channels/search.ts
View file

@ -11,6 +11,7 @@ import type { ChannelsRepository } from '@/models/_.js';
import { ChannelEntityService } from '@/core/entities/ChannelEntityService.js';
import { DI } from '@/di-symbols.js';
import { sqlLikeEscape } from '@/misc/sql-like-escape.js';
import { promiseMap } from '@/misc/promise-map.js';
export const meta = {
tags: ['channels'],
@ -75,7 +76,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
.limit(ps.limit)
.getMany();
return await Promise.all(channels.map(x => this.channelEntityService.pack(x, me)));
return await promiseMap(channels, async x => await this.channelEntityService.pack(x, me), { limit: 4 });
});
}
}

3
packages/backend/src/server/api/endpoints/drive/folders.ts
View file

@ -10,6 +10,7 @@ import { QueryService } from '@/core/QueryService.js';
import { DriveFolderEntityService } from '@/core/entities/DriveFolderEntityService.js';
import { DI } from '@/di-symbols.js';
import { sqlLikeEscape } from '@/misc/sql-like-escape.js';
import { promiseMap } from '@/misc/promise-map.js';
export const meta = {
tags: ['drive'],
@ -71,7 +72,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
}
const folders = await query.limit(ps.limit).getMany();
return await Promise.all(folders.map(folder => this.driveFolderEntityService.pack(folder)));
return await promiseMap(folders, async folder => await this.driveFolderEntityService.pack(folder), { limit: 4 });
});
}
}

3
packages/backend/src/server/api/endpoints/drive/folders/find.ts
View file

@ -9,6 +9,7 @@ import { Endpoint } from '@/server/api/endpoint-base.js';
import type { DriveFoldersRepository } from '@/models/_.js';
import { DriveFolderEntityService } from '@/core/entities/DriveFolderEntityService.js';
import { DI } from '@/di-symbols.js';
import { promiseMap } from '@/misc/promise-map.js';
export const meta = {
tags: ['drive'],
@ -58,7 +59,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
parentId: ps.parentId ?? IsNull(),
});
return await Promise.all(folders.map(folder => this.driveFolderEntityService.pack(folder)));
return await promiseMap(folders, async folder => await this.driveFolderEntityService.pack(folder), { limit: 4 });
});
}
}

11
packages/backend/src/server/api/endpoints/gallery/posts/create.ts
View file

@ -13,6 +13,7 @@ import { IdService } from '@/core/IdService.js';
import { GalleryPostEntityService } from '@/core/entities/GalleryPostEntityService.js';
import { DI } from '@/di-symbols.js';
import { TimeService } from '@/global/TimeService.js';
import { In } from 'typeorm';
export const meta = {
tags: ['gallery'],
@ -66,12 +67,10 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private readonly timeService: TimeService,
) {
super(meta, paramDef, async (ps, me) => {
const files = (await Promise.all(ps.fileIds.map(fileId =>
this.driveFilesRepository.findOneBy({
id: fileId,
userId: me.id,
}),
))).filter(x => x != null);
const files = await this.driveFilesRepository.findBy({
id: In(ps.fileIds),
userId: me.id,
});
if (files.length === 0) {
throw new Error('no files specified');

3
packages/backend/src/server/api/endpoints/hashtags/users.ts
View file

@ -12,6 +12,7 @@ import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { DI } from '@/di-symbols.js';
import { RoleService } from '@/core/RoleService.js';
import { TimeService } from '@/global/TimeService.js';
import { promiseMap } from '@/misc/promise-map.js';
export const meta = {
requireCredential: false,
@ -97,7 +98,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
// 2. A span of more than "limit" consecutive non-trendable users may cause the pagination to stop early.
// Unfortunately, there's no better solution unless we refactor role policies to be persisted to the DB.
if (ps.trending) {
const usersWithRoles = await Promise.all(users.map(async u => [u, await this.roleService.getUserPolicies(u)] as const));
const usersWithRoles = await promiseMap(users, async u => [u, await this.roleService.getUserPolicies(u)] as const, { limit: 4 });
users = usersWithRoles
.filter(([,p]) => p.canTrend)
.map(([u]) => u);

7
packages/backend/src/server/api/endpoints/i/authorized-apps.ts
View file

@ -9,6 +9,7 @@ import { Endpoint } from '@/server/api/endpoint-base.js';
import type { AccessTokensRepository } from '@/models/_.js';
import { AppEntityService } from '@/core/entities/AppEntityService.js';
import { DI } from '@/di-symbols.js';
import { promiseMap } from '@/misc/promise-map.js';
export const meta = {
requireCredential: true,
@ -88,9 +89,11 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
},
});
return await Promise.all(tokens.map(token => this.appEntityService.pack(token.appId!, me, {
return await promiseMap(tokens, async token => await this.appEntityService.pack(token.appId!, me, {
detail: true,
})));
}), {
limit: 4,
});
});
}
}

6
packages/backend/src/server/api/endpoints/i/favorites.ts
View file

@ -5,7 +5,7 @@
import { Inject, Injectable } from '@nestjs/common';
import { Endpoint } from '@/server/api/endpoint-base.js';
import type { NoteFavoritesRepository } from '@/models/_.js';
import type { MiNote, MiNoteFavorite, NoteFavoritesRepository } from '@/models/_.js';
import { QueryService } from '@/core/QueryService.js';
import { NoteFavoriteEntityService } from '@/core/entities/NoteFavoriteEntityService.js';
import { DI } from '@/di-symbols.js';
@ -56,11 +56,11 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
super(meta, paramDef, async (ps, me) => {
const query = this.queryService.makePaginationQuery(this.noteFavoritesRepository.createQueryBuilder('favorite'), ps.sinceId, ps.untilId)
.andWhere('favorite.userId = :meId', { meId: me.id })
.leftJoinAndSelect('favorite.note', 'note');
.innerJoinAndSelect('favorite.note', 'note');
const favorites = await query
.limit(ps.limit)
.getMany();
.getMany() as (MiNoteFavorite & { note: MiNote })[];
return await this.noteFavoriteEntityService.packMany(favorites, me);
});

3
packages/backend/src/server/api/endpoints/i/signin-history.ts
View file

@ -9,6 +9,7 @@ import type { SigninsRepository } from '@/models/_.js';
import { QueryService } from '@/core/QueryService.js';
import { SigninEntityService } from '@/core/entities/SigninEntityService.js';
import { DI } from '@/di-symbols.js';
import { promiseMap } from '@/misc/promise-map.js';
export const meta = {
requireCredential: true,
@ -56,7 +57,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
const history = await query.limit(ps.limit).getMany();
return await Promise.all(history.map(record => this.signinEntityService.pack(record)));
return await promiseMap(history, async record => await this.signinEntityService.pack(record), { limit: 4 });
});
}
}

7
packages/backend/src/server/api/endpoints/my/apps.ts
View file

@ -8,6 +8,7 @@ import { Endpoint } from '@/server/api/endpoint-base.js';
import type { AppsRepository } from '@/models/_.js';
import { AppEntityService } from '@/core/entities/AppEntityService.js';
import { DI } from '@/di-symbols.js';
import { promiseMap } from '@/misc/promise-map.js';
export const meta = {
tags: ['account', 'app'],
@ -60,9 +61,11 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
skip: ps.offset,
});
return await Promise.all(apps.map(app => this.appEntityService.pack(app, me, {
return await promiseMap(apps, async app => await this.appEntityService.pack(app, me, {
detail: true,
})));
}), {
limit: 4,
});
});
}
}

18
packages/backend/src/server/api/endpoints/notes/schedule/list.ts
View file

@ -15,6 +15,8 @@ import { QueryService } from '@/core/QueryService.js';
import { Packed } from '@/misc/json-schema.js';
import { noteVisibilities } from '@/types.js';
import { bindThis } from '@/decorators.js';
import { promiseMap } from '@/misc/promise-map.js';
import { In } from 'typeorm';
export const meta = {
tags: ['notes'],
@ -95,6 +97,9 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
const query = this.queryService.makePaginationQuery(this.noteScheduleRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId)
.andWhere('note.userId = :userId', { userId: me.id });
const scheduleNotes = await query.limit(ps.limit).getMany();
const refNoteIds = scheduleNotes.flatMap(s => [s.note.reply, s.note.renote]).filter(id => id != null);
const refNotesList = await this.notesRepository.findBy({ id: In(refNoteIds) });
const refNotesMap = new Map(refNotesList.map(n => [n.id, n]));
const user = await this.userEntityService.pack(me, me);
const scheduleNotesPack: {
id: string;
@ -111,9 +116,9 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
};
userId: string;
scheduledAt: string;
}[] = await Promise.all(scheduleNotes.map(async (item: MiNoteSchedule) => {
const renote = await this.fetchNote(item.note.renote, me);
const reply = await this.fetchNote(item.note.reply, me);
}[] = await promiseMap(scheduleNotes, async (item: MiNoteSchedule) => {
const renote = await this.fetchNote(item.note.renote, me, refNotesMap);
const reply = await this.fetchNote(item.note.reply, me, refNotesMap);
return {
...item,
@ -136,7 +141,9 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
poll: item.note.poll ? await this.fillPoll(item.note.poll) : undefined,
},
};
}));
}, {
limit: 4,
});
return scheduleNotesPack;
});
@ -146,9 +153,10 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private async fetchNote(
id: MiNote['id'] | null | undefined,
me: MiUser,
hint?: Map<string, MiNote>,
): Promise<Packed<'Note'> | null> {
if (id) {
const note = await this.notesRepository.findOneBy({ id });
const note = hint?.get(id) ?? await this.notesRepository.findOneBy({ id });
if (note) {
note.reactionAndUserPairCache ??= [];
return await this.noteEntityService.pack(note, me);

3
packages/backend/src/server/api/endpoints/users.ts
View file

@ -12,6 +12,7 @@ import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { DI } from '@/di-symbols.js';
import { RoleService } from '@/core/RoleService.js';
import { TimeService } from '@/global/TimeService.js';
import { promiseMap } from '@/misc/promise-map.js';
import type { SelectQueryBuilder } from 'typeorm';
export const meta = {
@ -113,7 +114,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
// 1. It may return less than "limit" results.
// 2. A span of more than "limit" consecutive non-trendable users may cause the pagination to stop early.
// Unfortunately, there's no better solution unless we refactor role policies to be persisted to the DB.
const usersWithRoles = await Promise.all(allUsers.map(async u => [u, await this.roleService.getUserPolicies(u)] as const));
const usersWithRoles = await promiseMap(allUsers, async u => [u, await this.roleService.getUserPolicies(u)] as const, { limit: 4 });
const users = usersWithRoles
.filter(([,p]) => p.canTrend)
.map(([u]) => u);

3
packages/backend/src/server/api/endpoints/users/lists/list.ts
View file

@ -9,6 +9,7 @@ import { Endpoint } from '@/server/api/endpoint-base.js';
import { UserListEntityService } from '@/core/entities/UserListEntityService.js';
import { ApiError } from '@/server/api/error.js';
import { DI } from '@/di-symbols.js';
import { promiseMap } from '@/misc/promise-map.js';
export const meta = {
tags: ['lists', 'account'],
@ -88,7 +89,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
isPublic: true,
});
return await Promise.all(userLists.map(x => this.userListEntityService.pack(x, me?.id)));
return await promiseMap(userLists, async x => await this.userListEntityService.pack(x, me?.id), { limit: 4 });
});
}
}

11
packages/backend/src/server/api/mastodon/MastodonApiServerService.ts
View file

@ -23,6 +23,7 @@ import { parseTimelineArgs, TimelineArgs, toBoolean } from './argsUtils.js';
import { convertAnnouncement, convertAttachment, MastodonConverters, convertRelationship } from './MastodonConverters.js';
import type { Entity } from 'megalodon';
import type { FastifyInstance, FastifyPluginOptions } from 'fastify';
import { promiseMap } from '@/misc/promise-map.js';
@Injectable()
export class MastodonApiServerService {
@ -178,7 +179,7 @@ export class MastodonApiServerService {
const { client, me } = await this.clientService.getAuthClient(_request);
const data = await client.getBookmarks(parseTimelineArgs(_request.query));
const response = await Promise.all(data.data.map((status) => this.mastoConverters.convertStatus(status, me)));
const response = await promiseMap(data.data, async (status) => await this.mastoConverters.convertStatus(status, me), { limit: 4 });
return reply.send(response);
});
@ -200,7 +201,7 @@ export class MastodonApiServerService {
userId: me.id,
};
const data = await client.getFavourites(args);
const response = await Promise.all(data.data.map((status) => this.mastoConverters.convertStatus(status, me)));
const response = await promiseMap(data.data, async (status) => await this.mastoConverters.convertStatus(status, me), { limit: 4 });
return reply.send(response);
});
@ -209,7 +210,7 @@ export class MastodonApiServerService {
const client = this.clientService.getClient(_request);
const data = await client.getMutes(parseTimelineArgs(_request.query));
const response = await Promise.all(data.data.map((account) => this.mastoConverters.convertAccount(account)));
const response = await promiseMap(data.data, async (account) => await this.mastoConverters.convertAccount(account), { limit: 4 });
return reply.send(response);
});
@ -218,7 +219,7 @@ export class MastodonApiServerService {
const client = this.clientService.getClient(_request);
const data = await client.getBlocks(parseTimelineArgs(_request.query));
const response = await Promise.all(data.data.map((account) => this.mastoConverters.convertAccount(account)));
const response = await promiseMap(data.data, async (account) => await this.mastoConverters.convertAccount(account), { limit: 4 });
return reply.send(response);
});
@ -228,7 +229,7 @@ export class MastodonApiServerService {
const limit = _request.query.limit ? parseInt(_request.query.limit) : 20;
const data = await client.getFollowRequests(limit);
const response = await Promise.all(data.data.map((account) => this.mastoConverters.convertAccount(account as Entity.Account)));
const response = await promiseMap(data.data, async (account) => await this.mastoConverters.convertAccount(account), { limit: 4 });
return reply.send(response);
});

17
packages/backend/src/server/api/mastodon/MastodonConverters.ts
View file

@ -24,6 +24,7 @@ import { GetterService } from '@/server/api/GetterService.js';
import { appendContentWarning } from '@/misc/append-content-warning.js';
import { isRenote } from '@/misc/is-renote.js';
import { FederatedInstanceService } from '@/core/FederatedInstanceService.js';
import { promiseMap } from '@/misc/promise-map.js';
// Missing from Megalodon apparently
// https://docs.joinmastodon.org/entities/StatusEdit/
@ -283,11 +284,15 @@ export class MastodonConverters {
});
});
const mentions = Promise.all(note.mentions.map(p =>
this.getUser(p)
.then(u => this.encode(u, mentionedRemoteUsers))
.catch(() => null)))
.then((p: (Entity.Mention | null)[]) => p.filter(m => m != null));
const mentions = promiseMap(note.mentions, async p => {
try {
const u = await this.getUser(p);
return this.encode(u, mentionedRemoteUsers);
} catch {
return null;
}
}, { limit: 4 })
.then((p: Entity.Mention[]) => p.filter(m => m));
const tags = note.tags.map(tag => {
return {
@ -363,7 +368,7 @@ export class MastodonConverters {
public async convertConversation(conversation: Entity.Conversation, me: MiLocalUser | null): Promise<MastodonEntity.Conversation> {
return {
id: conversation.id,
accounts: await Promise.all(conversation.accounts.map((a: Entity.Account) => this.convertAccount(a))),
accounts: await promiseMap(conversation.accounts, async (a: Entity.Account) => await this.convertAccount(a), { limit: 4 }),
last_status: conversation.last_status ? await this.convertStatus(conversation.last_status, me) : null,
unread: conversation.unread,
};

7
packages/backend/src/server/api/mastodon/endpoints/account.ts
View file

@ -12,6 +12,7 @@ import type { AccessTokensRepository, UserProfilesRepository } from '@/models/_.
import { attachMinMaxPagination } from '@/server/api/mastodon/pagination.js';
import { MastodonConverters, convertRelationship, convertFeaturedTag, convertList } from '../MastodonConverters.js';
import type { FastifyInstance } from 'fastify';
import { promiseMap } from '@/misc/promise-map.js';
interface ApiAccountMastodonRoute {
Params: { id?: string },
@ -188,7 +189,7 @@ export class ApiAccountMastodon {
const { client, me } = await this.clientService.getAuthClient(request);
const args = parseTimelineArgs(request.query);
const data = await client.getAccountStatuses(request.params.id, args);
const response = await Promise.all(data.data.map(async (status) => await this.mastoConverters.convertStatus(status, me)));
const response = await promiseMap(data.data, async status => await this.mastoConverters.convertStatus(status, me), { limit: 2 });
attachMinMaxPagination(request, reply, response);
return reply.send(response);
@ -212,7 +213,7 @@ export class ApiAccountMastodon {
request.params.id,
parseTimelineArgs(request.query),
);
const response = await Promise.all(data.data.map(async (account) => await this.mastoConverters.convertAccount(account)));
const response = await promiseMap(data.data, async account => await this.mastoConverters.convertAccount(account), { limit: 2 });
attachMinMaxPagination(request, reply, response);
return reply.send(response);
@ -226,7 +227,7 @@ export class ApiAccountMastodon {
request.params.id,
parseTimelineArgs(request.query),
);
const response = await Promise.all(data.data.map(async (account) => await this.mastoConverters.convertAccount(account)));
const response = await promiseMap(data.data, async account => await this.mastoConverters.convertAccount(account), { limit: 2 });
attachMinMaxPagination(request, reply, response);
return reply.send(response);

3
packages/backend/src/server/api/mastodon/endpoints/notifications.ts
View file

@ -10,6 +10,7 @@ import { MastodonConverters } from '@/server/api/mastodon/MastodonConverters.js'
import { attachMinMaxPagination } from '@/server/api/mastodon/pagination.js';
import { MastodonClientService } from '../MastodonClientService.js';
import type { FastifyInstance } from 'fastify';
import { promiseMap } from '@/misc/promise-map.js';
interface ApiNotifyMastodonRoute {
Params: {
@ -29,7 +30,7 @@ export class ApiNotificationsMastodon {
fastify.get<ApiNotifyMastodonRoute>('/v1/notifications', async (request, reply) => {
const { client, me } = await this.clientService.getAuthClient(request);
const data = await client.getNotifications(parseTimelineArgs(request.query));
const notifications = await Promise.all(data.data.map(n => this.mastoConverters.convertNotification(n, me)));
const notifications = await promiseMap(data.data, async n => await this.mastoConverters.convertNotification(n, me), { limit: 4 });
const response: MastodonEntity.Notification[] = [];
for (const notification of notifications) {
// Notifications for inaccessible notes will be null and should be ignored

23
packages/backend/src/server/api/mastodon/endpoints/search.ts
View file

@ -6,6 +6,7 @@
import { Injectable } from '@nestjs/common';
import { MastodonClientService } from '@/server/api/mastodon/MastodonClientService.js';
import { attachMinMaxPagination, attachOffsetPagination } from '@/server/api/mastodon/pagination.js';
import { promiseMap } from '@/misc/promise-map.js';
import { MastodonConverters } from '../MastodonConverters.js';
import { parseTimelineArgs, TimelineArgs, toBoolean, toInt } from '../argsUtils.js';
import { ApiError } from '../../error.js';
@ -52,8 +53,8 @@ export class ApiSearchMastodon {
const { data } = await client.search(request.query.q, { type, ...query });
const response = {
...data,
accounts: await Promise.all(data.accounts.map((account: Entity.Account) => this.mastoConverters.convertAccount(account))),
statuses: await Promise.all(data.statuses.map((status: Entity.Status) => this.mastoConverters.convertStatus(status, me))),
accounts: await promiseMap(data.accounts, (account: Entity.Account) => this.mastoConverters.convertAccount(account), { limit: 3 }),
statuses: await promiseMap(data.statuses, (status: Entity.Status) => this.mastoConverters.convertStatus(status, me), { limit: 3 }),
};
if (type === 'hashtags') {
@ -89,8 +90,8 @@ export class ApiSearchMastodon {
const stat = !type || type === 'statuses' ? await client.search(request.query.q, { type: 'statuses', ...query }) : null;
const tags = !type || type === 'hashtags' ? await client.search(request.query.q, { type: 'hashtags', ...query }) : null;
const response = {
accounts: await Promise.all(acct?.data.accounts.map((account: Entity.Account) => this.mastoConverters.convertAccount(account)) ?? []),
statuses: await Promise.all(stat?.data.statuses.map((status: Entity.Status) => this.mastoConverters.convertStatus(status, me)) ?? []),
accounts: acct ? await promiseMap(acct.data.accounts, async (account: Entity.Account) => await this.mastoConverters.convertAccount(account), { limit: 3 }) : [],
statuses: acct ? await promiseMap(acct.data.statuses, async (status: Entity.Status) => this.mastoConverters.convertStatus(status, me), { limit: 3 }) : [],
hashtags: tags?.data.hashtags ?? [],
};
@ -123,7 +124,7 @@ export class ApiSearchMastodon {
const data = await res.json() as Entity.Status[];
const me = await this.clientService.getAuth(request);
const response = await Promise.all(data.map(status => this.mastoConverters.convertStatus(status, me)));
const response = await promiseMap(data, async status => await this.mastoConverters.convertStatus(status, me), { limit: 4 });
attachMinMaxPagination(request, reply, response);
return reply.send(response);
@ -150,12 +151,12 @@ export class ApiSearchMastodon {
await verifyResponse(res);
const data = await res.json() as Entity.Account[];
const response = await Promise.all(data.map(async entry => {
return {
source: 'global',
account: await this.mastoConverters.convertAccount(entry),
};
}));
const response = await promiseMap(data, async entry => ({
source: 'global',
account: await this.mastoConverters.convertAccount(entry),
}), {
limit: 4,
});
attachOffsetPagination(request, reply, response);
return reply.send(response);

9
packages/backend/src/server/api/mastodon/endpoints/status.ts
View file

@ -11,6 +11,7 @@ import { MastodonClientService } from '@/server/api/mastodon/MastodonClientServi
import { MastodonDataService } from '@/server/api/mastodon/MastodonDataService.js';
import { getNoteSummary } from '@/misc/get-note-summary.js';
import { isPureRenote } from '@/misc/is-renote.js';
import { promiseMap } from '@/misc/promise-map.js';
import { convertAttachment, convertPoll, MastodonConverters } from '../MastodonConverters.js';
import type { Entity } from 'megalodon';
import type { FastifyInstance } from 'fastify';
@ -97,8 +98,8 @@ export class ApiStatusMastodon {
const { client, me } = await this.clientService.getAuthClient(_request);
const { data } = await client.getStatusContext(_request.params.id, parseTimelineArgs(_request.query));
const ancestors = await Promise.all(data.ancestors.map(async (status: Entity.Status) => await this.mastoConverters.convertStatus(status, me)));
const descendants = await Promise.all(data.descendants.map(async (status: Entity.Status) => await this.mastoConverters.convertStatus(status, me)));
const ancestors = await promiseMap(data.ancestors, async (status: Entity.Status) => await this.mastoConverters.convertStatus(status, me), { limit: 4 });
const descendants = await promiseMap(data.descendants, async (status: Entity.Status) => await this.mastoConverters.convertStatus(status, me), { limit: 4 });
const response = { ancestors, descendants };
return reply.send(response);
@ -118,7 +119,7 @@ export class ApiStatusMastodon {
const client = this.clientService.getClient(_request);
const data = await client.getStatusRebloggedBy(_request.params.id);
const response = await Promise.all(data.data.map((account: Entity.Account) => this.mastoConverters.convertAccount(account)));
const response = await promiseMap(data.data, async (account: Entity.Account) => await this.mastoConverters.convertAccount(account), { limit: 4 });
return reply.send(response);
});
@ -128,7 +129,7 @@ export class ApiStatusMastodon {
const client = this.clientService.getClient(_request);
const data = await client.getStatusFavouritedBy(_request.params.id);
const response = await Promise.all(data.data.map((account: Entity.Account) => this.mastoConverters.convertAccount(account)));
const response = await promiseMap(data.data, async (account: Entity.Account) => await this.mastoConverters.convertAccount(account), { limit: 4 });
return reply.send(response);
});

13
packages/backend/src/server/api/mastodon/endpoints/timeline.ts
View file

@ -6,6 +6,7 @@
import { Injectable } from '@nestjs/common';
import { MastodonClientService } from '@/server/api/mastodon/MastodonClientService.js';
import { attachMinMaxPagination } from '@/server/api/mastodon/pagination.js';
import { promiseMap } from '@/misc/promise-map.js';
import { convertList, MastodonConverters } from '../MastodonConverters.js';
import { parseTimelineArgs, TimelineArgs, toBoolean } from '../argsUtils.js';
import type { Entity } from 'megalodon';
@ -25,7 +26,7 @@ export class ApiTimelineMastodon {
const data = toBoolean(request.query.local)
? await client.getLocalTimeline(query)
: await client.getPublicTimeline(query);
const response = await Promise.all(data.data.map((status: Entity.Status) => this.mastoConverters.convertStatus(status, me)));
const response = await promiseMap(data.data, async (status: Entity.Status) => await this.mastoConverters.convertStatus(status, me), { limit: 4 });
attachMinMaxPagination(request, reply, response);
return reply.send(response);
@ -35,7 +36,7 @@ export class ApiTimelineMastodon {
const { client, me } = await this.clientService.getAuthClient(request);
const query = parseTimelineArgs(request.query);
const data = await client.getHomeTimeline(query);
const response = await Promise.all(data.data.map((status: Entity.Status) => this.mastoConverters.convertStatus(status, me)));
const response = await promiseMap(data.data, async (status: Entity.Status) => await this.mastoConverters.convertStatus(status, me), { limit: 4 });
attachMinMaxPagination(request, reply, response);
return reply.send(response);
@ -47,7 +48,7 @@ export class ApiTimelineMastodon {
const { client, me } = await this.clientService.getAuthClient(request);
const query = parseTimelineArgs(request.query);
const data = await client.getTagTimeline(request.params.hashtag, query);
const response = await Promise.all(data.data.map((status: Entity.Status) => this.mastoConverters.convertStatus(status, me)));
const response = await promiseMap(data.data, async (status: Entity.Status) => await this.mastoConverters.convertStatus(status, me), { limit: 4 });
attachMinMaxPagination(request, reply, response);
return reply.send(response);
@ -59,7 +60,7 @@ export class ApiTimelineMastodon {
const { client, me } = await this.clientService.getAuthClient(request);
const query = parseTimelineArgs(request.query);
const data = await client.getListTimeline(request.params.id, query);
const response = await Promise.all(data.data.map(async (status: Entity.Status) => await this.mastoConverters.convertStatus(status, me)));
const response = await promiseMap(data.data, async (status: Entity.Status) => await this.mastoConverters.convertStatus(status, me), { limit: 4 });
attachMinMaxPagination(request, reply, response);
return reply.send(response);
@ -69,7 +70,7 @@ export class ApiTimelineMastodon {
const { client, me } = await this.clientService.getAuthClient(request);
const query = parseTimelineArgs(request.query);
const data = await client.getConversationTimeline(query);
const response = await Promise.all(data.data.map((conversation: Entity.Conversation) => this.mastoConverters.convertConversation(conversation, me)));
const response = await promiseMap(data.data, async (conversation: Entity.Conversation) => await this.mastoConverters.convertConversation(conversation, me), { limit: 4 });
attachMinMaxPagination(request, reply, response);
return reply.send(response);
@ -99,7 +100,7 @@ export class ApiTimelineMastodon {
const client = this.clientService.getClient(request);
const data = await client.getAccountsInList(request.params.id, parseTimelineArgs(request.query));
const response = await Promise.all(data.data.map((account: Entity.Account) => this.mastoConverters.convertAccount(account)));
const response = await promiseMap(data.data, async (account: Entity.Account) => await this.mastoConverters.convertAccount(account), { limit: 4 });
attachMinMaxPagination(request, reply, response);
return reply.send(response);

170
packages/backend/test/unit/misc/promise-map.ts Normal file
View file

@ -0,0 +1,170 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { setTimeout } from 'node:timers/promises';
import promiseLimit from 'promise-limit';
import { promiseMap } from '@/misc/promise-map.js';
async function randomDelay() {
await setTimeout(10 * Math.abs(Math.random()));
}
describe(promiseMap, () => {
it('should return empty array with no input', async () => {
const result = await promiseMap([] as string[], async () => 'wrong');
expect(result).toHaveLength(0);
});
it('should map items in correct order', async () => {
const items = [1, 2, 3, 4, 5];
const results = await promiseMap(items, async i => {
await randomDelay();
return String(i);
});
expect(results).toEqual(['1', '2', '3', '4', '5']);
});
it('should accept async iterable input', async () => {
async function *generator() {
yield 1;
yield 2;
yield 3;
}
const results = await promiseMap(generator(), async i => String(i));
expect(results).toEqual(['1', '2', '3']);
});
it('should accept limit input', async () => {
const items = [1, 2, 3, 4, 5];
let inProgress = 0;
let maxProgress = 0;
const results = await promiseMap(items, async i => {
inProgress++;
maxProgress = Math.max(maxProgress, inProgress);
await randomDelay();
inProgress--;
return String(i);
}, {
limit: 2,
});
expect(results).toEqual(['1', '2', '3', '4', '5']);
expect(maxProgress).toEqual(2);
});
it('should accept limit as instance', async () => {
const items = [1, 2, 3, 4, 5];
const limit = promiseLimit<void>(2);
let inProgress = 0;
let maxProgress = 0;
const results = await promiseMap(items, async i => {
inProgress++;
maxProgress = Math.max(maxProgress, inProgress);
await randomDelay();
inProgress--;
return String(i);
}, {
limit,
});
expect(results).toEqual(['1', '2', '3', '4', '5']);
expect(maxProgress).toEqual(2);
});
it('should reject when signal aborts', async () => {
const items = [1, 2, 3, 4, 5];
const controller = new AbortController();
const promise = promiseMap(items, async i => {
if (i === 3) {
controller.abort(new Error('test abort'));
}
return String(i);
}, {
limit: 1,
signal: controller.signal,
});
await expect(promise).rejects.toThrow('abort');
});
it('should abort when signal aborts', async () => {
const items = [1, 2, 3, 4, 5];
const controller = new AbortController();
const processed: number[] = [];
await promiseMap(items, async i => {
if (i === 3) {
controller.abort('test abort');
}
processed.push(i);
return String(i);
}, {
limit: 1,
signal: controller.signal,
}).catch(() => null);
expect(processed).toEqual([1, 2, 3]);
});
it('should reject when promise rejects', async () => {
const items = [1, 2, 3, 4, 5];
const promise = promiseMap(items, async i => {
if (i === 3) {
throw new Error('test error');
}
return String(i);
});
await expect(promise).rejects.toThrow('test');
});
it('should abort when promise rejects', async () => {
const items = [1, 2, 3, 4, 5];
const processed: number[] = [];
await promiseMap(items, async i => {
if (i === 3) {
throw new Error('test error');
}
processed.push(i);
return String(i);
}).catch(() => null);
expect(processed).toEqual([1, 2]);
});
it('should aggregate all errors', async () => {
const items = [1, 2, 3, 4, 5];
const promise = promiseMap(items, async i => {
await setTimeout(10);
throw new Error(`test error: ${i}`);
});
await expect(promise).rejects.toThrow(AggregateError);
});
});