From 87f6895ca971f9cccfeb0fdd2113bd6dcb93bb49 Mon Sep 17 00:00:00 2001 From: dakkar Date: Fri, 13 Jun 2025 09:26:37 +0100 Subject: [PATCH 01/54] avoid pushing to timelines of remote users --- packages/backend/src/core/NoteCreateService.ts | 1 + packages/backend/src/core/NoteEditService.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index f8584a4a48..0dd0a9b822 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -964,6 +964,7 @@ export class NoteCreateService implements OnApplicationShutdown { // TODO: あまりにも数が多いと redisPipeline.exec に失敗する(理由は不明)ため、3万件程度を目安に分割して実行するようにする for (const following of followings) { + if (following.followerHost !== null) continue; // 基本的にvisibleUserIdsには自身のidが含まれている前提であること if (note.visibility === 'specified' && !note.visibleUserIds.some(v => v === following.followerId)) continue; diff --git a/packages/backend/src/core/NoteEditService.ts b/packages/backend/src/core/NoteEditService.ts index d963bf1945..533ee7942d 100644 --- a/packages/backend/src/core/NoteEditService.ts +++ b/packages/backend/src/core/NoteEditService.ts @@ -849,6 +849,7 @@ export class NoteEditService implements OnApplicationShutdown { // TODO: あまりにも数が多いと redisPipeline.exec に失敗する(理由は不明)ため、3万件程度を目安に分割して実行するようにする for (const following of followings) { + if (following.followerHost !== null) continue; // 基本的にvisibleUserIdsには自身のidが含まれている前提であること if (note.visibility === 'specified' && !note.visibleUserIds.some(v => v === following.followerId)) continue; From 01872419c3feae1cadd10aa299eba3802ec0b5b7 Mon Sep 17 00:00:00 2001 From: dakkar Date: Fri, 13 Jun 2025 10:06:35 +0100 Subject: [PATCH 02/54] fix `UserEntityService` packMany hints * there's no need to pre-load follow requests for many users, since at most we'll pack them for only 1 user (the one requesting the data) * similarly, it makes sense to preload security keys for many users if we're serving a moderator's request, but if not, we need at most 1 user's keys (the requesting user's), and we can let `.pack()` fetch those * we always need to preload relations when serving a detailed request, not only when the set of users to pack does not include the requesting user --- .../src/core/entities/UserEntityService.ts | 38 ++++--------------- 1 file changed, 8 insertions(+), 30 deletions(-) diff --git a/packages/backend/src/core/entities/UserEntityService.ts b/packages/backend/src/core/entities/UserEntityService.ts index 91bf258ff4..04b7cbc79b 100644 --- a/packages/backend/src/core/entities/UserEntityService.ts +++ b/packages/backend/src/core/entities/UserEntityService.ts @@ -432,8 +432,6 @@ export class UserEntityService implements OnModuleInit { userIdsByUri?: Map, instances?: Map, securityKeyCounts?: Map, - pendingReceivedFollows?: Set, - pendingSentFollows?: Set, }, ): Promise> { const opts = Object.assign({ @@ -679,8 +677,8 @@ export class UserEntityService implements OnModuleInit { hasUnreadAntenna: this.getHasUnreadAntenna(user.id), hasUnreadChannel: false, // 後方互換性のため hasUnreadNotification: notificationsInfo?.hasUnread, // 後方互換性のため - hasPendingReceivedFollowRequest: opts.pendingReceivedFollows?.has(user.id) ?? this.getHasPendingReceivedFollowRequest(user.id), - hasPendingSentFollowRequest: opts.pendingSentFollows?.has(user.id) ?? this.getHasPendingSentFollowRequest(user.id), + hasPendingReceivedFollowRequest: this.getHasPendingReceivedFollowRequest(user.id), + hasPendingSentFollowRequest: this.getHasPendingSentFollowRequest(user.id), unreadNotificationsCount: notificationsInfo?.unreadCount, mutedWords: profile!.mutedWords, hardMutedWords: profile!.hardMutedWords, @@ -764,7 +762,7 @@ export class UserEntityService implements OnModuleInit { const isMe = meId && _userIds.includes(meId); const isDetailed = options && options.schema !== 'UserLite'; const isDetailedAndMe = isDetailed && isMe; - const isDetailedAndMeOrMod = isDetailed && (isMe || iAmModerator); + const isDetailedAndMod = isDetailed && iAmModerator; const isDetailedAndNotMe = isDetailed && !isMe; const userUris = new Set(_users @@ -787,14 +785,14 @@ export class UserEntityService implements OnModuleInit { // -- 実行者の有無や指定スキーマの種別によって要否が異なる値群を取得 - const [profilesMap, userMemos, userRelations, pinNotes, userIdsByUri, instances, securityKeyCounts, pendingReceivedFollows, pendingSentFollows] = await Promise.all([ + const [profilesMap, userMemos, userRelations, pinNotes, userIdsByUri, instances, securityKeyCounts] = await Promise.all([ // profilesMap this.cacheService.userProfileCache.fetchMany(_profilesToFetch).then(profiles => new Map(profiles.concat(_profilesFromUsers))), // userMemos isDetailed && meId ? this.userMemosRepository.findBy({ userId: meId }) .then(memos => new Map(memos.map(memo => [memo.targetUserId, memo.memo]))) : new Map(), // userRelations - isDetailedAndNotMe && meId ? this.getRelations(meId, _userIds) : new Map(), + isDetailed && meId ? this.getRelations(meId, _userIds) : new Map(), // pinNotes isDetailed ? this.userNotePiningsRepository.createQueryBuilder('pin') .where('pin.userId IN (:...userIds)', { userIds: _userIds }) @@ -828,7 +826,7 @@ export class UserEntityService implements OnModuleInit { Promise.all(Array.from(userHosts).map(async host => [host, await this.federatedInstanceService.fetch(host)] as const)) .then(hosts => new Map(hosts)), // securityKeyCounts - isDetailedAndMeOrMod ? this.userSecurityKeysRepository.createQueryBuilder('key') + isDetailedAndMod ? this.userSecurityKeysRepository.createQueryBuilder('key') .select('key.userId', 'userId') .addSelect('count(key.id)', 'userCount') .where({ @@ -836,26 +834,8 @@ export class UserEntityService implements OnModuleInit { }) .groupBy('key.userId') .getRawMany<{ userId: string, userCount: number }>() - .then(counts => new Map(counts.map(c => [c.userId, c.userCount]))) : new Map(), - // TODO optimization: cache follow requests - // pendingReceivedFollows - isDetailedAndMe ? this.followRequestsRepository.createQueryBuilder('req') - .select('req.followeeId', 'followeeId') - .where({ - followeeId: In(_userIds), - }) - .groupBy('req.followeeId') - .getRawMany<{ followeeId: string }>() - .then(reqs => new Set(reqs.map(r => r.followeeId))) : new Set(), - // pendingSentFollows - isDetailedAndMe ? this.followRequestsRepository.createQueryBuilder('req') - .select('req.followerId', 'followerId') - .where({ - followerId: In(_userIds), - }) - .groupBy('req.followerId') - .getRawMany<{ followerId: string }>() - .then(reqs => new Set(reqs.map(r => r.followerId))) : new Set(), + .then(counts => new Map(counts.map(c => [c.userId, c.userCount]))) + : undefined, // .pack will fetch the keys for the requesting user if it's in the _userIds ]); return Promise.all( @@ -872,8 +852,6 @@ export class UserEntityService implements OnModuleInit { userIdsByUri, instances, securityKeyCounts, - pendingReceivedFollows, - pendingSentFollows, }, )), ); From db8f94b0fb5098f665c4d38e3699b3cb08346692 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 13 Jun 2025 11:42:07 -0400 Subject: [PATCH 03/54] correct type of IObject.attachment --- packages/backend/src/core/activitypub/type.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/core/activitypub/type.ts b/packages/backend/src/core/activitypub/type.ts index 554420d670..5d4b2b01c5 100644 --- a/packages/backend/src/core/activitypub/type.ts +++ b/packages/backend/src/core/activitypub/type.ts @@ -24,7 +24,7 @@ export interface IObject { cc?: ApObject; to?: ApObject; attributedTo?: ApObject; - attachment?: any[]; + attachment?: IApDocument[]; inReplyTo?: any; replies?: ICollection | IOrderedCollection | string; content?: string | null; From e5593af42219592932e567e3649a4261c58fd4c5 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 13 Jun 2025 11:42:31 -0400 Subject: [PATCH 04/54] improve alt text mapping in ApImageService.createImage --- packages/backend/src/core/activitypub/models/ApImageService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/core/activitypub/models/ApImageService.ts b/packages/backend/src/core/activitypub/models/ApImageService.ts index 7a16972ea4..782ee002ca 100644 --- a/packages/backend/src/core/activitypub/models/ApImageService.ts +++ b/packages/backend/src/core/activitypub/models/ApImageService.ts @@ -86,7 +86,7 @@ export class ApImageService { uri: image.url, sensitive: !!(image.sensitive), isLink: !shouldBeCached, - comment: truncate(image.name ?? undefined, this.config.maxRemoteAltTextLength), + comment: truncate(image.name || image.summary || undefined, this.config.maxRemoteAltTextLength), }); if (!file.isLink || file.url === image.url) return file; From eb22bc5f5dddfa35a803e06a14be3704299a8efe Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 13 Jun 2025 11:42:53 -0400 Subject: [PATCH 05/54] extract note attachments from inline HTML --- .../core/activitypub/models/ApNoteService.ts | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/packages/backend/src/core/activitypub/models/ApNoteService.ts b/packages/backend/src/core/activitypub/models/ApNoteService.ts index 2a28405121..300cdbd0a0 100644 --- a/packages/backend/src/core/activitypub/models/ApNoteService.ts +++ b/packages/backend/src/core/activitypub/models/ApNoteService.ts @@ -5,6 +5,7 @@ import { forwardRef, Inject, Injectable } from '@nestjs/common'; import { In } from 'typeorm'; +import { load as cheerio } from 'cheerio/slim'; import { UnrecoverableError } from 'bullmq'; import { DI } from '@/di-symbols.js'; import type { UsersRepository, PollsRepository, EmojisRepository, NotesRepository, MiMeta } from '@/models/_.js'; @@ -41,6 +42,7 @@ import { ApQuestionService } from './ApQuestionService.js'; import { ApImageService } from './ApImageService.js'; import type { Resolver } from '../ApResolverService.js'; import type { IObject, IPost } from '../type.js'; +import type { CheerioAPI } from 'cheerio/slim'; @Injectable() export class ApNoteService { @@ -265,6 +267,16 @@ export class ApNoteService { if (file) files.push(file); } + // Extract inline media from note content. + // Don't use source.content, _misskey_content, or anything else because those aren't HTML. + if (note.content) { + for (const attach of extractInlineMedia(note.content)) { + attach.sensitive ??= note.sensitive; + const file = await this.apImageService.resolveImage(actor, attach); + if (file) files.push(file); + } + } + // リプライ const reply: MiNote | null = note.inReplyTo ? await this.resolveNote(note.inReplyTo, { resolver }) @@ -463,6 +475,16 @@ export class ApNoteService { if (file) files.push(file); } + // Extract inline media from note content. + // Don't use source.content, _misskey_content, or anything else because those aren't HTML. + if (note.content) { + for (const attach of extractInlineMedia(note.content)) { + attach.sensitive ??= note.sensitive; + const file = await this.apImageService.resolveImage(actor, attach); + if (file) files.push(file); + } + } + // リプライ const reply: MiNote | null = note.inReplyTo ? await this.resolveNote(note.inReplyTo, { resolver }) @@ -741,3 +763,73 @@ function getBestIcon(note: IObject): IObject | null { return best; }, null as IApDocument | null) ?? null; } + +function extractInlineMedia(html: string): IApDocument[] { + const $ = parseHtml(html); + if (!$) return []; + + const attachments: IApDocument[] = []; + + // tags, including and fallback elements + // https://developer.mozilla.org/en-US/docs/Web/HTML/Reference/Elements/img + $('img[src]') + .toArray() + .forEach(img => attachments.push({ + type: 'Image', + url: img.attribs.src, + name: img.attribs.alt || img.attribs.title || null, + })); + + // tags + // https://developer.mozilla.org/en-US/docs/Web/HTML/Reference/Elements/object + $('object[data]') + .toArray() + .forEach(object => attachments.push({ + type: 'Document', + url: object.attribs.data, + name: object.attribs.alt || object.attribs.title || null, + })); + + // tags + // https://developer.mozilla.org/en-US/docs/Web/HTML/Reference/Elements/embed + $('embed[src]') + .toArray() + .forEach(embed => attachments.push({ + type: 'Document', + url: embed.attribs.src, + name: embed.attribs.alt || embed.attribs.title || null, + })); + + //