From aa7cadbb6c5068e231638ff25922c439ff949783 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Wed, 21 May 2025 21:51:36 -0400 Subject: [PATCH 01/54] implement InternalEventService --- packages/backend/src/core/CoreModule.ts | 6 ++ .../backend/src/core/GlobalEventService.ts | 9 +- .../backend/src/core/InternalEventService.ts | 102 ++++++++++++++++++ .../test/misc/FakeInternalEventService.ts | 92 ++++++++++++++++ 4 files changed, 207 insertions(+), 2 deletions(-) create mode 100644 packages/backend/src/core/InternalEventService.ts create mode 100644 packages/backend/test/misc/FakeInternalEventService.ts diff --git a/packages/backend/src/core/CoreModule.ts b/packages/backend/src/core/CoreModule.ts index dd8e61d322..6839ba0159 100644 --- a/packages/backend/src/core/CoreModule.ts +++ b/packages/backend/src/core/CoreModule.ts @@ -41,6 +41,7 @@ import { HttpRequestService } from './HttpRequestService.js'; import { IdService } from './IdService.js'; import { ImageProcessingService } from './ImageProcessingService.js'; import { SystemAccountService } from './SystemAccountService.js'; +import { InternalEventService } from './InternalEventService.js'; import { InternalStorageService } from './InternalStorageService.js'; import { MetaService } from './MetaService.js'; import { MfmService } from './MfmService.js'; @@ -186,6 +187,7 @@ const $HashtagService: Provider = { provide: 'HashtagService', useExisting: Hash const $HttpRequestService: Provider = { provide: 'HttpRequestService', useExisting: HttpRequestService }; const $IdService: Provider = { provide: 'IdService', useExisting: IdService }; const $ImageProcessingService: Provider = { provide: 'ImageProcessingService', useExisting: ImageProcessingService }; +const $InternalEventService: Provider = { provide: 'InternalEventService', useExisting: InternalEventService }; const $InternalStorageService: Provider = { provide: 'InternalStorageService', useExisting: InternalStorageService }; const $MetaService: Provider = { provide: 'MetaService', useExisting: MetaService }; const $MfmService: Provider = { provide: 'MfmService', useExisting: MfmService }; @@ -345,6 +347,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp HttpRequestService, IdService, ImageProcessingService, + InternalEventService, InternalStorageService, MetaService, MfmService, @@ -500,6 +503,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp $HttpRequestService, $IdService, $ImageProcessingService, + $InternalEventService, $InternalStorageService, $MetaService, $MfmService, @@ -656,6 +660,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp HttpRequestService, IdService, ImageProcessingService, + InternalEventService, InternalStorageService, MetaService, MfmService, @@ -810,6 +815,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp $HttpRequestService, $IdService, $ImageProcessingService, + $InternalEventService, $InternalStorageService, $MetaService, $MfmService, diff --git a/packages/backend/src/core/GlobalEventService.ts b/packages/backend/src/core/GlobalEventService.ts index c0027ae129..d1a5cabd85 100644 --- a/packages/backend/src/core/GlobalEventService.ts +++ b/packages/backend/src/core/GlobalEventService.ts @@ -353,12 +353,12 @@ export class GlobalEventService { } @bindThis - private publish(channel: StreamChannels, type: string | null, value?: any): void { + private async publish(channel: StreamChannels, type: string | null, value?: any): Promise { const message = type == null ? value : value == null ? { type: type, body: null } : { type: type, body: value }; - this.redisForPub.publish(this.config.host, JSON.stringify({ + await this.redisForPub.publish(this.config.host, JSON.stringify({ channel: channel, message: message, })); @@ -369,6 +369,11 @@ export class GlobalEventService { this.publish('internal', type, typeof value === 'undefined' ? null : value); } + @bindThis + public async publishInternalEventAsync(type: K, value?: InternalEventTypes[K]): Promise { + await this.publish('internal', type, typeof value === 'undefined' ? null : value); + } + @bindThis public publishBroadcastStream(type: K, value?: BroadcastTypes[K]): void { this.publish('broadcast', type, typeof value === 'undefined' ? null : value); diff --git a/packages/backend/src/core/InternalEventService.ts b/packages/backend/src/core/InternalEventService.ts new file mode 100644 index 0000000000..375ee928c4 --- /dev/null +++ b/packages/backend/src/core/InternalEventService.ts @@ -0,0 +1,102 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; +import Redis from 'ioredis'; +import { DI } from '@/di-symbols.js'; +import { GlobalEventService } from '@/core/GlobalEventService.js'; +import type { GlobalEvents, InternalEventTypes } from '@/core/GlobalEventService.js'; +import { bindThis } from '@/decorators.js'; + +export type Listener = (value: InternalEventTypes[K], key: K) => void | Promise; + +export interface ListenerProps { + ignoreLocal?: boolean, +} + +@Injectable() +export class InternalEventService implements OnApplicationShutdown { + private readonly listeners = new Map, ListenerProps>>(); + + constructor( + @Inject(DI.redisForSub) + private readonly redisForSub: Redis.Redis, + + private readonly globalEventService: GlobalEventService, + ) { + this.redisForSub.on('message', this.onMessage); + } + + @bindThis + public on(type: K, listener: Listener, props?: ListenerProps): void { + let set = this.listeners.get(type); + if (!set) { + set = new Map(); + this.listeners.set(type, set); + } + + // Functionally, this is just a set with metadata on the values. + set.set(listener as Listener, props ?? {}); + } + + @bindThis + public off(type: K, listener: Listener): void { + this.listeners.get(type)?.delete(listener as Listener); + } + + @bindThis + public async emit(type: K, value: InternalEventTypes[K]): Promise { + await this.emitInternal(type, value, true); + await this.globalEventService.publishInternalEventAsync(type, { ...value, _pid: process.pid }); + } + + @bindThis + private async emitInternal(type: K, value: InternalEventTypes[K], isLocal: boolean): Promise { + const listeners = this.listeners.get(type); + if (!listeners) { + return; + } + + const promises: Promise[] = []; + for (const [listener, props] of listeners) { + if (!isLocal || !props.ignoreLocal) { + const promise = Promise.resolve(listener(value, type)); + promises.push(promise); + } + } + await Promise.all(promises); + } + + @bindThis + private async onMessage(_: string, data: string): Promise { + const obj = JSON.parse(data); + + if (obj.channel === 'internal') { + const { type, body } = obj.message as GlobalEvents['internal']['payload']; + if (!isLocalInternalEvent(body) || body._pid !== process.pid) { + await this.emitInternal(type, body as InternalEventTypes[keyof InternalEventTypes], false); + } + } + } + + @bindThis + public dispose(): void { + this.redisForSub.off('message', this.onMessage); + this.listeners.clear(); + } + + @bindThis + public onApplicationShutdown(): void { + this.dispose(); + } +} + +interface LocalInternalEvent { + _pid: number; +} + +function isLocalInternalEvent(body: object): body is LocalInternalEvent { + return '_pid' in body && typeof(body._pid) === 'number'; +} diff --git a/packages/backend/test/misc/FakeInternalEventService.ts b/packages/backend/test/misc/FakeInternalEventService.ts new file mode 100644 index 0000000000..ffe8b81d78 --- /dev/null +++ b/packages/backend/test/misc/FakeInternalEventService.ts @@ -0,0 +1,92 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import type { Listener, ListenerProps } from '@/core/InternalEventService.js'; +import type Redis from 'ioredis'; +import type { GlobalEventService, InternalEventTypes } from '@/core/GlobalEventService.js'; +import { InternalEventService } from '@/core/InternalEventService.js'; +import { bindThis } from '@/decorators.js'; + +type FakeCall = [K, Parameters]; +type FakeListener = [K, Listener, ListenerProps]; + +/** + * Minimal implementation of InternalEventService meant for use in unit tests. + * There is no redis connection, and metadata is tracked in the public _calls and _listeners arrays. + * The on/off/emit methods are fully functional and can be called in tests to invoke any registered listeners. + */ +export class FakeInternalEventService extends InternalEventService { + /** + * List of calls to public methods, in chronological order. + */ + public _calls: FakeCall[] = []; + + /** + * List of currently registered listeners. + */ + public _listeners: FakeListener[] = []; + + /** + * Resets the mock. + * Clears all listeners and tracked calls. + */ + public _reset() { + this._calls = []; + this._listeners = []; + } + + /** + * Simulates a remote event sent from another process in the cluster via redis. + */ + @bindThis + public async _emitRedis(type: K, value: InternalEventTypes[K]): Promise { + await this.emit(type, value, false); + } + + constructor() { + super( + { on: () => {} } as unknown as Redis.Redis, + {} as unknown as GlobalEventService, + ); + } + + @bindThis + public on(type: K, listener: Listener, props?: ListenerProps): void { + if (!this._listeners.some(l => l[0] === type && l[1] === listener)) { + this._listeners.push([type, listener as Listener, props ?? {}]); + } + this._calls.push(['on', [type, listener as Listener, props]]); + } + + @bindThis + public off(type: K, listener: Listener): void { + this._listeners = this._listeners.filter(l => l[0] !== type || l[1] !== listener); + this._calls.push(['off', [type, listener as Listener]]); + } + + @bindThis + public async emit(type: K, value: InternalEventTypes[K], isLocal = true): Promise { + for (const listener of this._listeners) { + if (listener[0] === type) { + if (!isLocal || !listener[2].ignoreLocal) { + await listener[1](value, type); + } + } + } + this._calls.push(['emit', [type, value]]); + } + + @bindThis + public dispose(): void { + this._listeners = []; + this._calls.push(['dispose', []]); + } + + @bindThis + public onApplicationShutdown(): void { + this._calls.push(['onApplicationShutdown', []]); + } +} + From f446d77cb51af1f66a0042feec2f0907537a16ce Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Thu, 5 Jun 2025 10:49:16 -0400 Subject: [PATCH 02/54] implement QuantumKVCache --- .../backend/src/core/GlobalEventService.ts | 1 + packages/backend/src/misc/cache.ts | 233 ++++++++++ packages/backend/test/unit/misc/cache.ts | 431 ++++++++++++++++++ 3 files changed, 665 insertions(+) create mode 100644 packages/backend/test/unit/misc/cache.ts diff --git a/packages/backend/src/core/GlobalEventService.ts b/packages/backend/src/core/GlobalEventService.ts index d1a5cabd85..763ab8c086 100644 --- a/packages/backend/src/core/GlobalEventService.ts +++ b/packages/backend/src/core/GlobalEventService.ts @@ -265,6 +265,7 @@ export interface InternalEventTypes { unmute: { muterId: MiUser['id']; muteeId: MiUser['id']; }; userListMemberAdded: { userListId: MiUserList['id']; memberId: MiUser['id']; }; userListMemberRemoved: { userListId: MiUserList['id']; memberId: MiUser['id']; }; + quantumCacheUpdated: { name: string, key: string, op: 's' | 'd' }; } type EventTypesToEventPayload = EventUnionFromDictionary>>; diff --git a/packages/backend/src/misc/cache.ts b/packages/backend/src/misc/cache.ts index a6ab96c189..31e6f126b8 100644 --- a/packages/backend/src/misc/cache.ts +++ b/packages/backend/src/misc/cache.ts @@ -5,6 +5,8 @@ import * as Redis from 'ioredis'; import { bindThis } from '@/decorators.js'; +import { InternalEventService } from '@/core/InternalEventService.js'; +import { InternalEventTypes } from '@/core/GlobalEventService.js'; export class RedisKVCache { private readonly lifetime: number; @@ -322,6 +324,10 @@ export class MemoryKVCache { clearInterval(this.gcIntervalHandle); } + public get size() { + return this.cache.size; + } + public get entries() { return this.cache.entries(); } @@ -410,3 +416,230 @@ export class MemorySingleCache { return value; } } + +export interface QuantumKVOpts { + /** + * Memory cache lifetime in milliseconds. + */ + lifetime: number; + + /** + * Callback to fetch the value for a key that wasn't found in the cache. + * May be synchronous or async. + */ + fetcher: (key: string, cache: QuantumKVCache) => T | Promise; + + /** + * Optional callback when a value is created or changed in the cache, either locally or elsewhere in the cluster. + * This is called *after* the cache state is updated. + * May be synchronous or async. + */ + onSet?: (key: string, cache: QuantumKVCache) => void | Promise; + + /** + * Optional callback when a value is deleted from the cache, either locally or elsewhere in the cluster. + * This is called *after* the cache state is updated. + * May be synchronous or async. + */ + onDelete?: (key: string, cache: QuantumKVCache) => void | Promise; +} + +/** + * QuantumKVCache is a lifetime-bounded memory cache (like MemoryKVCache) with automatic cross-cluster synchronization via Redis. + * All nodes in the cluster are guaranteed to have a *subset* view of the current accurate state, though individual processes may have different items in their local cache. + * This ensures that a call to get() will never return stale data. + */ +export class QuantumKVCache implements Iterable<[key: string, value: T]> { + private readonly memoryCache: MemoryKVCache; + + private readonly fetcher: QuantumKVOpts['fetcher']; + private readonly onSet: QuantumKVOpts['onSet']; + private readonly onDelete: QuantumKVOpts['onDelete']; + + /** + * @param internalEventService Service bus to synchronize events. + * @param name Unique name of the cache - must be the same in all processes. + * @param opts Cache options + */ + constructor( + private readonly internalEventService: InternalEventService, + private readonly name: string, + opts: QuantumKVOpts, + ) { + this.memoryCache = new MemoryKVCache(opts.lifetime); + this.fetcher = opts.fetcher; + this.onSet = opts.onSet; + this.onDelete = opts.onDelete; + + this.internalEventService.on('quantumCacheUpdated', this.onQuantumCacheUpdated, { + // Ignore our own events, otherwise we'll immediately erase any set value. + ignoreLocal: true, + }); + } + + /** + * The number of items currently in memory. + * This applies to the local subset view, not the cross-cluster cache state. + */ + public get size() { + return this.memoryCache.size; + } + + /** + * Iterates all [key, value] pairs in memory. + * This applies to the local subset view, not the cross-cluster cache state. + */ + @bindThis + public *entries(): Generator<[key: string, value: T]> { + for (const entry of this.memoryCache.entries) { + yield [entry[0], entry[1].value]; + } + } + + /** + * Iterates all keys in memory. + * This applies to the local subset view, not the cross-cluster cache state. + */ + @bindThis + public *keys() { + for (const entry of this.memoryCache.entries) { + yield entry[0]; + } + } + + /** + * Iterates all values pairs in memory. + * This applies to the local subset view, not the cross-cluster cache state. + */ + @bindThis + public *values() { + for (const entry of this.memoryCache.entries) { + yield entry[1].value; + } + } + + /** + * Creates or updates a value in the cache, and erases any stale caches across the cluster. + * Fires an onSet event after the cache has been updated in all processes. + * Skips if the value is unchanged. + */ + @bindThis + public async set(key: string, value: T): Promise { + if (this.memoryCache.get(key) === value) { + return; + } + + this.memoryCache.set(key, value); + + await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, op: 's', key }); + + if (this.onSet) { + await this.onSet(key, this); + } + } + + /** + * Gets or fetches a value from the cache. + * Fires an onSet event, but does not emit an update event to other processes. + */ + @bindThis + public async get(key: string): Promise { + let value = this.memoryCache.get(key); + if (value === undefined) { + value = await this.fetcher(key, this); + this.memoryCache.set(key, value); + + if (this.onSet) { + await this.onSet(key, this); + } + } + return value; + } + + /** + * Alias to get(), included for backwards-compatibility with RedisKVCache. + * @deprecated use get() instead + */ + @bindThis + public async fetch(key: string): Promise { + return await this.get(key); + } + + /** + * Returns true is a key exists in memory. + * This applies to the local subset view, not the cross-cluster cache state. + */ + @bindThis + public has(key: string): boolean { + return this.memoryCache.get(key) !== undefined; + } + + /** + * Deletes a value from the cache, and erases any stale caches across the cluster. + * Fires an onDelete event after the cache has been updated in all processes. + */ + @bindThis + public async delete(key: string): Promise { + this.memoryCache.delete(key); + + await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, op: 'd', key }); + + if (this.onDelete) { + await this.onDelete(key, this); + } + } + + /** + * Refreshes the value of a key from the fetcher, and erases any stale caches across the cluster. + * Fires an onSet event after the cache has been updated in all processes. + */ + @bindThis + public async refresh(key: string): Promise { + const value = await this.fetcher(key, this); + await this.set(key, value); + return value; + } + + /** + * Erases all entries from the local memory cache. + * Does not send any events or update other processes. + */ + @bindThis + public gc() { + this.memoryCache.gc(); + } + + /** + * Erases all data and disconnects from the cluster. + * This *must* be called when shutting down to prevent memory leaks! + */ + @bindThis + public dispose() { + this.internalEventService.off('quantumCacheUpdated', this.onQuantumCacheUpdated); + + this.memoryCache.dispose(); + } + + @bindThis + private async onQuantumCacheUpdated(data: InternalEventTypes['quantumCacheUpdated']): Promise { + if (data.name === this.name) { + this.memoryCache.delete(data.key); + + if (data.op === 's' && this.onSet) { + await this.onSet(data.key, this); + } + + if (data.op === 'd' && this.onDelete) { + await this.onDelete(data.key, this); + } + } + } + + /** + * Iterates all [key, value] pairs in memory. + * This applies to the local subset view, not the cross-cluster cache state. + */ + [Symbol.iterator](): Iterator<[key: string, value: T]> { + return this.entries(); + } +} diff --git a/packages/backend/test/unit/misc/cache.ts b/packages/backend/test/unit/misc/cache.ts new file mode 100644 index 0000000000..5b242c47d4 --- /dev/null +++ b/packages/backend/test/unit/misc/cache.ts @@ -0,0 +1,431 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { jest } from '@jest/globals'; +import { FakeInternalEventService } from '../../misc/FakeInternalEventService.js'; +import { QuantumKVCache, QuantumKVOpts } from '@/misc/cache.js'; + +describe(QuantumKVCache, () => { + let fakeInternalEventService: FakeInternalEventService; + let madeCaches: { dispose: () => void }[]; + + function makeCache(opts?: Partial> & { name?: string }): QuantumKVCache { + const _opts = { + name: 'test', + lifetime: Infinity, + fetcher: () => { throw new Error('not implemented'); }, + } satisfies QuantumKVOpts & { name: string }; + + if (opts) { + Object.assign(_opts, opts); + } + + const cache = new QuantumKVCache(fakeInternalEventService, _opts.name, _opts); + madeCaches.push(cache); + return cache; + } + + beforeEach(() => { + madeCaches = []; + fakeInternalEventService = new FakeInternalEventService(); + }); + + afterEach(() => { + madeCaches.forEach(cache => { + cache.dispose(); + }); + }); + + it('should connect on construct', () => { + makeCache(); + + expect(fakeInternalEventService._calls).toContainEqual(['on', ['quantumCacheUpdated', expect.anything(), { ignoreLocal: true }]]); + }); + + it('should disconnect on dispose', () => { + const cache = makeCache(); + + cache.dispose(); + + const callback = fakeInternalEventService._calls + .find(c => c[0] === 'on' && c[1][0] === 'quantumCacheUpdated') + ?.[1][1]; + expect(fakeInternalEventService._calls).toContainEqual(['off', ['quantumCacheUpdated', callback]]); + }); + + it('should store in memory cache', async () => { + const cache = makeCache(); + + await cache.set('foo', 'bar'); + await cache.set('alpha', 'omega'); + + const result1 = await cache.get('foo'); + const result2 = await cache.get('alpha'); + + expect(result1).toBe('bar'); + expect(result2).toBe('omega'); + }); + + it('should emit event when storing', async () => { + const cache = makeCache({ name: 'fake' }); + + await cache.set('foo', 'bar'); + + expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', op: 's', key: 'foo' }]]); + }); + + it('should call onSet when storing', async () => { + const fakeOnSet = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + name: 'fake', + onSet: fakeOnSet, + }); + + await cache.set('foo', 'bar'); + + expect(fakeOnSet).toHaveBeenCalledWith('foo', cache); + }); + + it('should not emit event when storing unchanged value', async () => { + const cache = makeCache({ name: 'fake' }); + + await cache.set('foo', 'bar'); + await cache.set('foo', 'bar'); + + expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(1); + }); + + it('should not call onSet when storing unchanged value', async () => { + const fakeOnSet = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + name: 'fake', + onSet: fakeOnSet, + }); + + await cache.set('foo', 'bar'); + await cache.set('foo', 'bar'); + + expect(fakeOnSet).toHaveBeenCalledTimes(1); + }); + + it('should fetch when getting an unknown value', async () => { + const cache = makeCache({ + name: 'fake', + fetcher: key => `value#${key}`, + }); + + const result = await cache.get('foo'); + + expect(result).toBe('value#foo'); + }); + + it('should store fetched value in memory cache', async () => { + const cache = makeCache({ + name: 'fake', + fetcher: key => `value#${key}`, + }); + + await cache.get('foo'); + + const result = cache.has('foo'); + expect(result).toBe(true); + }); + + it('should call onSet when fetching', async () => { + const fakeOnSet = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + name: 'fake', + fetcher: key => `value#${key}`, + onSet: fakeOnSet, + }); + + await cache.get('foo'); + + expect(fakeOnSet).toHaveBeenCalledWith('foo', cache); + }); + + it('should not emit event when fetching', async () => { + const cache = makeCache({ + name: 'fake', + fetcher: key => `value#${key}`, + }); + + await cache.get('foo'); + + expect(fakeInternalEventService._calls).not.toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', op: 's', key: 'foo' }]]); + }); + + it('should delete from memory cache', async () => { + const cache = makeCache(); + + await cache.set('foo', 'bar'); + await cache.delete('foo'); + + const result = cache.has('foo'); + expect(result).toBe(false); + }); + + it('should call onDelete when deleting', async () => { + const fakeOnDelete = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + name: 'fake', + onDelete: fakeOnDelete, + }); + + await cache.set('foo', 'bar'); + await cache.delete('foo'); + + expect(fakeOnDelete).toHaveBeenCalledWith('foo', cache); + }); + + it('should emit event when deleting', async () => { + const cache = makeCache({ name: 'fake' }); + + await cache.set('foo', 'bar'); + await cache.delete('foo'); + + expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', op: 'd', key: 'foo' }]]); + }); + + it('should delete when receiving set event', async () => { + const cache = makeCache({ name: 'fake' }); + await cache.set('foo', 'bar'); + + await fakeInternalEventService._emitRedis('quantumCacheUpdated', { name: 'fake', op: 's', key: 'foo' }); + + const result = cache.has('foo'); + expect(result).toBe(false); + }); + + it('should call onSet when receiving set event', async () => { + const fakeOnSet = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + name: 'fake', + onSet: fakeOnSet, + }); + + await fakeInternalEventService._emitRedis('quantumCacheUpdated', { name: 'fake', op: 's', key: 'foo' }); + + expect(fakeOnSet).toHaveBeenCalledWith('foo', cache); + }); + + it('should delete when receiving delete event', async () => { + const cache = makeCache({ name: 'fake' }); + await cache.set('foo', 'bar'); + + await fakeInternalEventService._emitRedis('quantumCacheUpdated', { name: 'fake', op: 'd', key: 'foo' }); + + const result = cache.has('foo'); + expect(result).toBe(false); + }); + + it('should call onDelete when receiving delete event', async () => { + const fakeOnDelete = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + name: 'fake', + onDelete: fakeOnDelete, + }); + await cache.set('foo', 'bar'); + + await fakeInternalEventService._emitRedis('quantumCacheUpdated', { name: 'fake', op: 'd', key: 'foo' }); + + expect(fakeOnDelete).toHaveBeenCalledWith('foo', cache); + }); + + describe('fetch', () => { + it('should perform same logic as get', async () => { + const fakeOnSet = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + name: 'fake', + fetcher: key => `value#${key}`, + onSet: fakeOnSet, + }); + + // noinspection JSDeprecatedSymbols + const result = await cache.fetch('foo'); + + expect(result).toBe('value#foo'); + expect(fakeOnSet).toHaveBeenCalledWith('foo', cache); + expect(fakeInternalEventService._calls).not.toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', op: 's', key: 'foo' }]]); + }); + }); + + describe('refresh', () => { + it('should populate the value', async () => { + const cache = makeCache({ + name: 'fake', + fetcher: key => `value#${key}`, + }); + + await cache.refresh('foo'); + + const result = cache.has('foo'); + expect(result).toBe(true); + }); + + it('should return the value', async () => { + const cache = makeCache({ + name: 'fake', + fetcher: key => `value#${key}`, + }); + + const result = await cache.refresh('foo'); + + expect(result).toBe('value#foo'); + }); + + it('should replace the value if it exists', async () => { + const cache = makeCache({ + name: 'fake', + fetcher: key => `value#${key}`, + }); + + await cache.set('foo', 'bar'); + const result = await cache.refresh('foo'); + + expect(result).toBe('value#foo'); + }); + + it('should call onSet', async () => { + const fakeOnSet = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + name: 'fake', + fetcher: key => `value#${key}`, + onSet: fakeOnSet, + }); + + await cache.refresh('foo') + + expect(fakeOnSet).toHaveBeenCalledWith('foo', cache); + }); + + it('should emit event', async () => { + const cache = makeCache({ + name: 'fake', + fetcher: key => `value#${key}`, + }); + + await cache.refresh('foo'); + + expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', op: 's', key: 'foo' }]]); + }); + }); + + describe('has', () => { + it('should return false when empty', () => { + const cache = makeCache(); + const result = cache.has('foo'); + expect(result).toBe(false); + }); + + it('should return false when value is not in memory', async () => { + const cache = makeCache(); + await cache.set('foo', 'bar'); + + const result = cache.has('alpha'); + + expect(result).toBe(false); + }); + + it('should return true when value is in memory', async () => { + const cache = makeCache(); + await cache.set('foo', 'bar'); + + const result = cache.has('foo'); + + expect(result).toBe(true); + }); + }); + + describe('size', () => { + it('should return 0 when empty', () => { + const cache = makeCache(); + expect(cache.size).toBe(0); + }); + + it('should return correct size when populated', async () => { + const cache = makeCache(); + await cache.set('foo', 'bar'); + + expect(cache.size).toBe(1); + }); + }); + + describe('entries', () => { + it('should return empty when empty', () => { + const cache = makeCache(); + + const result = Array.from(cache.entries()); + + expect(result).toHaveLength(0); + }); + + it('should return all entries when populated', async () => { + const cache = makeCache(); + await cache.set('foo', 'bar'); + + const result = Array.from(cache.entries()); + + expect(result).toEqual([['foo', 'bar']]); + }); + }); + + describe('keys', () => { + it('should return empty when empty', () => { + const cache = makeCache(); + + const result = Array.from(cache.keys()); + + expect(result).toHaveLength(0); + }); + + it('should return all keys when populated', async () => { + const cache = makeCache(); + await cache.set('foo', 'bar'); + + const result = Array.from(cache.keys()); + + expect(result).toEqual(['foo']); + }); + }); + + describe('values', () => { + it('should return empty when empty', () => { + const cache = makeCache(); + + const result = Array.from(cache.values()); + + expect(result).toHaveLength(0); + }); + + it('should return all values when populated', async () => { + const cache = makeCache(); + await cache.set('foo', 'bar'); + + const result = Array.from(cache.values()); + + expect(result).toEqual(['bar']); + }); + }); + + describe('[Symbol.iterator]', () => { + it('should return empty when empty', () => { + const cache = makeCache(); + + const result = Array.from(cache); + + expect(result).toHaveLength(0); + }); + + it('should return all entries when populated', async () => { + const cache = makeCache(); + await cache.set('foo', 'bar'); + + const result = Array.from(cache); + + expect(result).toEqual([['foo', 'bar']]); + }); + }); +}); From 1f2742ddd7ba462b715a4fe2ae25d3dedebec3e0 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Thu, 5 Jun 2025 12:54:58 -0400 Subject: [PATCH 03/54] add ignoreRemote filter to InternalEventService --- packages/backend/src/core/InternalEventService.ts | 7 ++++--- packages/backend/test/misc/FakeInternalEventService.ts | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/backend/src/core/InternalEventService.ts b/packages/backend/src/core/InternalEventService.ts index 375ee928c4..5b164b605e 100644 --- a/packages/backend/src/core/InternalEventService.ts +++ b/packages/backend/src/core/InternalEventService.ts @@ -10,10 +10,11 @@ import { GlobalEventService } from '@/core/GlobalEventService.js'; import type { GlobalEvents, InternalEventTypes } from '@/core/GlobalEventService.js'; import { bindThis } from '@/decorators.js'; -export type Listener = (value: InternalEventTypes[K], key: K) => void | Promise; +export type Listener = (value: InternalEventTypes[K], key: K, isLocal: boolean) => void | Promise; export interface ListenerProps { ignoreLocal?: boolean, + ignoreRemote?: boolean, } @Injectable() @@ -61,8 +62,8 @@ export class InternalEventService implements OnApplicationShutdown { const promises: Promise[] = []; for (const [listener, props] of listeners) { - if (!isLocal || !props.ignoreLocal) { - const promise = Promise.resolve(listener(value, type)); + if ((isLocal && !props.ignoreLocal) || (!isLocal && !props.ignoreRemote)) { + const promise = Promise.resolve(listener(value, type, isLocal)); promises.push(promise); } } diff --git a/packages/backend/test/misc/FakeInternalEventService.ts b/packages/backend/test/misc/FakeInternalEventService.ts index ffe8b81d78..d18a080eaf 100644 --- a/packages/backend/test/misc/FakeInternalEventService.ts +++ b/packages/backend/test/misc/FakeInternalEventService.ts @@ -70,8 +70,8 @@ export class FakeInternalEventService extends InternalEventService { public async emit(type: K, value: InternalEventTypes[K], isLocal = true): Promise { for (const listener of this._listeners) { if (listener[0] === type) { - if (!isLocal || !listener[2].ignoreLocal) { - await listener[1](value, type); + if ((isLocal && !listener[2].ignoreLocal) || (!isLocal && !listener[2].ignoreRemote)) { + await listener[1](value, type, isLocal); } } } From 46a6612dc0e5eaa470170031012ae247f7a5eec5 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Thu, 5 Jun 2025 13:16:23 -0400 Subject: [PATCH 04/54] convert many RedisKVCaches to QuantumKVCache or MemoryKVCache --- packages/backend/src/core/CacheService.ts | 104 +++++++++--------- .../src/core/ChannelFollowingService.ts | 30 ++--- .../src/core/PushNotificationService.ts | 19 ++-- .../backend/src/core/UserBlockingService.ts | 12 +- .../backend/src/core/UserFollowingService.ts | 12 +- .../backend/src/core/UserKeypairService.ts | 14 +-- packages/backend/src/core/UserListService.ts | 34 +++--- .../backend/src/core/UserMutingService.ts | 9 +- .../src/core/UserRenoteMutingService.ts | 9 +- .../server/api/endpoints/admin/nsfw-user.ts | 2 +- .../src/server/api/endpoints/i/update.ts | 2 +- .../src/server/api/endpoints/sw/register.ts | 2 +- .../src/server/api/endpoints/sw/unregister.ts | 2 +- .../api/endpoints/sw/update-registration.ts | 2 +- 14 files changed, 126 insertions(+), 127 deletions(-) diff --git a/packages/backend/src/core/CacheService.ts b/packages/backend/src/core/CacheService.ts index 1cf63221f9..f04b18c02b 100644 --- a/packages/backend/src/core/CacheService.ts +++ b/packages/backend/src/core/CacheService.ts @@ -7,12 +7,13 @@ import { Inject, Injectable } from '@nestjs/common'; import * as Redis from 'ioredis'; import { IsNull } from 'typeorm'; import type { BlockingsRepository, FollowingsRepository, MutingsRepository, RenoteMutingsRepository, MiUserProfile, UserProfilesRepository, UsersRepository, MiFollowing, MiNote } from '@/models/_.js'; -import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js'; +import { MemoryKVCache, QuantumKVCache, RedisKVCache } from '@/misc/cache.js'; import type { MiLocalUser, MiUser } from '@/models/User.js'; import { DI } from '@/di-symbols.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { bindThis } from '@/decorators.js'; -import type { GlobalEvents } from '@/core/GlobalEventService.js'; +import type { GlobalEvents, InternalEventTypes } from '@/core/GlobalEventService.js'; +import { InternalEventService } from '@/core/InternalEventService.js'; import type { OnApplicationShutdown } from '@nestjs/common'; export interface FollowStats { @@ -39,12 +40,12 @@ export class CacheService implements OnApplicationShutdown { public localUserByNativeTokenCache: MemoryKVCache; public localUserByIdCache: MemoryKVCache; public uriPersonCache: MemoryKVCache; - public userProfileCache: RedisKVCache; - public userMutingsCache: RedisKVCache>; - public userBlockingCache: RedisKVCache>; - public userBlockedCache: RedisKVCache>; // NOTE: 「被」Blockキャッシュ - public renoteMutingsCache: RedisKVCache>; - public userFollowingsCache: RedisKVCache | undefined>>; + public userProfileCache: QuantumKVCache; + public userMutingsCache: QuantumKVCache>; + public userBlockingCache: QuantumKVCache>; + public userBlockedCache: QuantumKVCache>; // NOTE: 「被」Blockキャッシュ + public renoteMutingsCache: QuantumKVCache>; + public userFollowingsCache: QuantumKVCache | undefined>>; private readonly userFollowStatsCache = new MemoryKVCache(1000 * 60 * 10); // 10 minutes private readonly translationsCache: RedisKVCache; @@ -74,6 +75,7 @@ export class CacheService implements OnApplicationShutdown { private followingsRepository: FollowingsRepository, private userEntityService: UserEntityService, + private readonly internalEventService: InternalEventService, ) { //this.onMessage = this.onMessage.bind(this); @@ -82,49 +84,33 @@ export class CacheService implements OnApplicationShutdown { this.localUserByIdCache = new MemoryKVCache(1000 * 60 * 5); // 5m this.uriPersonCache = new MemoryKVCache(1000 * 60 * 5); // 5m - this.userProfileCache = new RedisKVCache(this.redisClient, 'userProfile', { + this.userProfileCache = new QuantumKVCache(this.internalEventService, 'userProfile', { lifetime: 1000 * 60 * 30, // 30m - memoryCacheLifetime: 1000 * 60, // 1m fetcher: (key) => this.userProfilesRepository.findOneByOrFail({ userId: key }), - toRedisConverter: (value) => JSON.stringify(value), - fromRedisConverter: (value) => JSON.parse(value), // TODO: date型の考慮 }); - this.userMutingsCache = new RedisKVCache>(this.redisClient, 'userMutings', { + this.userMutingsCache = new QuantumKVCache>(this.internalEventService, 'userMutings', { lifetime: 1000 * 60 * 30, // 30m - memoryCacheLifetime: 1000 * 60, // 1m fetcher: (key) => this.mutingsRepository.find({ where: { muterId: key }, select: ['muteeId'] }).then(xs => new Set(xs.map(x => x.muteeId))), - toRedisConverter: (value) => JSON.stringify(Array.from(value)), - fromRedisConverter: (value) => new Set(JSON.parse(value)), }); - this.userBlockingCache = new RedisKVCache>(this.redisClient, 'userBlocking', { + this.userBlockingCache = new QuantumKVCache>(this.internalEventService, 'userBlocking', { lifetime: 1000 * 60 * 30, // 30m - memoryCacheLifetime: 1000 * 60, // 1m fetcher: (key) => this.blockingsRepository.find({ where: { blockerId: key }, select: ['blockeeId'] }).then(xs => new Set(xs.map(x => x.blockeeId))), - toRedisConverter: (value) => JSON.stringify(Array.from(value)), - fromRedisConverter: (value) => new Set(JSON.parse(value)), }); - this.userBlockedCache = new RedisKVCache>(this.redisClient, 'userBlocked', { + this.userBlockedCache = new QuantumKVCache>(this.internalEventService, 'userBlocked', { lifetime: 1000 * 60 * 30, // 30m - memoryCacheLifetime: 1000 * 60, // 1m fetcher: (key) => this.blockingsRepository.find({ where: { blockeeId: key }, select: ['blockerId'] }).then(xs => new Set(xs.map(x => x.blockerId))), - toRedisConverter: (value) => JSON.stringify(Array.from(value)), - fromRedisConverter: (value) => new Set(JSON.parse(value)), }); - this.renoteMutingsCache = new RedisKVCache>(this.redisClient, 'renoteMutings', { + this.renoteMutingsCache = new QuantumKVCache>(this.internalEventService, 'renoteMutings', { lifetime: 1000 * 60 * 30, // 30m - memoryCacheLifetime: 1000 * 60, // 1m fetcher: (key) => this.renoteMutingsRepository.find({ where: { muterId: key }, select: ['muteeId'] }).then(xs => new Set(xs.map(x => x.muteeId))), - toRedisConverter: (value) => JSON.stringify(Array.from(value)), - fromRedisConverter: (value) => new Set(JSON.parse(value)), }); - this.userFollowingsCache = new RedisKVCache | undefined>>(this.redisClient, 'userFollowings', { + this.userFollowingsCache = new QuantumKVCache | undefined>>(this.internalEventService, 'userFollowings', { lifetime: 1000 * 60 * 30, // 30m - memoryCacheLifetime: 1000 * 60, // 1m fetcher: (key) => this.followingsRepository.find({ where: { followerId: key }, select: ['followeeId', 'withReplies'] }).then(xs => { const obj: Record | undefined> = {}; for (const x of xs) { @@ -132,8 +118,6 @@ export class CacheService implements OnApplicationShutdown { } return obj; }), - toRedisConverter: (value) => JSON.stringify(value), - fromRedisConverter: (value) => JSON.parse(value), }); this.translationsCache = new RedisKVCache(this.redisClient, 'translations', { @@ -143,20 +127,21 @@ export class CacheService implements OnApplicationShutdown { // NOTE: チャンネルのフォロー状況キャッシュはChannelFollowingServiceで行っている - this.redisForSub.on('message', this.onMessage); + this.internalEventService.on('userChangeSuspendedState', this.onUserEvent); + this.internalEventService.on('userChangeDeletedState', this.onUserEvent); + this.internalEventService.on('remoteUserUpdated', this.onUserEvent); + this.internalEventService.on('localUserUpdated', this.onUserEvent); + this.internalEventService.on('userChangeSuspendedState', this.onUserEvent); + this.internalEventService.on('userTokenRegenerated', this.onTokenEvent); + this.internalEventService.on('follow', this.onFollowEvent); + this.internalEventService.on('unfollow', this.onFollowEvent); } @bindThis - private async onMessage(_: string, data: string): Promise { - const obj = JSON.parse(data); - - if (obj.channel === 'internal') { - const { type, body } = obj.message as GlobalEvents['internal']['payload']; - switch (type) { - case 'userChangeSuspendedState': - case 'userChangeDeletedState': - case 'remoteUserUpdated': - case 'localUserUpdated': { + private async onUserEvent(body: InternalEventTypes[E]): Promise { + { + { + { const user = await this.usersRepository.findOneBy({ id: body.id }); if (user == null) { this.userByIdCache.delete(body.id); @@ -178,20 +163,32 @@ export class CacheService implements OnApplicationShutdown { this.localUserByIdCache.set(user.id, user); } } - break; } - case 'userTokenRegenerated': { + } + } + } + + private async onTokenEvent(body: InternalEventTypes[E]): Promise { + { + { + { const user = await this.usersRepository.findOneByOrFail({ id: body.id }) as MiLocalUser; this.localUserByNativeTokenCache.delete(body.oldToken); this.localUserByNativeTokenCache.set(body.newToken, user); - break; } + } + } + } + + private async onFollowEvent(body: InternalEventTypes[E], type: E): Promise { + { + switch (type) { case 'follow': { const follower = this.userByIdCache.get(body.followerId); if (follower) follower.followingCount++; const followee = this.userByIdCache.get(body.followeeId); if (followee) followee.followersCount++; - this.userFollowingsCache.delete(body.followerId); + await this.userFollowingsCache.delete(body.followerId); this.userFollowStatsCache.delete(body.followerId); this.userFollowStatsCache.delete(body.followeeId); break; @@ -201,13 +198,11 @@ export class CacheService implements OnApplicationShutdown { if (follower) follower.followingCount--; const followee = this.userByIdCache.get(body.followeeId); if (followee) followee.followersCount--; - this.userFollowingsCache.delete(body.followerId); + await this.userFollowingsCache.delete(body.followerId); this.userFollowStatsCache.delete(body.followerId); this.userFollowStatsCache.delete(body.followeeId); break; } - default: - break; } } } @@ -300,7 +295,14 @@ export class CacheService implements OnApplicationShutdown { @bindThis public dispose(): void { - this.redisForSub.off('message', this.onMessage); + this.internalEventService.off('userChangeSuspendedState', this.onUserEvent); + this.internalEventService.off('userChangeDeletedState', this.onUserEvent); + this.internalEventService.off('remoteUserUpdated', this.onUserEvent); + this.internalEventService.off('localUserUpdated', this.onUserEvent); + this.internalEventService.off('userChangeSuspendedState', this.onUserEvent); + this.internalEventService.off('userTokenRegenerated', this.onTokenEvent); + this.internalEventService.off('follow', this.onFollowEvent); + this.internalEventService.off('unfollow', this.onFollowEvent); this.userByIdCache.dispose(); this.localUserByNativeTokenCache.dispose(); this.localUserByIdCache.dispose(); diff --git a/packages/backend/src/core/ChannelFollowingService.ts b/packages/backend/src/core/ChannelFollowingService.ts index 12251595e2..869456998b 100644 --- a/packages/backend/src/core/ChannelFollowingService.ts +++ b/packages/backend/src/core/ChannelFollowingService.ts @@ -9,14 +9,16 @@ import { DI } from '@/di-symbols.js'; import type { ChannelFollowingsRepository } from '@/models/_.js'; import { MiChannel } from '@/models/_.js'; import { IdService } from '@/core/IdService.js'; -import { GlobalEvents, GlobalEventService } from '@/core/GlobalEventService.js'; +import { GlobalEvents, GlobalEventService, InternalEventTypes } from '@/core/GlobalEventService.js'; import { bindThis } from '@/decorators.js'; import type { MiLocalUser } from '@/models/User.js'; -import { RedisKVCache } from '@/misc/cache.js'; +import { QuantumKVCache, RedisKVCache } from '@/misc/cache.js'; +import { InternalEventService } from './InternalEventService.js'; @Injectable() export class ChannelFollowingService implements OnModuleInit { - public userFollowingChannelsCache: RedisKVCache>; + // TODO check for regs + public userFollowingChannelsCache: QuantumKVCache>; constructor( @Inject(DI.redis) @@ -27,19 +29,18 @@ export class ChannelFollowingService implements OnModuleInit { private channelFollowingsRepository: ChannelFollowingsRepository, private idService: IdService, private globalEventService: GlobalEventService, + private readonly internalEventService: InternalEventService, ) { - this.userFollowingChannelsCache = new RedisKVCache>(this.redisClient, 'userFollowingChannels', { + this.userFollowingChannelsCache = new QuantumKVCache>(this.internalEventService, 'userFollowingChannels', { lifetime: 1000 * 60 * 30, // 30m - memoryCacheLifetime: 1000 * 60, // 1m fetcher: (key) => this.channelFollowingsRepository.find({ where: { followerId: key }, select: ['followeeId'], }).then(xs => new Set(xs.map(x => x.followeeId))), - toRedisConverter: (value) => JSON.stringify(Array.from(value)), - fromRedisConverter: (value) => new Set(JSON.parse(value)), }); - this.redisForSub.on('message', this.onMessage); + this.internalEventService.on('followChannel', this.onMessage); + this.internalEventService.on('unfollowChannel', this.onMessage); } onModuleInit() { @@ -79,18 +80,15 @@ export class ChannelFollowingService implements OnModuleInit { } @bindThis - private async onMessage(_: string, data: string): Promise { - const obj = JSON.parse(data); - - if (obj.channel === 'internal') { - const { type, body } = obj.message as GlobalEvents['internal']['payload']; + private async onMessage(body: InternalEventTypes[E], type: E): Promise { + { switch (type) { case 'followChannel': { - this.userFollowingChannelsCache.refresh(body.userId); + await this.userFollowingChannelsCache.delete(body.userId); break; } case 'unfollowChannel': { - this.userFollowingChannelsCache.delete(body.userId); + await this.userFollowingChannelsCache.delete(body.userId); break; } } @@ -99,6 +97,8 @@ export class ChannelFollowingService implements OnModuleInit { @bindThis public dispose(): void { + this.internalEventService.off('followChannel', this.onMessage); + this.internalEventService.off('unfollowChannel', this.onMessage); this.userFollowingChannelsCache.dispose(); } diff --git a/packages/backend/src/core/PushNotificationService.ts b/packages/backend/src/core/PushNotificationService.ts index 9333c1ebc5..38bc5e3901 100644 --- a/packages/backend/src/core/PushNotificationService.ts +++ b/packages/backend/src/core/PushNotificationService.ts @@ -12,7 +12,8 @@ import type { Packed } from '@/misc/json-schema.js'; import { getNoteSummary } from '@/misc/get-note-summary.js'; import type { MiMeta, MiSwSubscription, SwSubscriptionsRepository } from '@/models/_.js'; import { bindThis } from '@/decorators.js'; -import { RedisKVCache } from '@/misc/cache.js'; +import { QuantumKVCache, RedisKVCache } from '@/misc/cache.js'; +import { InternalEventService } from '@/core/InternalEventService.js'; // Defined also packages/sw/types.ts#L13 type PushNotificationsTypes = { @@ -48,7 +49,7 @@ function truncateBody(type: T, body: Pus @Injectable() export class PushNotificationService implements OnApplicationShutdown { - private subscriptionsCache: RedisKVCache; + private subscriptionsCache: QuantumKVCache; constructor( @Inject(DI.config) @@ -62,13 +63,11 @@ export class PushNotificationService implements OnApplicationShutdown { @Inject(DI.swSubscriptionsRepository) private swSubscriptionsRepository: SwSubscriptionsRepository, + private readonly internalEventService: InternalEventService, ) { - this.subscriptionsCache = new RedisKVCache(this.redisClient, 'userSwSubscriptions', { + this.subscriptionsCache = new QuantumKVCache(this.internalEventService, 'userSwSubscriptions', { lifetime: 1000 * 60 * 60 * 1, // 1h - memoryCacheLifetime: 1000 * 60 * 3, // 3m fetcher: (key) => this.swSubscriptionsRepository.findBy({ userId: key }), - toRedisConverter: (value) => JSON.stringify(value), - fromRedisConverter: (value) => JSON.parse(value), }); } @@ -114,8 +113,8 @@ export class PushNotificationService implements OnApplicationShutdown { endpoint: subscription.endpoint, auth: subscription.auth, publickey: subscription.publickey, - }).then(() => { - this.refreshCache(userId); + }).then(async () => { + await this.refreshCache(userId); }); } }); @@ -123,8 +122,8 @@ export class PushNotificationService implements OnApplicationShutdown { } @bindThis - public refreshCache(userId: string): void { - this.subscriptionsCache.refresh(userId); + public async refreshCache(userId: string): Promise { + await this.subscriptionsCache.refresh(userId); } @bindThis diff --git a/packages/backend/src/core/UserBlockingService.ts b/packages/backend/src/core/UserBlockingService.ts index 8da1bb2092..1a1e7c4778 100644 --- a/packages/backend/src/core/UserBlockingService.ts +++ b/packages/backend/src/core/UserBlockingService.ts @@ -77,8 +77,10 @@ export class UserBlockingService implements OnModuleInit { await this.blockingsRepository.insert(blocking); - this.cacheService.userBlockingCache.refresh(blocker.id); - this.cacheService.userBlockedCache.refresh(blockee.id); + await Promise.all([ + this.cacheService.userBlockingCache.delete(blocker.id), + this.cacheService.userBlockedCache.delete(blockee.id), + ]); this.globalEventService.publishInternalEvent('blockingCreated', { blockerId: blocker.id, @@ -168,8 +170,10 @@ export class UserBlockingService implements OnModuleInit { await this.blockingsRepository.delete(blocking.id); - this.cacheService.userBlockingCache.refresh(blocker.id); - this.cacheService.userBlockedCache.refresh(blockee.id); + await Promise.all([ + this.cacheService.userBlockingCache.delete(blocker.id), + this.cacheService.userBlockedCache.delete(blockee.id), + ]); this.globalEventService.publishInternalEvent('blockingDeleted', { blockerId: blocker.id, diff --git a/packages/backend/src/core/UserFollowingService.ts b/packages/backend/src/core/UserFollowingService.ts index 897b950022..6a6c9a3000 100644 --- a/packages/backend/src/core/UserFollowingService.ts +++ b/packages/backend/src/core/UserFollowingService.ts @@ -29,6 +29,7 @@ import { AccountMoveService } from '@/core/AccountMoveService.js'; import { UtilityService } from '@/core/UtilityService.js'; import type { ThinUser } from '@/queue/types.js'; import { LoggerService } from '@/core/LoggerService.js'; +import { InternalEventService } from '@/core/InternalEventService.js'; import type Logger from '../logger.js'; type Local = MiLocalUser | { @@ -86,6 +87,7 @@ export class UserFollowingService implements OnModuleInit { private accountMoveService: AccountMoveService, private perUserFollowingChart: PerUserFollowingChart, private instanceChart: InstanceChart, + private readonly internalEventService: InternalEventService, loggerService: LoggerService, ) { @@ -264,7 +266,8 @@ export class UserFollowingService implements OnModuleInit { } }); - this.cacheService.userFollowingsCache.refresh(follower.id); + // Handled by CacheService + //this.cacheService.userFollowingsCache.refresh(follower.id); const requestExist = await this.followRequestsRepository.exists({ where: { @@ -291,7 +294,7 @@ export class UserFollowingService implements OnModuleInit { }, followee.id); } - this.globalEventService.publishInternalEvent('follow', { followerId: follower.id, followeeId: followee.id }); + await this.internalEventService.emit('follow', { followerId: follower.id, followeeId: followee.id }); const [followeeUser, followerUser] = await Promise.all([ this.usersRepository.findOneByOrFail({ id: followee.id }), @@ -381,7 +384,8 @@ export class UserFollowingService implements OnModuleInit { await this.followingsRepository.delete(following.id); - this.cacheService.userFollowingsCache.refresh(follower.id); + // Handled by CacheService + // this.cacheService.userFollowingsCache.refresh(follower.id); this.decrementFollowing(following.follower, following.followee); @@ -412,7 +416,7 @@ export class UserFollowingService implements OnModuleInit { follower: MiUser, followee: MiUser, ): Promise { - this.globalEventService.publishInternalEvent('unfollow', { followerId: follower.id, followeeId: followee.id }); + await this.internalEventService.emit('unfollow', { followerId: follower.id, followeeId: followee.id }); // Neither followee nor follower has moved. if (!follower.movedToUri && !followee.movedToUri) { diff --git a/packages/backend/src/core/UserKeypairService.ts b/packages/backend/src/core/UserKeypairService.ts index 92d61cd103..d8a67d273b 100644 --- a/packages/backend/src/core/UserKeypairService.ts +++ b/packages/backend/src/core/UserKeypairService.ts @@ -7,14 +7,14 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import * as Redis from 'ioredis'; import type { MiUser } from '@/models/User.js'; import type { UserKeypairsRepository } from '@/models/_.js'; -import { RedisKVCache } from '@/misc/cache.js'; +import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js'; import type { MiUserKeypair } from '@/models/UserKeypair.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; @Injectable() export class UserKeypairService implements OnApplicationShutdown { - private cache: RedisKVCache; + private cache: MemoryKVCache; constructor( @Inject(DI.redis) @@ -23,18 +23,12 @@ export class UserKeypairService implements OnApplicationShutdown { @Inject(DI.userKeypairsRepository) private userKeypairsRepository: UserKeypairsRepository, ) { - this.cache = new RedisKVCache(this.redisClient, 'userKeypair', { - lifetime: 1000 * 60 * 60 * 24, // 24h - memoryCacheLifetime: 1000 * 60 * 60, // 1h - fetcher: (key) => this.userKeypairsRepository.findOneByOrFail({ userId: key }), - toRedisConverter: (value) => JSON.stringify(value), - fromRedisConverter: (value) => JSON.parse(value), - }); + this.cache = new MemoryKVCache(1000 * 60 * 60 * 24); // 24h } @bindThis public async getUserKeypair(userId: MiUser['id']): Promise { - return await this.cache.fetch(userId); + return await this.cache.fetch(userId, () => this.userKeypairsRepository.findOneByOrFail({ userId })); } @bindThis diff --git a/packages/backend/src/core/UserListService.ts b/packages/backend/src/core/UserListService.ts index e7200ab1bf..0240184d13 100644 --- a/packages/backend/src/core/UserListService.ts +++ b/packages/backend/src/core/UserListService.ts @@ -11,21 +11,22 @@ import type { MiUser } from '@/models/User.js'; import type { MiUserList } from '@/models/UserList.js'; import type { MiUserListMembership } from '@/models/UserListMembership.js'; import { IdService } from '@/core/IdService.js'; -import type { GlobalEvents } from '@/core/GlobalEventService.js'; +import type { GlobalEvents, InternalEventTypes } from '@/core/GlobalEventService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import { DI } from '@/di-symbols.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { bindThis } from '@/decorators.js'; import { QueueService } from '@/core/QueueService.js'; -import { RedisKVCache } from '@/misc/cache.js'; +import { QuantumKVCache, RedisKVCache } from '@/misc/cache.js'; import { RoleService } from '@/core/RoleService.js'; import { SystemAccountService } from '@/core/SystemAccountService.js'; +import { InternalEventService } from '@/core/InternalEventService.js'; @Injectable() export class UserListService implements OnApplicationShutdown, OnModuleInit { public static TooManyUsersError = class extends Error {}; - public membersCache: RedisKVCache>; + public membersCache: QuantumKVCache>; private roleService: RoleService; constructor( @@ -48,16 +49,15 @@ export class UserListService implements OnApplicationShutdown, OnModuleInit { private globalEventService: GlobalEventService, private queueService: QueueService, private systemAccountService: SystemAccountService, + private readonly internalEventService: InternalEventService, ) { - this.membersCache = new RedisKVCache>(this.redisClient, 'userListMembers', { + this.membersCache = new QuantumKVCache>(this.internalEventService, 'userListMembers', { lifetime: 1000 * 60 * 30, // 30m - memoryCacheLifetime: 1000 * 60, // 1m fetcher: (key) => this.userListMembershipsRepository.find({ where: { userListId: key }, select: ['userId'] }).then(xs => new Set(xs.map(x => x.userId))), - toRedisConverter: (value) => JSON.stringify(Array.from(value)), - fromRedisConverter: (value) => new Set(JSON.parse(value)), }); - this.redisForSub.on('message', this.onMessage); + this.internalEventService.on('userListMemberAdded', this.onMessage); + this.internalEventService.on('userListMemberRemoved', this.onMessage); } async onModuleInit() { @@ -65,24 +65,21 @@ export class UserListService implements OnApplicationShutdown, OnModuleInit { } @bindThis - private async onMessage(_: string, data: string): Promise { - const obj = JSON.parse(data); - - if (obj.channel === 'internal') { - const { type, body } = obj.message as GlobalEvents['internal']['payload']; + private async onMessage(body: InternalEventTypes[E], type: E): Promise { + { switch (type) { case 'userListMemberAdded': { const { userListId, memberId } = body; - const members = await this.membersCache.get(userListId); - if (members) { + if (this.membersCache.has(userListId)) { + const members = await this.membersCache.get(userListId); members.add(memberId); } break; } case 'userListMemberRemoved': { const { userListId, memberId } = body; - const members = await this.membersCache.get(userListId); - if (members) { + if (this.membersCache.has(userListId)) { + const members = await this.membersCache.get(userListId); members.delete(memberId); } break; @@ -150,7 +147,8 @@ export class UserListService implements OnApplicationShutdown, OnModuleInit { @bindThis public dispose(): void { - this.redisForSub.off('message', this.onMessage); + this.internalEventService.off('userListMemberAdded', this.onMessage); + this.internalEventService.off('userListMemberRemoved', this.onMessage); this.membersCache.dispose(); } diff --git a/packages/backend/src/core/UserMutingService.ts b/packages/backend/src/core/UserMutingService.ts index 06643be5fb..4f72c1863b 100644 --- a/packages/backend/src/core/UserMutingService.ts +++ b/packages/backend/src/core/UserMutingService.ts @@ -32,7 +32,7 @@ export class UserMutingService { muteeId: target.id, }); - this.cacheService.userMutingsCache.refresh(user.id); + await this.cacheService.userMutingsCache.delete(user.id); } @bindThis @@ -43,9 +43,8 @@ export class UserMutingService { id: In(mutings.map(m => m.id)), }); - const muterIds = [...new Set(mutings.map(m => m.muterId))]; - for (const muterId of muterIds) { - this.cacheService.userMutingsCache.refresh(muterId); - } + await Promise.all(Array + .from(new Set(mutings.map(m => m.muterId))) + .map(muterId => this.cacheService.userMutingsCache.delete(muterId))); } } diff --git a/packages/backend/src/core/UserRenoteMutingService.ts b/packages/backend/src/core/UserRenoteMutingService.ts index bdc5e23f4b..9d5ec164c8 100644 --- a/packages/backend/src/core/UserRenoteMutingService.ts +++ b/packages/backend/src/core/UserRenoteMutingService.ts @@ -33,7 +33,7 @@ export class UserRenoteMutingService { muteeId: target.id, }); - await this.cacheService.renoteMutingsCache.refresh(user.id); + await this.cacheService.renoteMutingsCache.delete(user.id); } @bindThis @@ -44,9 +44,8 @@ export class UserRenoteMutingService { id: In(mutings.map(m => m.id)), }); - const muterIds = [...new Set(mutings.map(m => m.muterId))]; - for (const muterId of muterIds) { - await this.cacheService.renoteMutingsCache.refresh(muterId); - } + await Promise.all(Array + .from(new Set(mutings.map(m => m.muterId))) + .map(muterId => this.cacheService.renoteMutingsCache.delete(muterId))); } } diff --git a/packages/backend/src/server/api/endpoints/admin/nsfw-user.ts b/packages/backend/src/server/api/endpoints/admin/nsfw-user.ts index 194e793eda..f6c4f0b635 100644 --- a/packages/backend/src/server/api/endpoints/admin/nsfw-user.ts +++ b/packages/backend/src/server/api/endpoints/admin/nsfw-user.ts @@ -47,7 +47,7 @@ export default class extends Endpoint { // eslint- alwaysMarkNsfw: true, }); - await this.cacheService.userProfileCache.refresh(ps.userId); + await this.cacheService.userProfileCache.delete(ps.userId); }); } } diff --git a/packages/backend/src/server/api/endpoints/i/update.ts b/packages/backend/src/server/api/endpoints/i/update.ts index dda42ce0e4..e632915f62 100644 --- a/packages/backend/src/server/api/endpoints/i/update.ts +++ b/packages/backend/src/server/api/endpoints/i/update.ts @@ -617,7 +617,7 @@ export default class extends Endpoint { // eslint- const updatedProfile = await this.userProfilesRepository.findOneByOrFail({ userId: user.id }); - this.cacheService.userProfileCache.set(user.id, updatedProfile); + await this.cacheService.userProfileCache.set(user.id, updatedProfile); // Publish meUpdated event this.globalEventService.publishMainStream(user.id, 'meUpdated', iObj); diff --git a/packages/backend/src/server/api/endpoints/sw/register.ts b/packages/backend/src/server/api/endpoints/sw/register.ts index f447b5598b..2f72e6ce1d 100644 --- a/packages/backend/src/server/api/endpoints/sw/register.ts +++ b/packages/backend/src/server/api/endpoints/sw/register.ts @@ -104,7 +104,7 @@ export default class extends Endpoint { // eslint- sendReadMessage: ps.sendReadMessage, }); - this.pushNotificationService.refreshCache(me.id); + await this.pushNotificationService.refreshCache(me.id); return { state: 'subscribed' as const, diff --git a/packages/backend/src/server/api/endpoints/sw/unregister.ts b/packages/backend/src/server/api/endpoints/sw/unregister.ts index aa7e03dceb..f43a2cce28 100644 --- a/packages/backend/src/server/api/endpoints/sw/unregister.ts +++ b/packages/backend/src/server/api/endpoints/sw/unregister.ts @@ -46,7 +46,7 @@ export default class extends Endpoint { // eslint- }); if (me) { - this.pushNotificationService.refreshCache(me.id); + await this.pushNotificationService.refreshCache(me.id); } }); } diff --git a/packages/backend/src/server/api/endpoints/sw/update-registration.ts b/packages/backend/src/server/api/endpoints/sw/update-registration.ts index 78b9323b7b..0cbed273e8 100644 --- a/packages/backend/src/server/api/endpoints/sw/update-registration.ts +++ b/packages/backend/src/server/api/endpoints/sw/update-registration.ts @@ -86,7 +86,7 @@ export default class extends Endpoint { // eslint- sendReadMessage: swSubscription.sendReadMessage, }); - this.pushNotificationService.refreshCache(me.id); + await this.pushNotificationService.refreshCache(me.id); return { userId: swSubscription.userId, From 207abaff889291b3878984f81cfe37e9fc465133 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Thu, 5 Jun 2025 14:28:19 -0400 Subject: [PATCH 05/54] implement QuantumKVCache.setMany and QuantumKVCache.seleteMany --- .../src/core/ChannelFollowingService.ts | 1 - .../backend/src/core/GlobalEventService.ts | 2 +- packages/backend/src/core/UserListService.ts | 8 +- .../backend/src/core/UserMutingService.ts | 4 +- .../src/core/UserRenoteMutingService.ts | 4 +- packages/backend/src/misc/cache.ts | 87 ++++++++--- packages/backend/test/unit/misc/cache.ts | 146 +++++++++++++++--- 7 files changed, 201 insertions(+), 51 deletions(-) diff --git a/packages/backend/src/core/ChannelFollowingService.ts b/packages/backend/src/core/ChannelFollowingService.ts index 869456998b..26b023179c 100644 --- a/packages/backend/src/core/ChannelFollowingService.ts +++ b/packages/backend/src/core/ChannelFollowingService.ts @@ -17,7 +17,6 @@ import { InternalEventService } from './InternalEventService.js'; @Injectable() export class ChannelFollowingService implements OnModuleInit { - // TODO check for regs public userFollowingChannelsCache: QuantumKVCache>; constructor( diff --git a/packages/backend/src/core/GlobalEventService.ts b/packages/backend/src/core/GlobalEventService.ts index 763ab8c086..de35e9db19 100644 --- a/packages/backend/src/core/GlobalEventService.ts +++ b/packages/backend/src/core/GlobalEventService.ts @@ -265,7 +265,7 @@ export interface InternalEventTypes { unmute: { muterId: MiUser['id']; muteeId: MiUser['id']; }; userListMemberAdded: { userListId: MiUserList['id']; memberId: MiUser['id']; }; userListMemberRemoved: { userListId: MiUserList['id']; memberId: MiUser['id']; }; - quantumCacheUpdated: { name: string, key: string, op: 's' | 'd' }; + quantumCacheUpdated: { name: string, keys: string[], op: 's' | 'd' }; } type EventTypesToEventPayload = EventUnionFromDictionary>>; diff --git a/packages/backend/src/core/UserListService.ts b/packages/backend/src/core/UserListService.ts index 0240184d13..0d2220049a 100644 --- a/packages/backend/src/core/UserListService.ts +++ b/packages/backend/src/core/UserListService.ts @@ -70,16 +70,16 @@ export class UserListService implements OnApplicationShutdown, OnModuleInit { switch (type) { case 'userListMemberAdded': { const { userListId, memberId } = body; - if (this.membersCache.has(userListId)) { - const members = await this.membersCache.get(userListId); + const members = this.membersCache.get(userListId); + if (members) { members.add(memberId); } break; } case 'userListMemberRemoved': { const { userListId, memberId } = body; - if (this.membersCache.has(userListId)) { - const members = await this.membersCache.get(userListId); + const members = this.membersCache.get(userListId); + if (members) { members.delete(memberId); } break; diff --git a/packages/backend/src/core/UserMutingService.ts b/packages/backend/src/core/UserMutingService.ts index 4f72c1863b..c15a979d0f 100644 --- a/packages/backend/src/core/UserMutingService.ts +++ b/packages/backend/src/core/UserMutingService.ts @@ -43,8 +43,6 @@ export class UserMutingService { id: In(mutings.map(m => m.id)), }); - await Promise.all(Array - .from(new Set(mutings.map(m => m.muterId))) - .map(muterId => this.cacheService.userMutingsCache.delete(muterId))); + await this.cacheService.userMutingsCache.deleteMany(mutings.map(m => m.muterId)); } } diff --git a/packages/backend/src/core/UserRenoteMutingService.ts b/packages/backend/src/core/UserRenoteMutingService.ts index 9d5ec164c8..7c0693f216 100644 --- a/packages/backend/src/core/UserRenoteMutingService.ts +++ b/packages/backend/src/core/UserRenoteMutingService.ts @@ -44,8 +44,6 @@ export class UserRenoteMutingService { id: In(mutings.map(m => m.id)), }); - await Promise.all(Array - .from(new Set(mutings.map(m => m.muterId))) - .map(muterId => this.cacheService.renoteMutingsCache.delete(muterId))); + await this.cacheService.renoteMutingsCache.deleteMany(mutings.map(m => m.muterId)); } } diff --git a/packages/backend/src/misc/cache.ts b/packages/backend/src/misc/cache.ts index 31e6f126b8..22201e243f 100644 --- a/packages/backend/src/misc/cache.ts +++ b/packages/backend/src/misc/cache.ts @@ -531,19 +531,54 @@ export class QuantumKVCache implements Iterable<[key: string, value: T]> { this.memoryCache.set(key, value); - await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, op: 's', key }); + await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, op: 's', keys: [key] }); if (this.onSet) { await this.onSet(key, this); } } + /** + * Creates or updates multiple value in the cache, and erases any stale caches across the cluster. + * Fires an onSet for each changed item event after the cache has been updated in all processes. + * Skips if all values are unchanged. + */ + @bindThis + public async setMany(items: Iterable<[key: string, value: T]>): Promise { + const changedKeys: string[] = []; + + for (const item of items) { + if (this.memoryCache.get(item[0]) !== item[1]) { + changedKeys.push(item[0]); + this.memoryCache.set(item[0], item[1]); + } + } + + if (changedKeys.length > 0) { + await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, op: 's', keys: changedKeys }); + + if (this.onSet) { + for (const key of changedKeys) { + await this.onSet(key, this); + } + } + } + } + + /** + * Gets a value from the local memory cache, or returns undefined if not found. + */ + @bindThis + public get(key: string): T | undefined { + return this.memoryCache.get(key); + } + /** * Gets or fetches a value from the cache. * Fires an onSet event, but does not emit an update event to other processes. */ @bindThis - public async get(key: string): Promise { + public async fetch(key: string): Promise { let value = this.memoryCache.get(key); if (value === undefined) { value = await this.fetcher(key, this); @@ -556,15 +591,6 @@ export class QuantumKVCache implements Iterable<[key: string, value: T]> { return value; } - /** - * Alias to get(), included for backwards-compatibility with RedisKVCache. - * @deprecated use get() instead - */ - @bindThis - public async fetch(key: string): Promise { - return await this.get(key); - } - /** * Returns true is a key exists in memory. * This applies to the local subset view, not the cross-cluster cache state. @@ -582,12 +608,35 @@ export class QuantumKVCache implements Iterable<[key: string, value: T]> { public async delete(key: string): Promise { this.memoryCache.delete(key); - await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, op: 'd', key }); + await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, op: 'd', keys: [key] }); if (this.onDelete) { await this.onDelete(key, this); } } + /** + * Deletes multiple values from the cache, and erases any stale caches across the cluster. + * Fires an onDelete event for each key after the cache has been updated in all processes. + * Skips if the input is empty. + */ + @bindThis + public async deleteMany(keys: string[]): Promise { + if (keys.length === 0) { + return; + } + + for (const key of keys) { + this.memoryCache.delete(key); + } + + await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, op: 'd', keys }); + + if (this.onDelete) { + for (const key of keys) { + await this.onDelete(key, this); + } + } + } /** * Refreshes the value of a key from the fetcher, and erases any stale caches across the cluster. @@ -623,14 +672,16 @@ export class QuantumKVCache implements Iterable<[key: string, value: T]> { @bindThis private async onQuantumCacheUpdated(data: InternalEventTypes['quantumCacheUpdated']): Promise { if (data.name === this.name) { - this.memoryCache.delete(data.key); + for (const key of data.keys) { + this.memoryCache.delete(key); - if (data.op === 's' && this.onSet) { - await this.onSet(data.key, this); - } + if (data.op === 's' && this.onSet) { + await this.onSet(key, this); + } - if (data.op === 'd' && this.onDelete) { - await this.onDelete(data.key, this); + if (data.op === 'd' && this.onDelete) { + await this.onDelete(key, this); + } } } } diff --git a/packages/backend/test/unit/misc/cache.ts b/packages/backend/test/unit/misc/cache.ts index 5b242c47d4..0b658618e6 100644 --- a/packages/backend/test/unit/misc/cache.ts +++ b/packages/backend/test/unit/misc/cache.ts @@ -73,7 +73,7 @@ describe(QuantumKVCache, () => { await cache.set('foo', 'bar'); - expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', op: 's', key: 'foo' }]]); + expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', op: 's', keys: ['foo'] }]]); }); it('should call onSet when storing', async () => { @@ -110,13 +110,13 @@ describe(QuantumKVCache, () => { expect(fakeOnSet).toHaveBeenCalledTimes(1); }); - it('should fetch when getting an unknown value', async () => { + it('should fetch an unknown value', async () => { const cache = makeCache({ name: 'fake', fetcher: key => `value#${key}`, }); - const result = await cache.get('foo'); + const result = await cache.fetch('foo'); expect(result).toBe('value#foo'); }); @@ -127,7 +127,7 @@ describe(QuantumKVCache, () => { fetcher: key => `value#${key}`, }); - await cache.get('foo'); + await cache.fetch('foo'); const result = cache.has('foo'); expect(result).toBe(true); @@ -141,7 +141,7 @@ describe(QuantumKVCache, () => { onSet: fakeOnSet, }); - await cache.get('foo'); + await cache.fetch('foo'); expect(fakeOnSet).toHaveBeenCalledWith('foo', cache); }); @@ -152,9 +152,9 @@ describe(QuantumKVCache, () => { fetcher: key => `value#${key}`, }); - await cache.get('foo'); + await cache.fetch('foo'); - expect(fakeInternalEventService._calls).not.toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', op: 's', key: 'foo' }]]); + expect(fakeInternalEventService._calls).not.toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', op: 's', keys: ['foo'] }]]); }); it('should delete from memory cache', async () => { @@ -186,14 +186,14 @@ describe(QuantumKVCache, () => { await cache.set('foo', 'bar'); await cache.delete('foo'); - expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', op: 'd', key: 'foo' }]]); + expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', op: 'd', keys: ['foo'] }]]); }); it('should delete when receiving set event', async () => { const cache = makeCache({ name: 'fake' }); await cache.set('foo', 'bar'); - await fakeInternalEventService._emitRedis('quantumCacheUpdated', { name: 'fake', op: 's', key: 'foo' }); + await fakeInternalEventService._emitRedis('quantumCacheUpdated', { name: 'fake', op: 's', keys: ['foo'] }); const result = cache.has('foo'); expect(result).toBe(false); @@ -206,7 +206,7 @@ describe(QuantumKVCache, () => { onSet: fakeOnSet, }); - await fakeInternalEventService._emitRedis('quantumCacheUpdated', { name: 'fake', op: 's', key: 'foo' }); + await fakeInternalEventService._emitRedis('quantumCacheUpdated', { name: 'fake', op: 's', keys: ['foo'] }); expect(fakeOnSet).toHaveBeenCalledWith('foo', cache); }); @@ -215,7 +215,7 @@ describe(QuantumKVCache, () => { const cache = makeCache({ name: 'fake' }); await cache.set('foo', 'bar'); - await fakeInternalEventService._emitRedis('quantumCacheUpdated', { name: 'fake', op: 'd', key: 'foo' }); + await fakeInternalEventService._emitRedis('quantumCacheUpdated', { name: 'fake', op: 'd', keys: ['foo'] }); const result = cache.has('foo'); expect(result).toBe(false); @@ -229,26 +229,130 @@ describe(QuantumKVCache, () => { }); await cache.set('foo', 'bar'); - await fakeInternalEventService._emitRedis('quantumCacheUpdated', { name: 'fake', op: 'd', key: 'foo' }); + await fakeInternalEventService._emitRedis('quantumCacheUpdated', { name: 'fake', op: 'd', keys: ['foo'] }); expect(fakeOnDelete).toHaveBeenCalledWith('foo', cache); }); - describe('fetch', () => { - it('should perform same logic as get', async () => { + describe('get', () => { + it('should return value if present', async () => { + const cache = makeCache(); + await cache.set('foo', 'bar'); + + const result = cache.get('foo'); + + expect(result).toBe('bar'); + }); + it('should return undefined if missing', () => { + const cache = makeCache(); + + const result = cache.get('foo'); + + expect(result).toBe(undefined); + }); + }); + + describe('setMany', () => { + it('should populate all values', async () => { + const cache = makeCache(); + + await cache.setMany([['foo', 'bar'], ['alpha', 'omega']]); + + expect(cache.has('foo')).toBe(true); + expect(cache.has('alpha')).toBe(true); + }); + + it('should emit one event', async () => { + const cache = makeCache({ + name: 'fake', + }); + + await cache.setMany([['foo', 'bar'], ['alpha', 'omega']]); + + expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', op: 's', keys: ['foo', 'alpha'] }]]); + expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(1); + }); + + it('should call onSet for each item', async () => { const fakeOnSet = jest.fn(() => Promise.resolve()); const cache = makeCache({ name: 'fake', - fetcher: key => `value#${key}`, onSet: fakeOnSet, }); - // noinspection JSDeprecatedSymbols - const result = await cache.fetch('foo'); + await cache.setMany([['foo', 'bar'], ['alpha', 'omega']]); - expect(result).toBe('value#foo'); expect(fakeOnSet).toHaveBeenCalledWith('foo', cache); - expect(fakeInternalEventService._calls).not.toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', op: 's', key: 'foo' }]]); + expect(fakeOnSet).toHaveBeenCalledWith('alpha', cache); + }); + + it('should emit events only for changed items', async () => { + const fakeOnSet = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + name: 'fake', + onSet: fakeOnSet, + }); + + await cache.set('foo', 'bar'); + fakeOnSet.mockClear(); + fakeInternalEventService._reset(); + + await cache.setMany([['foo', 'bar'], ['alpha', 'omega']]); + + expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', op: 's', keys: ['alpha'] }]]); + expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(1); + expect(fakeOnSet).toHaveBeenCalledWith('alpha', cache); + expect(fakeOnSet).toHaveBeenCalledTimes(1); + }); + }); + + describe('deleteMany', () => { + it('should remove keys from memory cache', async () => { + const cache = makeCache(); + + await cache.set('foo', 'bar'); + await cache.set('alpha', 'omega'); + await cache.deleteMany(['foo', 'alpha']); + + expect(cache.has('foo')).toBe(false); + expect(cache.has('alpha')).toBe(false); + }); + + it('should emit only one event', async () => { + const cache = makeCache({ + name: 'fake', + }); + + await cache.deleteMany(['foo', 'alpha']); + + expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', op: 'd', keys: ['foo', 'alpha'] }]]); + expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(1); + }); + + it('should call onDelete for each key', async () => { + const fakeOnDelete = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + name: 'fake', + onDelete: fakeOnDelete, + }); + + await cache.deleteMany(['foo', 'alpha']); + + expect(fakeOnDelete).toHaveBeenCalledWith('foo', cache); + expect(fakeOnDelete).toHaveBeenCalledWith('alpha', cache); + }); + + it('should do nothing if no keys are provided', async () => { + const fakeOnDelete = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + name: 'fake', + onDelete: fakeOnDelete, + }); + + await cache.deleteMany([]); + + expect(fakeOnDelete).not.toHaveBeenCalled(); + expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0); }); }); @@ -296,7 +400,7 @@ describe(QuantumKVCache, () => { onSet: fakeOnSet, }); - await cache.refresh('foo') + await cache.refresh('foo'); expect(fakeOnSet).toHaveBeenCalledWith('foo', cache); }); @@ -309,7 +413,7 @@ describe(QuantumKVCache, () => { await cache.refresh('foo'); - expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', op: 's', key: 'foo' }]]); + expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', op: 's', keys: ['foo'] }]]); }); }); From bf1156426eea0bd11a5926ba7883c870bad2e144 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 6 Jun 2025 00:08:34 -0400 Subject: [PATCH 06/54] add CacheService.getUserFollowings and CacheService.getUserBlockers --- packages/backend/src/core/CacheService.ts | 110 +++++++++++++++++++++- 1 file changed, 109 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/core/CacheService.ts b/packages/backend/src/core/CacheService.ts index f04b18c02b..6e979130a0 100644 --- a/packages/backend/src/core/CacheService.ts +++ b/packages/backend/src/core/CacheService.ts @@ -5,7 +5,7 @@ import { Inject, Injectable } from '@nestjs/common'; import * as Redis from 'ioredis'; -import { IsNull } from 'typeorm'; +import { In, IsNull } from 'typeorm'; import type { BlockingsRepository, FollowingsRepository, MutingsRepository, RenoteMutingsRepository, MiUserProfile, UserProfilesRepository, UsersRepository, MiFollowing, MiNote } from '@/models/_.js'; import { MemoryKVCache, QuantumKVCache, RedisKVCache } from '@/misc/cache.js'; import type { MiLocalUser, MiUser } from '@/models/User.js'; @@ -293,6 +293,114 @@ export class CacheService implements OnApplicationShutdown { }); } + @bindThis + public async getUserFollowings(userIds: Iterable): Promise>> { + const followings = new Map>(); + + const toFetch: string[] = []; + for (const userId of userIds) { + const fromCache = this.userFollowingsCache.get(userId); + if (fromCache) { + followings.set(userId, new Set(Object.keys(fromCache))); + } else { + toFetch.push(userId); + } + } + + if (toFetch.length > 0) { + const fetchedFollowings = await this.followingsRepository + .createQueryBuilder('following') + .select([ + 'following.followerId', + 'following.followeeId', + 'following.withReplies', + ]) + .where({ + followerId: In(toFetch), + }) + .getMany(); + + const toCache = new Map | undefined>>(); + + // Pivot to a map + for (const { followerId, followeeId, withReplies } of fetchedFollowings) { + // Queue for cache + let cacheSet = toCache.get(followerId); + if (!cacheSet) { + cacheSet = {}; + toCache.set(followerId, cacheSet); + } + cacheSet[followeeId] = { withReplies }; + + // Queue for return + let returnSet = followings.get(followerId); + if (!returnSet) { + returnSet = new Set(); + followings.set(followerId, returnSet); + } + returnSet.add(followeeId); + } + + // Update cache to speed up future calls + await this.userFollowingsCache.setMany(toCache.entries()); + } + + return followings; + } + + @bindThis + public async getUserBlockers(userIds: Iterable): Promise>> { + const blockers = new Map>(); + + const toFetch: string[] = []; + for (const userId of userIds) { + const fromCache = this.userBlockedCache.get(userId); + if (fromCache) { + blockers.set(userId, fromCache); + } else { + toFetch.push(userId); + } + } + + if (toFetch.length > 0) { + const fetchedBlockers = await this.blockingsRepository.createQueryBuilder('blocking') + .select([ + 'blocking.blockerId', + 'blocking.blockeeId', + ]) + .where({ + blockeeId: In(toFetch), + }) + .getMany(); + + const toCache = new Map>(); + + // Pivot to a map + for (const { blockerId, blockeeId } of fetchedBlockers) { + // Queue for cache + let cacheSet = toCache.get(blockeeId); + if (!cacheSet) { + cacheSet = new Set(); + toCache.set(blockeeId, cacheSet); + } + cacheSet.add(blockerId); + + // Queue for return + let returnSet = blockers.get(blockeeId); + if (!returnSet) { + returnSet = new Set(); + blockers.set(blockeeId, returnSet); + } + returnSet.add(blockerId); + } + + // Update cache to speed up future calls + await this.userBlockedCache.setMany(toCache.entries()); + } + + return blockers; + } + @bindThis public dispose(): void { this.internalEventService.off('userChangeSuspendedState', this.onUserEvent); From b7624666d65d96e9b9db62439a164e9625883c55 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 6 Jun 2025 02:13:53 -0400 Subject: [PATCH 07/54] implement QuantumKVCache.add and QuantumKVCache.addMany --- packages/backend/src/misc/cache.ts | 22 +++++++++ packages/backend/test/unit/misc/cache.ts | 62 ++++++++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/packages/backend/src/misc/cache.ts b/packages/backend/src/misc/cache.ts index 22201e243f..3145550a44 100644 --- a/packages/backend/src/misc/cache.ts +++ b/packages/backend/src/misc/cache.ts @@ -565,6 +565,28 @@ export class QuantumKVCache implements Iterable<[key: string, value: T]> { } } + /** + * Adds a value to the local memory cache without notifying other process. + * Neither a Redis event nor onSet callback will be fired, as the value has not actually changed. + * This should only be used when the value is known to be current, like after fetching from the database. + */ + @bindThis + public add(key: string, value: T): void { + this.memoryCache.set(key, value); + } + + /** + * Adds multiple values to the local memory cache without notifying other process. + * Neither a Redis event nor onSet callback will be fired, as the value has not actually changed. + * This should only be used when the value is known to be current, like after fetching from the database. + */ + @bindThis + public addMany(items: Iterable<[key: string, value: T]>): void { + for (const [key, value] of items) { + this.memoryCache.set(key, value); + } + } + /** * Gets a value from the local memory cache, or returns undefined if not found. */ diff --git a/packages/backend/test/unit/misc/cache.ts b/packages/backend/test/unit/misc/cache.ts index 0b658618e6..e24f6d4dcc 100644 --- a/packages/backend/test/unit/misc/cache.ts +++ b/packages/backend/test/unit/misc/cache.ts @@ -417,6 +417,68 @@ describe(QuantumKVCache, () => { }); }); + describe('add', () => { + it('should add the item', () => { + const cache = makeCache(); + cache.add('foo', 'bar'); + expect(cache.has('foo')).toBe(true); + }); + + it('should not emit event', () => { + const cache = makeCache({ + name: 'fake', + }); + + cache.add('foo', 'bar'); + + expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0); + }); + + it('should not call onSet', () => { + const fakeOnSet = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + onSet: fakeOnSet, + }); + + cache.add('foo', 'bar'); + + expect(fakeOnSet).not.toHaveBeenCalled(); + }); + }); + + describe('addMany', () => { + it('should add all items', () => { + const cache = makeCache(); + + cache.addMany([['foo', 'bar'], ['alpha', 'omega']]); + + expect(cache.has('foo')).toBe(true); + expect(cache.has('alpha')).toBe(true); + }); + + + it('should not emit event', () => { + const cache = makeCache({ + name: 'fake', + }); + + cache.addMany([['foo', 'bar'], ['alpha', 'omega']]); + + expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0); + }); + + it('should not call onSet', () => { + const fakeOnSet = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + onSet: fakeOnSet, + }); + + cache.addMany([['foo', 'bar'], ['alpha', 'omega']]); + + expect(fakeOnSet).not.toHaveBeenCalled(); + }); + }); + describe('has', () => { it('should return false when empty', () => { const cache = makeCache(); From 9853a4f3bdf39e2e7b05765a5099e4d2554b1d2c Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 6 Jun 2025 02:14:20 -0400 Subject: [PATCH 08/54] use addMany instead of setMany when populating quantum caches from DB --- packages/backend/src/core/CacheService.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/core/CacheService.ts b/packages/backend/src/core/CacheService.ts index 6e979130a0..00d97c7e1d 100644 --- a/packages/backend/src/core/CacheService.ts +++ b/packages/backend/src/core/CacheService.ts @@ -342,7 +342,7 @@ export class CacheService implements OnApplicationShutdown { } // Update cache to speed up future calls - await this.userFollowingsCache.setMany(toCache.entries()); + this.userFollowingsCache.addMany(toCache); } return followings; @@ -395,7 +395,7 @@ export class CacheService implements OnApplicationShutdown { } // Update cache to speed up future calls - await this.userBlockedCache.setMany(toCache.entries()); + this.userBlockedCache.addMany(toCache); } return blockers; From 3d13860ec8fea79062ba0c498302cb19143e01f8 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 6 Jun 2025 02:15:31 -0400 Subject: [PATCH 09/54] update quantum caches when a user is deleted --- packages/backend/src/core/CacheService.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/core/CacheService.ts b/packages/backend/src/core/CacheService.ts index 00d97c7e1d..1fba4f32d0 100644 --- a/packages/backend/src/core/CacheService.ts +++ b/packages/backend/src/core/CacheService.ts @@ -138,7 +138,7 @@ export class CacheService implements OnApplicationShutdown { } @bindThis - private async onUserEvent(body: InternalEventTypes[E]): Promise { + private async onUserEvent(body: InternalEventTypes[E], _: E, isLocal: boolean): Promise { { { { @@ -151,6 +151,16 @@ export class CacheService implements OnApplicationShutdown { this.uriPersonCache.delete(k); } } + if (isLocal) { + await Promise.all([ + this.userProfileCache.delete(body.id), + this.userMutingsCache.delete(body.id), + this.userBlockingCache.delete(body.id), + this.userBlockedCache.delete(body.id), + this.renoteMutingsCache.delete(body.id), + this.userFollowingsCache.delete(body.id), + ]); + } } else { this.userByIdCache.set(user.id, user); for (const [k, v] of this.uriPersonCache.entries) { From 68b84b28dd23cd1afea286946c1ada7efc25fed0 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 6 Jun 2025 02:15:59 -0400 Subject: [PATCH 10/54] implement CacheService.getUsers and CacheService.getUserProfiles --- packages/backend/src/core/CacheService.ts | 56 +++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/packages/backend/src/core/CacheService.ts b/packages/backend/src/core/CacheService.ts index 1fba4f32d0..ae24a9721f 100644 --- a/packages/backend/src/core/CacheService.ts +++ b/packages/backend/src/core/CacheService.ts @@ -411,6 +411,62 @@ export class CacheService implements OnApplicationShutdown { return blockers; } + public async getUserProfiles(userIds: Iterable): Promise> { + const profiles = new Map; + + const toFetch: string[] = []; + for (const userId of userIds) { + const fromCache = this.userProfileCache.get(userId); + if (fromCache) { + profiles.set(userId, fromCache); + } else { + toFetch.push(userId); + } + } + + if (toFetch.length > 0) { + const fetched = await this.userProfilesRepository.findBy({ + userId: In(toFetch), + }); + + for (const profile of fetched) { + profiles.set(profile.userId, profile); + } + + const toCache = new Map(fetched.map(p => [p.userId, p])); + this.userProfileCache.addMany(toCache); + } + + return profiles; + } + + public async getUsers(userIds: Iterable): Promise> { + const users = new Map; + + const toFetch: string[] = []; + for (const userId of userIds) { + const fromCache = this.userByIdCache.get(userId); + if (fromCache) { + users.set(userId, fromCache); + } else { + toFetch.push(userId); + } + } + + if (toFetch.length > 0) { + const fetched = await this.usersRepository.findBy({ + id: In(toFetch), + }); + + for (const user of fetched) { + users.set(user.id, user); + this.userByIdCache.set(user.id, user); + } + } + + return users; + } + @bindThis public dispose(): void { this.internalEventService.off('userChangeSuspendedState', this.onUserEvent); From 5e7d0e9acc0762261c71734a61cd0788a63ae246 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 6 Jun 2025 02:32:32 -0400 Subject: [PATCH 11/54] fix typo in QueryService.generateBlockQueryForUsers --- packages/backend/src/core/QueryService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/core/QueryService.ts b/packages/backend/src/core/QueryService.ts index 4089fc080c..f9e4804e14 100644 --- a/packages/backend/src/core/QueryService.ts +++ b/packages/backend/src/core/QueryService.ts @@ -94,7 +94,7 @@ export class QueryService { @bindThis public generateBlockQueryForUsers(q: SelectQueryBuilder, me: { id: MiUser['id'] }): SelectQueryBuilder { this.andNotBlockingUser(q, ':meId', 'user.id'); - this.andNotBlockingUser(q, 'user.id', ':me.id'); + this.andNotBlockingUser(q, 'user.id', ':meId'); return q.setParameters({ meId: me.id }); } From bd8cd8c4e439e39b8253f9e63131eb50b82469c5 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 6 Jun 2025 02:33:38 -0400 Subject: [PATCH 12/54] use more bulk hints in NoteEntityService / UserEntityService, and run the packMany queries in parallel --- packages/backend/src/core/ReactionService.ts | 2 +- .../src/core/activitypub/ApInboxService.ts | 2 +- .../src/core/entities/NoteEntityService.ts | 270 +++++++++++----- .../src/core/entities/UserEntityService.ts | 301 ++++++++++-------- .../src/server/api/endpoints/notes/create.ts | 2 +- .../src/server/api/endpoints/notes/edit.ts | 2 +- .../server/api/endpoints/notes/translate.ts | 2 +- 7 files changed, 363 insertions(+), 218 deletions(-) diff --git a/packages/backend/src/core/ReactionService.ts b/packages/backend/src/core/ReactionService.ts index c23bb51178..8d2dc7d4e8 100644 --- a/packages/backend/src/core/ReactionService.ts +++ b/packages/backend/src/core/ReactionService.ts @@ -122,7 +122,7 @@ export class ReactionService { } // check visibility - if (!await this.noteEntityService.isVisibleForMe(note, user.id)) { + if (!await this.noteEntityService.isVisibleForMe(note, user.id, { me: user })) { throw new IdentifiableError('68e9d2d1-48bf-42c2-b90a-b20e09fd3d48', 'Note not accessible for you.'); } diff --git a/packages/backend/src/core/activitypub/ApInboxService.ts b/packages/backend/src/core/activitypub/ApInboxService.ts index b384ec58c5..7c26deb00f 100644 --- a/packages/backend/src/core/activitypub/ApInboxService.ts +++ b/packages/backend/src/core/activitypub/ApInboxService.ts @@ -365,7 +365,7 @@ export class ApInboxService { const renote = await this.apNoteService.resolveNote(target, { resolver, sentFrom: getApId(target) }); if (renote == null) return 'announce target is null'; - if (!await this.noteEntityService.isVisibleForMe(renote, actor.id)) { + if (!await this.noteEntityService.isVisibleForMe(renote, actor.id, { me: actor })) { return 'skip: invalid actor for this activity'; } diff --git a/packages/backend/src/core/entities/NoteEntityService.ts b/packages/backend/src/core/entities/NoteEntityService.ts index 2a93b678dc..7ddf0abc3e 100644 --- a/packages/backend/src/core/entities/NoteEntityService.ts +++ b/packages/backend/src/core/entities/NoteEntityService.ts @@ -11,7 +11,7 @@ import type { Packed } from '@/misc/json-schema.js'; import { awaitAll } from '@/misc/prelude/await-all.js'; import type { MiUser } from '@/models/User.js'; import type { MiNote } from '@/models/Note.js'; -import type { UsersRepository, NotesRepository, FollowingsRepository, PollsRepository, PollVotesRepository, NoteReactionsRepository, ChannelsRepository, MiMeta } from '@/models/_.js'; +import type { UsersRepository, NotesRepository, FollowingsRepository, PollsRepository, PollVotesRepository, NoteReactionsRepository, ChannelsRepository, MiMeta, MiPollVote, MiPoll, MiChannel } from '@/models/_.js'; import { bindThis } from '@/decorators.js'; import { DebounceLoader } from '@/misc/loader.js'; import { IdService } from '@/core/IdService.js'; @@ -26,13 +26,13 @@ import type { UserEntityService } from './UserEntityService.js'; import type { DriveFileEntityService } from './DriveFileEntityService.js'; // is-renote.tsとよしなにリンク -function isPureRenote(note: MiNote): note is MiNote & { renoteId: MiNote['id']; renote: MiNote } { +function isPureRenote(note: MiNote): note is MiNote & { renoteId: MiNote['id'] } { return ( - note.renote != null && - note.reply == null && + note.renoteId != null && + note.replyId == null && note.text == null && note.cw == null && - (note.fileIds == null || note.fileIds.length === 0) && + note.fileIds.length === 0 && !note.hasPoll ); } @@ -132,7 +132,10 @@ export class NoteEntityService implements OnModuleInit { } @bindThis - public async hideNote(packedNote: Packed<'Note'>, meId: MiUser['id'] | null): Promise { + public async hideNote(packedNote: Packed<'Note'>, meId: MiUser['id'] | null, hint?: { + myFollowing?: ReadonlySet, + myBlockers?: ReadonlySet, + }): Promise { if (meId === packedNote.userId) return; // TODO: isVisibleForMe を使うようにしても良さそう(型違うけど) @@ -188,14 +191,9 @@ export class NoteEntityService implements OnModuleInit { } else if (packedNote.renote && (meId === packedNote.renote.userId)) { hide = false; } else { - // フォロワーかどうか - // TODO: 当関数呼び出しごとにクエリが走るのは重そうだからなんとかする - const isFollowing = await this.followingsRepository.exists({ - where: { - followeeId: packedNote.userId, - followerId: meId, - }, - }); + const isFollowing = hint?.myFollowing + ? hint.myFollowing.has(packedNote.userId) + : (await this.cacheService.userFollowingsCache.fetch(meId))[packedNote.userId] != null; hide = !isFollowing; } @@ -211,7 +209,8 @@ export class NoteEntityService implements OnModuleInit { } if (!hide && meId && packedNote.userId !== meId) { - const isBlocked = (await this.cacheService.userBlockedCache.fetch(meId)).has(packedNote.userId); + const blockers = hint?.myBlockers ?? await this.cacheService.userBlockedCache.fetch(meId); + const isBlocked = blockers.has(packedNote.userId); if (isBlocked) hide = true; } @@ -235,8 +234,11 @@ export class NoteEntityService implements OnModuleInit { } @bindThis - private async populatePoll(note: MiNote, meId: MiUser['id'] | null) { - const poll = await this.pollsRepository.findOneByOrFail({ noteId: note.id }); + private async populatePoll(note: MiNote, meId: MiUser['id'] | null, hint?: { + poll?: MiPoll, + myVotes?: MiPollVote[], + }) { + const poll = hint?.poll ?? await this.pollsRepository.findOneByOrFail({ noteId: note.id }); const choices = poll.choices.map(c => ({ text: c, votes: poll.votes[poll.choices.indexOf(c)], @@ -245,7 +247,7 @@ export class NoteEntityService implements OnModuleInit { if (meId) { if (poll.multiple) { - const votes = await this.pollVotesRepository.findBy({ + const votes = hint?.myVotes ?? await this.pollVotesRepository.findBy({ userId: meId, noteId: note.id, }); @@ -255,7 +257,7 @@ export class NoteEntityService implements OnModuleInit { choices[myChoice].isVoted = true; } } else { - const vote = await this.pollVotesRepository.findOneBy({ + const vote = hint?.myVotes ? hint.myVotes[0] : await this.pollVotesRepository.findOneBy({ userId: meId, noteId: note.id, }); @@ -317,7 +319,12 @@ export class NoteEntityService implements OnModuleInit { } @bindThis - public async isVisibleForMe(note: MiNote, meId: MiUser['id'] | null): Promise { + public async isVisibleForMe(note: MiNote, meId: MiUser['id'] | null, hint?: { + myFollowing?: ReadonlySet, + myBlocking?: ReadonlySet, + myBlockers?: ReadonlySet, + me?: Pick | null, + }): Promise { // This code must always be synchronized with the checks in generateVisibilityQuery. // visibility が specified かつ自分が指定されていなかったら非表示 if (note.visibility === 'specified') { @@ -345,16 +352,20 @@ export class NoteEntityService implements OnModuleInit { return true; } else { // フォロワーかどうか - const [blocked, following, user] = await Promise.all([ - this.cacheService.userBlockingCache.fetch(meId).then((ids) => ids.has(note.userId)), - this.followingsRepository.count({ - where: { + const [blocked, following, userHost] = await Promise.all([ + hint?.myBlocking + ? hint.myBlocking.has(note.userId) + : this.cacheService.userBlockingCache.fetch(meId).then((ids) => ids.has(note.userId)), + hint?.myFollowing + ? hint.myFollowing.has(note.userId) + : this.followingsRepository.existsBy({ followeeId: note.userId, followerId: meId, - }, - take: 1, - }), - this.usersRepository.findOneByOrFail({ id: meId }), + }), + hint?.me !== undefined + ? (hint.me?.host ?? null) + : this.cacheService.userByIdCache.fetch(meId, () => this.usersRepository.findOneByOrFail({ id: meId })) + .then(me => me.host), ]); if (blocked) return false; @@ -366,12 +377,13 @@ export class NoteEntityService implements OnModuleInit { in which case we can never know the following. Instead we have to assume that the users are following each other. */ - return following > 0 || (note.userHost != null && user.host != null); + return following || (note.userHost != null && userHost != null); } } if (meId != null) { - const isBlocked = (await this.cacheService.userBlockedCache.fetch(meId)).has(note.userId); + const blockers = hint?.myBlockers ?? await this.cacheService.userBlockedCache.fetch(meId); + const isBlocked = blockers.has(note.userId); if (isBlocked) return false; } @@ -408,6 +420,11 @@ export class NoteEntityService implements OnModuleInit { packedFiles: Map | null>; packedUsers: Map>; mentionHandles: Record; + userFollowings: Map>; + userBlockers: Map>; + polls: Map; + pollVotes: Map>; + channels: Map; }; }, ): Promise> { @@ -437,9 +454,7 @@ export class NoteEntityService implements OnModuleInit { } const channel = note.channelId - ? note.channel - ? note.channel - : await this.channelsRepository.findOneBy({ id: note.channelId }) + ? (opts._hint_?.channels.get(note.channelId) ?? note.channel ?? await this.channelsRepository.findOneBy({ id: note.channelId })) : null; const reactionEmojiNames = Object.keys(reactions) @@ -485,7 +500,10 @@ export class NoteEntityService implements OnModuleInit { mentionHandles: note.mentions.length > 0 ? this.getUserHandles(note.mentions, options?._hint_?.mentionHandles) : undefined, uri: note.uri ?? undefined, url: note.url ?? undefined, - poll: note.hasPoll ? this.populatePoll(note, meId) : undefined, + poll: note.hasPoll ? this.populatePoll(note, meId, { + poll: opts._hint_?.polls.get(note.id), + myVotes: opts._hint_?.pollVotes.get(note.id)?.get(note.userId), + }) : undefined, ...(meId && Object.keys(reactions).length > 0 ? { myReaction: this.populateMyReaction({ @@ -518,7 +536,10 @@ export class NoteEntityService implements OnModuleInit { this.treatVisibility(packed); if (!opts.skipHide) { - await this.hideNote(packed, meId); + await this.hideNote(packed, meId, meId == null ? undefined : { + myFollowing: opts._hint_?.userFollowings.get(meId), + myBlockers: opts._hint_?.userBlockers.get(meId), + }); } return packed; @@ -536,69 +557,53 @@ export class NoteEntityService implements OnModuleInit { if (notes.length === 0) return []; const targetNotes: MiNote[] = []; + const targetNotesToFetch: string[] = []; for (const note of notes) { if (isPureRenote(note)) { // we may need to fetch 'my reaction' for renote target. - targetNotes.push(note.renote); - if (note.renote.reply) { - // idem if the renote is also a reply. - targetNotes.push(note.renote.reply); + if (note.renote) { + targetNotes.push(note.renote); + if (note.renote.reply) { + // idem if the renote is also a reply. + targetNotes.push(note.renote.reply); + } + } else if (options?.detail) { + targetNotesToFetch.push(note.renoteId); } } else { if (note.reply) { // idem for OP of a regular reply. targetNotes.push(note.reply); + } else if (note.replyId && options?.detail) { + targetNotesToFetch.push(note.replyId); } targetNotes.push(note); } } - const bufferedReactions = this.meta.enableReactionsBuffering ? await this.reactionsBufferingService.getMany([...getAppearNoteIds(notes)]) : null; - - const meId = me ? me.id : null; - const myReactionsMap = new Map(); - if (meId) { - const idsNeedFetchMyReaction = new Set(); - - for (const note of targetNotes) { - const reactionsCount = Object.values(this.reactionsBufferingService.mergeReactions(note.reactions, bufferedReactions?.get(note.id)?.deltas ?? {})).reduce((a, b) => a + b, 0); - if (reactionsCount === 0) { - myReactionsMap.set(note.id, null); - } else if (reactionsCount <= note.reactionAndUserPairCache.length + (bufferedReactions?.get(note.id)?.pairs.length ?? 0)) { - const pairInBuffer = bufferedReactions?.get(note.id)?.pairs.find(p => p[0] === meId); - if (pairInBuffer) { - myReactionsMap.set(note.id, pairInBuffer[1]); - } else { - const pair = note.reactionAndUserPairCache.find(p => p.startsWith(meId)); - myReactionsMap.set(note.id, pair ? pair.split('/')[1] : null); - } - } else { - idsNeedFetchMyReaction.add(note.id); - } - } - - const myReactions = idsNeedFetchMyReaction.size > 0 ? await this.noteReactionsRepository.findBy({ - userId: meId, - noteId: In(Array.from(idsNeedFetchMyReaction)), - }) : []; - - for (const id of idsNeedFetchMyReaction) { - myReactionsMap.set(id, myReactions.find(reaction => reaction.noteId === id)?.reaction ?? null); - } + // Populate any relations that weren't included in the source + if (targetNotesToFetch.length > 0) { + const newNotes = await this.notesRepository.find({ + where: { + id: In(targetNotesToFetch), + }, + relations: { + user: true, + reply: true, + renote: true, + channel: true, + }, + }); + targetNotes.push(...newNotes); } - await this.customEmojiService.prefetchEmojis(this.aggregateNoteEmojis(notes)); - // TODO: 本当は renote とか reply がないのに renoteId とか replyId があったらここで解決しておく const fileIds = notes.map(n => [n.fileIds, n.renote?.fileIds, n.reply?.fileIds]).flat(2).filter(x => x != null); - const packedFiles = fileIds.length > 0 ? await this.driveFileEntityService.packManyByIdsMap(fileIds) : new Map(); const users = [ ...notes.map(({ user, userId }) => user ?? userId), - ...notes.map(({ replyUserId }) => replyUserId).filter(x => x != null), - ...notes.map(({ renoteUserId }) => renoteUserId).filter(x => x != null), + ...notes.map(({ reply, replyUserId }) => reply?.user ?? replyUserId).filter(x => x != null), + ...notes.map(({ renote, renoteUserId }) => renote?.user ?? renoteUserId).filter(x => x != null), ]; - const packedUsers = await this.userEntityService.packMany(users, me) - .then(users => new Map(users.map(u => [u.id, u]))); // Recursively add all mentioned users from all notes + replies + renotes const allMentionedUsers = targetNotes.reduce((users, note) => { @@ -607,7 +612,47 @@ export class NoteEntityService implements OnModuleInit { } return users; }, new Set()); - const mentionHandles = await this.getUserHandles(Array.from(allMentionedUsers)); + + const userIds = Array.from(new Set(users.map(u => typeof(u) === 'string' ? u : u.id))); + const noteIds = Array.from(new Set(targetNotes.map(n => n.id))); + const [{ bufferedReactions, myReactionsMap }, packedFiles, packedUsers, mentionHandles, userFollowings, userBlockers, polls, pollVotes, channels] = await Promise.all([ + // bufferedReactions & myReactionsMap + this.getReactions(targetNotes, me), + // packedFiles + this.driveFileEntityService.packManyByIdsMap(fileIds), + // packedUsers + this.userEntityService.packMany(users, me) + .then(users => new Map(users.map(u => [u.id, u]))), + // mentionHandles + this.getUserHandles(Array.from(allMentionedUsers)), + // userFollowings + this.cacheService.getUserFollowings(userIds), + // userBlockers + this.cacheService.getUserBlockers(userIds), + // polls + this.pollsRepository.findBy({ noteId: In(noteIds) }) + .then(polls => new Map(polls.map(p => [p.noteId, p]))), + // pollVotes + this.pollVotesRepository.findBy({ noteId: In(noteIds), userId: In(userIds) }) + .then(votes => votes.reduce((noteMap, vote) => { + let userMap = noteMap.get(vote.noteId); + if (!userMap) { + userMap = new Map(); + noteMap.set(vote.noteId, userMap); + } + let voteList = userMap.get(vote.userId); + if (!voteList) { + voteList = []; + userMap.set(vote.userId, voteList); + } + voteList.push(vote); + return noteMap; + }, new Map>)), + // channels + this.getChannels(targetNotes), + // (not returned) + this.customEmojiService.prefetchEmojis(this.aggregateNoteEmojis(notes)), + ]); return await Promise.all(notes.map(n => this.pack(n, me, { ...options, @@ -617,6 +662,11 @@ export class NoteEntityService implements OnModuleInit { packedFiles, packedUsers, mentionHandles, + userFollowings, + userBlockers, + polls, + pollVotes, + channels, }, }))); } @@ -685,6 +735,68 @@ export class NoteEntityService implements OnModuleInit { }, {} as Record); } + private async getChannels(notes: MiNote[]): Promise> { + const channels = new Map(); + const channelsToFetch = new Set(); + + for (const note of notes) { + if (note.channel) { + channels.set(note.channel.id, note.channel); + } else if (note.channelId) { + channelsToFetch.add(note.channelId); + } + } + + if (channelsToFetch.size > 0) { + const newChannels = await this.channelsRepository.findBy({ + id: In(Array.from(channelsToFetch)), + }); + for (const channel of newChannels) { + channels.set(channel.id, channel); + } + } + + return channels; + } + + private async getReactions(notes: MiNote[], me: { id: string } | null | undefined) { + const bufferedReactions = this.meta.enableReactionsBuffering ? await this.reactionsBufferingService.getMany([...getAppearNoteIds(notes)]) : null; + + const meId = me ? me.id : null; + const myReactionsMap = new Map(); + if (meId) { + const idsNeedFetchMyReaction = new Set(); + + for (const note of notes) { + const reactionsCount = Object.values(this.reactionsBufferingService.mergeReactions(note.reactions, bufferedReactions?.get(note.id)?.deltas ?? {})).reduce((a, b) => a + b, 0); + if (reactionsCount === 0) { + myReactionsMap.set(note.id, null); + } else if (reactionsCount <= note.reactionAndUserPairCache.length + (bufferedReactions?.get(note.id)?.pairs.length ?? 0)) { + const pairInBuffer = bufferedReactions?.get(note.id)?.pairs.find(p => p[0] === meId); + if (pairInBuffer) { + myReactionsMap.set(note.id, pairInBuffer[1]); + } else { + const pair = note.reactionAndUserPairCache.find(p => p.startsWith(meId)); + myReactionsMap.set(note.id, pair ? pair.split('/')[1] : null); + } + } else { + idsNeedFetchMyReaction.add(note.id); + } + } + + const myReactions = idsNeedFetchMyReaction.size > 0 ? await this.noteReactionsRepository.findBy({ + userId: meId, + noteId: In(Array.from(idsNeedFetchMyReaction)), + }) : []; + + for (const id of idsNeedFetchMyReaction) { + myReactionsMap.set(id, myReactions.find(reaction => reaction.noteId === id)?.reaction ?? null); + } + } + + return { bufferedReactions, myReactionsMap }; + } + @bindThis public genLocalNoteUri(noteId: string): string { return `${this.config.url}/notes/${noteId}`; diff --git a/packages/backend/src/core/entities/UserEntityService.ts b/packages/backend/src/core/entities/UserEntityService.ts index 326baaefd4..7e3a55e155 100644 --- a/packages/backend/src/core/entities/UserEntityService.ts +++ b/packages/backend/src/core/entities/UserEntityService.ts @@ -30,6 +30,7 @@ import type { FollowingsRepository, FollowRequestsRepository, MiFollowing, + MiInstance, MiMeta, MiUserNotePining, MiUserProfile, @@ -42,7 +43,7 @@ import type { UsersRepository, } from '@/models/_.js'; import { bindThis } from '@/decorators.js'; -import { RoleService } from '@/core/RoleService.js'; +import { RolePolicies, RoleService } from '@/core/RoleService.js'; import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; import { IdService } from '@/core/IdService.js'; @@ -52,6 +53,7 @@ import { AvatarDecorationService } from '@/core/AvatarDecorationService.js'; import { ChatService } from '@/core/ChatService.js'; import { isSystemAccount } from '@/misc/is-system-account.js'; import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js'; +import type { CacheService } from '@/core/CacheService.js'; import type { OnModuleInit } from '@nestjs/common'; import type { NoteEntityService } from './NoteEntityService.js'; import type { PageEntityService } from './PageEntityService.js'; @@ -103,6 +105,7 @@ export class UserEntityService implements OnModuleInit { private idService: IdService; private avatarDecorationService: AvatarDecorationService; private chatService: ChatService; + private cacheService: CacheService; constructor( private moduleRef: ModuleRef, @@ -163,6 +166,7 @@ export class UserEntityService implements OnModuleInit { this.idService = this.moduleRef.get('IdService'); this.avatarDecorationService = this.moduleRef.get('AvatarDecorationService'); this.chatService = this.moduleRef.get('ChatService'); + this.cacheService = this.moduleRef.get('CacheService'); } //#region Validators @@ -215,45 +219,23 @@ export class UserEntityService implements OnModuleInit { followeeId: me, }, }), - this.blockingsRepository.exists({ - where: { - blockerId: me, - blockeeId: target, - }, - }), - this.blockingsRepository.exists({ - where: { - blockerId: target, - blockeeId: me, - }, - }), - this.mutingsRepository.exists({ - where: { - muterId: me, - muteeId: target, - }, - }), - this.renoteMutingsRepository.exists({ - where: { - muterId: me, - muteeId: target, - }, - }), - this.usersRepository.createQueryBuilder('u') - .select('u.host') - .where({ id: target }) - .getRawOne<{ u_host: string }>() - .then(it => it?.u_host ?? null), + this.cacheService.userBlockedCache.fetch(me) + .then(blockers => blockers.size > 0), + this.cacheService.userBlockingCache.fetch(me) + .then(blockees => blockees.size > 0), + this.cacheService.userMutingsCache.fetch(me) + .then(mutings => mutings.size > 0), + this.cacheService.renoteMutingsCache.fetch(me) + .then(mutings => mutings.size > 0), + this.cacheService.userByIdCache.fetch(target, () => this.usersRepository.findOneByOrFail({ id: target })) + .then(user => user.host), this.userMemosRepository.createQueryBuilder('m') .select('m.memo') .where({ userId: me, targetUserId: target }) .getRawOne<{ m_memo: string | null }>() .then(it => it?.m_memo ?? null), - this.userProfilesRepository.createQueryBuilder('p') - .select('p.mutedInstances') - .where({ userId: me }) - .getRawOne<{ p_mutedInstances: string[] }>() - .then(it => it?.p_mutedInstances ?? []), + this.cacheService.userProfileCache.fetch(me) + .then(profile => profile.mutedInstances), ]); const isInstanceMuted = !!host && mutedInstances.includes(host); @@ -306,34 +288,18 @@ export class UserEntityService implements OnModuleInit { .where('f.followeeId = :me', { me }) .getRawMany<{ f_followerId: string }>() .then(it => it.map(it => it.f_followerId)), - this.blockingsRepository.createQueryBuilder('b') - .select('b.blockeeId') - .where('b.blockerId = :me', { me }) - .getRawMany<{ b_blockeeId: string }>() - .then(it => it.map(it => it.b_blockeeId)), - this.blockingsRepository.createQueryBuilder('b') - .select('b.blockerId') - .where('b.blockeeId = :me', { me }) - .getRawMany<{ b_blockerId: string }>() - .then(it => it.map(it => it.b_blockerId)), - this.mutingsRepository.createQueryBuilder('m') - .select('m.muteeId') - .where('m.muterId = :me', { me }) - .getRawMany<{ m_muteeId: string }>() - .then(it => it.map(it => it.m_muteeId)), - this.renoteMutingsRepository.createQueryBuilder('m') - .select('m.muteeId') - .where('m.muterId = :me', { me }) - .getRawMany<{ m_muteeId: string }>() - .then(it => it.map(it => it.m_muteeId)), - this.usersRepository.createQueryBuilder('u') - .select(['u.id', 'u.host']) - .where({ id: In(targets) } ) - .getRawMany<{ m_id: string, m_host: string }>() - .then(it => it.reduce((map, it) => { - map[it.m_id] = it.m_host; - return map; - }, {} as Record)), + this.cacheService.userBlockedCache.fetch(me), + this.cacheService.userBlockingCache.fetch(me), + this.cacheService.userMutingsCache.fetch(me), + this.cacheService.renoteMutingsCache.fetch(me), + this.cacheService.getUsers(targets) + .then(users => { + const record: Record = {}; + for (const [id, user] of users) { + record[id] = user.host; + } + return record; + }), this.userMemosRepository.createQueryBuilder('m') .select(['m.targetUserId', 'm.memo']) .where({ userId: me, targetUserId: In(targets) }) @@ -342,11 +308,8 @@ export class UserEntityService implements OnModuleInit { map[it.m_targetUserId] = it.m_memo; return map; }, {} as Record)), - this.userProfilesRepository.createQueryBuilder('p') - .select('p.mutedInstances') - .where({ userId: me }) - .getRawOne<{ p_mutedInstances: string[] }>() - .then(it => it?.p_mutedInstances ?? []), + this.cacheService.userProfileCache.fetch(me) + .then(p => p.mutedInstances), ]); return new Map( @@ -362,11 +325,11 @@ export class UserEntityService implements OnModuleInit { isFollowed: followees.includes(target), hasPendingFollowRequestFromYou: followersRequests.includes(target), hasPendingFollowRequestToYou: followeesRequests.includes(target), - isBlocking: blockers.includes(target), - isBlocked: blockees.includes(target), - isMuted: muters.includes(target), - isRenoteMuted: renoteMuters.includes(target), - isInstanceMuted: mutedInstances.includes(hosts[target]), + isBlocking: blockers.has(target), + isBlocked: blockees.has(target), + isMuted: muters.has(target), + isRenoteMuted: renoteMuters.has(target), + isInstanceMuted: hosts[target] != null && mutedInstances.includes(hosts[target]), memo: memos[target] ?? null, }, ]; @@ -391,6 +354,7 @@ export class UserEntityService implements OnModuleInit { return false; // TODO } + // TODO make redis calls in MULTI? @bindThis public async getNotificationsInfo(userId: MiUser['id']): Promise<{ hasUnread: boolean; @@ -424,16 +388,14 @@ export class UserEntityService implements OnModuleInit { @bindThis public async getHasPendingReceivedFollowRequest(userId: MiUser['id']): Promise { - const count = await this.followRequestsRepository.countBy({ + return await this.followRequestsRepository.existsBy({ followeeId: userId, }); - - return count > 0; } @bindThis public async getHasPendingSentFollowRequest(userId: MiUser['id']): Promise { - return this.followRequestsRepository.existsBy({ + return await this.followRequestsRepository.existsBy({ followerId: userId, }); } @@ -480,6 +442,12 @@ export class UserEntityService implements OnModuleInit { userRelations?: Map, userMemos?: Map, pinNotes?: Map, + iAmModerator?: boolean, + userIdsByUri?: Map, + instances?: Map, + securityKeyCounts?: Map, + pendingReceivedFollows?: Set, + pendingSentFollows?: Set, }, ): Promise> { const opts = Object.assign({ @@ -521,7 +489,7 @@ export class UserEntityService implements OnModuleInit { const isDetailed = opts.schema !== 'UserLite'; const meId = me ? me.id : null; const isMe = meId === user.id; - const iAmModerator = me ? await this.roleService.isModerator(me as MiUser) : false; + const iAmModerator = opts.iAmModerator ?? (me ? await this.roleService.isModerator(me as MiUser) : false); const profile = isDetailed ? (opts.userProfile ?? user.userProfile ?? await this.userProfilesRepository.findOneByOrFail({ userId: user.id })) @@ -582,6 +550,9 @@ export class UserEntityService implements OnModuleInit { const checkHost = user.host == null ? this.config.host : user.host; const notificationsInfo = isMe && isDetailed ? await this.getNotificationsInfo(user.id) : null; + let fetchPoliciesPromise: Promise | null = null; + const fetchPolicies = () => fetchPoliciesPromise ??= this.roleService.getUserPolicies(user); + const packed = { id: user.id, name: user.name, @@ -607,13 +578,13 @@ export class UserEntityService implements OnModuleInit { mandatoryCW: user.mandatoryCW, rejectQuotes: user.rejectQuotes, attributionDomains: user.attributionDomains, - isSilenced: user.isSilenced || this.roleService.getUserPolicies(user.id).then(r => !r.canPublicNote), + isSilenced: user.isSilenced || fetchPolicies().then(r => !r.canPublicNote), speakAsCat: user.speakAsCat ?? false, approved: user.approved, requireSigninToViewContents: user.requireSigninToViewContents === false ? undefined : true, makeNotesFollowersOnlyBefore: user.makeNotesFollowersOnlyBefore ?? undefined, makeNotesHiddenBefore: user.makeNotesHiddenBefore ?? undefined, - instance: user.host ? this.federatedInstanceService.fetch(user.host).then(instance => instance ? { + instance: user.host ? Promise.resolve(opts.instances?.has(user.host) ? opts.instances.get(user.host) : this.federatedInstanceService.fetch(user.host)).then(instance => instance ? { name: instance.name, softwareName: instance.softwareName, softwareVersion: instance.softwareVersion, @@ -628,7 +599,7 @@ export class UserEntityService implements OnModuleInit { emojis: this.customEmojiService.populateEmojis(user.emojis, checkHost), onlineStatus: this.getOnlineStatus(user), // パフォーマンス上の理由でローカルユーザーのみ - badgeRoles: user.host == null ? this.roleService.getUserBadgeRoles(user.id).then((rs) => rs + badgeRoles: user.host == null ? this.roleService.getUserBadgeRoles(user).then((rs) => rs .filter((r) => r.isPublic || iAmModerator) .sort((a, b) => b.displayOrder - a.displayOrder) .map((r) => ({ @@ -641,9 +612,9 @@ export class UserEntityService implements OnModuleInit { ...(isDetailed ? { url: profile!.url, uri: user.uri, - movedTo: user.movedToUri ? this.apPersonService.resolvePerson(user.movedToUri).then(user => user.id).catch(() => null) : null, + movedTo: user.movedToUri ? Promise.resolve(opts.userIdsByUri?.get(user.movedToUri) ?? this.apPersonService.resolvePerson(user.movedToUri).then(user => user.id).catch(() => null)) : null, alsoKnownAs: user.alsoKnownAs - ? Promise.all(user.alsoKnownAs.map(uri => this.apPersonService.fetchPerson(uri).then(user => user?.id).catch(() => null))) + ? Promise.all(user.alsoKnownAs.map(uri => Promise.resolve(opts.userIdsByUri?.get(uri) ?? this.apPersonService.fetchPerson(uri).then(user => user?.id).catch(() => null)))) .then(xs => xs.length === 0 ? null : xs.filter(x => x != null)) : null, updatedAt: user.updatedAt ? user.updatedAt.toISOString() : null, @@ -670,8 +641,8 @@ export class UserEntityService implements OnModuleInit { followersVisibility: profile!.followersVisibility, followingVisibility: profile!.followingVisibility, chatScope: user.chatScope, - canChat: this.roleService.getUserPolicies(user.id).then(r => r.chatAvailability === 'available'), - roles: this.roleService.getUserRoles(user.id).then(roles => roles.filter(role => role.isPublic).sort((a, b) => b.displayOrder - a.displayOrder).map(role => ({ + canChat: fetchPolicies().then(r => r.chatAvailability === 'available'), + roles: this.roleService.getUserRoles(user).then(roles => roles.filter(role => role.isPublic).sort((a, b) => b.displayOrder - a.displayOrder).map(role => ({ id: role.id, name: role.name, color: role.color, @@ -689,7 +660,7 @@ export class UserEntityService implements OnModuleInit { twoFactorEnabled: profile!.twoFactorEnabled, usePasswordLessLogin: profile!.usePasswordLessLogin, securityKeys: profile!.twoFactorEnabled - ? this.userSecurityKeysRepository.countBy({ userId: user.id }).then(result => result >= 1) + ? Promise.resolve(opts.securityKeyCounts?.get(user.id) ?? this.userSecurityKeysRepository.countBy({ userId: user.id })).then(result => result >= 1) : false, } : {}), @@ -722,8 +693,8 @@ export class UserEntityService implements OnModuleInit { hasUnreadAntenna: this.getHasUnreadAntenna(user.id), hasUnreadChannel: false, // 後方互換性のため hasUnreadNotification: notificationsInfo?.hasUnread, // 後方互換性のため - hasPendingReceivedFollowRequest: this.getHasPendingReceivedFollowRequest(user.id), - hasPendingSentFollowRequest: this.getHasPendingSentFollowRequest(user.id), + hasPendingReceivedFollowRequest: opts.pendingReceivedFollows?.has(user.id) ?? this.getHasPendingReceivedFollowRequest(user.id), + hasPendingSentFollowRequest: opts.pendingSentFollows?.has(user.id) ?? this.getHasPendingSentFollowRequest(user.id), unreadNotificationsCount: notificationsInfo?.unreadCount, mutedWords: profile!.mutedWords, hardMutedWords: profile!.hardMutedWords, @@ -733,7 +704,7 @@ export class UserEntityService implements OnModuleInit { emailNotificationTypes: profile!.emailNotificationTypes, achievements: profile!.achievements, loggedInDays: profile!.loggedInDates.length, - policies: this.roleService.getUserPolicies(user.id), + policies: fetchPolicies(), defaultCW: profile!.defaultCW, defaultCWPriority: profile!.defaultCWPriority, allowUnsignedFetch: user.allowUnsignedFetch, @@ -783,6 +754,8 @@ export class UserEntityService implements OnModuleInit { includeSecrets?: boolean, }, ): Promise[]> { + if (users.length === 0) return []; + // -- IDのみの要素を補完して完全なエンティティ一覧を作る const _users = users.filter((user): user is MiUser => typeof user !== 'string'); @@ -800,57 +773,111 @@ export class UserEntityService implements OnModuleInit { } const _userIds = _users.map(u => u.id); - // -- 実行者の有無や指定スキーマの種別によって要否が異なる値群を取得 + const iAmModerator = await this.roleService.isModerator(me as MiUser); + const meId = me ? me.id : null; + const isMe = meId && _userIds.includes(meId); + const isDetailed = options && options.schema !== 'UserLite'; + const isDetailedAndMe = isDetailed && isMe; + const isDetailedAndMeOrMod = isDetailed && (isMe || iAmModerator); + const isDetailedAndNotMe = isDetailed && !isMe; - let profilesMap: Map = new Map(); - let userRelations: Map = new Map(); - let userMemos: Map = new Map(); - let pinNotes: Map = new Map(); + const userUris = new Set(_users + .flatMap(user => [user.uri, user.movedToUri]) + .filter((uri): uri is string => uri != null)); - if (options?.schema !== 'UserLite') { - const _profiles: MiUserProfile[] = []; - const _profilesToFetch: string[] = []; - for (const user of _users) { - if (user.userProfile) { - _profiles.push(user.userProfile); - } else { - _profilesToFetch.push(user.id); - } - } - if (_profilesToFetch.length > 0) { - const fetched = await this.userProfilesRepository.findBy({ userId: In(_profilesToFetch) }); - _profiles.push(...fetched); - } - profilesMap = new Map(_profiles.map(p => [p.userId, p])); + const userHosts = new Set(_users + .map(user => user.host) + .filter((host): host is string => host != null)); - const meId = me ? me.id : null; - if (meId) { - userMemos = await this.userMemosRepository.findBy({ userId: meId }) - .then(memos => new Map(memos.map(memo => [memo.targetUserId, memo.memo]))); - - if (_userIds.length > 0) { - userRelations = await this.getRelations(meId, _userIds); - pinNotes = await this.userNotePiningsRepository.createQueryBuilder('pin') - .where('pin.userId IN (:...userIds)', { userIds: _userIds }) - .innerJoinAndSelect('pin.note', 'note') - .getMany() - .then(pinsNotes => { - const map = new Map(); - for (const note of pinsNotes) { - const notes = map.get(note.userId) ?? []; - notes.push(note); - map.set(note.userId, notes); - } - for (const [, notes] of map.entries()) { - // pack側ではDESCで取得しているので、それに合わせて降順に並び替えておく - notes.sort((a, b) => b.id.localeCompare(a.id)); - } - return map; - }); - } + const _profiles: MiUserProfile[] = []; + const _profilesToFetch: string[] = []; + for (const user of _users) { + if (user.userProfile) { + _profiles.push(user.userProfile); + } else { + _profilesToFetch.push(user.id); } } + // -- 実行者の有無や指定スキーマの種別によって要否が異なる値群を取得 + + const [profilesMap, userMemos, userRelations, pinNotes, userIdsByUri, instances, securityKeyCounts, pendingReceivedFollows, pendingSentFollows] = await Promise.all([ + // profilesMap + this.cacheService.getUserProfiles(_profilesToFetch) + .then(profiles => { + for (const profile of _profiles) { + profiles.set(profile.userId, profile); + } + return profiles; + }), + // userMemos + isDetailed && meId ? this.userMemosRepository.findBy({ userId: meId }) + .then(memos => new Map(memos.map(memo => [memo.targetUserId, memo.memo]))) : new Map(), + // userRelations + isDetailedAndNotMe && meId ? this.getRelations(meId, _userIds) : new Map(), + // pinNotes + isDetailed ? this.userNotePiningsRepository.createQueryBuilder('pin') + .where('pin.userId IN (:...userIds)', { userIds: _userIds }) + .innerJoinAndSelect('pin.note', 'note') + .getMany() + .then(pinsNotes => { + const map = new Map(); + for (const note of pinsNotes) { + const notes = map.get(note.userId) ?? []; + notes.push(note); + map.set(note.userId, notes); + } + for (const [, notes] of map.entries()) { + // pack側ではDESCで取得しているので、それに合わせて降順に並び替えておく + notes.sort((a, b) => b.id.localeCompare(a.id)); + } + return map; + }) : new Map(), + // userIdsByUrl + isDetailed ? this.usersRepository.createQueryBuilder('user') + .select([ + 'user.id', + 'user.uri', + ]) + .where({ + uri: In(Array.from(userUris)), + }) + .getRawMany<{ user_uri: string, user_id: string }>() + .then(users => new Map(users.map(u => [u.user_uri, u.user_id]))) : new Map(), + // instances + Promise.all(Array.from(userHosts).map(async host => [host, await this.federatedInstanceService.fetch(host)] as const)) + .then(hosts => new Map(hosts)), + // securityKeyCounts + isDetailedAndMeOrMod ? this.userSecurityKeysRepository.createQueryBuilder('key') + .select('key.userId', 'userId') + .addSelect('count(key.id)', 'userCount') + .where({ + userId: In(_userIds), + }) + .groupBy('key.userId') + .getRawMany<{ userId: string, userCount: number }>() + .then(counts => new Map(counts.map(c => [c.userId, c.userCount]))) : new Map(), + // TODO check query performance + // pendingReceivedFollows + isDetailedAndMe ? this.followRequestsRepository.createQueryBuilder('req') + .select('req.followeeId', 'followeeId') + .where({ + followeeId: In(_userIds), + }) + .groupBy('req.followeeId') + .getRawMany<{ followeeId: string }>() + .then(reqs => new Set(reqs.map(r => r.followeeId))) : new Set(), + // pendingSentFollows + isDetailedAndMe ? this.followRequestsRepository.createQueryBuilder('req') + .select('req.followerId', 'followerId') + .where({ + followerId: In(_userIds), + }) + .groupBy('req.followerId') + .getRawMany<{ followerId: string }>() + .then(reqs => new Set(reqs.map(r => r.followerId))) : new Set(), + ]); + return Promise.all( _users.map(u => this.pack( u, @@ -861,6 +888,12 @@ export class UserEntityService implements OnModuleInit { userRelations: userRelations, userMemos: userMemos, pinNotes: pinNotes, + iAmModerator, + userIdsByUri, + instances, + securityKeyCounts, + pendingReceivedFollows, + pendingSentFollows, }, )), ); diff --git a/packages/backend/src/server/api/endpoints/notes/create.ts b/packages/backend/src/server/api/endpoints/notes/create.ts index 3dd90c3dca..461910543f 100644 --- a/packages/backend/src/server/api/endpoints/notes/create.ts +++ b/packages/backend/src/server/api/endpoints/notes/create.ts @@ -348,7 +348,7 @@ export default class extends Endpoint { // eslint- throw new ApiError(meta.errors.noSuchReplyTarget); } else if (isRenote(reply) && !isQuote(reply)) { throw new ApiError(meta.errors.cannotReplyToPureRenote); - } else if (!await this.noteEntityService.isVisibleForMe(reply, me.id)) { + } else if (!await this.noteEntityService.isVisibleForMe(reply, me.id, { me })) { throw new ApiError(meta.errors.cannotReplyToInvisibleNote); } else if (reply.visibility === 'specified' && ps.visibility !== 'specified') { throw new ApiError(meta.errors.cannotReplyToSpecifiedVisibilityNoteWithExtendedVisibility); diff --git a/packages/backend/src/server/api/endpoints/notes/edit.ts b/packages/backend/src/server/api/endpoints/notes/edit.ts index 2c01b26584..bd70cb7835 100644 --- a/packages/backend/src/server/api/endpoints/notes/edit.ts +++ b/packages/backend/src/server/api/endpoints/notes/edit.ts @@ -402,7 +402,7 @@ export default class extends Endpoint { // eslint- throw new ApiError(meta.errors.noSuchReplyTarget); } else if (isRenote(reply) && !isQuote(reply)) { throw new ApiError(meta.errors.cannotReplyToPureRenote); - } else if (!await this.noteEntityService.isVisibleForMe(reply, me.id)) { + } else if (!await this.noteEntityService.isVisibleForMe(reply, me.id, { me })) { throw new ApiError(meta.errors.cannotReplyToInvisibleNote); } else if (reply.visibility === 'specified' && ps.visibility !== 'specified') { throw new ApiError(meta.errors.cannotReplyToSpecifiedVisibilityNoteWithExtendedVisibility); diff --git a/packages/backend/src/server/api/endpoints/notes/translate.ts b/packages/backend/src/server/api/endpoints/notes/translate.ts index e55168e296..5ebd5ef362 100644 --- a/packages/backend/src/server/api/endpoints/notes/translate.ts +++ b/packages/backend/src/server/api/endpoints/notes/translate.ts @@ -91,7 +91,7 @@ export default class extends Endpoint { // eslint- throw err; }); - if (!(await this.noteEntityService.isVisibleForMe(note, me?.id ?? null))) { + if (!(await this.noteEntityService.isVisibleForMe(note, me?.id ?? null, { me }))) { throw new ApiError(meta.errors.cannotTranslateInvisibleNote); } From c9884a74fc5038d35652d02c78a2672be15b7a57 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 6 Jun 2025 02:48:49 -0400 Subject: [PATCH 13/54] populate userProfile in more places --- .../src/core/entities/NoteEntityService.ts | 21 ++++++++++++++++--- .../src/core/entities/UserEntityService.ts | 6 +++--- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/core/entities/NoteEntityService.ts b/packages/backend/src/core/entities/NoteEntityService.ts index 7ddf0abc3e..327b702e7e 100644 --- a/packages/backend/src/core/entities/NoteEntityService.ts +++ b/packages/backend/src/core/entities/NoteEntityService.ts @@ -589,9 +589,24 @@ export class NoteEntityService implements OnModuleInit { id: In(targetNotesToFetch), }, relations: { - user: true, - reply: true, - renote: true, + user: { + userProfile: true, + }, + reply: { + user: { + userProfile: true, + }, + }, + renote: { + user: { + userProfile: true, + }, + reply: { + user: { + userProfile: true, + }, + }, + }, channel: true, }, }); diff --git a/packages/backend/src/core/entities/UserEntityService.ts b/packages/backend/src/core/entities/UserEntityService.ts index 7e3a55e155..84b6a4a021 100644 --- a/packages/backend/src/core/entities/UserEntityService.ts +++ b/packages/backend/src/core/entities/UserEntityService.ts @@ -789,11 +789,11 @@ export class UserEntityService implements OnModuleInit { .map(user => user.host) .filter((host): host is string => host != null)); - const _profiles: MiUserProfile[] = []; + const _profilesFromUsers: MiUserProfile[] = []; const _profilesToFetch: string[] = []; for (const user of _users) { if (user.userProfile) { - _profiles.push(user.userProfile); + _profilesFromUsers.push(user.userProfile); } else { _profilesToFetch.push(user.id); } @@ -805,7 +805,7 @@ export class UserEntityService implements OnModuleInit { // profilesMap this.cacheService.getUserProfiles(_profilesToFetch) .then(profiles => { - for (const profile of _profiles) { + for (const profile of _profilesFromUsers) { profiles.set(profile.userId, profile); } return profiles; From 223cdc9ea9d7fe58b1f7501cecb24a0f7a062dea Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 6 Jun 2025 03:11:20 -0400 Subject: [PATCH 14/54] fix note/user accumulation in NoteEntityService.packMany, improving performance and avoiding extra sub-packs and fetches --- .../src/core/entities/NoteEntityService.ts | 73 ++++++++++++------- 1 file changed, 48 insertions(+), 25 deletions(-) diff --git a/packages/backend/src/core/entities/NoteEntityService.ts b/packages/backend/src/core/entities/NoteEntityService.ts index 327b702e7e..1b3920e13f 100644 --- a/packages/backend/src/core/entities/NoteEntityService.ts +++ b/packages/backend/src/core/entities/NoteEntityService.ts @@ -425,6 +425,7 @@ export class NoteEntityService implements OnModuleInit { polls: Map; pollVotes: Map>; channels: Map; + notes: Map; }; }, ): Promise> { @@ -517,14 +518,14 @@ export class NoteEntityService implements OnModuleInit { clippedCount: note.clippedCount, processErrors: note.processErrors, - reply: note.replyId ? this.pack(note.reply ?? note.replyId, me, { + reply: note.replyId ? this.pack(note.reply ?? opts._hint_?.notes.get(note.replyId) ?? note.replyId, me, { detail: false, skipHide: opts.skipHide, withReactionAndUserPairCache: opts.withReactionAndUserPairCache, _hint_: options?._hint_, }) : undefined, - renote: note.renoteId ? this.pack(note.renote ?? note.renoteId, me, { + renote: note.renoteId ? this.pack(note.renote ?? opts._hint_?.notes.get(note.renoteId) ?? note.renoteId, me, { detail: true, skipHide: opts.skipHide, withReactionAndUserPairCache: opts.withReactionAndUserPairCache, @@ -556,16 +557,16 @@ export class NoteEntityService implements OnModuleInit { ) { if (notes.length === 0) return []; - const targetNotes: MiNote[] = []; - const targetNotesToFetch: string[] = []; + const targetNotesMap = new Map(); + const targetNotesToFetch : string[] = []; for (const note of notes) { if (isPureRenote(note)) { // we may need to fetch 'my reaction' for renote target. if (note.renote) { - targetNotes.push(note.renote); + targetNotesMap.set(note.renote.id, note.renote); if (note.renote.reply) { // idem if the renote is also a reply. - targetNotes.push(note.renote.reply); + targetNotesMap.set(note.renote.reply.id, note.renote.reply); } } else if (options?.detail) { targetNotesToFetch.push(note.renoteId); @@ -573,12 +574,19 @@ export class NoteEntityService implements OnModuleInit { } else { if (note.reply) { // idem for OP of a regular reply. - targetNotes.push(note.reply); + targetNotesMap.set(note.reply.id, note.reply); } else if (note.replyId && options?.detail) { targetNotesToFetch.push(note.replyId); } - targetNotes.push(note); + targetNotesMap.set(note.id, note); + } + } + + // Don't fetch notes that were added by ID and then found inline in another note. + for (let i = targetNotesToFetch.length - 1; i >= 0; i--) { + if (targetNotesMap.has(targetNotesToFetch[i])) { + targetNotesToFetch.splice(i, 1); } } @@ -610,36 +618,50 @@ export class NoteEntityService implements OnModuleInit { channel: true, }, }); - targetNotes.push(...newNotes); + + for (const note of newNotes) { + targetNotesMap.set(note.id, note); + } } - const fileIds = notes.map(n => [n.fileIds, n.renote?.fileIds, n.reply?.fileIds]).flat(2).filter(x => x != null); - const users = [ - ...notes.map(({ user, userId }) => user ?? userId), - ...notes.map(({ reply, replyUserId }) => reply?.user ?? replyUserId).filter(x => x != null), - ...notes.map(({ renote, renoteUserId }) => renote?.user ?? renoteUserId).filter(x => x != null), - ]; + const targetNotes = Array.from(targetNotesMap.values()); + const noteIds = Array.from(targetNotesMap.keys()); - // Recursively add all mentioned users from all notes + replies + renotes - const allMentionedUsers = targetNotes.reduce((users, note) => { - for (const user of note.mentions) { - users.add(user); + const usersMap = new Map(); + const allUsers = notes.flatMap(note => [ + note.user ?? note.userId, + note.reply?.user ?? note.replyUserId, + note.renote?.user ?? note.renoteUserId, + ]); + + for (const user of allUsers) { + if (!user) continue; + + if (typeof(user) === 'object') { + // ID -> Entity + usersMap.set(user.id, user); + } else if (!usersMap.has(user)) { + // ID -> ID + usersMap.set(user, user); } - return users; - }, new Set()); + } + + const users = Array.from(usersMap.values()); + const userIds = Array.from(usersMap.keys()); + + const fileIds = new Set(targetNotes.flatMap(n => n.fileIds)); + const mentionedUsers = new Set(targetNotes.flatMap(note => note.mentions)); - const userIds = Array.from(new Set(users.map(u => typeof(u) === 'string' ? u : u.id))); - const noteIds = Array.from(new Set(targetNotes.map(n => n.id))); const [{ bufferedReactions, myReactionsMap }, packedFiles, packedUsers, mentionHandles, userFollowings, userBlockers, polls, pollVotes, channels] = await Promise.all([ // bufferedReactions & myReactionsMap this.getReactions(targetNotes, me), // packedFiles - this.driveFileEntityService.packManyByIdsMap(fileIds), + this.driveFileEntityService.packManyByIdsMap(Array.from(fileIds)), // packedUsers this.userEntityService.packMany(users, me) .then(users => new Map(users.map(u => [u.id, u]))), // mentionHandles - this.getUserHandles(Array.from(allMentionedUsers)), + this.getUserHandles(Array.from(mentionedUsers)), // userFollowings this.cacheService.getUserFollowings(userIds), // userBlockers @@ -682,6 +704,7 @@ export class NoteEntityService implements OnModuleInit { polls, pollVotes, channels, + notes: new Map(targetNotes.map(n => [n.id, n])), }, }))); } From 20cc73645765fc7434525e29647eafe98bf84926 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 6 Jun 2025 03:20:27 -0400 Subject: [PATCH 15/54] fix single-user relation calculations --- packages/backend/src/core/entities/UserEntityService.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/backend/src/core/entities/UserEntityService.ts b/packages/backend/src/core/entities/UserEntityService.ts index 84b6a4a021..5aaa345e75 100644 --- a/packages/backend/src/core/entities/UserEntityService.ts +++ b/packages/backend/src/core/entities/UserEntityService.ts @@ -220,13 +220,13 @@ export class UserEntityService implements OnModuleInit { }, }), this.cacheService.userBlockedCache.fetch(me) - .then(blockers => blockers.size > 0), + .then(blockers => blockers.has(target)), this.cacheService.userBlockingCache.fetch(me) - .then(blockees => blockees.size > 0), + .then(blockees => blockees.has(target)), this.cacheService.userMutingsCache.fetch(me) - .then(mutings => mutings.size > 0), + .then(mutings => mutings.has(target)), this.cacheService.renoteMutingsCache.fetch(me) - .then(mutings => mutings.size > 0), + .then(mutings => mutings.has(target)), this.cacheService.userByIdCache.fetch(target, () => this.usersRepository.findOneByOrFail({ id: target })) .then(user => user.host), this.userMemosRepository.createQueryBuilder('m') From 1d06ac4824d66624a57112747914e6bfe0b73b90 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 6 Jun 2025 12:16:49 -0400 Subject: [PATCH 16/54] fix reversed isBlocking / isBlocked in UserEntityService.getRelation --- packages/backend/src/core/entities/UserEntityService.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/backend/src/core/entities/UserEntityService.ts b/packages/backend/src/core/entities/UserEntityService.ts index 5aaa345e75..8ed482af6f 100644 --- a/packages/backend/src/core/entities/UserEntityService.ts +++ b/packages/backend/src/core/entities/UserEntityService.ts @@ -219,10 +219,10 @@ export class UserEntityService implements OnModuleInit { followeeId: me, }, }), - this.cacheService.userBlockedCache.fetch(me) - .then(blockers => blockers.has(target)), this.cacheService.userBlockingCache.fetch(me) .then(blockees => blockees.has(target)), + this.cacheService.userBlockedCache.fetch(me) + .then(blockers => blockers.has(target)), this.cacheService.userMutingsCache.fetch(me) .then(mutings => mutings.has(target)), this.cacheService.renoteMutingsCache.fetch(me) @@ -325,8 +325,8 @@ export class UserEntityService implements OnModuleInit { isFollowed: followees.includes(target), hasPendingFollowRequestFromYou: followersRequests.includes(target), hasPendingFollowRequestToYou: followeesRequests.includes(target), - isBlocking: blockers.has(target), - isBlocked: blockees.has(target), + isBlocking: blockees.has(target), + isBlocked: blockers.has(target), isMuted: muters.has(target), isRenoteMuted: renoteMuters.has(target), isInstanceMuted: hosts[target] != null && mutedInstances.includes(hosts[target]), From 2e486f02ffb83192fd46fd231fb3604e1b982416 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 6 Jun 2025 12:17:04 -0400 Subject: [PATCH 17/54] implement no-op caches for testing --- packages/backend/src/core/CacheService.ts | 22 ++- packages/backend/src/misc/cache.ts | 41 +++-- packages/backend/test/misc/noOpCaches.ts | 196 ++++++++++++++++++++++ 3 files changed, 244 insertions(+), 15 deletions(-) create mode 100644 packages/backend/test/misc/noOpCaches.ts diff --git a/packages/backend/src/core/CacheService.ts b/packages/backend/src/core/CacheService.ts index ae24a9721f..2c136eac2b 100644 --- a/packages/backend/src/core/CacheService.ts +++ b/packages/backend/src/core/CacheService.ts @@ -28,7 +28,7 @@ export interface CachedTranslation { text: string | undefined; } -interface CachedTranslationEntity { +export interface CachedTranslationEntity { l?: string; t?: string; u?: number; @@ -46,8 +46,8 @@ export class CacheService implements OnApplicationShutdown { public userBlockedCache: QuantumKVCache>; // NOTE: 「被」Blockキャッシュ public renoteMutingsCache: QuantumKVCache>; public userFollowingsCache: QuantumKVCache | undefined>>; - private readonly userFollowStatsCache = new MemoryKVCache(1000 * 60 * 10); // 10 minutes - private readonly translationsCache: RedisKVCache; + protected userFollowStatsCache = new MemoryKVCache(1000 * 60 * 10); // 10 minutes + protected translationsCache: RedisKVCache; constructor( @Inject(DI.redis) @@ -467,6 +467,22 @@ export class CacheService implements OnApplicationShutdown { return users; } + @bindThis + public clear(): void { + this.userByIdCache.clear(); + this.localUserByNativeTokenCache.clear(); + this.localUserByIdCache.clear(); + this.uriPersonCache.clear(); + this.userProfileCache.clear(); + this.userMutingsCache.clear(); + this.userBlockingCache.clear(); + this.userBlockedCache.clear(); + this.renoteMutingsCache.clear(); + this.userFollowingsCache.clear(); + this.userFollowStatsCache.clear(); + this.translationsCache.clear(); + } + @bindThis public dispose(): void { this.internalEventService.off('userChangeSuspendedState', this.onUserEvent); diff --git a/packages/backend/src/misc/cache.ts b/packages/backend/src/misc/cache.ts index 3145550a44..0a1cf6adb4 100644 --- a/packages/backend/src/misc/cache.ts +++ b/packages/backend/src/misc/cache.ts @@ -11,9 +11,9 @@ import { InternalEventTypes } from '@/core/GlobalEventService.js'; export class RedisKVCache { private readonly lifetime: number; private readonly memoryCache: MemoryKVCache; - private readonly fetcher: (key: string) => Promise; - private readonly toRedisConverter: (value: T) => string; - private readonly fromRedisConverter: (value: string) => T | undefined; + public readonly fetcher: (key: string) => Promise; + public readonly toRedisConverter: (value: T) => string; + public readonly fromRedisConverter: (value: string) => T | undefined; constructor( private redisClient: Redis.Redis, @@ -101,6 +101,11 @@ export class RedisKVCache { // TODO: イベント発行して他プロセスのメモリキャッシュも更新できるようにする } + @bindThis + public clear() { + this.memoryCache.clear(); + } + @bindThis public gc() { this.memoryCache.gc(); @@ -125,16 +130,17 @@ export class RedisSingleCache { opts: { lifetime: number; memoryCacheLifetime: number; - fetcher: RedisSingleCache['fetcher']; - toRedisConverter: RedisSingleCache['toRedisConverter']; - fromRedisConverter: RedisSingleCache['fromRedisConverter']; + fetcher?: RedisSingleCache['fetcher']; + toRedisConverter?: RedisSingleCache['toRedisConverter']; + fromRedisConverter?: RedisSingleCache['fromRedisConverter']; }, ) { this.lifetime = opts.lifetime; this.memoryCache = new MemorySingleCache(opts.memoryCacheLifetime); - this.fetcher = opts.fetcher; - this.toRedisConverter = opts.toRedisConverter; - this.fromRedisConverter = opts.fromRedisConverter; + + this.fetcher = opts.fetcher ?? (() => { throw new Error('fetch not supported - use get/set directly'); }); + this.toRedisConverter = opts.toRedisConverter ?? ((value) => JSON.stringify(value)); + this.fromRedisConverter = opts.fromRedisConverter ?? ((value) => JSON.parse(value)); } @bindThis @@ -417,6 +423,8 @@ export class MemorySingleCache { } } +// TODO move to separate file + export interface QuantumKVOpts { /** * Memory cache lifetime in milliseconds. @@ -452,9 +460,9 @@ export interface QuantumKVOpts { export class QuantumKVCache implements Iterable<[key: string, value: T]> { private readonly memoryCache: MemoryKVCache; - private readonly fetcher: QuantumKVOpts['fetcher']; - private readonly onSet: QuantumKVOpts['onSet']; - private readonly onDelete: QuantumKVOpts['onDelete']; + public readonly fetcher: QuantumKVOpts['fetcher']; + public readonly onSet: QuantumKVOpts['onSet']; + public readonly onDelete: QuantumKVOpts['onDelete']; /** * @param internalEventService Service bus to synchronize events. @@ -676,6 +684,15 @@ export class QuantumKVCache implements Iterable<[key: string, value: T]> { * Does not send any events or update other processes. */ @bindThis + public clear() { + this.memoryCache.clear(); + } + + /** + * Removes expired cache entries from the local view. + * Does not send any events or update other processes. + */ + @bindThis public gc() { this.memoryCache.gc(); } diff --git a/packages/backend/test/misc/noOpCaches.ts b/packages/backend/test/misc/noOpCaches.ts new file mode 100644 index 0000000000..373c7bddcc --- /dev/null +++ b/packages/backend/test/misc/noOpCaches.ts @@ -0,0 +1,196 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import * as Redis from 'ioredis'; +import { Inject } from '@nestjs/common'; +import { FakeInternalEventService } from './FakeInternalEventService.js'; +import type { BlockingsRepository, FollowingsRepository, MiFollowing, MiUser, MiUserProfile, MutingsRepository, RenoteMutingsRepository, UserProfilesRepository, UsersRepository } from '@/models/_.js'; +import type { MiLocalUser } from '@/models/User.js'; +import { MemoryKVCache, MemorySingleCache, QuantumKVCache, QuantumKVOpts, RedisKVCache, RedisSingleCache } from '@/misc/cache.js'; +import { CacheService, CachedTranslationEntity, FollowStats } from '@/core/CacheService.js'; +import { DI } from '@/di-symbols.js'; +import { UserEntityService } from '@/core/entities/UserEntityService.js'; + +export function noOpRedis() { + return { + set: () => Promise.resolve(), + get: () => Promise.resolve(null), + del: () => Promise.resolve(), + on: () => {}, + off: () => {}, + } as unknown as Redis.Redis; +} + +export class NoOpCacheService extends CacheService { + public readonly fakeRedis: { + [K in keyof Redis.Redis]: Redis.Redis[K]; + }; + public readonly fakeInternalEventService: FakeInternalEventService; + + constructor( + @Inject(DI.usersRepository) + usersRepository: UsersRepository, + + @Inject(DI.userProfilesRepository) + userProfilesRepository: UserProfilesRepository, + + @Inject(DI.mutingsRepository) + mutingsRepository: MutingsRepository, + + @Inject(DI.blockingsRepository) + blockingsRepository: BlockingsRepository, + + @Inject(DI.renoteMutingsRepository) + renoteMutingsRepository: RenoteMutingsRepository, + + @Inject(DI.followingsRepository) + followingsRepository: FollowingsRepository, + + @Inject(UserEntityService) + userEntityService: UserEntityService, + ) { + const fakeRedis = noOpRedis(); + const fakeInternalEventService = new FakeInternalEventService(); + + super( + fakeRedis, + fakeRedis, + usersRepository, + userProfilesRepository, + mutingsRepository, + blockingsRepository, + renoteMutingsRepository, + followingsRepository, + userEntityService, + fakeInternalEventService, + ); + + this.fakeRedis = fakeRedis; + this.fakeInternalEventService = fakeInternalEventService; + + // Override caches + this.userByIdCache = new NoOpMemoryKVCache(); + this.localUserByNativeTokenCache = new NoOpMemoryKVCache(); + this.localUserByIdCache = new NoOpMemoryKVCache(); + this.uriPersonCache = new NoOpMemoryKVCache(); + this.userProfileCache = new NoOpQuantumKVCache({ + internalEventService: fakeInternalEventService, + fetcher: this.userProfileCache.fetcher, + onSet: this.userProfileCache.onSet, + onDelete: this.userProfileCache.onDelete, + }); + this.userMutingsCache = new NoOpQuantumKVCache>({ + internalEventService: fakeInternalEventService, + fetcher: this.userMutingsCache.fetcher, + onSet: this.userMutingsCache.onSet, + onDelete: this.userMutingsCache.onDelete, + }); + this.userBlockingCache = new NoOpQuantumKVCache>({ + internalEventService: fakeInternalEventService, + fetcher: this.userBlockingCache.fetcher, + onSet: this.userBlockingCache.onSet, + onDelete: this.userBlockingCache.onDelete, + }); + this.userBlockedCache = new NoOpQuantumKVCache>({ + internalEventService: fakeInternalEventService, + fetcher: this.userBlockedCache.fetcher, + onSet: this.userBlockedCache.onSet, + onDelete: this.userBlockedCache.onDelete, + }); + this.renoteMutingsCache = new NoOpQuantumKVCache>({ + internalEventService: fakeInternalEventService, + fetcher: this.renoteMutingsCache.fetcher, + onSet: this.renoteMutingsCache.onSet, + onDelete: this.renoteMutingsCache.onDelete, + }); + this.userFollowingsCache = new NoOpQuantumKVCache | undefined>>({ + internalEventService: fakeInternalEventService, + fetcher: this.userFollowingsCache.fetcher, + onSet: this.userFollowingsCache.onSet, + onDelete: this.userFollowingsCache.onDelete, + }); + this.userFollowStatsCache = new NoOpMemoryKVCache(); + this.translationsCache = new NoOpRedisKVCache({ + redis: fakeRedis, + fetcher: this.translationsCache.fetcher, + toRedisConverter: this.translationsCache.toRedisConverter, + fromRedisConverter: this.translationsCache.fromRedisConverter, + }); + } +} + +export class NoOpMemoryKVCache extends MemoryKVCache { + constructor() { + super(-1); + } +} + +export class NoOpMemorySingleCache extends MemorySingleCache { + constructor() { + super(-1); + } +} + +export class NoOpRedisKVCache extends RedisKVCache { + constructor(opts?: { + redis?: Redis.Redis; + fetcher?: RedisKVCache['fetcher']; + toRedisConverter?: RedisKVCache['toRedisConverter']; + fromRedisConverter?: RedisKVCache['fromRedisConverter']; + }) { + super( + opts?.redis ?? noOpRedis(), + 'no-op', + { + lifetime: -1, + memoryCacheLifetime: -1, + fetcher: opts?.fetcher, + toRedisConverter: opts?.toRedisConverter, + fromRedisConverter: opts?.fromRedisConverter, + }, + ); + } +} + +export class NoOpRedisSingleCache extends RedisSingleCache { + constructor(opts?: { + fakeRedis?: Redis.Redis; + fetcher?: RedisSingleCache['fetcher']; + toRedisConverter?: RedisSingleCache['toRedisConverter']; + fromRedisConverter?: RedisSingleCache['fromRedisConverter']; + }) { + super( + opts?.fakeRedis ?? noOpRedis(), + 'no-op', + { + lifetime: -1, + memoryCacheLifetime: -1, + fetcher: opts?.fetcher, + toRedisConverter: opts?.toRedisConverter, + fromRedisConverter: opts?.fromRedisConverter, + }, + ); + } +} + +export class NoOpQuantumKVCache extends QuantumKVCache { + constructor(opts: { + internalEventService?: FakeInternalEventService, + fetcher: QuantumKVOpts['fetcher'], + onSet?: QuantumKVOpts['onSet'], + onDelete?: QuantumKVOpts['onDelete'], + }) { + super( + opts.internalEventService ?? new FakeInternalEventService(), + 'no-op', + { + lifetime: -1, + fetcher: opts.fetcher, + onSet: opts.onSet, + onDelete: opts.onDelete, + }, + ); + } +} From 633b64e5c3dc807835cab0c1866ae66513b2a07b Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 6 Jun 2025 12:18:31 -0400 Subject: [PATCH 18/54] disable caches in unit tests --- .../backend/test/unit/AnnouncementService.ts | 26 ++++++++++++------- packages/backend/test/unit/RoleService.ts | 16 +++++++++--- packages/backend/test/unit/activitypub.ts | 10 +++++-- .../test/unit/entities/UserEntityService.ts | 9 ++++++- 4 files changed, 44 insertions(+), 17 deletions(-) diff --git a/packages/backend/test/unit/AnnouncementService.ts b/packages/backend/test/unit/AnnouncementService.ts index a79655c9aa..32d7df05bf 100644 --- a/packages/backend/test/unit/AnnouncementService.ts +++ b/packages/backend/test/unit/AnnouncementService.ts @@ -8,9 +8,12 @@ process.env.NODE_ENV = 'test'; import { jest } from '@jest/globals'; import { ModuleMocker } from 'jest-mock'; import { Test } from '@nestjs/testing'; +import { NoOpCacheService } from '../misc/noOpCaches.js'; +import { FakeInternalEventService } from '../misc/FakeInternalEventService.js'; import { GlobalModule } from '@/GlobalModule.js'; import { AnnouncementService } from '@/core/AnnouncementService.js'; import { AnnouncementEntityService } from '@/core/entities/AnnouncementEntityService.js'; +import { InternalEventService } from '@/core/InternalEventService.js'; import type { AnnouncementReadsRepository, AnnouncementsRepository, @@ -71,24 +74,27 @@ describe('AnnouncementService', () => { AnnouncementEntityService, CacheService, IdService, + InternalEventService, + GlobalEventService, + ModerationLogService, ], }) .useMocker((token) => { - if (token === GlobalEventService) { - return { - publishMainStream: jest.fn(), - publishBroadcastStream: jest.fn(), - }; - } else if (token === ModerationLogService) { - return { - log: jest.fn(), - }; - } else if (typeof token === 'function') { + if (typeof token === 'function') { const mockMetadata = moduleMocker.getMetadata(token) as MockFunctionMetadata; const Mock = moduleMocker.generateFromMetadata(mockMetadata); return new Mock(); } }) + .overrideProvider(GlobalEventService).useValue({ + publishMainStream: jest.fn(), + publishBroadcastStream: jest.fn(), + } as unknown as GlobalEventService) + .overrideProvider(ModerationLogService).useValue({ + log: jest.fn(), + }) + .overrideProvider(InternalEventService).useClass(FakeInternalEventService) + .overrideProvider(CacheService).useClass(NoOpCacheService) .compile(); app.enableShutdownHooks(); diff --git a/packages/backend/test/unit/RoleService.ts b/packages/backend/test/unit/RoleService.ts index 839402418e..2afe22618d 100644 --- a/packages/backend/test/unit/RoleService.ts +++ b/packages/backend/test/unit/RoleService.ts @@ -10,12 +10,15 @@ import { jest } from '@jest/globals'; import { ModuleMocker } from 'jest-mock'; import { Test } from '@nestjs/testing'; import * as lolex from '@sinonjs/fake-timers'; +import { NoOpCacheService } from '../misc/noOpCaches.js'; +import { FakeInternalEventService } from '../misc/FakeInternalEventService.js'; import type { TestingModule } from '@nestjs/testing'; import type { MockFunctionMetadata } from 'jest-mock'; import { GlobalModule } from '@/GlobalModule.js'; import { RoleService } from '@/core/RoleService.js'; import { InstancesRepository, + MetasRepository, MiMeta, MiRole, MiRoleAssignment, @@ -34,6 +37,7 @@ import { secureRndstr } from '@/misc/secure-rndstr.js'; import { NotificationService } from '@/core/NotificationService.js'; import { RoleCondFormulaValue } from '@/models/Role.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; +import { InternalEventService } from '@/core/InternalEventService.js'; const moduleMocker = new ModuleMocker(global); @@ -45,6 +49,7 @@ describe('RoleService', () => { let rolesRepository: RolesRepository; let roleAssignmentsRepository: RoleAssignmentsRepository; let meta: jest.Mocked; + let metasRepository: MetasRepository; let notificationService: jest.Mocked; let clock: lolex.InstalledClock; @@ -143,18 +148,20 @@ describe('RoleService', () => { provide: NotificationService.name, useExisting: NotificationService, }, + MetaService, + InternalEventService, ], }) .useMocker((token) => { - if (token === MetaService) { - return { fetch: jest.fn() }; - } if (typeof token === 'function') { const mockMetadata = moduleMocker.getMetadata(token) as MockFunctionMetadata; const Mock = moduleMocker.generateFromMetadata(mockMetadata); return new Mock(); } }) + .overrideProvider(MetaService).useValue({ fetch: jest.fn() }) + .overrideProvider(InternalEventService).useClass(FakeInternalEventService) + .overrideProvider(CacheService).useClass(NoOpCacheService) .compile(); app.enableShutdownHooks(); @@ -164,6 +171,7 @@ describe('RoleService', () => { usersRepository = app.get(DI.usersRepository); rolesRepository = app.get(DI.rolesRepository); roleAssignmentsRepository = app.get(DI.roleAssignmentsRepository); + metasRepository = app.get(DI.metasRepository); meta = app.get(DI.meta) as jest.Mocked; notificationService = app.get(NotificationService) as jest.Mocked; @@ -175,7 +183,7 @@ describe('RoleService', () => { clock.uninstall(); await Promise.all([ - app.get(DI.metasRepository).delete({}), + metasRepository.delete({}), usersRepository.delete({}), rolesRepository.delete({}), roleAssignmentsRepository.delete({}), diff --git a/packages/backend/test/unit/activitypub.ts b/packages/backend/test/unit/activitypub.ts index 94dec16401..45975275d4 100644 --- a/packages/backend/test/unit/activitypub.ts +++ b/packages/backend/test/unit/activitypub.ts @@ -9,8 +9,12 @@ import { generateKeyPair } from 'crypto'; import { Test } from '@nestjs/testing'; import { jest } from '@jest/globals'; +import { NoOpCacheService } from '../misc/noOpCaches.js'; +import { FakeInternalEventService } from '../misc/FakeInternalEventService.js'; import type { Config } from '@/config.js'; import type { MiLocalUser, MiRemoteUser } from '@/models/User.js'; +import { InternalEventService } from '@/core/InternalEventService.js'; +import { CacheService } from '@/core/CacheService.js'; import { ApImageService } from '@/core/activitypub/models/ApImageService.js'; import { ApNoteService } from '@/core/activitypub/models/ApNoteService.js'; import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js'; @@ -30,7 +34,7 @@ import { genAidx } from '@/misc/id/aidx.js'; import { IdService } from '@/core/IdService.js'; import { MockResolver } from '../misc/mock-resolver.js'; import { UserKeypairService } from '@/core/UserKeypairService.js'; -import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js'; +import { MemoryKVCache } from '@/misc/cache.js'; const host = 'https://host1.test'; @@ -154,6 +158,8 @@ describe('ActivityPub', () => { }, }) .overrideProvider(DI.meta).useFactory({ factory: () => meta }) + .overrideProvider(CacheService).useClass(NoOpCacheService) + .overrideProvider(InternalEventService).useClass(FakeInternalEventService) .compile(); await app.init(); @@ -556,7 +562,7 @@ describe('ActivityPub', () => { publicKey, privateKey, }); - ((userKeypairService as unknown as { cache: RedisKVCache }).cache as unknown as { memoryCache: MemoryKVCache }).memoryCache.set(author.id, keypair); + (userKeypairService as unknown as { cache: MemoryKVCache }).cache.set(author.id, keypair); note = new MiNote({ id: idService.gen(), diff --git a/packages/backend/test/unit/entities/UserEntityService.ts b/packages/backend/test/unit/entities/UserEntityService.ts index ce3f931bb0..4f45f3216d 100644 --- a/packages/backend/test/unit/entities/UserEntityService.ts +++ b/packages/backend/test/unit/entities/UserEntityService.ts @@ -4,6 +4,8 @@ */ import { Test, TestingModule } from '@nestjs/testing'; +import { FakeInternalEventService } from '../../misc/FakeInternalEventService.js'; +import { NoOpCacheService } from '../../misc/noOpCaches.js'; import type { MiUser } from '@/models/User.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { GlobalModule } from '@/GlobalModule.js'; @@ -51,6 +53,7 @@ import { ReactionService } from '@/core/ReactionService.js'; import { NotificationService } from '@/core/NotificationService.js'; import { ReactionsBufferingService } from '@/core/ReactionsBufferingService.js'; import { ChatService } from '@/core/ChatService.js'; +import { InternalEventService } from '@/core/InternalEventService.js'; process.env.NODE_ENV = 'test'; @@ -174,6 +177,7 @@ describe('UserEntityService', () => { ReactionsBufferingService, NotificationService, ChatService, + InternalEventService, ]; app = await Test.createTestingModule({ @@ -182,7 +186,10 @@ describe('UserEntityService', () => { ...services, ...services.map(x => ({ provide: x.name, useExisting: x })), ], - }).compile(); + }) + .overrideProvider(InternalEventService).useClass(FakeInternalEventService) + .overrideProvider(CacheService).useClass(NoOpCacheService) + .compile(); await app.init(); app.enableShutdownHooks(); From 0c84d73294cb85a2126696abadb37003f3c08d7b Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 6 Jun 2025 12:26:43 -0400 Subject: [PATCH 19/54] move QuantumKVCache to a separate file --- packages/backend/src/core/CacheService.ts | 3 +- .../src/core/ChannelFollowingService.ts | 2 +- .../src/core/PushNotificationService.ts | 2 +- packages/backend/src/core/UserListService.ts | 2 +- packages/backend/src/misc/QuantumKVCache.ts | 318 ++++++++++++++++++ packages/backend/src/misc/cache.ts | 311 ----------------- packages/backend/test/misc/noOpCaches.ts | 3 +- .../unit/misc/{cache.ts => QuantumKVCache.ts} | 3 +- 8 files changed, 326 insertions(+), 318 deletions(-) create mode 100644 packages/backend/src/misc/QuantumKVCache.ts rename packages/backend/test/unit/misc/{cache.ts => QuantumKVCache.ts} (99%) diff --git a/packages/backend/src/core/CacheService.ts b/packages/backend/src/core/CacheService.ts index 2c136eac2b..e59857b4ce 100644 --- a/packages/backend/src/core/CacheService.ts +++ b/packages/backend/src/core/CacheService.ts @@ -7,7 +7,8 @@ import { Inject, Injectable } from '@nestjs/common'; import * as Redis from 'ioredis'; import { In, IsNull } from 'typeorm'; import type { BlockingsRepository, FollowingsRepository, MutingsRepository, RenoteMutingsRepository, MiUserProfile, UserProfilesRepository, UsersRepository, MiFollowing, MiNote } from '@/models/_.js'; -import { MemoryKVCache, QuantumKVCache, RedisKVCache } from '@/misc/cache.js'; +import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js'; +import { QuantumKVCache } from '@/misc/QuantumKVCache.js'; import type { MiLocalUser, MiUser } from '@/models/User.js'; import { DI } from '@/di-symbols.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; diff --git a/packages/backend/src/core/ChannelFollowingService.ts b/packages/backend/src/core/ChannelFollowingService.ts index 26b023179c..430711fef1 100644 --- a/packages/backend/src/core/ChannelFollowingService.ts +++ b/packages/backend/src/core/ChannelFollowingService.ts @@ -12,7 +12,7 @@ import { IdService } from '@/core/IdService.js'; import { GlobalEvents, GlobalEventService, InternalEventTypes } from '@/core/GlobalEventService.js'; import { bindThis } from '@/decorators.js'; import type { MiLocalUser } from '@/models/User.js'; -import { QuantumKVCache, RedisKVCache } from '@/misc/cache.js'; +import { QuantumKVCache } from '@/misc/QuantumKVCache.js'; import { InternalEventService } from './InternalEventService.js'; @Injectable() diff --git a/packages/backend/src/core/PushNotificationService.ts b/packages/backend/src/core/PushNotificationService.ts index 38bc5e3901..e3f10d4504 100644 --- a/packages/backend/src/core/PushNotificationService.ts +++ b/packages/backend/src/core/PushNotificationService.ts @@ -12,7 +12,7 @@ import type { Packed } from '@/misc/json-schema.js'; import { getNoteSummary } from '@/misc/get-note-summary.js'; import type { MiMeta, MiSwSubscription, SwSubscriptionsRepository } from '@/models/_.js'; import { bindThis } from '@/decorators.js'; -import { QuantumKVCache, RedisKVCache } from '@/misc/cache.js'; +import { QuantumKVCache } from '@/misc/QuantumKVCache.js'; import { InternalEventService } from '@/core/InternalEventService.js'; // Defined also packages/sw/types.ts#L13 diff --git a/packages/backend/src/core/UserListService.ts b/packages/backend/src/core/UserListService.ts index 0d2220049a..b4486b9808 100644 --- a/packages/backend/src/core/UserListService.ts +++ b/packages/backend/src/core/UserListService.ts @@ -17,7 +17,7 @@ import { DI } from '@/di-symbols.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { bindThis } from '@/decorators.js'; import { QueueService } from '@/core/QueueService.js'; -import { QuantumKVCache, RedisKVCache } from '@/misc/cache.js'; +import { QuantumKVCache } from '@/misc/QuantumKVCache.js'; import { RoleService } from '@/core/RoleService.js'; import { SystemAccountService } from '@/core/SystemAccountService.js'; import { InternalEventService } from '@/core/InternalEventService.js'; diff --git a/packages/backend/src/misc/QuantumKVCache.ts b/packages/backend/src/misc/QuantumKVCache.ts new file mode 100644 index 0000000000..6b36789f5e --- /dev/null +++ b/packages/backend/src/misc/QuantumKVCache.ts @@ -0,0 +1,318 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { InternalEventService } from '@/core/InternalEventService.js'; +import { bindThis } from '@/decorators.js'; +import { InternalEventTypes } from '@/core/GlobalEventService.js'; +import { MemoryKVCache } from '@/misc/cache.js'; + +export interface QuantumKVOpts { + /** + * Memory cache lifetime in milliseconds. + */ + lifetime: number; + + /** + * Callback to fetch the value for a key that wasn't found in the cache. + * May be synchronous or async. + */ + fetcher: (key: string, cache: QuantumKVCache) => T | Promise; + + /** + * Optional callback when a value is created or changed in the cache, either locally or elsewhere in the cluster. + * This is called *after* the cache state is updated. + * May be synchronous or async. + */ + onSet?: (key: string, cache: QuantumKVCache) => void | Promise; + + /** + * Optional callback when a value is deleted from the cache, either locally or elsewhere in the cluster. + * This is called *after* the cache state is updated. + * May be synchronous or async. + */ + onDelete?: (key: string, cache: QuantumKVCache) => void | Promise; +} + +/** + * QuantumKVCache is a lifetime-bounded memory cache (like MemoryKVCache) with automatic cross-cluster synchronization via Redis. + * All nodes in the cluster are guaranteed to have a *subset* view of the current accurate state, though individual processes may have different items in their local cache. + * This ensures that a call to get() will never return stale data. + */ +export class QuantumKVCache implements Iterable<[key: string, value: T]> { + private readonly memoryCache: MemoryKVCache; + + public readonly fetcher: QuantumKVOpts['fetcher']; + public readonly onSet: QuantumKVOpts['onSet']; + public readonly onDelete: QuantumKVOpts['onDelete']; + + /** + * @param internalEventService Service bus to synchronize events. + * @param name Unique name of the cache - must be the same in all processes. + * @param opts Cache options + */ + constructor( + private readonly internalEventService: InternalEventService, + private readonly name: string, + opts: QuantumKVOpts, + ) { + this.memoryCache = new MemoryKVCache(opts.lifetime); + this.fetcher = opts.fetcher; + this.onSet = opts.onSet; + this.onDelete = opts.onDelete; + + this.internalEventService.on('quantumCacheUpdated', this.onQuantumCacheUpdated, { + // Ignore our own events, otherwise we'll immediately erase any set value. + ignoreLocal: true, + }); + } + + /** + * The number of items currently in memory. + * This applies to the local subset view, not the cross-cluster cache state. + */ + public get size() { + return this.memoryCache.size; + } + + /** + * Iterates all [key, value] pairs in memory. + * This applies to the local subset view, not the cross-cluster cache state. + */ + @bindThis + public *entries(): Generator<[key: string, value: T]> { + for (const entry of this.memoryCache.entries) { + yield [entry[0], entry[1].value]; + } + } + + /** + * Iterates all keys in memory. + * This applies to the local subset view, not the cross-cluster cache state. + */ + @bindThis + public *keys() { + for (const entry of this.memoryCache.entries) { + yield entry[0]; + } + } + + /** + * Iterates all values pairs in memory. + * This applies to the local subset view, not the cross-cluster cache state. + */ + @bindThis + public *values() { + for (const entry of this.memoryCache.entries) { + yield entry[1].value; + } + } + + /** + * Creates or updates a value in the cache, and erases any stale caches across the cluster. + * Fires an onSet event after the cache has been updated in all processes. + * Skips if the value is unchanged. + */ + @bindThis + public async set(key: string, value: T): Promise { + if (this.memoryCache.get(key) === value) { + return; + } + + this.memoryCache.set(key, value); + + await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, op: 's', keys: [key] }); + + if (this.onSet) { + await this.onSet(key, this); + } + } + + /** + * Creates or updates multiple value in the cache, and erases any stale caches across the cluster. + * Fires an onSet for each changed item event after the cache has been updated in all processes. + * Skips if all values are unchanged. + */ + @bindThis + public async setMany(items: Iterable<[key: string, value: T]>): Promise { + const changedKeys: string[] = []; + + for (const item of items) { + if (this.memoryCache.get(item[0]) !== item[1]) { + changedKeys.push(item[0]); + this.memoryCache.set(item[0], item[1]); + } + } + + if (changedKeys.length > 0) { + await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, op: 's', keys: changedKeys }); + + if (this.onSet) { + for (const key of changedKeys) { + await this.onSet(key, this); + } + } + } + } + + /** + * Adds a value to the local memory cache without notifying other process. + * Neither a Redis event nor onSet callback will be fired, as the value has not actually changed. + * This should only be used when the value is known to be current, like after fetching from the database. + */ + @bindThis + public add(key: string, value: T): void { + this.memoryCache.set(key, value); + } + + /** + * Adds multiple values to the local memory cache without notifying other process. + * Neither a Redis event nor onSet callback will be fired, as the value has not actually changed. + * This should only be used when the value is known to be current, like after fetching from the database. + */ + @bindThis + public addMany(items: Iterable<[key: string, value: T]>): void { + for (const [key, value] of items) { + this.memoryCache.set(key, value); + } + } + + /** + * Gets a value from the local memory cache, or returns undefined if not found. + */ + @bindThis + public get(key: string): T | undefined { + return this.memoryCache.get(key); + } + + /** + * Gets or fetches a value from the cache. + * Fires an onSet event, but does not emit an update event to other processes. + */ + @bindThis + public async fetch(key: string): Promise { + let value = this.memoryCache.get(key); + if (value === undefined) { + value = await this.fetcher(key, this); + this.memoryCache.set(key, value); + + if (this.onSet) { + await this.onSet(key, this); + } + } + return value; + } + + /** + * Returns true is a key exists in memory. + * This applies to the local subset view, not the cross-cluster cache state. + */ + @bindThis + public has(key: string): boolean { + return this.memoryCache.get(key) !== undefined; + } + + /** + * Deletes a value from the cache, and erases any stale caches across the cluster. + * Fires an onDelete event after the cache has been updated in all processes. + */ + @bindThis + public async delete(key: string): Promise { + this.memoryCache.delete(key); + + await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, op: 'd', keys: [key] }); + + if (this.onDelete) { + await this.onDelete(key, this); + } + } + /** + * Deletes multiple values from the cache, and erases any stale caches across the cluster. + * Fires an onDelete event for each key after the cache has been updated in all processes. + * Skips if the input is empty. + */ + @bindThis + public async deleteMany(keys: string[]): Promise { + if (keys.length === 0) { + return; + } + + for (const key of keys) { + this.memoryCache.delete(key); + } + + await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, op: 'd', keys }); + + if (this.onDelete) { + for (const key of keys) { + await this.onDelete(key, this); + } + } + } + + /** + * Refreshes the value of a key from the fetcher, and erases any stale caches across the cluster. + * Fires an onSet event after the cache has been updated in all processes. + */ + @bindThis + public async refresh(key: string): Promise { + const value = await this.fetcher(key, this); + await this.set(key, value); + return value; + } + + /** + * Erases all entries from the local memory cache. + * Does not send any events or update other processes. + */ + @bindThis + public clear() { + this.memoryCache.clear(); + } + + /** + * Removes expired cache entries from the local view. + * Does not send any events or update other processes. + */ + @bindThis + public gc() { + this.memoryCache.gc(); + } + + /** + * Erases all data and disconnects from the cluster. + * This *must* be called when shutting down to prevent memory leaks! + */ + @bindThis + public dispose() { + this.internalEventService.off('quantumCacheUpdated', this.onQuantumCacheUpdated); + + this.memoryCache.dispose(); + } + + @bindThis + private async onQuantumCacheUpdated(data: InternalEventTypes['quantumCacheUpdated']): Promise { + if (data.name === this.name) { + for (const key of data.keys) { + this.memoryCache.delete(key); + + if (data.op === 's' && this.onSet) { + await this.onSet(key, this); + } + + if (data.op === 'd' && this.onDelete) { + await this.onDelete(key, this); + } + } + } + } + + /** + * Iterates all [key, value] pairs in memory. + * This applies to the local subset view, not the cross-cluster cache state. + */ + [Symbol.iterator](): Iterator<[key: string, value: T]> { + return this.entries(); + } +} diff --git a/packages/backend/src/misc/cache.ts b/packages/backend/src/misc/cache.ts index 0a1cf6adb4..932c0b409a 100644 --- a/packages/backend/src/misc/cache.ts +++ b/packages/backend/src/misc/cache.ts @@ -422,314 +422,3 @@ export class MemorySingleCache { return value; } } - -// TODO move to separate file - -export interface QuantumKVOpts { - /** - * Memory cache lifetime in milliseconds. - */ - lifetime: number; - - /** - * Callback to fetch the value for a key that wasn't found in the cache. - * May be synchronous or async. - */ - fetcher: (key: string, cache: QuantumKVCache) => T | Promise; - - /** - * Optional callback when a value is created or changed in the cache, either locally or elsewhere in the cluster. - * This is called *after* the cache state is updated. - * May be synchronous or async. - */ - onSet?: (key: string, cache: QuantumKVCache) => void | Promise; - - /** - * Optional callback when a value is deleted from the cache, either locally or elsewhere in the cluster. - * This is called *after* the cache state is updated. - * May be synchronous or async. - */ - onDelete?: (key: string, cache: QuantumKVCache) => void | Promise; -} - -/** - * QuantumKVCache is a lifetime-bounded memory cache (like MemoryKVCache) with automatic cross-cluster synchronization via Redis. - * All nodes in the cluster are guaranteed to have a *subset* view of the current accurate state, though individual processes may have different items in their local cache. - * This ensures that a call to get() will never return stale data. - */ -export class QuantumKVCache implements Iterable<[key: string, value: T]> { - private readonly memoryCache: MemoryKVCache; - - public readonly fetcher: QuantumKVOpts['fetcher']; - public readonly onSet: QuantumKVOpts['onSet']; - public readonly onDelete: QuantumKVOpts['onDelete']; - - /** - * @param internalEventService Service bus to synchronize events. - * @param name Unique name of the cache - must be the same in all processes. - * @param opts Cache options - */ - constructor( - private readonly internalEventService: InternalEventService, - private readonly name: string, - opts: QuantumKVOpts, - ) { - this.memoryCache = new MemoryKVCache(opts.lifetime); - this.fetcher = opts.fetcher; - this.onSet = opts.onSet; - this.onDelete = opts.onDelete; - - this.internalEventService.on('quantumCacheUpdated', this.onQuantumCacheUpdated, { - // Ignore our own events, otherwise we'll immediately erase any set value. - ignoreLocal: true, - }); - } - - /** - * The number of items currently in memory. - * This applies to the local subset view, not the cross-cluster cache state. - */ - public get size() { - return this.memoryCache.size; - } - - /** - * Iterates all [key, value] pairs in memory. - * This applies to the local subset view, not the cross-cluster cache state. - */ - @bindThis - public *entries(): Generator<[key: string, value: T]> { - for (const entry of this.memoryCache.entries) { - yield [entry[0], entry[1].value]; - } - } - - /** - * Iterates all keys in memory. - * This applies to the local subset view, not the cross-cluster cache state. - */ - @bindThis - public *keys() { - for (const entry of this.memoryCache.entries) { - yield entry[0]; - } - } - - /** - * Iterates all values pairs in memory. - * This applies to the local subset view, not the cross-cluster cache state. - */ - @bindThis - public *values() { - for (const entry of this.memoryCache.entries) { - yield entry[1].value; - } - } - - /** - * Creates or updates a value in the cache, and erases any stale caches across the cluster. - * Fires an onSet event after the cache has been updated in all processes. - * Skips if the value is unchanged. - */ - @bindThis - public async set(key: string, value: T): Promise { - if (this.memoryCache.get(key) === value) { - return; - } - - this.memoryCache.set(key, value); - - await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, op: 's', keys: [key] }); - - if (this.onSet) { - await this.onSet(key, this); - } - } - - /** - * Creates or updates multiple value in the cache, and erases any stale caches across the cluster. - * Fires an onSet for each changed item event after the cache has been updated in all processes. - * Skips if all values are unchanged. - */ - @bindThis - public async setMany(items: Iterable<[key: string, value: T]>): Promise { - const changedKeys: string[] = []; - - for (const item of items) { - if (this.memoryCache.get(item[0]) !== item[1]) { - changedKeys.push(item[0]); - this.memoryCache.set(item[0], item[1]); - } - } - - if (changedKeys.length > 0) { - await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, op: 's', keys: changedKeys }); - - if (this.onSet) { - for (const key of changedKeys) { - await this.onSet(key, this); - } - } - } - } - - /** - * Adds a value to the local memory cache without notifying other process. - * Neither a Redis event nor onSet callback will be fired, as the value has not actually changed. - * This should only be used when the value is known to be current, like after fetching from the database. - */ - @bindThis - public add(key: string, value: T): void { - this.memoryCache.set(key, value); - } - - /** - * Adds multiple values to the local memory cache without notifying other process. - * Neither a Redis event nor onSet callback will be fired, as the value has not actually changed. - * This should only be used when the value is known to be current, like after fetching from the database. - */ - @bindThis - public addMany(items: Iterable<[key: string, value: T]>): void { - for (const [key, value] of items) { - this.memoryCache.set(key, value); - } - } - - /** - * Gets a value from the local memory cache, or returns undefined if not found. - */ - @bindThis - public get(key: string): T | undefined { - return this.memoryCache.get(key); - } - - /** - * Gets or fetches a value from the cache. - * Fires an onSet event, but does not emit an update event to other processes. - */ - @bindThis - public async fetch(key: string): Promise { - let value = this.memoryCache.get(key); - if (value === undefined) { - value = await this.fetcher(key, this); - this.memoryCache.set(key, value); - - if (this.onSet) { - await this.onSet(key, this); - } - } - return value; - } - - /** - * Returns true is a key exists in memory. - * This applies to the local subset view, not the cross-cluster cache state. - */ - @bindThis - public has(key: string): boolean { - return this.memoryCache.get(key) !== undefined; - } - - /** - * Deletes a value from the cache, and erases any stale caches across the cluster. - * Fires an onDelete event after the cache has been updated in all processes. - */ - @bindThis - public async delete(key: string): Promise { - this.memoryCache.delete(key); - - await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, op: 'd', keys: [key] }); - - if (this.onDelete) { - await this.onDelete(key, this); - } - } - /** - * Deletes multiple values from the cache, and erases any stale caches across the cluster. - * Fires an onDelete event for each key after the cache has been updated in all processes. - * Skips if the input is empty. - */ - @bindThis - public async deleteMany(keys: string[]): Promise { - if (keys.length === 0) { - return; - } - - for (const key of keys) { - this.memoryCache.delete(key); - } - - await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, op: 'd', keys }); - - if (this.onDelete) { - for (const key of keys) { - await this.onDelete(key, this); - } - } - } - - /** - * Refreshes the value of a key from the fetcher, and erases any stale caches across the cluster. - * Fires an onSet event after the cache has been updated in all processes. - */ - @bindThis - public async refresh(key: string): Promise { - const value = await this.fetcher(key, this); - await this.set(key, value); - return value; - } - - /** - * Erases all entries from the local memory cache. - * Does not send any events or update other processes. - */ - @bindThis - public clear() { - this.memoryCache.clear(); - } - - /** - * Removes expired cache entries from the local view. - * Does not send any events or update other processes. - */ - @bindThis - public gc() { - this.memoryCache.gc(); - } - - /** - * Erases all data and disconnects from the cluster. - * This *must* be called when shutting down to prevent memory leaks! - */ - @bindThis - public dispose() { - this.internalEventService.off('quantumCacheUpdated', this.onQuantumCacheUpdated); - - this.memoryCache.dispose(); - } - - @bindThis - private async onQuantumCacheUpdated(data: InternalEventTypes['quantumCacheUpdated']): Promise { - if (data.name === this.name) { - for (const key of data.keys) { - this.memoryCache.delete(key); - - if (data.op === 's' && this.onSet) { - await this.onSet(key, this); - } - - if (data.op === 'd' && this.onDelete) { - await this.onDelete(key, this); - } - } - } - } - - /** - * Iterates all [key, value] pairs in memory. - * This applies to the local subset view, not the cross-cluster cache state. - */ - [Symbol.iterator](): Iterator<[key: string, value: T]> { - return this.entries(); - } -} diff --git a/packages/backend/test/misc/noOpCaches.ts b/packages/backend/test/misc/noOpCaches.ts index 373c7bddcc..c05632239b 100644 --- a/packages/backend/test/misc/noOpCaches.ts +++ b/packages/backend/test/misc/noOpCaches.ts @@ -8,7 +8,8 @@ import { Inject } from '@nestjs/common'; import { FakeInternalEventService } from './FakeInternalEventService.js'; import type { BlockingsRepository, FollowingsRepository, MiFollowing, MiUser, MiUserProfile, MutingsRepository, RenoteMutingsRepository, UserProfilesRepository, UsersRepository } from '@/models/_.js'; import type { MiLocalUser } from '@/models/User.js'; -import { MemoryKVCache, MemorySingleCache, QuantumKVCache, QuantumKVOpts, RedisKVCache, RedisSingleCache } from '@/misc/cache.js'; +import { MemoryKVCache, MemorySingleCache, RedisKVCache, RedisSingleCache } from '@/misc/cache.js'; +import { QuantumKVCache, QuantumKVOpts } from '@/misc/QuantumKVCache.js'; import { CacheService, CachedTranslationEntity, FollowStats } from '@/core/CacheService.js'; import { DI } from '@/di-symbols.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; diff --git a/packages/backend/test/unit/misc/cache.ts b/packages/backend/test/unit/misc/QuantumKVCache.ts similarity index 99% rename from packages/backend/test/unit/misc/cache.ts rename to packages/backend/test/unit/misc/QuantumKVCache.ts index e24f6d4dcc..72997494ce 100644 --- a/packages/backend/test/unit/misc/cache.ts +++ b/packages/backend/test/unit/misc/QuantumKVCache.ts @@ -5,7 +5,7 @@ import { jest } from '@jest/globals'; import { FakeInternalEventService } from '../../misc/FakeInternalEventService.js'; -import { QuantumKVCache, QuantumKVOpts } from '@/misc/cache.js'; +import { QuantumKVCache, QuantumKVOpts } from '@/misc/QuantumKVCache.js'; describe(QuantumKVCache, () => { let fakeInternalEventService: FakeInternalEventService; @@ -456,7 +456,6 @@ describe(QuantumKVCache, () => { expect(cache.has('alpha')).toBe(true); }); - it('should not emit event', () => { const cache = makeCache({ name: 'fake', From 853b548a4369051b8fdaabbda80d7d6ed52adb77 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sat, 7 Jun 2025 21:27:25 -0400 Subject: [PATCH 20/54] re-type userFollowingsCache to match the others --- packages/backend/src/core/AntennaService.ts | 3 +- packages/backend/src/core/CacheService.ts | 38 ++++++++----------- .../backend/src/core/NotificationService.ts | 12 +++--- .../src/core/entities/NoteEntityService.ts | 14 +++---- .../api/endpoints/notes/hybrid-timeline.ts | 2 +- .../server/api/endpoints/notes/timeline.ts | 2 +- .../src/server/api/endpoints/users/notes.ts | 2 +- .../src/server/api/stream/Connection.ts | 2 +- .../backend/src/server/api/stream/channel.ts | 6 +-- .../api/stream/channels/bubble-timeline.ts | 2 +- .../api/stream/channels/global-timeline.ts | 2 +- .../api/stream/channels/home-timeline.ts | 4 +- .../api/stream/channels/hybrid-timeline.ts | 4 +- .../api/stream/channels/local-timeline.ts | 2 +- .../api/stream/channels/role-timeline.ts | 2 +- .../server/api/stream/channels/user-list.ts | 2 +- packages/backend/test/misc/noOpCaches.ts | 4 +- 17 files changed, 47 insertions(+), 56 deletions(-) diff --git a/packages/backend/src/core/AntennaService.ts b/packages/backend/src/core/AntennaService.ts index cf696e3599..667df57943 100644 --- a/packages/backend/src/core/AntennaService.ts +++ b/packages/backend/src/core/AntennaService.ts @@ -130,7 +130,8 @@ export class AntennaService implements OnApplicationShutdown { } if (note.visibility === 'followers') { - const isFollowing = Object.hasOwn(await this.cacheService.userFollowingsCache.fetch(antenna.userId), note.userId); + const followings = await this.cacheService.userFollowingsCache.fetch(antenna.userId); + const isFollowing = followings.has(note.userId); if (!isFollowing && antenna.userId !== note.userId) return false; } diff --git a/packages/backend/src/core/CacheService.ts b/packages/backend/src/core/CacheService.ts index e59857b4ce..38a93e57f4 100644 --- a/packages/backend/src/core/CacheService.ts +++ b/packages/backend/src/core/CacheService.ts @@ -6,14 +6,14 @@ import { Inject, Injectable } from '@nestjs/common'; import * as Redis from 'ioredis'; import { In, IsNull } from 'typeorm'; -import type { BlockingsRepository, FollowingsRepository, MutingsRepository, RenoteMutingsRepository, MiUserProfile, UserProfilesRepository, UsersRepository, MiFollowing, MiNote } from '@/models/_.js'; +import type { BlockingsRepository, FollowingsRepository, MutingsRepository, RenoteMutingsRepository, MiUserProfile, UserProfilesRepository, UsersRepository, MiNote } from '@/models/_.js'; import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js'; import { QuantumKVCache } from '@/misc/QuantumKVCache.js'; import type { MiLocalUser, MiUser } from '@/models/User.js'; import { DI } from '@/di-symbols.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { bindThis } from '@/decorators.js'; -import type { GlobalEvents, InternalEventTypes } from '@/core/GlobalEventService.js'; +import type { InternalEventTypes } from '@/core/GlobalEventService.js'; import { InternalEventService } from '@/core/InternalEventService.js'; import type { OnApplicationShutdown } from '@nestjs/common'; @@ -46,7 +46,7 @@ export class CacheService implements OnApplicationShutdown { public userBlockingCache: QuantumKVCache>; public userBlockedCache: QuantumKVCache>; // NOTE: 「被」Blockキャッシュ public renoteMutingsCache: QuantumKVCache>; - public userFollowingsCache: QuantumKVCache | undefined>>; + public userFollowingsCache: QuantumKVCache>; protected userFollowStatsCache = new MemoryKVCache(1000 * 60 * 10); // 10 minutes protected translationsCache: RedisKVCache; @@ -110,15 +110,9 @@ export class CacheService implements OnApplicationShutdown { fetcher: (key) => this.renoteMutingsRepository.find({ where: { muterId: key }, select: ['muteeId'] }).then(xs => new Set(xs.map(x => x.muteeId))), }); - this.userFollowingsCache = new QuantumKVCache | undefined>>(this.internalEventService, 'userFollowings', { + this.userFollowingsCache = new QuantumKVCache>(this.internalEventService, 'userFollowings', { lifetime: 1000 * 60 * 30, // 30m - fetcher: (key) => this.followingsRepository.find({ where: { followerId: key }, select: ['followeeId', 'withReplies'] }).then(xs => { - const obj: Record | undefined> = {}; - for (const x of xs) { - obj[x.followeeId] = { withReplies: x.withReplies }; - } - return obj; - }), + fetcher: (key) => this.followingsRepository.find({ where: { followerId: key }, select: ['followeeId', 'withReplies'] }).then(xs => new Map(xs.map(f => [f.followeeId, { withReplies: f.withReplies }]))), }); this.translationsCache = new RedisKVCache(this.redisClient, 'translations', { @@ -305,14 +299,14 @@ export class CacheService implements OnApplicationShutdown { } @bindThis - public async getUserFollowings(userIds: Iterable): Promise>> { - const followings = new Map>(); + public async getUserFollowings(userIds: Iterable): Promise>> { + const followings = new Map>(); const toFetch: string[] = []; for (const userId of userIds) { const fromCache = this.userFollowingsCache.get(userId); if (fromCache) { - followings.set(userId, new Set(Object.keys(fromCache))); + followings.set(userId, fromCache); } else { toFetch.push(userId); } @@ -331,25 +325,25 @@ export class CacheService implements OnApplicationShutdown { }) .getMany(); - const toCache = new Map | undefined>>(); + const toCache = new Map>(); // Pivot to a map for (const { followerId, followeeId, withReplies } of fetchedFollowings) { // Queue for cache - let cacheSet = toCache.get(followerId); - if (!cacheSet) { - cacheSet = {}; - toCache.set(followerId, cacheSet); + let cacheMap = toCache.get(followerId); + if (!cacheMap) { + cacheMap = new Map(); + toCache.set(followerId, cacheMap); } - cacheSet[followeeId] = { withReplies }; + cacheMap.set(followeeId, { withReplies }); // Queue for return let returnSet = followings.get(followerId); if (!returnSet) { - returnSet = new Set(); + returnSet = new Map(); followings.set(followerId, returnSet); } - returnSet.add(followeeId); + returnSet.set(followeeId, { withReplies }); } // Update cache to speed up future calls diff --git a/packages/backend/src/core/NotificationService.ts b/packages/backend/src/core/NotificationService.ts index 0f05f5425d..2ce7bdb5a9 100644 --- a/packages/backend/src/core/NotificationService.ts +++ b/packages/backend/src/core/NotificationService.ts @@ -113,27 +113,27 @@ export class NotificationService implements OnApplicationShutdown { } if (recieveConfig?.type === 'following') { - const isFollowing = await this.cacheService.userFollowingsCache.fetch(notifieeId).then(followings => Object.hasOwn(followings, notifierId)); + const isFollowing = await this.cacheService.userFollowingsCache.fetch(notifieeId).then(followings => followings.has(notifierId)); if (!isFollowing) { return null; } } else if (recieveConfig?.type === 'follower') { - const isFollower = await this.cacheService.userFollowingsCache.fetch(notifierId).then(followings => Object.hasOwn(followings, notifieeId)); + const isFollower = await this.cacheService.userFollowingsCache.fetch(notifierId).then(followings => followings.has(notifieeId)); if (!isFollower) { return null; } } else if (recieveConfig?.type === 'mutualFollow') { const [isFollowing, isFollower] = await Promise.all([ - this.cacheService.userFollowingsCache.fetch(notifieeId).then(followings => Object.hasOwn(followings, notifierId)), - this.cacheService.userFollowingsCache.fetch(notifierId).then(followings => Object.hasOwn(followings, notifieeId)), + this.cacheService.userFollowingsCache.fetch(notifieeId).then(followings => followings.has(notifierId)), + this.cacheService.userFollowingsCache.fetch(notifierId).then(followings => followings.has(notifieeId)), ]); if (!(isFollowing && isFollower)) { return null; } } else if (recieveConfig?.type === 'followingOrFollower') { const [isFollowing, isFollower] = await Promise.all([ - this.cacheService.userFollowingsCache.fetch(notifieeId).then(followings => Object.hasOwn(followings, notifierId)), - this.cacheService.userFollowingsCache.fetch(notifierId).then(followings => Object.hasOwn(followings, notifieeId)), + this.cacheService.userFollowingsCache.fetch(notifieeId).then(followings => followings.has(notifierId)), + this.cacheService.userFollowingsCache.fetch(notifierId).then(followings => followings.has(notifieeId)), ]); if (!isFollowing && !isFollower) { return null; diff --git a/packages/backend/src/core/entities/NoteEntityService.ts b/packages/backend/src/core/entities/NoteEntityService.ts index 1b3920e13f..3af66b220d 100644 --- a/packages/backend/src/core/entities/NoteEntityService.ts +++ b/packages/backend/src/core/entities/NoteEntityService.ts @@ -133,7 +133,7 @@ export class NoteEntityService implements OnModuleInit { @bindThis public async hideNote(packedNote: Packed<'Note'>, meId: MiUser['id'] | null, hint?: { - myFollowing?: ReadonlySet, + myFollowing?: ReadonlyMap, myBlockers?: ReadonlySet, }): Promise { if (meId === packedNote.userId) return; @@ -193,7 +193,7 @@ export class NoteEntityService implements OnModuleInit { } else { const isFollowing = hint?.myFollowing ? hint.myFollowing.has(packedNote.userId) - : (await this.cacheService.userFollowingsCache.fetch(meId))[packedNote.userId] != null; + : (await this.cacheService.userFollowingsCache.fetch(meId)).has(packedNote.userId); hide = !isFollowing; } @@ -358,14 +358,10 @@ export class NoteEntityService implements OnModuleInit { : this.cacheService.userBlockingCache.fetch(meId).then((ids) => ids.has(note.userId)), hint?.myFollowing ? hint.myFollowing.has(note.userId) - : this.followingsRepository.existsBy({ - followeeId: note.userId, - followerId: meId, - }), + : this.cacheService.userFollowingsCache.fetch(meId).then(ids => ids.has(note.userId)), hint?.me !== undefined ? (hint.me?.host ?? null) - : this.cacheService.userByIdCache.fetch(meId, () => this.usersRepository.findOneByOrFail({ id: meId })) - .then(me => me.host), + : this.cacheService.findUserById(meId).then(me => me.host), ]); if (blocked) return false; @@ -420,7 +416,7 @@ export class NoteEntityService implements OnModuleInit { packedFiles: Map | null>; packedUsers: Map>; mentionHandles: Record; - userFollowings: Map>; + userFollowings: Map>; userBlockers: Map>; polls: Map; pollVotes: Map>; diff --git a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts index a7b104e198..a5623d1f03 100644 --- a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts @@ -164,7 +164,7 @@ export default class extends Endpoint { // eslint- excludeBots: !ps.withBots, noteFilter: note => { if (note.reply && note.reply.visibility === 'followers') { - if (!Object.hasOwn(followings, note.reply.userId) && note.reply.userId !== me.id) return false; + if (!followings.has(note.reply.userId) && note.reply.userId !== me.id) return false; } return true; diff --git a/packages/backend/src/server/api/endpoints/notes/timeline.ts b/packages/backend/src/server/api/endpoints/notes/timeline.ts index 8cf7bb5795..44c539eaad 100644 --- a/packages/backend/src/server/api/endpoints/notes/timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/timeline.ts @@ -115,7 +115,7 @@ export default class extends Endpoint { // eslint- excludePureRenotes: !ps.withRenotes, noteFilter: note => { if (note.reply && note.reply.visibility === 'followers') { - if (!Object.hasOwn(followings, note.reply.userId) && note.reply.userId !== me.id) return false; + if (!followings.has(note.reply.userId) && note.reply.userId !== me.id) return false; } if (!ps.withBots && note.user?.isBot) return false; diff --git a/packages/backend/src/server/api/endpoints/users/notes.ts b/packages/backend/src/server/api/endpoints/users/notes.ts index 66b50e0633..4602709067 100644 --- a/packages/backend/src/server/api/endpoints/users/notes.ts +++ b/packages/backend/src/server/api/endpoints/users/notes.ts @@ -134,7 +134,7 @@ export default class extends Endpoint { // eslint- if (ps.withReplies) redisTimelines.push(`userTimelineWithReplies:${ps.userId}`); if (ps.withChannelNotes) redisTimelines.push(`userTimelineWithChannel:${ps.userId}`); - const isFollowing = me && Object.hasOwn(await this.cacheService.userFollowingsCache.fetch(me.id), ps.userId); + const isFollowing = me && (await this.cacheService.userFollowingsCache.fetch(me.id)).has(ps.userId); const timeline = await this.fanoutTimelineEndpointService.timeline({ untilId, diff --git a/packages/backend/src/server/api/stream/Connection.ts b/packages/backend/src/server/api/stream/Connection.ts index e0535a2f14..21437850d3 100644 --- a/packages/backend/src/server/api/stream/Connection.ts +++ b/packages/backend/src/server/api/stream/Connection.ts @@ -36,7 +36,7 @@ export default class Connection { private channels = new Map(); private subscribingNotes = new Map(); public userProfile: MiUserProfile | null = null; - public following: Record | undefined> = {}; + public following: Map = new Map(); public followingChannels: Set = new Set(); public userIdsWhoMeMuting: Set = new Set(); public userIdsWhoBlockingMe: Set = new Set(); diff --git a/packages/backend/src/server/api/stream/channel.ts b/packages/backend/src/server/api/stream/channel.ts index 3a82865577..40ad454adb 100644 --- a/packages/backend/src/server/api/stream/channel.ts +++ b/packages/backend/src/server/api/stream/channel.ts @@ -70,7 +70,7 @@ export default abstract class Channel { if (!this.user) return false; if (this.user.id === note.userId) return true; if (note.visibility === 'followers') { - return this.following[note.userId] != null; + return this.following.has(note.userId); } if (!note.visibleUserIds) return false; return note.visibleUserIds.includes(this.user.id); @@ -84,7 +84,7 @@ export default abstract class Channel { if (note.user.requireSigninToViewContents && !this.user) return true; // 流れてきたNoteがインスタンスミュートしたインスタンスが関わる - if (isInstanceMuted(note, this.userMutedInstances) && !this.following[note.userId]) return true; + if (isInstanceMuted(note, this.userMutedInstances) && !this.following.has(note.userId)) return true; // 流れてきたNoteがミュートしているユーザーが関わる if (isUserRelated(note, this.userIdsWhoMeMuting)) return true; @@ -101,7 +101,7 @@ export default abstract class Channel { if (note.user.isSilenced || note.user.instance?.isSilenced) { if (this.user == null) return true; if (this.user.id === note.userId) return false; - if (this.following[note.userId] == null) return true; + if (!this.following.has(note.userId)) return true; } // TODO muted threads diff --git a/packages/backend/src/server/api/stream/channels/bubble-timeline.ts b/packages/backend/src/server/api/stream/channels/bubble-timeline.ts index 393fe3883c..72f719b411 100644 --- a/packages/backend/src/server/api/stream/channels/bubble-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/bubble-timeline.ts @@ -62,7 +62,7 @@ class BubbleTimelineChannel extends Channel { const reply = note.reply; // 自分のフォローしていないユーザーの visibility: followers な投稿への返信は弾く if (!this.isNoteVisibleToMe(reply)) return; - if (!this.following[note.userId]?.withReplies) { + if (!this.following.get(note.userId)?.withReplies) { // 「チャンネル接続主への返信」でもなければ、「チャンネル接続主が行った返信」でもなければ、「投稿者の投稿者自身への返信」でもない場合 if (reply.userId !== this.user?.id && !isMe && reply.userId !== note.userId) return; } diff --git a/packages/backend/src/server/api/stream/channels/global-timeline.ts b/packages/backend/src/server/api/stream/channels/global-timeline.ts index bac0277538..5c73f637c7 100644 --- a/packages/backend/src/server/api/stream/channels/global-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/global-timeline.ts @@ -63,7 +63,7 @@ class GlobalTimelineChannel extends Channel { const reply = note.reply; // 自分のフォローしていないユーザーの visibility: followers な投稿への返信は弾く if (!this.isNoteVisibleToMe(reply)) return; - if (!this.following[note.userId]?.withReplies) { + if (!this.following.get(note.userId)?.withReplies) { // 「チャンネル接続主への返信」でもなければ、「チャンネル接続主が行った返信」でもなければ、「投稿者の投稿者自身への返信」でもない場合 if (reply.userId !== this.user?.id && !isMe && reply.userId !== note.userId) return; } diff --git a/packages/backend/src/server/api/stream/channels/home-timeline.ts b/packages/backend/src/server/api/stream/channels/home-timeline.ts index d1dcbd07e5..c7062c0394 100644 --- a/packages/backend/src/server/api/stream/channels/home-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/home-timeline.ts @@ -47,7 +47,7 @@ class HomeTimelineChannel extends Channel { if (!this.followingChannels.has(note.channelId)) return; } else { // その投稿のユーザーをフォローしていなかったら弾く - if (!isMe && !Object.hasOwn(this.following, note.userId)) return; + if (!isMe && !this.following.has(note.userId)) return; } if (this.isNoteMutedOrBlocked(note)) return; @@ -57,7 +57,7 @@ class HomeTimelineChannel extends Channel { const reply = note.reply; // 自分のフォローしていないユーザーの visibility: followers な投稿への返信は弾く if (!this.isNoteVisibleToMe(reply)) return; - if (!this.following[note.userId]?.withReplies) { + if (!this.following.get(note.userId)?.withReplies) { // 「チャンネル接続主への返信」でもなければ、「チャンネル接続主が行った返信」でもなければ、「投稿者の投稿者自身への返信」でもない場合 if (reply.userId !== this.user!.id && !isMe && reply.userId !== note.userId) return; } diff --git a/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts b/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts index d923167e04..7cb64c9f89 100644 --- a/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts @@ -62,7 +62,7 @@ class HybridTimelineChannel extends Channel { // フォローしているチャンネルの投稿 の場合だけ if (!( (note.channelId == null && isMe) || - (note.channelId == null && Object.hasOwn(this.following, note.userId)) || + (note.channelId == null && this.following.has(note.userId)) || (note.channelId == null && (note.user.host == null && note.visibility === 'public')) || (note.channelId != null && this.followingChannels.has(note.channelId)) )) return; @@ -74,7 +74,7 @@ class HybridTimelineChannel extends Channel { const reply = note.reply; // 自分のフォローしていないユーザーの visibility: followers な投稿への返信は弾く if (!this.isNoteVisibleToMe(reply)) return; - if (!this.following[note.userId]?.withReplies && !this.withReplies) { + if (!this.following.get(note.userId)?.withReplies && !this.withReplies) { // 「チャンネル接続主への返信」でもなければ、「チャンネル接続主が行った返信」でもなければ、「投稿者の投稿者自身への返信」でもない場合 if (reply.userId !== this.user!.id && !isMe && reply.userId !== note.userId) return; } diff --git a/packages/backend/src/server/api/stream/channels/local-timeline.ts b/packages/backend/src/server/api/stream/channels/local-timeline.ts index 2eb3460efa..4869d871d6 100644 --- a/packages/backend/src/server/api/stream/channels/local-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/local-timeline.ts @@ -67,7 +67,7 @@ class LocalTimelineChannel extends Channel { const reply = note.reply; // 自分のフォローしていないユーザーの visibility: followers な投稿への返信は弾く if (!this.isNoteVisibleToMe(reply)) return; - if (!this.following[note.userId]?.withReplies) { + if (!this.following.get(note.userId)?.withReplies) { // 「チャンネル接続主への返信」でもなければ、「チャンネル接続主が行った返信」でもなければ、「投稿者の投稿者自身への返信」でもない場合 if (reply.userId !== this.user?.id && !isMe && reply.userId !== note.userId) return; } diff --git a/packages/backend/src/server/api/stream/channels/role-timeline.ts b/packages/backend/src/server/api/stream/channels/role-timeline.ts index f5984b5ae9..a3886618f1 100644 --- a/packages/backend/src/server/api/stream/channels/role-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/role-timeline.ts @@ -55,7 +55,7 @@ class RoleTimelineChannel extends Channel { const reply = note.reply; // 自分のフォローしていないユーザーの visibility: followers な投稿への返信は弾く if (!this.isNoteVisibleToMe(reply)) return; - if (!this.following[note.userId]?.withReplies) { + if (!this.following.get(note.userId)?.withReplies) { // 「チャンネル接続主への返信」でもなければ、「チャンネル接続主が行った返信」でもなければ、「投稿者の投稿者自身への返信」でもない場合 if (reply.userId !== this.user?.id && !isMe && reply.userId !== note.userId) return; } diff --git a/packages/backend/src/server/api/stream/channels/user-list.ts b/packages/backend/src/server/api/stream/channels/user-list.ts index 3f1a5a8f8f..4dae24a696 100644 --- a/packages/backend/src/server/api/stream/channels/user-list.ts +++ b/packages/backend/src/server/api/stream/channels/user-list.ts @@ -98,7 +98,7 @@ class UserListChannel extends Channel { const reply = note.reply; // 自分のフォローしていないユーザーの visibility: followers な投稿への返信は弾く if (!this.isNoteVisibleToMe(reply)) return; - if (!this.following[note.userId]?.withReplies) { + if (!this.following.get(note.userId)?.withReplies) { // 「チャンネル接続主への返信」でもなければ、「チャンネル接続主が行った返信」でもなければ、「投稿者の投稿者自身への返信」でもない場合 if (reply.userId !== this.user!.id && !isMe && reply.userId !== note.userId) return; } diff --git a/packages/backend/test/misc/noOpCaches.ts b/packages/backend/test/misc/noOpCaches.ts index c05632239b..7e8c27503c 100644 --- a/packages/backend/test/misc/noOpCaches.ts +++ b/packages/backend/test/misc/noOpCaches.ts @@ -6,7 +6,7 @@ import * as Redis from 'ioredis'; import { Inject } from '@nestjs/common'; import { FakeInternalEventService } from './FakeInternalEventService.js'; -import type { BlockingsRepository, FollowingsRepository, MiFollowing, MiUser, MiUserProfile, MutingsRepository, RenoteMutingsRepository, UserProfilesRepository, UsersRepository } from '@/models/_.js'; +import type { BlockingsRepository, FollowingsRepository, MiUser, MiUserProfile, MutingsRepository, RenoteMutingsRepository, UserProfilesRepository, UsersRepository } from '@/models/_.js'; import type { MiLocalUser } from '@/models/User.js'; import { MemoryKVCache, MemorySingleCache, RedisKVCache, RedisSingleCache } from '@/misc/cache.js'; import { QuantumKVCache, QuantumKVOpts } from '@/misc/QuantumKVCache.js'; @@ -106,7 +106,7 @@ export class NoOpCacheService extends CacheService { onSet: this.renoteMutingsCache.onSet, onDelete: this.renoteMutingsCache.onDelete, }); - this.userFollowingsCache = new NoOpQuantumKVCache | undefined>>({ + this.userFollowingsCache = new NoOpQuantumKVCache>({ internalEventService: fakeInternalEventService, fetcher: this.userFollowingsCache.fetcher, onSet: this.userFollowingsCache.onSet, From 372714c9b603e6e06ac1f0792c96d4066b7413d5 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sat, 7 Jun 2025 21:32:55 -0400 Subject: [PATCH 21/54] implement userFollowersCache --- packages/backend/src/core/CacheService.ts | 17 +++++++++++++++-- packages/backend/test/misc/noOpCaches.ts | 6 ++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/core/CacheService.ts b/packages/backend/src/core/CacheService.ts index 38a93e57f4..e8b26f8b9b 100644 --- a/packages/backend/src/core/CacheService.ts +++ b/packages/backend/src/core/CacheService.ts @@ -47,6 +47,7 @@ export class CacheService implements OnApplicationShutdown { public userBlockedCache: QuantumKVCache>; // NOTE: 「被」Blockキャッシュ public renoteMutingsCache: QuantumKVCache>; public userFollowingsCache: QuantumKVCache>; + public userFollowersCache: QuantumKVCache>; protected userFollowStatsCache = new MemoryKVCache(1000 * 60 * 10); // 10 minutes protected translationsCache: RedisKVCache; @@ -115,6 +116,11 @@ export class CacheService implements OnApplicationShutdown { fetcher: (key) => this.followingsRepository.find({ where: { followerId: key }, select: ['followeeId', 'withReplies'] }).then(xs => new Map(xs.map(f => [f.followeeId, { withReplies: f.withReplies }]))), }); + this.userFollowersCache = new QuantumKVCache>(this.internalEventService, 'userFollowers', { + lifetime: 1000 * 60 * 30, // 30m + fetcher: (key) => this.followingsRepository.find({ where: { followeeId: key }, select: ['followerId'] }).then(xs => new Set(xs.map(x => x.followerId))), + }); + this.translationsCache = new RedisKVCache(this.redisClient, 'translations', { lifetime: 1000 * 60 * 60 * 24 * 7, // 1 week, memoryCacheLifetime: 1000 * 60, // 1 minute @@ -154,6 +160,7 @@ export class CacheService implements OnApplicationShutdown { this.userBlockedCache.delete(body.id), this.renoteMutingsCache.delete(body.id), this.userFollowingsCache.delete(body.id), + this.userFollowersCache.delete(body.id), ]); } } else { @@ -193,7 +200,10 @@ export class CacheService implements OnApplicationShutdown { if (follower) follower.followingCount++; const followee = this.userByIdCache.get(body.followeeId); if (followee) followee.followersCount++; - await this.userFollowingsCache.delete(body.followerId); + await Promise.all([ + this.userFollowingsCache.delete(body.followerId), + this.userFollowersCache.delete(body.followeeId), + ]); this.userFollowStatsCache.delete(body.followerId); this.userFollowStatsCache.delete(body.followeeId); break; @@ -203,7 +213,10 @@ export class CacheService implements OnApplicationShutdown { if (follower) follower.followingCount--; const followee = this.userByIdCache.get(body.followeeId); if (followee) followee.followersCount--; - await this.userFollowingsCache.delete(body.followerId); + await Promise.all([ + this.userFollowingsCache.delete(body.followerId), + this.userFollowersCache.delete(body.followeeId), + ]); this.userFollowStatsCache.delete(body.followerId); this.userFollowStatsCache.delete(body.followeeId); break; diff --git a/packages/backend/test/misc/noOpCaches.ts b/packages/backend/test/misc/noOpCaches.ts index 7e8c27503c..40c5d2dc65 100644 --- a/packages/backend/test/misc/noOpCaches.ts +++ b/packages/backend/test/misc/noOpCaches.ts @@ -112,6 +112,12 @@ export class NoOpCacheService extends CacheService { onSet: this.userFollowingsCache.onSet, onDelete: this.userFollowingsCache.onDelete, }); + this.userFollowersCache = new NoOpQuantumKVCache>({ + internalEventService: fakeInternalEventService, + fetcher: this.userFollowersCache.fetcher, + onSet: this.userFollowersCache.onSet, + onDelete: this.userFollowersCache.onDelete, + }); this.userFollowStatsCache = new NoOpMemoryKVCache(); this.translationsCache = new NoOpRedisKVCache({ redis: fakeRedis, From fa68751a19877474bf78a80ef7204102296f0f17 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sun, 8 Jun 2025 19:52:59 -0400 Subject: [PATCH 22/54] normalize userFollowingsCache / userFollowersCache and add hibernatedUserCache to reduce the number of cache-clears and allow use of caching in many more places --- .../backend/src/core/AccountMoveService.ts | 24 +- packages/backend/src/core/CacheService.ts | 316 +++++++++-------- .../backend/src/core/GlobalEventService.ts | 2 +- .../backend/src/core/NoteCreateService.ts | 43 +-- packages/backend/src/core/NoteEditService.ts | 33 +- .../backend/src/core/UserFollowingService.ts | 119 +++---- packages/backend/src/core/UserService.ts | 21 +- .../backend/src/core/UserSuspendService.ts | 10 +- .../activitypub/ApDeliverManagerService.ts | 54 ++- .../src/core/activitypub/ApInboxService.ts | 16 +- .../activitypub/models/ApPersonService.ts | 15 +- .../src/core/chart/charts/federation.ts | 1 + .../core/chart/charts/per-user-following.ts | 19 +- .../src/core/entities/NoteEntityService.ts | 10 +- .../src/core/entities/UserEntityService.ts | 44 +-- packages/backend/src/misc/QuantumKVCache.ts | 145 ++++++-- packages/backend/src/misc/cache.ts | 18 +- .../DeleteAccountProcessorService.ts | 18 + .../server/api/endpoints/following/delete.ts | 9 +- .../api/endpoints/following/invalidate.ts | 9 +- .../api/endpoints/following/update-all.ts | 4 + .../server/api/endpoints/following/update.ts | 9 +- .../server/api/endpoints/users/followers.ts | 9 +- .../server/api/endpoints/users/following.ts | 9 +- .../api/endpoints/users/recommendation.ts | 1 + .../src/server/api/stream/Connection.ts | 2 +- packages/backend/test/misc/noOpCaches.ts | 104 +++--- .../backend/test/unit/misc/QuantumKVCache.ts | 333 ++++++++++++++---- 28 files changed, 816 insertions(+), 581 deletions(-) diff --git a/packages/backend/src/core/AccountMoveService.ts b/packages/backend/src/core/AccountMoveService.ts index 738026f753..e107f02796 100644 --- a/packages/backend/src/core/AccountMoveService.ts +++ b/packages/backend/src/core/AccountMoveService.ts @@ -26,6 +26,7 @@ import PerUserFollowingChart from '@/core/chart/charts/per-user-following.js'; import { SystemAccountService } from '@/core/SystemAccountService.js'; import { RoleService } from '@/core/RoleService.js'; import { AntennaService } from '@/core/AntennaService.js'; +import { CacheService } from '@/core/CacheService.js'; @Injectable() export class AccountMoveService { @@ -68,6 +69,7 @@ export class AccountMoveService { private systemAccountService: SystemAccountService, private roleService: RoleService, private antennaService: AntennaService, + private readonly cacheService: CacheService, ) { } @@ -107,12 +109,10 @@ export class AccountMoveService { this.globalEventService.publishMainStream(src.id, 'meUpdated', iObj); // Unfollow after 24 hours - const followings = await this.followingsRepository.findBy({ - followerId: src.id, - }); - this.queueService.createDelayedUnfollowJob(followings.map(following => ({ + const followings = await this.cacheService.userFollowingsCache.fetch(src.id); + this.queueService.createDelayedUnfollowJob(Array.from(followings.keys()).map(followeeId => ({ from: { id: src.id }, - to: { id: following.followeeId }, + to: { id: followeeId }, })), process.env.NODE_ENV === 'test' ? 10000 : 1000 * 60 * 60 * 24); await this.postMoveProcess(src, dst); @@ -138,11 +138,9 @@ export class AccountMoveService { // follow the new account const proxy = await this.systemAccountService.fetch('proxy'); - const followings = await this.followingsRepository.findBy({ - followeeId: src.id, - followerHost: IsNull(), // follower is local - followerId: Not(proxy.id), - }); + const followings = await this.cacheService.userFollowersCache.fetch(src.id) + .then(fs => Array.from(fs.values()) + .filter(f => f.followerHost == null && f.followerId !== proxy.id)); const followJobs = followings.map(following => ({ from: { id: following.followerId }, to: { id: dst.id }, @@ -318,9 +316,9 @@ export class AccountMoveService { await this.usersRepository.decrement({ id: In(localFollowerIds) }, 'followingCount', 1); // Decrease follower counts of local followees by 1. - const oldFollowings = await this.followingsRepository.findBy({ followerId: oldAccount.id }); - if (oldFollowings.length > 0) { - await this.usersRepository.decrement({ id: In(oldFollowings.map(following => following.followeeId)) }, 'followersCount', 1); + const oldFollowings = await this.cacheService.userFollowingsCache.fetch(oldAccount.id); + if (oldFollowings.size > 0) { + await this.usersRepository.decrement({ id: In(Array.from(oldFollowings.keys())) }, 'followersCount', 1); } // Update instance stats by decreasing remote followers count by the number of local followers who were following the old account. diff --git a/packages/backend/src/core/CacheService.ts b/packages/backend/src/core/CacheService.ts index e8b26f8b9b..9c68597441 100644 --- a/packages/backend/src/core/CacheService.ts +++ b/packages/backend/src/core/CacheService.ts @@ -6,7 +6,7 @@ import { Inject, Injectable } from '@nestjs/common'; import * as Redis from 'ioredis'; import { In, IsNull } from 'typeorm'; -import type { BlockingsRepository, FollowingsRepository, MutingsRepository, RenoteMutingsRepository, MiUserProfile, UserProfilesRepository, UsersRepository, MiNote } from '@/models/_.js'; +import type { BlockingsRepository, FollowingsRepository, MutingsRepository, RenoteMutingsRepository, MiUserProfile, UserProfilesRepository, UsersRepository, MiNote, MiFollowing } from '@/models/_.js'; import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js'; import { QuantumKVCache } from '@/misc/QuantumKVCache.js'; import type { MiLocalUser, MiUser } from '@/models/User.js'; @@ -46,8 +46,9 @@ export class CacheService implements OnApplicationShutdown { public userBlockingCache: QuantumKVCache>; public userBlockedCache: QuantumKVCache>; // NOTE: 「被」Blockキャッシュ public renoteMutingsCache: QuantumKVCache>; - public userFollowingsCache: QuantumKVCache>; - public userFollowersCache: QuantumKVCache>; + public userFollowingsCache: QuantumKVCache>>; + public userFollowersCache: QuantumKVCache>>; + public hibernatedUserCache: QuantumKVCache; protected userFollowStatsCache = new MemoryKVCache(1000 * 60 * 10); // 10 minutes protected translationsCache: RedisKVCache; @@ -89,36 +90,145 @@ export class CacheService implements OnApplicationShutdown { this.userProfileCache = new QuantumKVCache(this.internalEventService, 'userProfile', { lifetime: 1000 * 60 * 30, // 30m fetcher: (key) => this.userProfilesRepository.findOneByOrFail({ userId: key }), + bulkFetcher: userIds => this.userProfilesRepository.findBy({ userId: In(userIds) }).then(ps => ps.map(p => [p.userId, p])), }); this.userMutingsCache = new QuantumKVCache>(this.internalEventService, 'userMutings', { lifetime: 1000 * 60 * 30, // 30m fetcher: (key) => this.mutingsRepository.find({ where: { muterId: key }, select: ['muteeId'] }).then(xs => new Set(xs.map(x => x.muteeId))), + bulkFetcher: muterIds => this.mutingsRepository + .createQueryBuilder('muting') + .select('"muting"."muterId"', 'muterId') + .addSelect('array_agg("muting"."muteeId")', 'muteeIds') + .where({ muterId: In(muterIds) }) + .groupBy('muting.muterId') + .getRawMany<{ muterId: string, muteeIds: string[] }>() + .then(ms => ms.map(m => [m.muterId, new Set(m.muteeIds)])), }); this.userBlockingCache = new QuantumKVCache>(this.internalEventService, 'userBlocking', { lifetime: 1000 * 60 * 30, // 30m fetcher: (key) => this.blockingsRepository.find({ where: { blockerId: key }, select: ['blockeeId'] }).then(xs => new Set(xs.map(x => x.blockeeId))), + bulkFetcher: blockerIds => this.blockingsRepository + .createQueryBuilder('blocking') + .select('"blocking"."blockerId"', 'blockerId') + .addSelect('array_agg("blocking"."blockeeId")', 'blockeeIds') + .where({ blockerId: In(blockerIds) }) + .groupBy('blocking.blockerId') + .getRawMany<{ blockerId: string, blockeeIds: string[] }>() + .then(ms => ms.map(m => [m.blockerId, new Set(m.blockeeIds)])), }); this.userBlockedCache = new QuantumKVCache>(this.internalEventService, 'userBlocked', { lifetime: 1000 * 60 * 30, // 30m fetcher: (key) => this.blockingsRepository.find({ where: { blockeeId: key }, select: ['blockerId'] }).then(xs => new Set(xs.map(x => x.blockerId))), + bulkFetcher: blockeeIds => this.blockingsRepository + .createQueryBuilder('blocking') + .select('"blocking"."blockeeId"', 'blockeeId') + .addSelect('array_agg("blocking"."blockeeId")', 'blockeeIds') + .where({ blockeeId: In(blockeeIds) }) + .groupBy('blocking.blockeeId') + .getRawMany<{ blockeeId: string, blockerIds: string[] }>() + .then(ms => ms.map(m => [m.blockeeId, new Set(m.blockerIds)])), }); this.renoteMutingsCache = new QuantumKVCache>(this.internalEventService, 'renoteMutings', { lifetime: 1000 * 60 * 30, // 30m fetcher: (key) => this.renoteMutingsRepository.find({ where: { muterId: key }, select: ['muteeId'] }).then(xs => new Set(xs.map(x => x.muteeId))), + bulkFetcher: muterIds => this.renoteMutingsRepository + .createQueryBuilder('muting') + .select('"muting"."muterId"', 'muterId') + .addSelect('array_agg("muting"."muteeId")', 'muteeIds') + .where({ muterId: In(muterIds) }) + .groupBy('muting.muterId') + .getRawMany<{ muterId: string, muteeIds: string[] }>() + .then(ms => ms.map(m => [m.muterId, new Set(m.muteeIds)])), }); - this.userFollowingsCache = new QuantumKVCache>(this.internalEventService, 'userFollowings', { + this.userFollowingsCache = new QuantumKVCache>>(this.internalEventService, 'userFollowings', { lifetime: 1000 * 60 * 30, // 30m - fetcher: (key) => this.followingsRepository.find({ where: { followerId: key }, select: ['followeeId', 'withReplies'] }).then(xs => new Map(xs.map(f => [f.followeeId, { withReplies: f.withReplies }]))), + fetcher: (key) => this.followingsRepository.findBy({ followerId: key }).then(xs => new Map(xs.map(f => [f.followeeId, f]))), + bulkFetcher: followerIds => this.followingsRepository + .findBy({ followerId: In(followerIds) }) + .then(fs => fs + .reduce((groups, f) => { + let group = groups.get(f.followerId); + if (!group) { + group = new Map(); + groups.set(f.followerId, group); + } + group.set(f.followeeId, f); + return groups; + }, {} as Map>>)), }); - this.userFollowersCache = new QuantumKVCache>(this.internalEventService, 'userFollowers', { + this.userFollowersCache = new QuantumKVCache>>(this.internalEventService, 'userFollowers', { lifetime: 1000 * 60 * 30, // 30m - fetcher: (key) => this.followingsRepository.find({ where: { followeeId: key }, select: ['followerId'] }).then(xs => new Set(xs.map(x => x.followerId))), + fetcher: followeeId => this.followingsRepository.findBy({ followeeId: followeeId }).then(xs => new Map(xs.map(x => [x.followerId, x]))), + bulkFetcher: followeeIds => this.followingsRepository + .findBy({ followeeId: In(followeeIds) }) + .then(fs => fs + .reduce((groups, f) => { + let group = groups.get(f.followeeId); + if (!group) { + group = new Map(); + groups.set(f.followeeId, group); + } + group.set(f.followerId, f); + return groups; + }, {} as Map>>)), + }); + + this.hibernatedUserCache = new QuantumKVCache(this.internalEventService, 'hibernatedUsers', { + lifetime: 1000 * 60 * 30, // 30m + fetcher: async userId => { + const { isHibernated } = await this.usersRepository.findOneOrFail({ + where: { id: userId }, + select: { isHibernated: true }, + }); + return isHibernated; + }, + bulkFetcher: async userIds => { + const results = await this.usersRepository.find({ + where: { id: In(userIds) }, + select: { id: true, isHibernated: true }, + }); + return results.map(({ id, isHibernated }) => [id, isHibernated]); + }, + onChanged: async userIds => { + // We only update local copies since each process will get this event, but we can have user objects in multiple different caches. + // Before doing anything else we must "find" all the objects to update. + const userObjects = new Map(); + const toUpdate: string[] = []; + for (const uid of userIds) { + const toAdd: MiUser[] = []; + + const localUserById = this.localUserByIdCache.get(uid); + if (localUserById) toAdd.push(localUserById); + + const userById = this.userByIdCache.get(uid); + if (userById) toAdd.push(userById); + + if (toAdd.length > 0) { + toUpdate.push(uid); + userObjects.set(uid, toAdd); + } + } + + // In many cases, we won't have to do anything. + // Skipping the DB fetch ensures that this remains a single-step synchronous process. + if (toUpdate.length > 0) { + const hibernations = await this.usersRepository.find({ where: { id: In(toUpdate) }, select: { id: true, isHibernated: true } }); + for (const { id, isHibernated } of hibernations) { + const users = userObjects.get(id); + if (users) { + for (const u of users) { + u.isHibernated = isHibernated; + } + } + } + } + }, }); this.translationsCache = new RedisKVCache(this.redisClient, 'translations', { @@ -161,6 +271,7 @@ export class CacheService implements OnApplicationShutdown { this.renoteMutingsCache.delete(body.id), this.userFollowingsCache.delete(body.id), this.userFollowersCache.delete(body.id), + this.hibernatedUserCache.delete(body.id), ]); } } else { @@ -312,142 +423,6 @@ export class CacheService implements OnApplicationShutdown { } @bindThis - public async getUserFollowings(userIds: Iterable): Promise>> { - const followings = new Map>(); - - const toFetch: string[] = []; - for (const userId of userIds) { - const fromCache = this.userFollowingsCache.get(userId); - if (fromCache) { - followings.set(userId, fromCache); - } else { - toFetch.push(userId); - } - } - - if (toFetch.length > 0) { - const fetchedFollowings = await this.followingsRepository - .createQueryBuilder('following') - .select([ - 'following.followerId', - 'following.followeeId', - 'following.withReplies', - ]) - .where({ - followerId: In(toFetch), - }) - .getMany(); - - const toCache = new Map>(); - - // Pivot to a map - for (const { followerId, followeeId, withReplies } of fetchedFollowings) { - // Queue for cache - let cacheMap = toCache.get(followerId); - if (!cacheMap) { - cacheMap = new Map(); - toCache.set(followerId, cacheMap); - } - cacheMap.set(followeeId, { withReplies }); - - // Queue for return - let returnSet = followings.get(followerId); - if (!returnSet) { - returnSet = new Map(); - followings.set(followerId, returnSet); - } - returnSet.set(followeeId, { withReplies }); - } - - // Update cache to speed up future calls - this.userFollowingsCache.addMany(toCache); - } - - return followings; - } - - @bindThis - public async getUserBlockers(userIds: Iterable): Promise>> { - const blockers = new Map>(); - - const toFetch: string[] = []; - for (const userId of userIds) { - const fromCache = this.userBlockedCache.get(userId); - if (fromCache) { - blockers.set(userId, fromCache); - } else { - toFetch.push(userId); - } - } - - if (toFetch.length > 0) { - const fetchedBlockers = await this.blockingsRepository.createQueryBuilder('blocking') - .select([ - 'blocking.blockerId', - 'blocking.blockeeId', - ]) - .where({ - blockeeId: In(toFetch), - }) - .getMany(); - - const toCache = new Map>(); - - // Pivot to a map - for (const { blockerId, blockeeId } of fetchedBlockers) { - // Queue for cache - let cacheSet = toCache.get(blockeeId); - if (!cacheSet) { - cacheSet = new Set(); - toCache.set(blockeeId, cacheSet); - } - cacheSet.add(blockerId); - - // Queue for return - let returnSet = blockers.get(blockeeId); - if (!returnSet) { - returnSet = new Set(); - blockers.set(blockeeId, returnSet); - } - returnSet.add(blockerId); - } - - // Update cache to speed up future calls - this.userBlockedCache.addMany(toCache); - } - - return blockers; - } - - public async getUserProfiles(userIds: Iterable): Promise> { - const profiles = new Map; - - const toFetch: string[] = []; - for (const userId of userIds) { - const fromCache = this.userProfileCache.get(userId); - if (fromCache) { - profiles.set(userId, fromCache); - } else { - toFetch.push(userId); - } - } - - if (toFetch.length > 0) { - const fetched = await this.userProfilesRepository.findBy({ - userId: In(toFetch), - }); - - for (const profile of fetched) { - profiles.set(profile.userId, profile); - } - - const toCache = new Map(fetched.map(p => [p.userId, p])); - this.userProfileCache.addMany(toCache); - } - - return profiles; - } - public async getUsers(userIds: Iterable): Promise> { const users = new Map; @@ -475,6 +450,61 @@ export class CacheService implements OnApplicationShutdown { return users; } + @bindThis + public async isFollowing(follower: string | { id: string }, followee: string | { id: string }): Promise { + const followerId = typeof(follower) === 'string' ? follower : follower.id; + const followeeId = typeof(followee) === 'string' ? followee : followee.id; + + // This lets us use whichever one is in memory, falling back to DB fetch via userFollowingsCache. + return this.userFollowersCache.get(followeeId)?.has(followerId) + ?? (await this.userFollowingsCache.fetch(followerId)).has(followeeId); + } + + /** + * Returns all hibernated followers. + */ + @bindThis + public async getHibernatedFollowers(followeeId: string): Promise { + const followers = await this.getFollowersWithHibernation(followeeId); + return followers.filter(f => f.isFollowerHibernated); + } + + /** + * Returns all non-hibernated followers. + */ + @bindThis + public async getNonHibernatedFollowers(followeeId: string): Promise { + const followers = await this.getFollowersWithHibernation(followeeId); + return followers.filter(f => !f.isFollowerHibernated); + } + + /** + * Returns follower relations with populated isFollowerHibernated. + * If you don't need this field, then please use userFollowersCache directly for reduced overhead. + */ + @bindThis + public async getFollowersWithHibernation(followeeId: string): Promise { + const followers = await this.userFollowersCache.fetch(followeeId); + const hibernations = await this.hibernatedUserCache.fetchMany(followers.keys()).then(fs => fs.reduce((map, f) => { + map.set(f[0], f[1]); + return map; + }, new Map)); + return Array.from(followers.values()).map(following => ({ + ...following, + isFollowerHibernated: hibernations.get(following.followerId) ?? false, + })); + } + + /** + * Refreshes follower and following relations for the given user. + */ + @bindThis + public async refreshFollowRelationsFor(userId: string): Promise { + const followings = await this.userFollowingsCache.refresh(userId); + const followees = Array.from(followings.values()).map(f => f.followeeId); + await this.userFollowersCache.deleteMany(followees); + } + @bindThis public clear(): void { this.userByIdCache.clear(); diff --git a/packages/backend/src/core/GlobalEventService.ts b/packages/backend/src/core/GlobalEventService.ts index de35e9db19..c146811331 100644 --- a/packages/backend/src/core/GlobalEventService.ts +++ b/packages/backend/src/core/GlobalEventService.ts @@ -265,7 +265,7 @@ export interface InternalEventTypes { unmute: { muterId: MiUser['id']; muteeId: MiUser['id']; }; userListMemberAdded: { userListId: MiUserList['id']; memberId: MiUser['id']; }; userListMemberRemoved: { userListId: MiUserList['id']; memberId: MiUser['id']; }; - quantumCacheUpdated: { name: string, keys: string[], op: 's' | 'd' }; + quantumCacheUpdated: { name: string, keys: string[] }; } type EventTypesToEventPayload = EventUnionFromDictionary>>; diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 4dceb6e953..a9f4083446 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -606,11 +606,11 @@ export class NoteCreateService implements OnApplicationShutdown { } if (data.reply == null) { - // TODO: キャッシュ - this.followingsRepository.findBy({ - followeeId: user.id, - notify: 'normal', - }).then(async followings => { + this.cacheService.userFollowersCache.fetch(user.id).then(async followingsMap => { + const followings = Array + .from(followingsMap.values()) + .filter(f => f.notify === 'normal'); + if (note.visibility !== 'specified') { const isPureRenote = this.isRenote(data) && !this.isQuote(data) ? true : false; for (const following of followings) { @@ -948,14 +948,7 @@ export class NoteCreateService implements OnApplicationShutdown { // TODO: キャッシュ？ // eslint-disable-next-line prefer-const let [followings, userListMemberships] = await Promise.all([ - this.followingsRepository.find({ - where: { - followeeId: user.id, - followerHost: IsNull(), - isFollowerHibernated: false, - }, - select: ['followerId', 'withReplies'], - }), + this.cacheService.getNonHibernatedFollowers(user.id), this.userListMembershipsRepository.find({ where: { userId: user.id, @@ -1072,17 +1065,19 @@ export class NoteCreateService implements OnApplicationShutdown { }); if (hibernatedUsers.length > 0) { - this.usersRepository.update({ - id: In(hibernatedUsers.map(x => x.id)), - }, { - isHibernated: true, - }); - - this.followingsRepository.update({ - followerId: In(hibernatedUsers.map(x => x.id)), - }, { - isFollowerHibernated: true, - }); + await Promise.all([ + this.usersRepository.update({ + id: In(hibernatedUsers.map(x => x.id)), + }, { + isHibernated: true, + }), + this.followingsRepository.update({ + followerId: In(hibernatedUsers.map(x => x.id)), + }, { + isFollowerHibernated: true, + }), + this.cacheService.hibernatedUserCache.setMany(hibernatedUsers.map(x => [x.id, true])), + ]); } } diff --git a/packages/backend/src/core/NoteEditService.ts b/packages/backend/src/core/NoteEditService.ts index 34af1c76dd..a359381573 100644 --- a/packages/backend/src/core/NoteEditService.ts +++ b/packages/backend/src/core/NoteEditService.ts @@ -833,14 +833,7 @@ export class NoteEditService implements OnApplicationShutdown { // TODO: キャッシュ？ // eslint-disable-next-line prefer-const let [followings, userListMemberships] = await Promise.all([ - this.followingsRepository.find({ - where: { - followeeId: user.id, - followerHost: IsNull(), - isFollowerHibernated: false, - }, - select: ['followerId', 'withReplies'], - }), + this.cacheService.getNonHibernatedFollowers(user.id), this.userListMembershipsRepository.find({ where: { userId: user.id, @@ -957,17 +950,19 @@ export class NoteEditService implements OnApplicationShutdown { }); if (hibernatedUsers.length > 0) { - this.usersRepository.update({ - id: In(hibernatedUsers.map(x => x.id)), - }, { - isHibernated: true, - }); - - this.followingsRepository.update({ - followerId: In(hibernatedUsers.map(x => x.id)), - }, { - isFollowerHibernated: true, - }); + await Promise.all([ + this.usersRepository.update({ + id: In(hibernatedUsers.map(x => x.id)), + }, { + isHibernated: true, + }), + this.followingsRepository.update({ + followerId: In(hibernatedUsers.map(x => x.id)), + }, { + isFollowerHibernated: true, + }), + this.cacheService.hibernatedUserCache.setMany(hibernatedUsers.map(x => [x.id, true])), + ]); } } diff --git a/packages/backend/src/core/UserFollowingService.ts b/packages/backend/src/core/UserFollowingService.ts index 6a6c9a3000..8470872eac 100644 --- a/packages/backend/src/core/UserFollowingService.ts +++ b/packages/backend/src/core/UserFollowingService.ts @@ -147,12 +147,7 @@ export class UserFollowingService implements OnModuleInit { if (blocked) throw new IdentifiableError('3338392a-f764-498d-8855-db939dcf8c48', 'blocked'); } - if (await this.followingsRepository.exists({ - where: { - followerId: follower.id, - followeeId: followee.id, - }, - })) { + if (await this.cacheService.isFollowing(follower, followee)) { // すでにフォロー関係が存在している場合 if (this.userEntityService.isRemoteUser(follower) && this.userEntityService.isLocalUser(followee)) { // リモート → ローカル: acceptを送り返しておしまい @@ -180,24 +175,14 @@ export class UserFollowingService implements OnModuleInit { let autoAccept = false; // 鍵アカウントであっても、既にフォローされていた場合はスルー - const isFollowing = await this.followingsRepository.exists({ - where: { - followerId: follower.id, - followeeId: followee.id, - }, - }); + const isFollowing = await this.cacheService.isFollowing(follower, followee); if (isFollowing) { autoAccept = true; } // フォローしているユーザーは自動承認オプション if (!autoAccept && (this.userEntityService.isLocalUser(followee) && followeeProfile.autoAcceptFollowed)) { - const isFollowed = await this.followingsRepository.exists({ - where: { - followerId: followee.id, - followeeId: follower.id, - }, - }); + const isFollowed = await this.cacheService.isFollowing(followee, follower); // intentionally reversed parameters if (isFollowed) autoAccept = true; } @@ -206,12 +191,7 @@ export class UserFollowingService implements OnModuleInit { if (followee.isLocked && !autoAccept) { autoAccept = !!(await this.accountMoveService.validateAlsoKnownAs( follower, - (oldSrc, newSrc) => this.followingsRepository.exists({ - where: { - followeeId: followee.id, - followerId: newSrc.id, - }, - }), + (oldSrc, newSrc) => this.cacheService.isFollowing(newSrc, followee), true, )); } @@ -366,32 +346,29 @@ export class UserFollowingService implements OnModuleInit { }, silent = false, ): Promise { - const following = await this.followingsRepository.findOne({ - relations: { - follower: true, - followee: true, - }, - where: { - followerId: follower.id, - followeeId: followee.id, - }, - }); + const [ + followerUser, + followeeUser, + following, + ] = await Promise.all([ + this.cacheService.findUserById(follower.id), + this.cacheService.findUserById(followee.id), + this.cacheService.userFollowingsCache.fetch(follower.id).then(fs => fs.get(followee.id)), + ]); - if (following === null || !following.follower || !following.followee) { + if (following == null) { this.logger.warn('フォロー解除がリクエストされましたがフォローしていませんでした'); return; } await this.followingsRepository.delete(following.id); + await this.internalEventService.emit('unfollow', { followerId: follower.id, followeeId: followee.id }); - // Handled by CacheService - // this.cacheService.userFollowingsCache.refresh(follower.id); - - this.decrementFollowing(following.follower, following.followee); + this.decrementFollowing(followerUser, followeeUser); if (!silent && this.userEntityService.isLocalUser(follower)) { // Publish unfollow event - this.userEntityService.pack(followee.id, follower, { + this.userEntityService.pack(followeeUser, follower, { schema: 'UserDetailedNotMe', }).then(async packed => { this.globalEventService.publishMainStream(follower.id, 'unfollow', packed); @@ -416,8 +393,6 @@ export class UserFollowingService implements OnModuleInit { follower: MiUser, followee: MiUser, ): Promise { - await this.internalEventService.emit('unfollow', { followerId: follower.id, followeeId: followee.id }); - // Neither followee nor follower has moved. if (!follower.movedToUri && !followee.movedToUri) { //#region Decrement following / followers counts @@ -691,22 +666,22 @@ export class UserFollowingService implements OnModuleInit { */ @bindThis private async removeFollow(followee: Both, follower: Both): Promise { - const following = await this.followingsRepository.findOne({ - relations: { - followee: true, - follower: true, - }, - where: { - followeeId: followee.id, - followerId: follower.id, - }, - }); + const [ + followerUser, + followeeUser, + following, + ] = await Promise.all([ + this.cacheService.findUserById(follower.id), + this.cacheService.findUserById(followee.id), + this.cacheService.userFollowingsCache.fetch(follower.id).then(fs => fs.get(followee.id)), + ]); - if (!following || !following.followee || !following.follower) return; + if (!following) return; await this.followingsRepository.delete(following.id); + await this.internalEventService.emit('unfollow', { followerId: follower.id, followeeId: followee.id }); - this.decrementFollowing(following.follower, following.followee); + this.decrementFollowing(followerUser, followeeUser); } /** @@ -737,36 +712,26 @@ export class UserFollowingService implements OnModuleInit { } @bindThis - public getFollowees(userId: MiUser['id']) { - return this.followingsRepository.createQueryBuilder('following') - .select('following.followeeId') - .where('following.followerId = :followerId', { followerId: userId }) - .getMany(); + public async getFollowees(userId: MiUser['id']) { + const followings = await this.cacheService.userFollowingsCache.fetch(userId); + return Array.from(followings.values()); } @bindThis - public isFollowing(followerId: MiUser['id'], followeeId: MiUser['id']) { - return this.followingsRepository.exists({ - where: { - followerId, - followeeId, - }, - }); + public async isFollowing(followerId: MiUser['id'], followeeId: MiUser['id']) { + return this.cacheService.isFollowing(followerId, followeeId); } @bindThis public async isMutual(aUserId: MiUser['id'], bUserId: MiUser['id']) { - const count = await this.followingsRepository.createQueryBuilder('following') - .where(new Brackets(qb => { - qb.where('following.followerId = :aUserId', { aUserId }) - .andWhere('following.followeeId = :bUserId', { bUserId }); - })) - .orWhere(new Brackets(qb => { - qb.where('following.followerId = :bUserId', { bUserId }) - .andWhere('following.followeeId = :aUserId', { aUserId }); - })) - .getCount(); + const [ + isFollowing, + isFollowed, + ] = await Promise.all([ + this.isFollowing(aUserId, bUserId), + this.isFollowing(bUserId, aUserId), + ]); - return count === 2; + return isFollowing && isFollowed; } } diff --git a/packages/backend/src/core/UserService.ts b/packages/backend/src/core/UserService.ts index 1f471513f3..4a04910105 100644 --- a/packages/backend/src/core/UserService.ts +++ b/packages/backend/src/core/UserService.ts @@ -10,6 +10,7 @@ import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; import { SystemWebhookService } from '@/core/SystemWebhookService.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; +import { CacheService } from '@/core/CacheService.js'; @Injectable() export class UserService { @@ -20,6 +21,7 @@ export class UserService { private followingsRepository: FollowingsRepository, private systemWebhookService: SystemWebhookService, private userEntityService: UserEntityService, + private readonly cacheService: CacheService, ) { } @@ -38,14 +40,17 @@ export class UserService { }); const wokeUp = result.isHibernated; if (wokeUp) { - this.usersRepository.update(user.id, { - isHibernated: false, - }); - this.followingsRepository.update({ - followerId: user.id, - }, { - isFollowerHibernated: false, - }); + await Promise.all([ + this.usersRepository.update(user.id, { + isHibernated: false, + }), + this.followingsRepository.update({ + followerId: user.id, + }, { + isFollowerHibernated: false, + }), + this.cacheService.hibernatedUserCache.set(user.id, false), + ]); } } else { this.usersRepository.update(user.id, { diff --git a/packages/backend/src/core/UserSuspendService.ts b/packages/backend/src/core/UserSuspendService.ts index 30dcaa6f7d..f375dff862 100644 --- a/packages/backend/src/core/UserSuspendService.ts +++ b/packages/backend/src/core/UserSuspendService.ts @@ -16,6 +16,7 @@ import { bindThis } from '@/decorators.js'; import { RelationshipJobData } from '@/queue/types.js'; import { ModerationLogService } from '@/core/ModerationLogService.js'; import { isSystemAccount } from '@/misc/is-system-account.js'; +import { CacheService } from '@/core/CacheService.js'; @Injectable() export class UserSuspendService { @@ -34,6 +35,7 @@ export class UserSuspendService { private globalEventService: GlobalEventService, private apRendererService: ApRendererService, private moderationLogService: ModerationLogService, + private readonly cacheService: CacheService, ) { } @@ -143,12 +145,8 @@ export class UserSuspendService { @bindThis private async unFollowAll(follower: MiUser) { - const followings = await this.followingsRepository.find({ - where: { - followerId: follower.id, - followeeId: Not(IsNull()), - }, - }); + const followings = await this.cacheService.userFollowingsCache.fetch(follower.id) + .then(fs => Array.from(fs.values()).filter(f => f.followeeHost != null)); const jobs: RelationshipJobData[] = []; for (const following of followings) { diff --git a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts index 746af41f55..86747f2508 100644 --- a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts +++ b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts @@ -5,7 +5,6 @@ import { Inject, Injectable } from '@nestjs/common'; import { IsNull, Not } from 'typeorm'; -import { UnrecoverableError } from 'bullmq'; import { DI } from '@/di-symbols.js'; import type { FollowingsRepository } from '@/models/_.js'; import type { MiLocalUser, MiRemoteUser, MiUser } from '@/models/User.js'; @@ -14,6 +13,7 @@ import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { bindThis } from '@/decorators.js'; import type { IActivity } from '@/core/activitypub/type.js'; import { ThinUser } from '@/queue/types.js'; +import { CacheService } from '@/core/CacheService.js'; interface IRecipe { type: string; @@ -41,16 +41,14 @@ class DeliverManager { /** * Constructor - * @param userEntityService - * @param followingsRepository * @param queueService + * @param cacheService * @param actor Actor * @param activity Activity to deliver */ constructor( - private userEntityService: UserEntityService, - private followingsRepository: FollowingsRepository, private queueService: QueueService, + private readonly cacheService: CacheService, actor: { id: MiUser['id']; host: null; }, activity: IActivity | null, @@ -114,24 +112,23 @@ class DeliverManager { // Process follower recipes first to avoid duplication when processing direct recipes later. if (this.recipes.some(r => isFollowers(r))) { // followers deliver - // TODO: SELECT DISTINCT ON ("followerSharedInbox") "followerSharedInbox" みたいな問い合わせにすればよりパフォーマンス向上できそう // ただ、sharedInboxがnullなリモートユーザーも稀におり、その対応ができなさそう？ - const followers = await this.followingsRepository.find({ - where: { - followeeId: this.actor.id, - followerHost: Not(IsNull()), - }, - select: { - followerSharedInbox: true, - followerInbox: true, - followerId: true, - }, - }); + const followers = await this.cacheService.userFollowingsCache + .fetch(this.actor.id) + .then(f => Array + .from(f.values()) + .filter(f => f.followerHost != null) + .map(f => ({ + followerInbox: f.followerInbox, + followerSharedInbox: f.followerSharedInbox, + }))); for (const following of followers) { - const inbox = following.followerSharedInbox ?? following.followerInbox; - if (inbox === null) throw new UnrecoverableError(`deliver failed for ${this.actor.id}: follower ${following.followerId} inbox is null`); - inboxes.set(inbox, following.followerSharedInbox != null); + if (following.followerSharedInbox) { + inboxes.set(following.followerSharedInbox, true); + } else if (following.followerInbox) { + inboxes.set(following.followerInbox, false); + } } } @@ -153,11 +150,8 @@ class DeliverManager { @Injectable() export class ApDeliverManagerService { constructor( - @Inject(DI.followingsRepository) - private followingsRepository: FollowingsRepository, - - private userEntityService: UserEntityService, private queueService: QueueService, + private readonly cacheService: CacheService, ) { } @@ -169,9 +163,8 @@ export class ApDeliverManagerService { @bindThis public async deliverToFollowers(actor: { id: MiLocalUser['id']; host: null; }, activity: IActivity): Promise { const manager = new DeliverManager( - this.userEntityService, - this.followingsRepository, this.queueService, + this.cacheService, actor, activity, ); @@ -188,9 +181,8 @@ export class ApDeliverManagerService { @bindThis public async deliverToUser(actor: { id: MiLocalUser['id']; host: null; }, activity: IActivity, to: MiRemoteUser): Promise { const manager = new DeliverManager( - this.userEntityService, - this.followingsRepository, this.queueService, + this.cacheService, actor, activity, ); @@ -207,9 +199,8 @@ export class ApDeliverManagerService { @bindThis public async deliverToUsers(actor: { id: MiLocalUser['id']; host: null; }, activity: IActivity, targets: MiRemoteUser[]): Promise { const manager = new DeliverManager( - this.userEntityService, - this.followingsRepository, this.queueService, + this.cacheService, actor, activity, ); @@ -220,9 +211,8 @@ export class ApDeliverManagerService { @bindThis public createDeliverManager(actor: { id: MiUser['id']; host: null; }, activity: IActivity | null): DeliverManager { return new DeliverManager( - this.userEntityService, - this.followingsRepository, this.queueService, + this.cacheService, actor, activity, diff --git a/packages/backend/src/core/activitypub/ApInboxService.ts b/packages/backend/src/core/activitypub/ApInboxService.ts index 7c26deb00f..009d4cbd39 100644 --- a/packages/backend/src/core/activitypub/ApInboxService.ts +++ b/packages/backend/src/core/activitypub/ApInboxService.ts @@ -37,6 +37,7 @@ import InstanceChart from '@/core/chart/charts/instance.js'; import FederationChart from '@/core/chart/charts/federation.js'; import { FetchInstanceMetadataService } from '@/core/FetchInstanceMetadataService.js'; import { UpdateInstanceQueue } from '@/core/UpdateInstanceQueue.js'; +import { CacheService } from '@/core/CacheService.js'; import { getApHrefNullable, getApId, getApIds, getApType, getNullableApId, isAccept, isActor, isAdd, isAnnounce, isApObject, isBlock, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isDislike, isMove, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost, isActivity, IObjectWithId } from './type.js'; import { ApNoteService } from './models/ApNoteService.js'; import { ApLoggerService } from './ApLoggerService.js'; @@ -98,6 +99,7 @@ export class ApInboxService { private readonly instanceChart: InstanceChart, private readonly federationChart: FederationChart, private readonly updateInstanceQueue: UpdateInstanceQueue, + private readonly cacheService: CacheService, ) { this.logger = this.apLoggerService.logger; } @@ -766,12 +768,7 @@ export class ApInboxService { return 'skip: follower not found'; } - const isFollowing = await this.followingsRepository.exists({ - where: { - followerId: follower.id, - followeeId: actor.id, - }, - }); + const isFollowing = await this.cacheService.userFollowingsCache.fetch(follower.id).then(f => f.has(actor.id)); if (isFollowing) { await this.userFollowingService.unfollow(follower, actor); @@ -830,12 +827,7 @@ export class ApInboxService { }, }); - const isFollowing = await this.followingsRepository.exists({ - where: { - followerId: actor.id, - followeeId: followee.id, - }, - }); + const isFollowing = await this.cacheService.userFollowingsCache.fetch(actor.id).then(f => f.has(followee.id)); if (requestExist) { await this.userFollowingService.cancelFollowRequest(followee, actor); diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index b7aa036068..29f7459219 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -741,10 +741,17 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { this.hashtagService.updateUsertags(exist, tags); // 該当ユーザーが既にフォロワーになっていた場合はFollowingもアップデートする - await this.followingsRepository.update( - { followerId: exist.id }, - { followerSharedInbox: person.sharedInbox ?? person.endpoints?.sharedInbox ?? null }, - ); + if (exist.inbox !== person.inbox || exist.sharedInbox !== (person.sharedInbox ?? person.endpoints?.sharedInbox)) { + await this.followingsRepository.update( + { followerId: exist.id }, + { + followerInbox: person.inbox, + followerSharedInbox: person.sharedInbox ?? person.endpoints?.sharedInbox ?? null, + }, + ); + + await this.cacheService.refreshFollowRelationsFor(exist.id); + } await this.updateFeatured(exist.id, resolver).catch(err => { // Permanent error implies hidden or inaccessible, which is a normal thing. diff --git a/packages/backend/src/core/chart/charts/federation.ts b/packages/backend/src/core/chart/charts/federation.ts index b6db6f5454..4bbb5437cc 100644 --- a/packages/backend/src/core/chart/charts/federation.ts +++ b/packages/backend/src/core/chart/charts/federation.ts @@ -44,6 +44,7 @@ export default class FederationChart extends Chart { // eslint-di } protected async tickMinor(): Promise>> { + // TODO optimization: replace these with exists() const pubsubSubQuery = this.followingsRepository.createQueryBuilder('f') .select('f.followerHost') .where('f.followerHost IS NOT NULL'); diff --git a/packages/backend/src/core/chart/charts/per-user-following.ts b/packages/backend/src/core/chart/charts/per-user-following.ts index 588ac638de..8d75a30e9a 100644 --- a/packages/backend/src/core/chart/charts/per-user-following.ts +++ b/packages/backend/src/core/chart/charts/per-user-following.ts @@ -15,6 +15,7 @@ import Chart from '../core.js'; import { ChartLoggerService } from '../ChartLoggerService.js'; import { name, schema } from './entities/per-user-following.js'; import type { KVs } from '../core.js'; +import { CacheService } from '@/core/CacheService.js'; /** * ユーザーごとのフォローに関するチャート @@ -31,23 +32,25 @@ export default class PerUserFollowingChart extends Chart { // esl private appLockService: AppLockService, private userEntityService: UserEntityService, private chartLoggerService: ChartLoggerService, + private readonly cacheService: CacheService, ) { super(db, (k) => appLockService.getChartInsertLock(k), chartLoggerService.logger, name, schema, true); } protected async tickMajor(group: string): Promise>> { const [ - localFollowingsCount, - localFollowersCount, - remoteFollowingsCount, - remoteFollowersCount, + followees, + followers, ] = await Promise.all([ - this.followingsRepository.countBy({ followerId: group, followeeHost: IsNull() }), - this.followingsRepository.countBy({ followeeId: group, followerHost: IsNull() }), - this.followingsRepository.countBy({ followerId: group, followeeHost: Not(IsNull()) }), - this.followingsRepository.countBy({ followeeId: group, followerHost: Not(IsNull()) }), + this.cacheService.userFollowingsCache.fetch(group).then(fs => Array.from(fs.values())), + this.cacheService.userFollowersCache.fetch(group).then(fs => Array.from(fs.values())), ]); + const localFollowingsCount = followees.reduce((sum, f) => sum + (f.followeeHost == null ? 1 : 0), 0); + const localFollowersCount = followers.reduce((sum, f) => sum + (f.followerHost == null ? 1 : 0), 0); + const remoteFollowingsCount = followees.reduce((sum, f) => sum + (f.followeeHost == null ? 0 : 1), 0); + const remoteFollowersCount = followers.reduce((sum, f) => sum + (f.followerHost == null ? 0 : 1), 0); + return { 'local.followings.total': localFollowingsCount, 'local.followers.total': localFollowersCount, diff --git a/packages/backend/src/core/entities/NoteEntityService.ts b/packages/backend/src/core/entities/NoteEntityService.ts index 3af66b220d..9b447a4064 100644 --- a/packages/backend/src/core/entities/NoteEntityService.ts +++ b/packages/backend/src/core/entities/NoteEntityService.ts @@ -11,7 +11,7 @@ import type { Packed } from '@/misc/json-schema.js'; import { awaitAll } from '@/misc/prelude/await-all.js'; import type { MiUser } from '@/models/User.js'; import type { MiNote } from '@/models/Note.js'; -import type { UsersRepository, NotesRepository, FollowingsRepository, PollsRepository, PollVotesRepository, NoteReactionsRepository, ChannelsRepository, MiMeta, MiPollVote, MiPoll, MiChannel } from '@/models/_.js'; +import type { UsersRepository, NotesRepository, FollowingsRepository, PollsRepository, PollVotesRepository, NoteReactionsRepository, ChannelsRepository, MiMeta, MiPollVote, MiPoll, MiChannel, MiFollowing } from '@/models/_.js'; import { bindThis } from '@/decorators.js'; import { DebounceLoader } from '@/misc/loader.js'; import { IdService } from '@/core/IdService.js'; @@ -133,7 +133,7 @@ export class NoteEntityService implements OnModuleInit { @bindThis public async hideNote(packedNote: Packed<'Note'>, meId: MiUser['id'] | null, hint?: { - myFollowing?: ReadonlyMap, + myFollowing?: ReadonlyMap, myBlockers?: ReadonlySet, }): Promise { if (meId === packedNote.userId) return; @@ -416,7 +416,7 @@ export class NoteEntityService implements OnModuleInit { packedFiles: Map | null>; packedUsers: Map>; mentionHandles: Record; - userFollowings: Map>; + userFollowings: Map>>; userBlockers: Map>; polls: Map; pollVotes: Map>; @@ -659,9 +659,9 @@ export class NoteEntityService implements OnModuleInit { // mentionHandles this.getUserHandles(Array.from(mentionedUsers)), // userFollowings - this.cacheService.getUserFollowings(userIds), + this.cacheService.userFollowingsCache.fetchMany(userIds).then(fs => new Map(fs)), // userBlockers - this.cacheService.getUserBlockers(userIds), + this.cacheService.userBlockedCache.fetchMany(userIds).then(bs => new Map(bs)), // polls this.pollsRepository.findBy({ noteId: In(noteIds) }) .then(polls => new Map(polls.map(p => [p.noteId, p]))), diff --git a/packages/backend/src/core/entities/UserEntityService.ts b/packages/backend/src/core/entities/UserEntityService.ts index 8ed482af6f..aecbaa7fd5 100644 --- a/packages/backend/src/core/entities/UserEntityService.ts +++ b/packages/backend/src/core/entities/UserEntityService.ts @@ -79,7 +79,7 @@ function isRemoteUser(user: MiUser | { host: MiUser['host'] }): boolean { export type UserRelation = { id: MiUser['id'] - following: MiFollowing | null, + following: Omit | null, isFollowing: boolean isFollowed: boolean hasPendingFollowRequestFromYou: boolean @@ -197,16 +197,8 @@ export class UserEntityService implements OnModuleInit { memo, mutedInstances, ] = await Promise.all([ - this.followingsRepository.findOneBy({ - followerId: me, - followeeId: target, - }), - this.followingsRepository.exists({ - where: { - followerId: target, - followeeId: me, - }, - }), + this.cacheService.userFollowingsCache.fetch(me).then(f => f.get(target) ?? null), + this.cacheService.userFollowingsCache.fetch(target).then(f => f.has(me)), this.followRequestsRepository.exists({ where: { followerId: me, @@ -227,8 +219,7 @@ export class UserEntityService implements OnModuleInit { .then(mutings => mutings.has(target)), this.cacheService.renoteMutingsCache.fetch(me) .then(mutings => mutings.has(target)), - this.cacheService.userByIdCache.fetch(target, () => this.usersRepository.findOneByOrFail({ id: target })) - .then(user => user.host), + this.cacheService.findUserById(target).then(u => u.host), this.userMemosRepository.createQueryBuilder('m') .select('m.memo') .where({ userId: me, targetUserId: target }) @@ -271,13 +262,8 @@ export class UserEntityService implements OnModuleInit { memos, mutedInstances, ] = await Promise.all([ - this.followingsRepository.findBy({ followerId: me }) - .then(f => new Map(f.map(it => [it.followeeId, it]))), - this.followingsRepository.createQueryBuilder('f') - .select('f.followerId') - .where('f.followeeId = :me', { me }) - .getRawMany<{ f_followerId: string }>() - .then(it => it.map(it => it.f_followerId)), + this.cacheService.userFollowingsCache.fetch(me), + this.cacheService.userFollowersCache.fetch(me), this.followRequestsRepository.createQueryBuilder('f') .select('f.followeeId') .where('f.followerId = :me', { me }) @@ -322,7 +308,7 @@ export class UserEntityService implements OnModuleInit { id: target, following: following, isFollowing: following != null, - isFollowed: followees.includes(target), + isFollowed: followees.has(target), hasPendingFollowRequestFromYou: followersRequests.includes(target), hasPendingFollowRequestToYou: followeesRequests.includes(target), isBlocking: blockees.has(target), @@ -354,7 +340,7 @@ export class UserEntityService implements OnModuleInit { return false; // TODO } - // TODO make redis calls in MULTI? + // TODO optimization: make redis calls in MULTI @bindThis public async getNotificationsInfo(userId: MiUser['id']): Promise<{ hasUnread: boolean; @@ -789,11 +775,11 @@ export class UserEntityService implements OnModuleInit { .map(user => user.host) .filter((host): host is string => host != null)); - const _profilesFromUsers: MiUserProfile[] = []; + const _profilesFromUsers: [string, MiUserProfile][] = []; const _profilesToFetch: string[] = []; for (const user of _users) { if (user.userProfile) { - _profilesFromUsers.push(user.userProfile); + _profilesFromUsers.push([user.id, user.userProfile]); } else { _profilesToFetch.push(user.id); } @@ -803,13 +789,7 @@ export class UserEntityService implements OnModuleInit { const [profilesMap, userMemos, userRelations, pinNotes, userIdsByUri, instances, securityKeyCounts, pendingReceivedFollows, pendingSentFollows] = await Promise.all([ // profilesMap - this.cacheService.getUserProfiles(_profilesToFetch) - .then(profiles => { - for (const profile of _profilesFromUsers) { - profiles.set(profile.userId, profile); - } - return profiles; - }), + this.cacheService.userProfileCache.fetchMany(_profilesToFetch).then(profiles => new Map(profiles.concat(_profilesFromUsers))), // userMemos isDetailed && meId ? this.userMemosRepository.findBy({ userId: meId }) .then(memos => new Map(memos.map(memo => [memo.targetUserId, memo.memo]))) : new Map(), @@ -857,7 +837,7 @@ export class UserEntityService implements OnModuleInit { .groupBy('key.userId') .getRawMany<{ userId: string, userCount: number }>() .then(counts => new Map(counts.map(c => [c.userId, c.userCount]))) : new Map(), - // TODO check query performance + // TODO optimization: cache follow requests // pendingReceivedFollows isDetailedAndMe ? this.followRequestsRepository.createQueryBuilder('req') .select('req.followeeId', 'followeeId') diff --git a/packages/backend/src/misc/QuantumKVCache.ts b/packages/backend/src/misc/QuantumKVCache.ts index 6b36789f5e..b96937d6f2 100644 --- a/packages/backend/src/misc/QuantumKVCache.ts +++ b/packages/backend/src/misc/QuantumKVCache.ts @@ -21,18 +21,18 @@ export interface QuantumKVOpts { fetcher: (key: string, cache: QuantumKVCache) => T | Promise; /** - * Optional callback when a value is created or changed in the cache, either locally or elsewhere in the cluster. - * This is called *after* the cache state is updated. + * Optional callback to fetch the value for multiple keys that weren't found in the cache. * May be synchronous or async. + * If not provided, then the implementation will fall back on repeated calls to fetcher(). */ - onSet?: (key: string, cache: QuantumKVCache) => void | Promise; + bulkFetcher?: (keys: string[], cache: QuantumKVCache) => Iterable<[key: string, value: T]> | Promise>; /** - * Optional callback when a value is deleted from the cache, either locally or elsewhere in the cluster. + * Optional callback when one or more values are changed (created, updated, or deleted) in the cache, either locally or elsewhere in the cluster. * This is called *after* the cache state is updated. - * May be synchronous or async. + * Implementations may be synchronous or async. */ - onDelete?: (key: string, cache: QuantumKVCache) => void | Promise; + onChanged?: (keys: string[], cache: QuantumKVCache) => void | Promise; } /** @@ -44,8 +44,8 @@ export class QuantumKVCache implements Iterable<[key: string, value: T]> { private readonly memoryCache: MemoryKVCache; public readonly fetcher: QuantumKVOpts['fetcher']; - public readonly onSet: QuantumKVOpts['onSet']; - public readonly onDelete: QuantumKVOpts['onDelete']; + public readonly bulkFetcher: QuantumKVOpts['bulkFetcher']; + public readonly onChanged: QuantumKVOpts['onChanged']; /** * @param internalEventService Service bus to synchronize events. @@ -59,8 +59,8 @@ export class QuantumKVCache implements Iterable<[key: string, value: T]> { ) { this.memoryCache = new MemoryKVCache(opts.lifetime); this.fetcher = opts.fetcher; - this.onSet = opts.onSet; - this.onDelete = opts.onDelete; + this.bulkFetcher = opts.bulkFetcher; + this.onChanged = opts.onChanged; this.internalEventService.on('quantumCacheUpdated', this.onQuantumCacheUpdated, { // Ignore our own events, otherwise we'll immediately erase any set value. @@ -122,10 +122,10 @@ export class QuantumKVCache implements Iterable<[key: string, value: T]> { this.memoryCache.set(key, value); - await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, op: 's', keys: [key] }); + await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, keys: [key] }); - if (this.onSet) { - await this.onSet(key, this); + if (this.onChanged) { + await this.onChanged([key], this); } } @@ -146,12 +146,10 @@ export class QuantumKVCache implements Iterable<[key: string, value: T]> { } if (changedKeys.length > 0) { - await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, op: 's', keys: changedKeys }); + await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, keys: changedKeys }); - if (this.onSet) { - for (const key of changedKeys) { - await this.onSet(key, this); - } + if (this.onChanged) { + await this.onChanged(changedKeys, this); } } } @@ -180,12 +178,26 @@ export class QuantumKVCache implements Iterable<[key: string, value: T]> { /** * Gets a value from the local memory cache, or returns undefined if not found. + * Returns cached data only - does not make any fetches. */ @bindThis public get(key: string): T | undefined { return this.memoryCache.get(key); } + /** + * Gets multiple values from the local memory cache; returning undefined for any missing keys. + * Returns cached data only - does not make any fetches. + */ + @bindThis + public getMany(keys: Iterable): [key: string, value: T | undefined][] { + const results: [key: string, value: T | undefined][] = []; + for (const key of keys) { + results.push([key, this.get(key)]); + } + return results; + } + /** * Gets or fetches a value from the cache. * Fires an onSet event, but does not emit an update event to other processes. @@ -197,13 +209,49 @@ export class QuantumKVCache implements Iterable<[key: string, value: T]> { value = await this.fetcher(key, this); this.memoryCache.set(key, value); - if (this.onSet) { - await this.onSet(key, this); + if (this.onChanged) { + await this.onChanged([key], this); } } return value; } + /** + * Gets or fetches multiple values from the cache. + * Fires onSet events, but does not emit any update events to other processes. + */ + @bindThis + public async fetchMany(keys: Iterable): Promise<[key: string, value: T][]> { + const results: [key: string, value: T][] = []; + const toFetch: string[] = []; + + // Spliterate into cached results / uncached keys. + for (const key of keys) { + const fromCache = this.get(key); + if (fromCache) { + results.push([key, fromCache]); + } else { + toFetch.push(key); + } + } + + // Fetch any uncached keys + if (toFetch.length > 0) { + const fetched = await this.bulkFetch(toFetch); + + // Add to cache and return set + this.addMany(fetched); + results.push(...fetched); + + // Emit event + if (this.onChanged) { + await this.onChanged(toFetch, this); + } + } + + return results; + } + /** * Returns true is a key exists in memory. * This applies to the local subset view, not the cross-cluster cache state. @@ -221,10 +269,10 @@ export class QuantumKVCache implements Iterable<[key: string, value: T]> { public async delete(key: string): Promise { this.memoryCache.delete(key); - await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, op: 'd', keys: [key] }); + await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, keys: [key] }); - if (this.onDelete) { - await this.onDelete(key, this); + if (this.onChanged) { + await this.onChanged([key], this); } } /** @@ -233,21 +281,22 @@ export class QuantumKVCache implements Iterable<[key: string, value: T]> { * Skips if the input is empty. */ @bindThis - public async deleteMany(keys: string[]): Promise { - if (keys.length === 0) { - return; - } + public async deleteMany(keys: Iterable): Promise { + const deleted: string[] = []; for (const key of keys) { this.memoryCache.delete(key); + deleted.push(key); } - await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, op: 'd', keys }); + if (deleted.length === 0) { + return; + } - if (this.onDelete) { - for (const key of keys) { - await this.onDelete(key, this); - } + await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, keys: deleted }); + + if (this.onChanged) { + await this.onChanged(deleted, this); } } @@ -262,6 +311,13 @@ export class QuantumKVCache implements Iterable<[key: string, value: T]> { return value; } + @bindThis + public async refreshMany(keys: Iterable): Promise<[key: string, value: T][]> { + const values = await this.bulkFetch(keys); + await this.setMany(values); + return values; + } + /** * Erases all entries from the local memory cache. * Does not send any events or update other processes. @@ -291,19 +347,30 @@ export class QuantumKVCache implements Iterable<[key: string, value: T]> { this.memoryCache.dispose(); } + @bindThis + private async bulkFetch(keys: Iterable): Promise<[key: string, value: T][]> { + if (this.bulkFetcher) { + const results = await this.bulkFetcher(Array.from(keys), this); + return Array.from(results); + } + + const results: [key: string, value: T][] = []; + for (const key of keys) { + const value = await this.fetcher(key, this); + results.push([key, value]); + } + return results; + } + @bindThis private async onQuantumCacheUpdated(data: InternalEventTypes['quantumCacheUpdated']): Promise { if (data.name === this.name) { for (const key of data.keys) { this.memoryCache.delete(key); + } - if (data.op === 's' && this.onSet) { - await this.onSet(key, this); - } - - if (data.op === 'd' && this.onDelete) { - await this.onDelete(key, this); - } + if (this.onChanged) { + await this.onChanged(data.keys, this); } } } diff --git a/packages/backend/src/misc/cache.ts b/packages/backend/src/misc/cache.ts index 932c0b409a..666e684c1c 100644 --- a/packages/backend/src/misc/cache.ts +++ b/packages/backend/src/misc/cache.ts @@ -5,8 +5,6 @@ import * as Redis from 'ioredis'; import { bindThis } from '@/decorators.js'; -import { InternalEventService } from '@/core/InternalEventService.js'; -import { InternalEventTypes } from '@/core/GlobalEventService.js'; export class RedisKVCache { private readonly lifetime: number; @@ -120,9 +118,9 @@ export class RedisKVCache { export class RedisSingleCache { private readonly lifetime: number; private readonly memoryCache: MemorySingleCache; - private readonly fetcher: () => Promise; - private readonly toRedisConverter: (value: T) => string; - private readonly fromRedisConverter: (value: string) => T | undefined; + public readonly fetcher: () => Promise; + public readonly toRedisConverter: (value: T) => string; + public readonly fromRedisConverter: (value: string) => T | undefined; constructor( private redisClient: Redis.Redis, @@ -245,6 +243,16 @@ export class MemoryKVCache { return cached.value; } + public has(key: string): boolean { + const cached = this.cache.get(key); + if (cached == null) return false; + if ((Date.now() - cached.date) > this.lifetime) { + this.cache.delete(key); + return false; + } + return true; + } + @bindThis public delete(key: string): void { this.cache.delete(key); diff --git a/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts b/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts index 6a1a8bcc66..5bf64e4f04 100644 --- a/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts +++ b/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts @@ -18,6 +18,7 @@ import { SearchService } from '@/core/SearchService.js'; import { ApLogService } from '@/core/ApLogService.js'; import { ReactionService } from '@/core/ReactionService.js'; import { QueueService } from '@/core/QueueService.js'; +import { CacheService } from '@/core/CacheService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; import type { DbUserDeleteJobData } from '../types.js'; @@ -94,6 +95,7 @@ export class DeleteAccountProcessorService { private searchService: SearchService, private reactionService: ReactionService, private readonly apLogService: ApLogService, + private readonly cacheService: CacheService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('delete-account'); } @@ -140,6 +142,22 @@ export class DeleteAccountProcessorService { } { // Delete user relations + await this.cacheService.refreshFollowRelationsFor(user.id); + await this.cacheService.userFollowingsCache.delete(user.id); + await this.cacheService.userFollowingsCache.delete(user.id); + await this.cacheService.userBlockingCache.delete(user.id); + await this.cacheService.userBlockedCache.delete(user.id); + await this.cacheService.userMutingsCache.delete(user.id); + await this.cacheService.userMutingsCache.delete(user.id); + await this.cacheService.hibernatedUserCache.delete(user.id); + await this.cacheService.renoteMutingsCache.delete(user.id); + await this.cacheService.userProfileCache.delete(user.id); + this.cacheService.userByIdCache.delete(user.id); + this.cacheService.localUserByIdCache.delete(user.id); + if (user.token) { + this.cacheService.localUserByNativeTokenCache.delete(user.token); + } + await this.followingsRepository.delete({ followerId: user.id, }); diff --git a/packages/backend/src/server/api/endpoints/following/delete.ts b/packages/backend/src/server/api/endpoints/following/delete.ts index ba146b6703..442352a4d2 100644 --- a/packages/backend/src/server/api/endpoints/following/delete.ts +++ b/packages/backend/src/server/api/endpoints/following/delete.ts @@ -12,6 +12,7 @@ import { UserFollowingService } from '@/core/UserFollowingService.js'; import { DI } from '@/di-symbols.js'; import { GetterService } from '@/server/api/GetterService.js'; import { ApiError } from '../../error.js'; +import { CacheService } from '@/core/CacheService.js'; export const meta = { tags: ['following', 'users'], @@ -69,6 +70,7 @@ export default class extends Endpoint { // eslint- private userEntityService: UserEntityService, private getterService: GetterService, private userFollowingService: UserFollowingService, + private readonly cacheService: CacheService, ) { super(meta, paramDef, async (ps, me) => { const follower = me; @@ -85,12 +87,7 @@ export default class extends Endpoint { // eslint- }); // Check not following - const exist = await this.followingsRepository.exists({ - where: { - followerId: follower.id, - followeeId: followee.id, - }, - }); + const exist = await this.cacheService.userFollowingsCache.fetch(follower.id).then(f => f.has(followee.id)); if (!exist) { throw new ApiError(meta.errors.notFollowing); diff --git a/packages/backend/src/server/api/endpoints/following/invalidate.ts b/packages/backend/src/server/api/endpoints/following/invalidate.ts index b45d21410b..3809bf29b0 100644 --- a/packages/backend/src/server/api/endpoints/following/invalidate.ts +++ b/packages/backend/src/server/api/endpoints/following/invalidate.ts @@ -11,6 +11,7 @@ import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { UserFollowingService } from '@/core/UserFollowingService.js'; import { DI } from '@/di-symbols.js'; import { GetterService } from '@/server/api/GetterService.js'; +import { CacheService } from '@/core/CacheService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -69,6 +70,7 @@ export default class extends Endpoint { // eslint- private userEntityService: UserEntityService, private getterService: GetterService, private userFollowingService: UserFollowingService, + private readonly cacheService: CacheService, ) { super(meta, paramDef, async (ps, me) => { const followee = me; @@ -85,12 +87,9 @@ export default class extends Endpoint { // eslint- }); // Check not following - const exist = await this.followingsRepository.findOneBy({ - followerId: follower.id, - followeeId: followee.id, - }); + const isFollowing = await this.cacheService.userFollowingsCache.fetch(follower.id).then(f => f.has(followee.id)); - if (exist == null) { + if (!isFollowing) { throw new ApiError(meta.errors.notFollowing); } diff --git a/packages/backend/src/server/api/endpoints/following/update-all.ts b/packages/backend/src/server/api/endpoints/following/update-all.ts index c953feb393..a02b51cc79 100644 --- a/packages/backend/src/server/api/endpoints/following/update-all.ts +++ b/packages/backend/src/server/api/endpoints/following/update-all.ts @@ -12,6 +12,7 @@ import { UserFollowingService } from '@/core/UserFollowingService.js'; import { DI } from '@/di-symbols.js'; import { GetterService } from '@/server/api/GetterService.js'; import { ApiError } from '../../error.js'; +import { CacheService } from '@/core/CacheService.js'; export const meta = { tags: ['following', 'users'], @@ -39,6 +40,7 @@ export default class extends Endpoint { // eslint- constructor( @Inject(DI.followingsRepository) private followingsRepository: FollowingsRepository, + private readonly cacheService: CacheService, ) { super(meta, paramDef, async (ps, me) => { await this.followingsRepository.update({ @@ -48,6 +50,8 @@ export default class extends Endpoint { // eslint- withReplies: ps.withReplies != null ? ps.withReplies : undefined, }); + await this.cacheService.refreshFollowRelationsFor(me.id); + return; }); } diff --git a/packages/backend/src/server/api/endpoints/following/update.ts b/packages/backend/src/server/api/endpoints/following/update.ts index d62cf210ed..f4ca21856f 100644 --- a/packages/backend/src/server/api/endpoints/following/update.ts +++ b/packages/backend/src/server/api/endpoints/following/update.ts @@ -11,6 +11,7 @@ import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { UserFollowingService } from '@/core/UserFollowingService.js'; import { DI } from '@/di-symbols.js'; import { GetterService } from '@/server/api/GetterService.js'; +import { CacheService } from '@/core/CacheService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -71,6 +72,7 @@ export default class extends Endpoint { // eslint- private userEntityService: UserEntityService, private getterService: GetterService, private userFollowingService: UserFollowingService, + private readonly cacheService: CacheService, ) { super(meta, paramDef, async (ps, me) => { const follower = me; @@ -87,10 +89,7 @@ export default class extends Endpoint { // eslint- }); // Check not following - const exist = await this.followingsRepository.findOneBy({ - followerId: follower.id, - followeeId: followee.id, - }); + const exist = await this.cacheService.userFollowingsCache.fetch(follower.id).then(f => f.get(followee.id)); if (exist == null) { throw new ApiError(meta.errors.notFollowing); @@ -103,6 +102,8 @@ export default class extends Endpoint { // eslint- withReplies: ps.withReplies != null ? ps.withReplies : undefined, }); + await this.cacheService.refreshFollowRelationsFor(follower.id); + return await this.userEntityService.pack(follower.id, me); }); } diff --git a/packages/backend/src/server/api/endpoints/users/followers.ts b/packages/backend/src/server/api/endpoints/users/followers.ts index c1617e14e5..82ce282bfc 100644 --- a/packages/backend/src/server/api/endpoints/users/followers.ts +++ b/packages/backend/src/server/api/endpoints/users/followers.ts @@ -12,6 +12,7 @@ import { FollowingEntityService } from '@/core/entities/FollowingEntityService.j import { UtilityService } from '@/core/UtilityService.js'; import { DI } from '@/di-symbols.js'; import { RoleService } from '@/core/RoleService.js'; +import { CacheService } from '@/core/CacheService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -89,6 +90,7 @@ export default class extends Endpoint { // eslint- private followingEntityService: FollowingEntityService, private queryService: QueryService, private roleService: RoleService, + private readonly cacheService: CacheService, ) { super(meta, paramDef, async (ps, me) => { const user = await this.usersRepository.findOneBy(ps.userId != null @@ -110,12 +112,7 @@ export default class extends Endpoint { // eslint- if (me == null) { throw new ApiError(meta.errors.forbidden); } else if (me.id !== user.id) { - const isFollowing = await this.followingsRepository.exists({ - where: { - followeeId: user.id, - followerId: me.id, - }, - }); + const isFollowing = await this.cacheService.userFollowingsCache.fetch(me.id).then(f => f.has(user.id)); if (!isFollowing) { throw new ApiError(meta.errors.forbidden); } diff --git a/packages/backend/src/server/api/endpoints/users/following.ts b/packages/backend/src/server/api/endpoints/users/following.ts index c292c6d6a3..80f0b0c484 100644 --- a/packages/backend/src/server/api/endpoints/users/following.ts +++ b/packages/backend/src/server/api/endpoints/users/following.ts @@ -13,6 +13,7 @@ import { FollowingEntityService } from '@/core/entities/FollowingEntityService.j import { UtilityService } from '@/core/UtilityService.js'; import { DI } from '@/di-symbols.js'; import { RoleService } from '@/core/RoleService.js'; +import { CacheService } from '@/core/CacheService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -98,6 +99,7 @@ export default class extends Endpoint { // eslint- private followingEntityService: FollowingEntityService, private queryService: QueryService, private roleService: RoleService, + private readonly cacheService: CacheService, ) { super(meta, paramDef, async (ps, me) => { const user = await this.usersRepository.findOneBy(ps.userId != null @@ -119,12 +121,7 @@ export default class extends Endpoint { // eslint- if (me == null) { throw new ApiError(meta.errors.forbidden); } else if (me.id !== user.id) { - const isFollowing = await this.followingsRepository.exists({ - where: { - followeeId: user.id, - followerId: me.id, - }, - }); + const isFollowing = await this.cacheService.userFollowingsCache.fetch(me.id).then(f => f.has(user.id)); if (!isFollowing) { throw new ApiError(meta.errors.forbidden); } diff --git a/packages/backend/src/server/api/endpoints/users/recommendation.ts b/packages/backend/src/server/api/endpoints/users/recommendation.ts index 642d788459..52dd2197b2 100644 --- a/packages/backend/src/server/api/endpoints/users/recommendation.ts +++ b/packages/backend/src/server/api/endpoints/users/recommendation.ts @@ -71,6 +71,7 @@ export default class extends Endpoint { // eslint- this.queryService.generateBlockQueryForUsers(query, me); this.queryService.generateBlockedUserQueryForNotes(query, me); + // TODO optimization: replace with exists() const followingQuery = this.followingsRepository.createQueryBuilder('following') .select('following.followeeId') .where('following.followerId = :followerId', { followerId: me.id }); diff --git a/packages/backend/src/server/api/stream/Connection.ts b/packages/backend/src/server/api/stream/Connection.ts index 21437850d3..0ee7078eb2 100644 --- a/packages/backend/src/server/api/stream/Connection.ts +++ b/packages/backend/src/server/api/stream/Connection.ts @@ -36,7 +36,7 @@ export default class Connection { private channels = new Map(); private subscribingNotes = new Map(); public userProfile: MiUserProfile | null = null; - public following: Map = new Map(); + public following: Map> = new Map(); public followingChannels: Set = new Set(); public userIdsWhoMeMuting: Set = new Set(); public userIdsWhoBlockingMe: Set = new Set(); diff --git a/packages/backend/test/misc/noOpCaches.ts b/packages/backend/test/misc/noOpCaches.ts index 40c5d2dc65..f3cc1e2ba2 100644 --- a/packages/backend/test/misc/noOpCaches.ts +++ b/packages/backend/test/misc/noOpCaches.ts @@ -6,13 +6,14 @@ import * as Redis from 'ioredis'; import { Inject } from '@nestjs/common'; import { FakeInternalEventService } from './FakeInternalEventService.js'; -import type { BlockingsRepository, FollowingsRepository, MiUser, MiUserProfile, MutingsRepository, RenoteMutingsRepository, UserProfilesRepository, UsersRepository } from '@/models/_.js'; +import type { BlockingsRepository, FollowingsRepository, MiUser, MutingsRepository, RenoteMutingsRepository, UserProfilesRepository, UsersRepository } from '@/models/_.js'; import type { MiLocalUser } from '@/models/User.js'; import { MemoryKVCache, MemorySingleCache, RedisKVCache, RedisSingleCache } from '@/misc/cache.js'; import { QuantumKVCache, QuantumKVOpts } from '@/misc/QuantumKVCache.js'; -import { CacheService, CachedTranslationEntity, FollowStats } from '@/core/CacheService.js'; +import { CacheService, FollowStats } from '@/core/CacheService.js'; import { DI } from '@/di-symbols.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; +import { InternalEventService } from '@/core/InternalEventService.js'; export function noOpRedis() { return { @@ -76,55 +77,16 @@ export class NoOpCacheService extends CacheService { this.localUserByNativeTokenCache = new NoOpMemoryKVCache(); this.localUserByIdCache = new NoOpMemoryKVCache(); this.uriPersonCache = new NoOpMemoryKVCache(); - this.userProfileCache = new NoOpQuantumKVCache({ - internalEventService: fakeInternalEventService, - fetcher: this.userProfileCache.fetcher, - onSet: this.userProfileCache.onSet, - onDelete: this.userProfileCache.onDelete, - }); - this.userMutingsCache = new NoOpQuantumKVCache>({ - internalEventService: fakeInternalEventService, - fetcher: this.userMutingsCache.fetcher, - onSet: this.userMutingsCache.onSet, - onDelete: this.userMutingsCache.onDelete, - }); - this.userBlockingCache = new NoOpQuantumKVCache>({ - internalEventService: fakeInternalEventService, - fetcher: this.userBlockingCache.fetcher, - onSet: this.userBlockingCache.onSet, - onDelete: this.userBlockingCache.onDelete, - }); - this.userBlockedCache = new NoOpQuantumKVCache>({ - internalEventService: fakeInternalEventService, - fetcher: this.userBlockedCache.fetcher, - onSet: this.userBlockedCache.onSet, - onDelete: this.userBlockedCache.onDelete, - }); - this.renoteMutingsCache = new NoOpQuantumKVCache>({ - internalEventService: fakeInternalEventService, - fetcher: this.renoteMutingsCache.fetcher, - onSet: this.renoteMutingsCache.onSet, - onDelete: this.renoteMutingsCache.onDelete, - }); - this.userFollowingsCache = new NoOpQuantumKVCache>({ - internalEventService: fakeInternalEventService, - fetcher: this.userFollowingsCache.fetcher, - onSet: this.userFollowingsCache.onSet, - onDelete: this.userFollowingsCache.onDelete, - }); - this.userFollowersCache = new NoOpQuantumKVCache>({ - internalEventService: fakeInternalEventService, - fetcher: this.userFollowersCache.fetcher, - onSet: this.userFollowersCache.onSet, - onDelete: this.userFollowersCache.onDelete, - }); + this.userProfileCache = NoOpQuantumKVCache.copy(this.userProfileCache, fakeInternalEventService); + this.userMutingsCache = NoOpQuantumKVCache.copy(this.userMutingsCache, fakeInternalEventService); + this.userBlockingCache = NoOpQuantumKVCache.copy(this.userBlockingCache, fakeInternalEventService); + this.userBlockedCache = NoOpQuantumKVCache.copy(this.userBlockedCache, fakeInternalEventService); + this.renoteMutingsCache = NoOpQuantumKVCache.copy(this.renoteMutingsCache, fakeInternalEventService); + this.userFollowingsCache = NoOpQuantumKVCache.copy(this.userFollowingsCache, fakeInternalEventService); + this.userFollowersCache = NoOpQuantumKVCache.copy(this.userFollowersCache, fakeInternalEventService); + this.hibernatedUserCache = NoOpQuantumKVCache.copy(this.hibernatedUserCache, fakeInternalEventService); this.userFollowStatsCache = new NoOpMemoryKVCache(); - this.translationsCache = new NoOpRedisKVCache({ - redis: fakeRedis, - fetcher: this.translationsCache.fetcher, - toRedisConverter: this.translationsCache.toRedisConverter, - fromRedisConverter: this.translationsCache.fromRedisConverter, - }); + this.translationsCache = NoOpRedisKVCache.copy(this.translationsCache, fakeRedis); } } @@ -159,17 +121,26 @@ export class NoOpRedisKVCache extends RedisKVCache { }, ); } + + public static copy(cache: RedisKVCache, redis?: Redis.Redis): NoOpRedisKVCache { + return new NoOpRedisKVCache({ + redis, + fetcher: cache.fetcher, + toRedisConverter: cache.toRedisConverter, + fromRedisConverter: cache.fromRedisConverter, + }); + } } export class NoOpRedisSingleCache extends RedisSingleCache { constructor(opts?: { - fakeRedis?: Redis.Redis; + redis?: Redis.Redis; fetcher?: RedisSingleCache['fetcher']; toRedisConverter?: RedisSingleCache['toRedisConverter']; fromRedisConverter?: RedisSingleCache['fromRedisConverter']; }) { super( - opts?.fakeRedis ?? noOpRedis(), + opts?.redis ?? noOpRedis(), 'no-op', { lifetime: -1, @@ -180,24 +151,37 @@ export class NoOpRedisSingleCache extends RedisSingleCache { }, ); } + + public static copy(cache: RedisSingleCache, redis?: Redis.Redis): NoOpRedisSingleCache { + return new NoOpRedisSingleCache({ + redis, + fetcher: cache.fetcher, + toRedisConverter: cache.toRedisConverter, + fromRedisConverter: cache.fromRedisConverter, + }); + } } export class NoOpQuantumKVCache extends QuantumKVCache { - constructor(opts: { - internalEventService?: FakeInternalEventService, - fetcher: QuantumKVOpts['fetcher'], - onSet?: QuantumKVOpts['onSet'], - onDelete?: QuantumKVOpts['onDelete'], + constructor(opts: Omit, 'lifetime'> & { + internalEventService?: InternalEventService, }) { super( opts.internalEventService ?? new FakeInternalEventService(), 'no-op', { + ...opts, lifetime: -1, - fetcher: opts.fetcher, - onSet: opts.onSet, - onDelete: opts.onDelete, }, ); } + + public static copy(cache: QuantumKVCache, internalEventService?: InternalEventService): NoOpQuantumKVCache { + return new NoOpQuantumKVCache({ + internalEventService, + fetcher: cache.fetcher, + bulkFetcher: cache.bulkFetcher, + onChanged: cache.onChanged, + }); + } } diff --git a/packages/backend/test/unit/misc/QuantumKVCache.ts b/packages/backend/test/unit/misc/QuantumKVCache.ts index 72997494ce..92792171be 100644 --- a/packages/backend/test/unit/misc/QuantumKVCache.ts +++ b/packages/backend/test/unit/misc/QuantumKVCache.ts @@ -73,19 +73,19 @@ describe(QuantumKVCache, () => { await cache.set('foo', 'bar'); - expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', op: 's', keys: ['foo'] }]]); + expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo'] }]]); }); - it('should call onSet when storing', async () => { - const fakeOnSet = jest.fn(() => Promise.resolve()); + it('should call onChanged when storing', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); const cache = makeCache({ name: 'fake', - onSet: fakeOnSet, + onChanged: fakeOnChanged, }); await cache.set('foo', 'bar'); - expect(fakeOnSet).toHaveBeenCalledWith('foo', cache); + expect(fakeOnChanged).toHaveBeenCalledWith(['foo'], cache); }); it('should not emit event when storing unchanged value', async () => { @@ -97,17 +97,17 @@ describe(QuantumKVCache, () => { expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(1); }); - it('should not call onSet when storing unchanged value', async () => { - const fakeOnSet = jest.fn(() => Promise.resolve()); + it('should not call onChanged when storing unchanged value', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); const cache = makeCache({ name: 'fake', - onSet: fakeOnSet, + onChanged: fakeOnChanged, }); await cache.set('foo', 'bar'); await cache.set('foo', 'bar'); - expect(fakeOnSet).toHaveBeenCalledTimes(1); + expect(fakeOnChanged).toHaveBeenCalledTimes(1); }); it('should fetch an unknown value', async () => { @@ -133,17 +133,17 @@ describe(QuantumKVCache, () => { expect(result).toBe(true); }); - it('should call onSet when fetching', async () => { - const fakeOnSet = jest.fn(() => Promise.resolve()); + it('should call onChanged when fetching', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); const cache = makeCache({ name: 'fake', fetcher: key => `value#${key}`, - onSet: fakeOnSet, + onChanged: fakeOnChanged, }); await cache.fetch('foo'); - expect(fakeOnSet).toHaveBeenCalledWith('foo', cache); + expect(fakeOnChanged).toHaveBeenCalledWith(['foo'], cache); }); it('should not emit event when fetching', async () => { @@ -154,7 +154,7 @@ describe(QuantumKVCache, () => { await cache.fetch('foo'); - expect(fakeInternalEventService._calls).not.toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', op: 's', keys: ['foo'] }]]); + expect(fakeInternalEventService._calls).not.toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo'] }]]); }); it('should delete from memory cache', async () => { @@ -167,17 +167,17 @@ describe(QuantumKVCache, () => { expect(result).toBe(false); }); - it('should call onDelete when deleting', async () => { - const fakeOnDelete = jest.fn(() => Promise.resolve()); + it('should call onChanged when deleting', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); const cache = makeCache({ name: 'fake', - onDelete: fakeOnDelete, + onChanged: fakeOnChanged, }); await cache.set('foo', 'bar'); await cache.delete('foo'); - expect(fakeOnDelete).toHaveBeenCalledWith('foo', cache); + expect(fakeOnChanged).toHaveBeenCalledWith(['foo'], cache); }); it('should emit event when deleting', async () => { @@ -186,52 +186,52 @@ describe(QuantumKVCache, () => { await cache.set('foo', 'bar'); await cache.delete('foo'); - expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', op: 'd', keys: ['foo'] }]]); + expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo'] }]]); }); it('should delete when receiving set event', async () => { const cache = makeCache({ name: 'fake' }); await cache.set('foo', 'bar'); - await fakeInternalEventService._emitRedis('quantumCacheUpdated', { name: 'fake', op: 's', keys: ['foo'] }); + await fakeInternalEventService._emitRedis('quantumCacheUpdated', { name: 'fake', keys: ['foo'] }); const result = cache.has('foo'); expect(result).toBe(false); }); - it('should call onSet when receiving set event', async () => { - const fakeOnSet = jest.fn(() => Promise.resolve()); + it('should call onChanged when receiving set event', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); const cache = makeCache({ name: 'fake', - onSet: fakeOnSet, + onChanged: fakeOnChanged, }); - await fakeInternalEventService._emitRedis('quantumCacheUpdated', { name: 'fake', op: 's', keys: ['foo'] }); + await fakeInternalEventService._emitRedis('quantumCacheUpdated', { name: 'fake', keys: ['foo'] }); - expect(fakeOnSet).toHaveBeenCalledWith('foo', cache); + expect(fakeOnChanged).toHaveBeenCalledWith(['foo'], cache); }); it('should delete when receiving delete event', async () => { const cache = makeCache({ name: 'fake' }); await cache.set('foo', 'bar'); - await fakeInternalEventService._emitRedis('quantumCacheUpdated', { name: 'fake', op: 'd', keys: ['foo'] }); + await fakeInternalEventService._emitRedis('quantumCacheUpdated', { name: 'fake', keys: ['foo'] }); const result = cache.has('foo'); expect(result).toBe(false); }); - it('should call onDelete when receiving delete event', async () => { - const fakeOnDelete = jest.fn(() => Promise.resolve()); + it('should call onChanged when receiving delete event', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); const cache = makeCache({ name: 'fake', - onDelete: fakeOnDelete, + onChanged: fakeOnChanged, }); await cache.set('foo', 'bar'); - await fakeInternalEventService._emitRedis('quantumCacheUpdated', { name: 'fake', op: 'd', keys: ['foo'] }); + await fakeInternalEventService._emitRedis('quantumCacheUpdated', { name: 'fake', keys: ['foo'] }); - expect(fakeOnDelete).toHaveBeenCalledWith('foo', cache); + expect(fakeOnChanged).toHaveBeenCalledWith(['foo'], cache); }); describe('get', () => { @@ -269,40 +269,243 @@ describe(QuantumKVCache, () => { await cache.setMany([['foo', 'bar'], ['alpha', 'omega']]); - expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', op: 's', keys: ['foo', 'alpha'] }]]); + expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo', 'alpha'] }]]); expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(1); }); - it('should call onSet for each item', async () => { - const fakeOnSet = jest.fn(() => Promise.resolve()); + it('should call onChanged once with all items', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); const cache = makeCache({ name: 'fake', - onSet: fakeOnSet, + onChanged: fakeOnChanged, }); await cache.setMany([['foo', 'bar'], ['alpha', 'omega']]); - expect(fakeOnSet).toHaveBeenCalledWith('foo', cache); - expect(fakeOnSet).toHaveBeenCalledWith('alpha', cache); + expect(fakeOnChanged).toHaveBeenCalledWith(['foo', 'alpha'], cache); + expect(fakeOnChanged).toHaveBeenCalledTimes(1); }); it('should emit events only for changed items', async () => { - const fakeOnSet = jest.fn(() => Promise.resolve()); + const fakeOnChanged = jest.fn(() => Promise.resolve()); const cache = makeCache({ name: 'fake', - onSet: fakeOnSet, + onChanged: fakeOnChanged, }); await cache.set('foo', 'bar'); - fakeOnSet.mockClear(); + fakeOnChanged.mockClear(); fakeInternalEventService._reset(); await cache.setMany([['foo', 'bar'], ['alpha', 'omega']]); - expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', op: 's', keys: ['alpha'] }]]); + expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['alpha'] }]]); + expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(1); + expect(fakeOnChanged).toHaveBeenCalledWith(['alpha'], cache); + expect(fakeOnChanged).toHaveBeenCalledTimes(1); + }); + }); + + describe('getMany', () => { + it('should return empty for empty input', () => { + const cache = makeCache(); + const result = cache.getMany([]); + expect(result).toEqual([]); + }); + + it('should return the value for all keys', () => { + const cache = makeCache(); + cache.add('foo', 'bar'); + cache.add('alpha', 'omega'); + + const result = cache.getMany(['foo', 'alpha']); + + expect(result).toEqual([['foo', 'bar'], ['alpha', 'omega']]); + }); + + it('should return undefined for missing keys', () => { + const cache = makeCache(); + cache.add('foo', 'bar'); + + const result = cache.getMany(['foo', 'alpha']); + + expect(result).toEqual([['foo', 'bar'], ['alpha', undefined]]); + }); + }); + + describe('fetchMany', () => { + it('should do nothing for empty input', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + onChanged: fakeOnChanged, + }); + + await cache.fetchMany([]); + + expect(fakeOnChanged).not.toHaveBeenCalled(); + expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0); + }); + + it('should return existing items', async () => { + const cache = makeCache(); + cache.add('foo', 'bar'); + cache.add('alpha', 'omega'); + + const result = await cache.fetchMany(['foo', 'alpha']); + + expect(result).toEqual([['foo', 'bar'], ['alpha', 'omega']]); + }); + + it('should return existing items without events', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + onChanged: fakeOnChanged, + }); + cache.add('foo', 'bar'); + cache.add('alpha', 'omega'); + + await cache.fetchMany(['foo', 'alpha']); + + expect(fakeOnChanged).not.toHaveBeenCalled(); + expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0); + }); + + it('should call bulkFetcher for missing items', async () => { + const cache = makeCache({ + bulkFetcher: keys => keys.map(k => [k, `${k}#many`]), + fetcher: key => `${key}#single`, + }); + + const results = await cache.fetchMany(['foo', 'alpha']); + + expect(results).toEqual([['foo', 'foo#many'], ['alpha', 'alpha#many']]); + }); + + it('should call bulkFetcher only once', async () => { + const mockBulkFetcher = jest.fn((keys: string[]) => keys.map(k => [k, `${k}#value`] as [string, string])); + const cache = makeCache({ + bulkFetcher: mockBulkFetcher, + }); + + await cache.fetchMany(['foo', 'bar']); + + expect(mockBulkFetcher).toHaveBeenCalledTimes(1); + }); + + it('should call fetcher when fetchMany is undefined', async () => { + const cache = makeCache({ + fetcher: key => `${key}#single`, + }); + + const results = await cache.fetchMany(['foo', 'alpha']); + + expect(results).toEqual([['foo', 'foo#single'], ['alpha', 'alpha#single']]); + }); + + it('should call onChanged', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + onChanged: fakeOnChanged, + fetcher: k => k, + }); + + await cache.fetchMany(['foo', 'alpha']); + + expect(fakeOnChanged).toHaveBeenCalledWith(['foo', 'alpha'], cache); + expect(fakeOnChanged).toHaveBeenCalledTimes(1); + }); + + it('should call onChanged only for changed', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + onChanged: fakeOnChanged, + fetcher: k => k, + }); + cache.add('foo', 'bar'); + + await cache.fetchMany(['foo', 'alpha']); + + expect(fakeOnChanged).toHaveBeenCalledWith(['alpha'], cache); + expect(fakeOnChanged).toHaveBeenCalledTimes(1); + }); + + it('should not emit event', async () => { + const cache = makeCache({ + fetcher: k => k, + }); + + await cache.fetchMany(['foo', 'alpha']); + + expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0); + }); + }); + + describe('refreshMany', () => { + it('should do nothing for empty input', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + onChanged: fakeOnChanged, + }); + + const result = await cache.refreshMany([]); + + expect(result).toEqual([]); + expect(fakeOnChanged).not.toHaveBeenCalled(); + expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0); + }); + + it('should call bulkFetcher for all keys', async () => { + const mockBulkFetcher = jest.fn((keys: string[]) => keys.map(k => [k, `${k}#value`] as [string, string])); + const cache = makeCache({ + bulkFetcher: mockBulkFetcher, + }); + + const result = await cache.refreshMany(['foo', 'alpha']); + + expect(result).toEqual([['foo', 'foo#value'], ['alpha', 'alpha#value']]); + expect(mockBulkFetcher).toHaveBeenCalledWith(['foo', 'alpha'], cache); + expect(mockBulkFetcher).toHaveBeenCalledTimes(1); + }); + + it('should replace any existing keys', async () => { + const mockBulkFetcher = jest.fn((keys: string[]) => keys.map(k => [k, `${k}#value`] as [string, string])); + const cache = makeCache({ + bulkFetcher: mockBulkFetcher, + }); + cache.add('foo', 'bar'); + + const result = await cache.refreshMany(['foo', 'alpha']); + + expect(result).toEqual([['foo', 'foo#value'], ['alpha', 'alpha#value']]); + expect(mockBulkFetcher).toHaveBeenCalledWith(['foo', 'alpha'], cache); + expect(mockBulkFetcher).toHaveBeenCalledTimes(1); + }); + + it('should call onChanged for all keys', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + bulkFetcher: keys => keys.map(k => [k, `${k}#value`]), + onChanged: fakeOnChanged, + }); + cache.add('foo', 'bar'); + + await cache.refreshMany(['foo', 'alpha']); + + expect(fakeOnChanged).toHaveBeenCalledWith(['foo', 'alpha'], cache); + expect(fakeOnChanged).toHaveBeenCalledTimes(1); + }); + + it('should emit event for all keys', async () => { + const cache = makeCache({ + name: 'fake', + bulkFetcher: keys => keys.map(k => [k, `${k}#value`]), + }); + cache.add('foo', 'bar'); + + await cache.refreshMany(['foo', 'alpha']); + + expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo', 'alpha'] }]]); expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(1); - expect(fakeOnSet).toHaveBeenCalledWith('alpha', cache); - expect(fakeOnSet).toHaveBeenCalledTimes(1); }); }); @@ -325,33 +528,33 @@ describe(QuantumKVCache, () => { await cache.deleteMany(['foo', 'alpha']); - expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', op: 'd', keys: ['foo', 'alpha'] }]]); + expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo', 'alpha'] }]]); expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(1); }); - it('should call onDelete for each key', async () => { - const fakeOnDelete = jest.fn(() => Promise.resolve()); + it('should call onChanged once with all items', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); const cache = makeCache({ name: 'fake', - onDelete: fakeOnDelete, + onChanged: fakeOnChanged, }); await cache.deleteMany(['foo', 'alpha']); - expect(fakeOnDelete).toHaveBeenCalledWith('foo', cache); - expect(fakeOnDelete).toHaveBeenCalledWith('alpha', cache); + expect(fakeOnChanged).toHaveBeenCalledWith(['foo', 'alpha'], cache); + expect(fakeOnChanged).toHaveBeenCalledTimes(1); }); it('should do nothing if no keys are provided', async () => { - const fakeOnDelete = jest.fn(() => Promise.resolve()); + const fakeOnChanged = jest.fn(() => Promise.resolve()); const cache = makeCache({ name: 'fake', - onDelete: fakeOnDelete, + onChanged: fakeOnChanged, }); await cache.deleteMany([]); - expect(fakeOnDelete).not.toHaveBeenCalled(); + expect(fakeOnChanged).not.toHaveBeenCalled(); expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0); }); }); @@ -392,17 +595,17 @@ describe(QuantumKVCache, () => { expect(result).toBe('value#foo'); }); - it('should call onSet', async () => { - const fakeOnSet = jest.fn(() => Promise.resolve()); + it('should call onChanged', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); const cache = makeCache({ name: 'fake', fetcher: key => `value#${key}`, - onSet: fakeOnSet, + onChanged: fakeOnChanged, }); await cache.refresh('foo'); - expect(fakeOnSet).toHaveBeenCalledWith('foo', cache); + expect(fakeOnChanged).toHaveBeenCalledWith(['foo'], cache); }); it('should emit event', async () => { @@ -413,7 +616,7 @@ describe(QuantumKVCache, () => { await cache.refresh('foo'); - expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', op: 's', keys: ['foo'] }]]); + expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo'] }]]); }); }); @@ -434,15 +637,15 @@ describe(QuantumKVCache, () => { expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0); }); - it('should not call onSet', () => { - const fakeOnSet = jest.fn(() => Promise.resolve()); + it('should not call onChanged', () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); const cache = makeCache({ - onSet: fakeOnSet, + onChanged: fakeOnChanged, }); cache.add('foo', 'bar'); - expect(fakeOnSet).not.toHaveBeenCalled(); + expect(fakeOnChanged).not.toHaveBeenCalled(); }); }); @@ -466,15 +669,15 @@ describe(QuantumKVCache, () => { expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0); }); - it('should not call onSet', () => { - const fakeOnSet = jest.fn(() => Promise.resolve()); + it('should not call onChanged', () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); const cache = makeCache({ - onSet: fakeOnSet, + onChanged: fakeOnChanged, }); cache.addMany([['foo', 'bar'], ['alpha', 'omega']]); - expect(fakeOnSet).not.toHaveBeenCalled(); + expect(fakeOnChanged).not.toHaveBeenCalled(); }); }); From e7feca87847cbc165cc7d376d757448e926ab725 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sun, 8 Jun 2025 21:58:07 -0400 Subject: [PATCH 23/54] document cache implementations --- CONTRIBUTING.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ef08d5275f..e1a12926cc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -622,6 +622,35 @@ marginはそのコンポーネントを使う側が設定する ### indexというファイル名を使うな ESMではディレクトリインポートは廃止されているのと、ディレクトリインポートせずともファイル名が index だと何故か一部のライブラリ？でディレクトリインポートだと見做されてエラーになる +### Memory Caches + +Sharkey offers multiple memory cache implementations, each meant for a different use case. +The following table compares the available options: + +| Cache | Type | Consistency | Persistence | Data Source | Cardinality | Eviction | Description | +|---------------------|-----------|-------------|-------------|-------------|-------------|----------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `MemoryKVCache` | Key-Value | None | None | Caller | Single | Lifetime | Implements a basic in-memory Key-Value store. The implementation is entirely synchronous, except for user-provided data sources. | +| `MemorySingleCache` | Single | None | None | Caller | Single | Lifetime | Implements a basic in-memory Single Value store. The implementation is entirely synchronous, except for user-provided data sources. | +| `RedisKVCache` | Key-Value | Eventual | Redis | Callback | Single | Lifetime | Extends `MemoryKVCache` with Redis-backed persistence and a pre-defined callback data source. This provides eventual consistency guarantees based on the memory cache lifetime. | +| `RedisSingleCache` | Single | Eventual | Redis | Callback | Single | Lifetime | Extends `MemorySingleCache` with Redis-backed persistence and a pre-defined callback data source. This provides eventual consistency guarantees based on the memory cache lifetime. | +| `QuantumKVCache` | Key-Value | Immediate | None | Callback | Multiple | Lifetime | Combines `MemoryKVCache` with a pre-defined callback data source and immediate consistency via Redis sync events. The implementation offers multi-item batch overloads for efficient bulk operations. **This is the recommended cache implementation for most use cases.** | + +Key-Value caches store multiple entries per cache, while Single caches store a single value that can be accessed directly. +Consistency refers to the consistency of cached data between different processes in the instance cluster: "None" means no consistency guarantees, "Eventual" caches will gradually become consistent after some unknown time, and "Immediate" consistency ensures accurate data ASAP after the update. +Caches with persistence can retain their data after a reboot through an external service such as Redis. +If a data source is supported, then this allows the cache to directly load missing data in response to a fetch. +"Caller" data sources are passed into the fetch method(s) directly, while "Callback" sources are passed in as a function when the cache is first initialized. +The cardinality of a cache refers to the number of items that can be updated in a single operation, and eviction, finally, is the method that the cache uses to evict stale data. + +#### Selecting a cache implementation + +For most cache uses, `QuantumKVCache` should be considered first. +It offers strong consistency guarantees, multiple cardinality, and a cleaner API surface than the older caches. +An alternate cache implementation should be considered if any of the following apply: +* The data is particularly slow to calculate or difficult to access. In these cases, either `RedisKVCache` or `RedisSingleCache` should be considered. +* If stale data is acceptable, then consider `MemoryKVCache` or `MemorySingleCache`. These synchronous implementations have much less overhead than the other options. +* There is only one data item, or all data items must be fetched together. Using `MemorySingleCache` or `RedisSingleCache` could provide a cleaner implementation without resorting to hacks like a fixed key. + ## CSS Recipe ### Lighten CSS vars From 2aaed4764c76240352732d0a68985cd1367b8f85 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 6 Jun 2025 13:53:02 -0400 Subject: [PATCH 24/54] add fep-044f "quote" to context and model --- packages/backend/src/core/activitypub/misc/contexts.ts | 4 ++++ packages/backend/src/core/activitypub/type.ts | 1 + 2 files changed, 5 insertions(+) diff --git a/packages/backend/src/core/activitypub/misc/contexts.ts b/packages/backend/src/core/activitypub/misc/contexts.ts index cedd1d8dd5..fa003b1791 100644 --- a/packages/backend/src/core/activitypub/misc/contexts.ts +++ b/packages/backend/src/core/activitypub/misc/contexts.ts @@ -540,6 +540,10 @@ const extension_context_definition = { quoteUrl: 'as:quoteUrl', fedibird: 'http://fedibird.com/ns#', quoteUri: 'fedibird:quoteUri', + quote: { + '@id': 'https://w3id.org/fep/044f#quote', + '@type': '@id', + }, // Mastodon toot: 'http://joinmastodon.org/ns#', Emoji: 'toot:Emoji', diff --git a/packages/backend/src/core/activitypub/type.ts b/packages/backend/src/core/activitypub/type.ts index cc7599d394..cdf4a65baa 100644 --- a/packages/backend/src/core/activitypub/type.ts +++ b/packages/backend/src/core/activitypub/type.ts @@ -204,6 +204,7 @@ export interface IPost extends IObject { _misskey_content?: string; quoteUrl?: string; quoteUri?: string; + quote?: string; updated?: string; } From 5a99e74ee3630270cff7e4c9f24c0e2376821b81 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 6 Jun 2025 13:53:32 -0400 Subject: [PATCH 25/54] render fep-044f "quote" in notes --- packages/backend/src/core/activitypub/ApRendererService.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts index 46a78687f3..079023284b 100644 --- a/packages/backend/src/core/activitypub/ApRendererService.ts +++ b/packages/backend/src/core/activitypub/ApRendererService.ts @@ -419,7 +419,7 @@ export class ApRendererService { inReplyTo = null; } - let quote; + let quote: string | undefined = undefined; if (note.renoteId) { const renote = await this.notesRepository.findOneBy({ id: note.renoteId }); @@ -537,6 +537,7 @@ export class ApRendererService { _misskey_quote: quote, quoteUrl: quote, quoteUri: quote, + quote: quote, published: this.idService.parse(note.id).date.toISOString(), to, cc, @@ -774,7 +775,7 @@ export class ApRendererService { inReplyTo = null; } - let quote; + let quote: string | undefined = undefined; if (note.renoteId) { const renote = await this.notesRepository.findOneBy({ id: note.renoteId }); @@ -886,6 +887,7 @@ export class ApRendererService { _misskey_quote: quote, quoteUrl: quote, quoteUri: quote, + quote: quote, published: this.idService.parse(note.id).date.toISOString(), to, cc, From 15baf78ad719b1a57a382ae8c50f7d85c4201cce Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 6 Jun 2025 13:53:51 -0400 Subject: [PATCH 26/54] accept fep-044f "quote" in objects --- .../backend/src/core/activitypub/models/ApNoteService.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/core/activitypub/models/ApNoteService.ts b/packages/backend/src/core/activitypub/models/ApNoteService.ts index 57d4303982..bb0bbe374f 100644 --- a/packages/backend/src/core/activitypub/models/ApNoteService.ts +++ b/packages/backend/src/core/activitypub/models/ApNoteService.ts @@ -657,9 +657,10 @@ export class ApNoteService { */ private async getQuote(note: IPost, entryUri: string, resolver: Resolver): Promise { const quoteUris = new Set(); - if (note._misskey_quote) quoteUris.add(note._misskey_quote); - if (note.quoteUrl) quoteUris.add(note.quoteUrl); - if (note.quoteUri) quoteUris.add(note.quoteUri); + if (note._misskey_quote && typeof(note._misskey_quote as unknown) === 'string') quoteUris.add(note._misskey_quote); + if (note.quoteUrl && typeof(note.quoteUrl as unknown) === 'string') quoteUris.add(note.quoteUrl); + if (note.quoteUri && typeof(note.quoteUri as unknown) === 'string') quoteUris.add(note.quoteUri); + if (note.quote && typeof(note.quote as unknown) === 'string') quoteUris.add(note.quote); // No quote, return undefined if (quoteUris.size < 1) return undefined; From b2c72da96c35e630f14f311e13564487d87a1d5c Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 6 Jun 2025 17:24:03 -0400 Subject: [PATCH 27/54] pull out ILink interface --- packages/backend/src/core/activitypub/type.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/core/activitypub/type.ts b/packages/backend/src/core/activitypub/type.ts index cdf4a65baa..554420d670 100644 --- a/packages/backend/src/core/activitypub/type.ts +++ b/packages/backend/src/core/activitypub/type.ts @@ -35,6 +35,7 @@ export interface IObject { mediaType?: string; url?: ApObject | string; href?: string; + rel?: string | string[]; tag?: IObject | IObject[]; sensitive?: boolean; } @@ -55,6 +56,16 @@ export function isAnonymousObject(object: IObject): object is IAnonymousObject { return object.id === undefined; } +export interface ILink extends IObject { + '@context'?: string | string[] | Obj | Obj[]; + type: 'Link' | 'Mention'; + href: string; +} + +export const isLink = (object: IObject): object is ILink => + (getApType(object) === 'Link' || getApType(object) === 'Link') && + typeof object.href === 'string'; + /** * Get array of ActivityStreams Objects id */ @@ -307,9 +318,8 @@ export const isPropertyValue = (object: IObject): object is IApPropertyValue => 'value' in object && typeof object.value === 'string'; -export interface IApMention extends IObject { +export interface IApMention extends ILink { type: 'Mention'; - href: string; name: string; } From aefd94c0b0e2c7a9e6fd34c85fbdb0f2ca8c276c Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 6 Jun 2025 17:24:42 -0400 Subject: [PATCH 28/54] render fep-e232 "tag" quotes in notes --- .../src/core/activitypub/ApRendererService.ts | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts index 079023284b..23a52a248a 100644 --- a/packages/backend/src/core/activitypub/ApRendererService.ts +++ b/packages/backend/src/core/activitypub/ApRendererService.ts @@ -34,7 +34,7 @@ import { UtilityService } from '@/core/UtilityService.js'; import { JsonLdService } from './JsonLdService.js'; import { ApMfmService } from './ApMfmService.js'; import { CONTEXT } from './misc/contexts.js'; -import { getApId, IOrderedCollection, IOrderedCollectionPage } from './type.js'; +import { getApId, ILink, IOrderedCollection, IOrderedCollectionPage } from './type.js'; import type { IAccept, IActivity, IAdd, IAnnounce, IApDocument, IApEmoji, IApHashtag, IApImage, IApMention, IBlock, ICreate, IDelete, IFlag, IFollow, IKey, ILike, IMove, IObject, IPost, IQuestion, IReject, IRemove, ITombstone, IUndo, IUpdate } from './type.js'; @Injectable() @@ -500,12 +500,22 @@ export class ApRendererService { const emojis = await this.getEmojis(note.emojis); const apemojis = emojis.filter(emoji => !emoji.localOnly).map(emoji => this.renderEmoji(emoji)); - const tag = [ + const tag: IObject[] = [ ...hashtagTags, ...mentionTags, ...apemojis, ]; + // https://codeberg.org/fediverse/fep/src/branch/main/fep/e232/fep-e232.md + if (quote) { + tag.push({ + type: 'Link', + mediaType: 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"', + rel: 'https://misskey-hub.net/ns#_misskey_quote', + href: quote, + } satisfies ILink); + } + const asPoll = poll ? { type: 'Question', [poll.expiresAt && poll.expiresAt < new Date() ? 'closed' : 'endTime']: poll.expiresAt, @@ -537,6 +547,7 @@ export class ApRendererService { _misskey_quote: quote, quoteUrl: quote, quoteUri: quote, + // https://codeberg.org/fediverse/fep/src/branch/main/fep/044f/fep-044f.md quote: quote, published: this.idService.parse(note.id).date.toISOString(), to, @@ -853,12 +864,22 @@ export class ApRendererService { const emojis = await this.getEmojis(note.emojis); const apemojis = emojis.filter(emoji => !emoji.localOnly).map(emoji => this.renderEmoji(emoji)); - const tag = [ + const tag: IObject[] = [ ...hashtagTags, ...mentionTags, ...apemojis, ]; + // https://codeberg.org/fediverse/fep/src/branch/main/fep/e232/fep-e232.md + if (quote) { + tag.push({ + type: 'Link', + mediaType: 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"', + rel: 'https://misskey-hub.net/ns#_misskey_quote', + href: quote, + } satisfies ILink); + } + const asPoll = poll ? { type: 'Question', [poll.expiresAt && poll.expiresAt < new Date() ? 'closed' : 'endTime']: poll.expiresAt, @@ -887,6 +908,7 @@ export class ApRendererService { _misskey_quote: quote, quoteUrl: quote, quoteUri: quote, + // https://codeberg.org/fediverse/fep/src/branch/main/fep/044f/fep-044f.md quote: quote, published: this.idService.parse(note.id).date.toISOString(), to, From cbb1ed3eeda4ec7e2e35bf7d36c8a2992b7b227a Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 6 Jun 2025 17:25:10 -0400 Subject: [PATCH 29/54] accept fep-e232 "tag" quotes in objects --- .../core/activitypub/models/ApNoteService.ts | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/core/activitypub/models/ApNoteService.ts b/packages/backend/src/core/activitypub/models/ApNoteService.ts index bb0bbe374f..2a28405121 100644 --- a/packages/backend/src/core/activitypub/models/ApNoteService.ts +++ b/packages/backend/src/core/activitypub/models/ApNoteService.ts @@ -27,7 +27,7 @@ import { checkHttps } from '@/misc/check-https.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; import { isRetryableError } from '@/misc/is-retryable-error.js'; import { renderInlineError } from '@/misc/render-inline-error.js'; -import { getOneApId, getApId, validPost, isEmoji, getApType, isApObject, isDocument, IApDocument } from '../type.js'; +import { getOneApId, getApId, validPost, isEmoji, getApType, isApObject, isDocument, IApDocument, isLink } from '../type.js'; import { ApLoggerService } from '../ApLoggerService.js'; import { ApMfmService } from '../ApMfmService.js'; import { ApDbResolverService } from '../ApDbResolverService.js'; @@ -660,8 +660,27 @@ export class ApNoteService { if (note._misskey_quote && typeof(note._misskey_quote as unknown) === 'string') quoteUris.add(note._misskey_quote); if (note.quoteUrl && typeof(note.quoteUrl as unknown) === 'string') quoteUris.add(note.quoteUrl); if (note.quoteUri && typeof(note.quoteUri as unknown) === 'string') quoteUris.add(note.quoteUri); + + // https://codeberg.org/fediverse/fep/src/branch/main/fep/044f/fep-044f.md if (note.quote && typeof(note.quote as unknown) === 'string') quoteUris.add(note.quote); + // https://codeberg.org/fediverse/fep/src/branch/main/fep/e232/fep-e232.md + const tags = toArray(note.tag).filter(tag => typeof(tag) === 'object' && isLink(tag)); + for (const tag of tags) { + if (!tag.href || typeof (tag.href as unknown) !== 'string') continue; + + const mediaTypes = toArray(tag.mediaType); + if ( + !mediaTypes.includes('application/ld+json; profile="https://www.w3.org/ns/activitystreams"') && + !mediaTypes.includes('application/activity+json') + ) continue; + + const rels = toArray(tag.rel); + if (!rels.includes('https://misskey-hub.net/ns#_misskey_quote')) continue; + + quoteUris.add(tag.href); + } + // No quote, return undefined if (quoteUris.size < 1) return undefined; From d0ae76214c647d44c45647263661622128073be6 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 6 Jun 2025 13:36:33 -0400 Subject: [PATCH 30/54] output log messages with correct level --- .../backend/src/core/activitypub/JsonLdService.ts | 15 ++++++++++----- packages/backend/src/logger.ts | 10 +++++++++- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/core/activitypub/JsonLdService.ts b/packages/backend/src/core/activitypub/JsonLdService.ts index 1db5df6ad9..08ebeb6707 100644 --- a/packages/backend/src/core/activitypub/JsonLdService.ts +++ b/packages/backend/src/core/activitypub/JsonLdService.ts @@ -8,6 +8,8 @@ import { Injectable } from '@nestjs/common'; import { UnrecoverableError } from 'bullmq'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { bindThis } from '@/decorators.js'; +import Logger from '@/logger.js'; +import { LoggerService } from '@/core/LoggerService.js'; import { StatusError } from '@/misc/status-error.js'; import { CONTEXT, PRELOADED_CONTEXTS } from './misc/contexts.js'; import { validateContentTypeSetAsJsonLD } from './misc/validator.js'; @@ -17,12 +19,12 @@ import type { JsonLd as JsonLdObject, RemoteDocument } from 'jsonld/jsonld-spec. // RsaSignature2017 implementation is based on https://github.com/transmute-industries/RsaSignature2017 class JsonLd { - public debug = false; public preLoad = true; public loderTimeout = 5000; constructor( private httpRequestService: HttpRequestService, + private readonly logger: Logger, ) { } @@ -84,7 +86,7 @@ class JsonLd { const transformedData = { ...data }; delete transformedData['signature']; const cannonidedData = await this.normalize(transformedData); - if (this.debug) console.debug(`cannonidedData: ${cannonidedData}`); + this.logger.debug('cannonidedData', cannonidedData); const documentHash = this.sha256(cannonidedData.toString()); const verifyData = `${optionsHash}${documentHash}`; return verifyData; @@ -115,7 +117,7 @@ class JsonLd { if (this.preLoad) { if (url in PRELOADED_CONTEXTS) { - if (this.debug) console.debug(`HIT: ${url}`); + this.logger.debug(`Preload HIT: ${url}`); return { contextUrl: undefined, document: PRELOADED_CONTEXTS[url], @@ -124,7 +126,7 @@ class JsonLd { } } - if (this.debug) console.debug(`MISS: ${url}`); + this.logger.debug(`Preload MISS: ${url}`); const document = await this.fetchDocument(url); return { contextUrl: undefined, @@ -169,13 +171,16 @@ class JsonLd { @Injectable() export class JsonLdService { + private readonly logger: Logger; constructor( private httpRequestService: HttpRequestService, + loggerService: LoggerService, ) { + this.logger = loggerService.getLogger('json-ld'); } @bindThis public use(): JsonLd { - return new JsonLd(this.httpRequestService); + return new JsonLd(this.httpRequestService, this.logger); } } diff --git a/packages/backend/src/logger.ts b/packages/backend/src/logger.ts index ca9b494ff2..4bf45fc76b 100644 --- a/packages/backend/src/logger.ts +++ b/packages/backend/src/logger.ts @@ -23,6 +23,14 @@ export type DataElement = DataObject | Error | string | null; // https://stackoverflow.com/questions/61148466/typescript-type-that-matches-any-object-but-not-arrays export type DataObject = Record | (object & { length?: never; }); +const levelFuncs = { + error: 'error', + warning: 'warn', + success: 'info', + info: 'log', + debug: 'debug', +} as const satisfies Record; + // eslint-disable-next-line import/no-default-export export default class Logger { private context: Context; @@ -86,7 +94,7 @@ export default class Logger { } else if (data != null) { args.push(data); } - console.log(...args); + console[levelFuncs[level]](...args); } @bindThis From 424e163c6f305830fe2b8aeb6c9fecc2bf93c61a Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 6 Jun 2025 22:03:53 -0400 Subject: [PATCH 31/54] fix type errors with JsonLdService and remove unused factory pattern --- .../src/core/activitypub/ApRendererService.ts | 4 +- .../src/core/activitypub/JsonLdService.ts | 75 ++++++++++++------- .../queue/processors/InboxProcessorService.ts | 15 ++-- packages/backend/test/unit/activitypub.ts | 4 +- 4 files changed, 54 insertions(+), 44 deletions(-) diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts index 46a78687f3..6771d84bdd 100644 --- a/packages/backend/src/core/activitypub/ApRendererService.ts +++ b/packages/backend/src/core/activitypub/ApRendererService.ts @@ -936,9 +936,7 @@ export class ApRendererService { const keypair = await this.userKeypairService.getUserKeypair(user.id); - const jsonLd = this.jsonLdService.use(); - jsonLd.debug = false; - activity = await jsonLd.signRsaSignature2017(activity, keypair.privateKey, `${this.config.url}/users/${user.id}#main-key`); + activity = await this.jsonLdService.signRsaSignature2017(activity, keypair.privateKey, `${this.config.url}/users/${user.id}#main-key`); return activity; } diff --git a/packages/backend/src/core/activitypub/JsonLdService.ts b/packages/backend/src/core/activitypub/JsonLdService.ts index 08ebeb6707..8f150ab201 100644 --- a/packages/backend/src/core/activitypub/JsonLdService.ts +++ b/packages/backend/src/core/activitypub/JsonLdService.ts @@ -13,23 +13,56 @@ import { LoggerService } from '@/core/LoggerService.js'; import { StatusError } from '@/misc/status-error.js'; import { CONTEXT, PRELOADED_CONTEXTS } from './misc/contexts.js'; import { validateContentTypeSetAsJsonLD } from './misc/validator.js'; -import type { JsonLdDocument } from 'jsonld'; +import type { ContextDefinition, JsonLdDocument } from 'jsonld'; import type { JsonLd as JsonLdObject, RemoteDocument } from 'jsonld/jsonld-spec.js'; +// https://stackoverflow.com/a/66252656 +type RemoveIndex = { + [ K in keyof T as string extends K + ? never + : number extends K + ? never + : symbol extends K + ? never + : K + ] : T[K]; +}; + +export type Document = RemoveIndex; + +export type Signature = { + id?: string; + type: string; + creator: string; + domain?: string; + nonce: string; + created: string; + signatureValue: string; +}; + +export type Signed = T & { + signature: Signature; +}; + +export function isSigned(doc: T): doc is Signed { + return 'signature' in doc && typeof(doc.signature) === 'object'; +} + // RsaSignature2017 implementation is based on https://github.com/transmute-industries/RsaSignature2017 -class JsonLd { - public preLoad = true; - public loderTimeout = 5000; +@Injectable() +export class JsonLdService { + private readonly logger: Logger; constructor( private httpRequestService: HttpRequestService, - private readonly logger: Logger, + loggerService: LoggerService, ) { + this.logger = loggerService.getLogger('json-ld'); } @bindThis - public async signRsaSignature2017(data: any, privateKey: string, creator: string, domain?: string, created?: Date): Promise { + public async signRsaSignature2017(data: T, privateKey: string, creator: string, domain?: string, created?: Date): Promise> { const options: { type: string; creator: string; @@ -65,7 +98,7 @@ class JsonLd { } @bindThis - public async verifyRsaSignature2017(data: any, publicKey: string): Promise { + public async verifyRsaSignature2017(data: Signed, publicKey: string): Promise { const toBeSigned = await this.createVerifyData(data, data.signature); const verifier = crypto.createVerify('sha256'); verifier.update(toBeSigned); @@ -73,7 +106,7 @@ class JsonLd { } @bindThis - public async createVerifyData(data: any, options: any): Promise { + public async createVerifyData(data: T, options: Partial): Promise { const transformedOptions = { ...options, '@context': 'https://w3id.org/identity/v1', @@ -83,7 +116,7 @@ class JsonLd { delete transformedOptions['signatureValue']; const canonizedOptions = await this.normalize(transformedOptions); const optionsHash = this.sha256(canonizedOptions.toString()); - const transformedData = { ...data }; + const transformedData = { ...data } as T & { signature?: unknown }; delete transformedData['signature']; const cannonidedData = await this.normalize(transformedData); this.logger.debug('cannonidedData', cannonidedData); @@ -93,7 +126,8 @@ class JsonLd { } @bindThis - public async compact(data: any, context: any = CONTEXT): Promise { + // TODO our default CONTEXT isn't valid for the library, is this a bug? + public async compact(data: Document, context: ContextDefinition = CONTEXT as unknown as ContextDefinition): Promise { const customLoader = this.getLoader(); // XXX: Importing jsonld dynamically since Jest frequently fails to import it statically // https://github.com/misskey-dev/misskey/pull/9894#discussion_r1103753595 @@ -103,7 +137,7 @@ class JsonLd { } @bindThis - public async normalize(data: JsonLdDocument): Promise { + public async normalize(data: Document): Promise { const customLoader = this.getLoader(); return (await import('jsonld')).default.normalize(data, { documentLoader: customLoader, @@ -115,7 +149,7 @@ class JsonLd { return async (url: string): Promise => { if (!/^https?:\/\//.test(url)) throw new UnrecoverableError(`Invalid URL: ${url}`); - if (this.preLoad) { + { if (url in PRELOADED_CONTEXTS) { this.logger.debug(`Preload HIT: ${url}`); return { @@ -144,7 +178,6 @@ class JsonLd { headers: { Accept: 'application/ld+json, application/json', }, - timeout: this.loderTimeout, }, { throwErrorWhenResponseNotOk: false, @@ -168,19 +201,3 @@ class JsonLd { return hash.digest('hex'); } } - -@Injectable() -export class JsonLdService { - private readonly logger: Logger; - constructor( - private httpRequestService: HttpRequestService, - loggerService: LoggerService, - ) { - this.logger = loggerService.getLogger('json-ld'); - } - - @bindThis - public use(): JsonLd { - return new JsonLd(this.httpRequestService, this.logger); - } -} diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index 612b16dfbf..5f82d558b3 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -21,7 +21,7 @@ import { ApDbResolverService } from '@/core/activitypub/ApDbResolverService.js'; import { StatusError } from '@/misc/status-error.js'; import { UtilityService } from '@/core/UtilityService.js'; import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js'; -import { JsonLdService } from '@/core/activitypub/JsonLdService.js'; +import { isSigned, JsonLdService } from '@/core/activitypub/JsonLdService.js'; import { ApInboxService } from '@/core/activitypub/ApInboxService.js'; import { bindThis } from '@/decorators.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; @@ -179,8 +179,8 @@ export class InboxProcessorService implements OnApplicationShutdown { // また、signatureのsignerは、activity.actorと一致する必要がある if (!httpSignatureValidated || authUser.user.uri !== actorId) { // 一致しなくても、でもLD-Signatureがありそうならそっちも見る - const ldSignature = activity.signature; - if (ldSignature) { + if (isSigned(activity)) { + const ldSignature = activity.signature; if (ldSignature.type !== 'RsaSignature2017') { throw new Bull.UnrecoverableError(`skip: unsupported LD-signature type ${ldSignature.type}`); } @@ -202,24 +202,21 @@ export class InboxProcessorService implements OnApplicationShutdown { throw new Bull.UnrecoverableError('skip: LD-SignatureのユーザーはpublicKeyを持っていませんでした'); } - const jsonLd = this.jsonLdService.use(); - // LD-Signature検証 - const verified = await jsonLd.verifyRsaSignature2017(activity, authUser.key.keyPem).catch(() => false); + const verified = await this.jsonLdService.verifyRsaSignature2017(activity, authUser.key.keyPem).catch(() => false); if (!verified) { throw new Bull.UnrecoverableError('skip: LD-Signatureの検証に失敗しました'); } // アクティビティを正規化 - delete activity.signature; + const copy = { ...activity, signature: undefined }; try { - activity = await jsonLd.compact(activity) as IActivity; + activity = await this.jsonLdService.compact(copy) as IActivity; } catch (e) { throw new Bull.UnrecoverableError(`skip: failed to compact activity: ${e}`); } // TODO: 元のアクティビティと非互換な形に正規化される場合は転送をスキップする // https://github.com/mastodon/mastodon/blob/664b0ca/app/services/activitypub/process_collection_service.rb#L24-L29 - activity.signature = ldSignature; // もう一度actorチェック if (authUser.user.uri !== actorId) { diff --git a/packages/backend/test/unit/activitypub.ts b/packages/backend/test/unit/activitypub.ts index 94dec16401..9abbb3e7a6 100644 --- a/packages/backend/test/unit/activitypub.ts +++ b/packages/backend/test/unit/activitypub.ts @@ -473,8 +473,6 @@ describe('ActivityPub', () => { describe('JSON-LD', () => { test('Compaction', async () => { - const jsonLd = jsonLdService.use(); - const object = { '@context': [ 'https://www.w3.org/ns/activitystreams', @@ -493,7 +491,7 @@ describe('ActivityPub', () => { unknown: 'test test bar', undefined: 'test test baz', }; - const compacted = await jsonLd.compact(object); + const compacted = await jsonLdService.compact(object); assert.deepStrictEqual(compacted, { '@context': CONTEXT, From 51572b731426080da51b8e14e0ad8d28b2e25c40 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 9 Jun 2025 11:26:21 -0400 Subject: [PATCH 32/54] fix refactoring mistake in CacheService.ts --- packages/backend/src/core/CacheService.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/core/CacheService.ts b/packages/backend/src/core/CacheService.ts index 9c68597441..48aa464e82 100644 --- a/packages/backend/src/core/CacheService.ts +++ b/packages/backend/src/core/CacheService.ts @@ -159,7 +159,7 @@ export class CacheService implements OnApplicationShutdown { } group.set(f.followeeId, f); return groups; - }, {} as Map>>)), + }, new Map>>)), }); this.userFollowersCache = new QuantumKVCache>>(this.internalEventService, 'userFollowers', { @@ -176,7 +176,7 @@ export class CacheService implements OnApplicationShutdown { } group.set(f.followerId, f); return groups; - }, {} as Map>>)), + }, new Map>>)), }); this.hibernatedUserCache = new QuantumKVCache(this.internalEventService, 'hibernatedUsers', { From ed766e98e1cdc20839a633e4a4784592a5b0e9f9 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 9 Jun 2025 11:37:51 -0400 Subject: [PATCH 33/54] fix mixed-up userFollowingsCache / userFollowersCache --- .../src/core/activitypub/ApDeliverManagerService.ts | 2 +- packages/backend/src/core/entities/UserEntityService.ts | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts index 86747f2508..91f6f2d9fc 100644 --- a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts +++ b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts @@ -113,7 +113,7 @@ class DeliverManager { if (this.recipes.some(r => isFollowers(r))) { // followers deliver // ただ、sharedInboxがnullなリモートユーザーも稀におり、その対応ができなさそう？ - const followers = await this.cacheService.userFollowingsCache + const followers = await this.cacheService.userFollowersCache .fetch(this.actor.id) .then(f => Array .from(f.values()) diff --git a/packages/backend/src/core/entities/UserEntityService.ts b/packages/backend/src/core/entities/UserEntityService.ts index aecbaa7fd5..91bf258ff4 100644 --- a/packages/backend/src/core/entities/UserEntityService.ts +++ b/packages/backend/src/core/entities/UserEntityService.ts @@ -250,8 +250,8 @@ export class UserEntityService implements OnModuleInit { @bindThis public async getRelations(me: MiUser['id'], targets: MiUser['id'][]): Promise> { const [ - followers, - followees, + myFollowing, + myFollowers, followersRequests, followeesRequests, blockers, @@ -300,7 +300,7 @@ export class UserEntityService implements OnModuleInit { return new Map( targets.map(target => { - const following = followers.get(target) ?? null; + const following = myFollowing.get(target) ?? null; return [ target, @@ -308,7 +308,7 @@ export class UserEntityService implements OnModuleInit { id: target, following: following, isFollowing: following != null, - isFollowed: followees.has(target), + isFollowed: myFollowers.has(target), hasPendingFollowRequestFromYou: followersRequests.includes(target), hasPendingFollowRequestToYou: followeesRequests.includes(target), isBlocking: blockees.has(target), From 646ffa7b622ff816a3c0e18322c73a04c4c5f48d Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 9 Jun 2025 11:45:20 -0400 Subject: [PATCH 34/54] fix missing @bindThis in CacheService --- packages/backend/src/core/CacheService.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/backend/src/core/CacheService.ts b/packages/backend/src/core/CacheService.ts index 48aa464e82..2d37cd6bab 100644 --- a/packages/backend/src/core/CacheService.ts +++ b/packages/backend/src/core/CacheService.ts @@ -291,6 +291,7 @@ export class CacheService implements OnApplicationShutdown { } } + @bindThis private async onTokenEvent(body: InternalEventTypes[E]): Promise { { { @@ -303,6 +304,7 @@ export class CacheService implements OnApplicationShutdown { } } + @bindThis private async onFollowEvent(body: InternalEventTypes[E], type: E): Promise { { switch (type) { From c200ed366671101992d3613be834071ebc217c00 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 9 Jun 2025 12:48:32 -0400 Subject: [PATCH 35/54] fix relations in MastodonDataService.ts --- .../server/api/mastodon/MastodonDataService.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/backend/src/server/api/mastodon/MastodonDataService.ts b/packages/backend/src/server/api/mastodon/MastodonDataService.ts index 73cd553b9a..e080cb10bd 100644 --- a/packages/backend/src/server/api/mastodon/MastodonDataService.ts +++ b/packages/backend/src/server/api/mastodon/MastodonDataService.ts @@ -57,19 +57,19 @@ export class MastodonDataService { if (relations.reply) { query.leftJoinAndSelect('note.reply', 'reply'); if (typeof(relations.reply) === 'object') { - if (relations.reply.reply) query.leftJoinAndSelect('note.reply.reply', 'replyReply'); - if (relations.reply.renote) query.leftJoinAndSelect('note.reply.renote', 'replyRenote'); - if (relations.reply.user) query.innerJoinAndSelect('note.reply.user', 'replyUser'); - if (relations.reply.channel) query.leftJoinAndSelect('note.reply.channel', 'replyChannel'); + if (relations.reply.reply) query.leftJoinAndSelect('reply.reply', 'replyReply'); + if (relations.reply.renote) query.leftJoinAndSelect('reply.renote', 'replyRenote'); + if (relations.reply.user) query.innerJoinAndSelect('reply.user', 'replyUser'); + if (relations.reply.channel) query.leftJoinAndSelect('reply.channel', 'replyChannel'); } } if (relations.renote) { query.leftJoinAndSelect('note.renote', 'renote'); if (typeof(relations.renote) === 'object') { - if (relations.renote.reply) query.leftJoinAndSelect('note.renote.reply', 'renoteReply'); - if (relations.renote.renote) query.leftJoinAndSelect('note.renote.renote', 'renoteRenote'); - if (relations.renote.user) query.innerJoinAndSelect('note.renote.user', 'renoteUser'); - if (relations.renote.channel) query.leftJoinAndSelect('note.renote.channel', 'renoteChannel'); + if (relations.renote.reply) query.leftJoinAndSelect('renote.reply', 'renoteReply'); + if (relations.renote.renote) query.leftJoinAndSelect('renote.renote', 'renoteRenote'); + if (relations.renote.user) query.innerJoinAndSelect('renote.user', 'renoteUser'); + if (relations.renote.channel) query.leftJoinAndSelect('renote.channel', 'renoteChannel'); } } if (relations.user) { From 27c27529f6b06374fed505ba87c6ee1165d23983 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Tue, 10 Jun 2025 19:52:47 -0400 Subject: [PATCH 36/54] enforce DM visibility in generateVisibilityQuery --- packages/backend/src/core/QueryService.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/packages/backend/src/core/QueryService.ts b/packages/backend/src/core/QueryService.ts index 4089fc080c..2d8ea51e65 100644 --- a/packages/backend/src/core/QueryService.ts +++ b/packages/backend/src/core/QueryService.ts @@ -157,15 +157,17 @@ export class QueryService { qb // My post .orWhere(':meId = note.userId') - // Reply to me - .orWhere(':meId = note.replyUserId') - // DM to me + // Visible to me .orWhere(':meIdAsList <@ note.visibleUserIds') - // Mentions me - .orWhere(':meIdAsList <@ note.mentions') // Followers-only post - .orWhere(new Brackets(qb => this - .andFollowingUser(qb, ':meId', 'note.userId') + .orWhere(new Brackets(qb => qb + .andWhere(new Brackets(qbb => this + // Following author + .orFollowingUser(qbb, ':meId', 'note.userId') + // Mentions me + .orWhere(':meIdAsList <@ note.mentions') + // Reply to me + .orWhere(':meId = note.replyUserId'))) .andWhere('note.visibility = \'followers\''))); q.setParameters({ meId: me.id, meIdAsList: [me.id] }); From 8f5212e453f7d20d1422fe5715e66d0b4a4610de Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Tue, 10 Jun 2025 19:56:36 -0400 Subject: [PATCH 37/54] enforce DM visibility in notes/mentions --- packages/backend/src/server/api/endpoints/notes/mentions.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/server/api/endpoints/notes/mentions.ts b/packages/backend/src/server/api/endpoints/notes/mentions.ts index a52f35cde6..f30e5a583f 100644 --- a/packages/backend/src/server/api/endpoints/notes/mentions.ts +++ b/packages/backend/src/server/api/endpoints/notes/mentions.ts @@ -79,14 +79,13 @@ export default class extends Endpoint { // eslint- , 'source') .innerJoin(MiNote, 'note', 'note.id = source.id'); - // Mentioned or visible users can always access - //this.queryService.generateVisibilityQuery(query, me); + this.queryService.generateVisibilityQuery(qb, me); this.queryService.generateBlockedHostQueryForNote(qb); this.queryService.generateMutedUserQueryForNotes(qb, me); this.queryService.generateMutedNoteThreadQuery(qb, me); this.queryService.generateBlockedUserQueryForNotes(qb, me); // A renote can't mention a user, so it will never appear here anyway. - //this.queryService.generateMutedUserRenotesQueryForNotes(query, me); + //this.queryService.generateMutedUserRenotesQueryForNotes(qb, me); if (ps.visibility) { qb.andWhere('note.visibility = :visibility', { visibility: ps.visibility }); From 3e9a2c36f135518cbf949505d630279cb874dd73 Mon Sep 17 00:00:00 2001 From: piuvas Date: Thu, 12 Jun 2025 20:46:52 -0300 Subject: [PATCH 38/54] make defederation rocket use theme colors. --- packages/frontend/src/components/MkPostForm.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/frontend/src/components/MkPostForm.vue b/packages/frontend/src/components/MkPostForm.vue index bbacdfda4d..e8effa1945 100644 --- a/packages/frontend/src/components/MkPostForm.vue +++ b/packages/frontend/src/components/MkPostForm.vue @@ -1359,7 +1359,7 @@ defineExpose({ } &.danger { - color: #ff2a2a; + color: var(--MI_THEME-warn); } } From 1a964cb6c037efe8e71c6e91e5c2ce032d22e107 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Wed, 21 May 2025 19:17:51 -0400 Subject: [PATCH 39/54] pcleanup dependencies: * Consolidate multiple different HTML/XML/RSS libraries to use the Cheerio stack * Remove unused deps * Move dev dependencies to correct section * Pin versions where missing --- packages/backend/package.json | 25 +- .../src/core/FetchInstanceMetadataService.ts | 4 +- packages/backend/src/core/MfmService.ts | 483 +++++------ packages/backend/src/core/WebfingerService.ts | 16 +- .../src/core/activitypub/ApRendererService.ts | 45 +- .../src/core/activitypub/ApRequestService.ts | 73 +- packages/backend/src/misc/truncate.ts | 4 +- .../backend/src/misc/verify-field-link.ts | 2 +- .../src/server/api/endpoints/fetch-rss.ts | 215 ++--- packages/backend/test/e2e/oauth.ts | 2 +- packages/backend/test/unit/MfmService.ts | 2 +- packages/backend/test/utils.ts | 4 +- packages/frontend-embed/package.json | 28 +- packages/frontend-shared/package.json | 1 - packages/frontend/package.json | 38 +- packages/misskey-js/src/autogen/types.ts | 48 +- pnpm-lock.yaml | 769 +++++++----------- 17 files changed, 755 insertions(+), 1004 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index bad6990ba5..69d38f3bfb 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -99,10 +99,8 @@ "archiver": "7.0.1", "argon2": "^0.40.1", "axios": "1.7.4", - "async-mutex": "0.5.0", "bcryptjs": "2.4.3", "blurhash": "2.0.5", - "body-parser": "1.20.3", "bullmq": "5.51.1", "cacheable-lookup": "7.0.0", "canvas": "^3.1.0", @@ -110,13 +108,14 @@ "chalk": "5.4.1", "chalk-template": "1.1.0", "cheerio": "1.0.0", - "chokidar": "3.6.0", "cli-highlight": "2.1.11", "color-convert": "2.0.1", "content-disposition": "0.5.4", "date-fns": "2.30.0", "deep-email-validator": "0.1.21", - "fast-xml-parser": "4.4.1", + "dom-serializer": "2.0.0", + "domhandler": "5.0.3", + "domutils": "3.2.2", "fastify": "5.3.2", "fastify-raw-body": "5.0.0", "feed": "4.2.2", @@ -125,10 +124,9 @@ "form-data": "4.0.2", "glob": "11.0.0", "got": "14.4.7", - "happy-dom": "16.8.1", "hpagent": "1.2.0", "htmlescape": "1.1.1", - "http-link-header": "1.1.3", + "htmlparser2": "10.0.0", "ioredis": "5.6.1", "ip-cidr": "4.0.2", "ipaddr.js": "2.2.0", @@ -136,26 +134,20 @@ "js-yaml": "4.1.0", "json5": "2.2.3", "jsonld": "8.3.3", - "jsrsasign": "11.1.0", "juice": "11.0.1", "megalodon": "workspace:*", "meilisearch": "0.50.0", - "microformats-parser": "2.0.2", "mime-types": "2.1.35", "misskey-js": "workspace:*", "misskey-reversi": "workspace:*", - "moment": "^2.30.1", + "moment": "2.30.1", "ms": "3.0.0-canary.1", "nanoid": "5.1.5", "nested-property": "4.0.0", "node-fetch": "3.3.2", "nodemailer": "6.10.1", - "oauth": "0.10.2", - "oauth2orize": "1.12.0", - "oauth2orize-pkce": "0.1.2", "os-utils": "0.0.14", "otpauth": "9.4.0", - "parse5": "7.3.0", "pg": "8.15.6", "pkce-challenge": "4.1.0", "probe-image-size": "7.2.3", @@ -165,20 +157,16 @@ "pug": "3.0.3", "qrcode": "1.5.4", "random-seed": "0.3.0", - "ratelimiter": "3.4.1", "re2": "1.21.4", "redis-info": "3.1.0", "redis-lock": "0.1.4", "reflect-metadata": "0.2.2", "rename": "1.0.4", - "rss-parser": "3.13.0", - "rxjs": "7.8.2", "sanitize-html": "2.16.0", "secure-json-parse": "3.0.2", "sharp": "0.34.1", "slacc": "0.0.10", "strict-event-emitter-types": "2.0.0", - "stringz": "2.1.0", "systeminformation": "5.25.11", "tinycolor2": "1.6.0", "tmp": "0.2.3", @@ -202,12 +190,10 @@ "@types/accepts": "1.3.7", "@types/archiver": "6.0.3", "@types/bcryptjs": "2.4.6", - "@types/body-parser": "1.19.5", "@types/color-convert": "2.0.4", "@types/content-disposition": "0.5.8", "@types/fluent-ffmpeg": "2.1.27", "@types/htmlescape": "1.1.3", - "@types/http-link-header": "1.0.7", "@types/jest": "29.5.14", "@types/js-yaml": "4.0.9", "@types/jsonld": "1.5.15", @@ -225,7 +211,6 @@ "@types/pug": "2.0.10", "@types/qrcode": "1.5.5", "@types/random-seed": "0.3.5", - "@types/ratelimiter": "3.4.6", "@types/redis-info": "3.0.3", "@types/rename": "1.0.7", "@types/sanitize-html": "2.15.0", diff --git a/packages/backend/src/core/FetchInstanceMetadataService.ts b/packages/backend/src/core/FetchInstanceMetadataService.ts index 9bfd7381f1..6fcfdfb596 100644 --- a/packages/backend/src/core/FetchInstanceMetadataService.ts +++ b/packages/backend/src/core/FetchInstanceMetadataService.ts @@ -7,7 +7,7 @@ import { URL } from 'node:url'; import { Inject, Injectable } from '@nestjs/common'; import tinycolor from 'tinycolor2'; import * as Redis from 'ioredis'; -import { load as cheerio } from 'cheerio'; +import { load as cheerio } from 'cheerio/slim'; import type { MiInstance } from '@/models/Instance.js'; import type Logger from '@/logger.js'; import { DI } from '@/di-symbols.js'; @@ -16,7 +16,7 @@ import { HttpRequestService } from '@/core/HttpRequestService.js'; import { bindThis } from '@/decorators.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; import { renderInlineError } from '@/misc/render-inline-error.js'; -import type { CheerioAPI } from 'cheerio'; +import type { CheerioAPI } from 'cheerio/slim'; type NodeInfo = { openRegistrations?: unknown; diff --git a/packages/backend/src/core/MfmService.ts b/packages/backend/src/core/MfmService.ts index 1ee3bd2275..d85ac7c807 100644 --- a/packages/backend/src/core/MfmService.ts +++ b/packages/backend/src/core/MfmService.ts @@ -5,25 +5,22 @@ import { URL } from 'node:url'; import { Inject, Injectable } from '@nestjs/common'; -import * as parse5 from 'parse5'; -import { type Document, type HTMLParagraphElement, Window } from 'happy-dom'; +import { isText, isTag, Text } from 'domhandler'; +import * as htmlparser2 from 'htmlparser2'; +import { Node, Document, ChildNode, Element, ParentNode } from 'domhandler'; +import * as domserializer from 'dom-serializer'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import { intersperse } from '@/misc/prelude/array.js'; import { normalizeForSearch } from '@/misc/normalize-for-search.js'; import type { IMentionedRemoteUsers } from '@/models/Note.js'; import { bindThis } from '@/decorators.js'; -import type { DefaultTreeAdapterMap } from 'parse5'; import type * as mfm from '@transfem-org/sfm-js'; -const treeAdapter = parse5.defaultTreeAdapter; -type Node = DefaultTreeAdapterMap['node']; -type ChildNode = DefaultTreeAdapterMap['childNode']; - const urlRegex = /^https?:\/\/[\w\/:%#@$&?!()\[\]~.,=+\-]+/; const urlRegexFull = /^https?:\/\/[\w\/:%#@$&?!()\[\]~.,=+\-]+$/; -export type Appender = (document: Document, body: HTMLParagraphElement) => void; +export type Appender = (document: Document, body: Element) => void; @Injectable() export class MfmService { @@ -40,7 +37,7 @@ export class MfmService { const normalizedHashtagNames = hashtagNames == null ? undefined : new Set(hashtagNames.map(x => normalizeForSearch(x))); - const dom = parse5.parseFragment(html); + const dom = htmlparser2.parseDocument(html); let text = ''; @@ -51,37 +48,31 @@ export class MfmService { return text.trim(); function getText(node: Node): string { - if (treeAdapter.isTextNode(node)) return node.value; - if (!treeAdapter.isElementNode(node)) return ''; - if (node.nodeName === 'br') return '\n'; + if (isText(node)) return node.data; + if (!isTag(node)) return ''; + if (node.tagName === 'br') return '\n'; - if (node.childNodes) { - return node.childNodes.map(n => getText(n)).join(''); - } - - return ''; + return node.childNodes.map(n => getText(n)).join(''); } function appendChildren(childNodes: ChildNode[]): void { - if (childNodes) { - for (const n of childNodes) { - analyze(n); - } + for (const n of childNodes) { + analyze(n); } } function analyze(node: Node) { - if (treeAdapter.isTextNode(node)) { - text += node.value; + if (isText(node)) { + text += node.data; return; } // Skip comment or document type node - if (!treeAdapter.isElementNode(node)) { + if (!isTag(node)) { return; } - switch (node.nodeName) { + switch (node.tagName) { case 'br': { text += '\n'; break; @@ -89,19 +80,19 @@ export class MfmService { case 'a': { const txt = getText(node); - const rel = node.attrs.find(x => x.name === 'rel'); - const href = node.attrs.find(x => x.name === 'href'); + const rel = node.attribs.rel; + const href = node.attribs.href; // ハッシュタグ if (normalizedHashtagNames && href && normalizedHashtagNames.has(normalizeForSearch(txt))) { text += txt; // メンション - } else if (txt.startsWith('@') && !(rel && rel.value.startsWith('me '))) { + } else if (txt.startsWith('@') && !(rel && rel.startsWith('me '))) { const part = txt.split('@'); if (part.length === 2 && href) { //#region ホスト名部分が省略されているので復元する - const acct = `${txt}@${(new URL(href.value)).hostname}`; + const acct = `${txt}@${(new URL(href)).hostname}`; text += acct; //#endregion } else if (part.length === 3) { @@ -116,17 +107,17 @@ export class MfmService { if (!href) { return txt; } - if (!txt || txt === href.value) { // #6383: Missing text node - if (href.value.match(urlRegexFull)) { - return href.value; + if (!txt || txt === href) { // #6383: Missing text node + if (href.match(urlRegexFull)) { + return href; } else { - return `<${href.value}>`; + return `<${href}>`; } } - if (href.value.match(urlRegex) && !href.value.match(urlRegexFull)) { - return `[${txt}](<${href.value}>)`; // #6846 + if (href.match(urlRegex) && !href.match(urlRegexFull)) { + return `[${txt}](<${href}>)`; // #6846 } else { - return `[${txt}](${href.value})`; + return `[${txt}](${href})`; } }; @@ -185,14 +176,17 @@ export class MfmService { case 'ruby--': { let ruby: [string, string][] = []; for (const child of node.childNodes) { - if (child.nodeName === 'rp') { + if (isText(child) && !/\s|\[|\]/.test(child.data)) { + ruby.push([child.data, '']); continue; } - if (treeAdapter.isTextNode(child) && !/\s|\[|\]/.test(child.value)) { - ruby.push([child.value, '']); + if (!isTag(child)) { continue; } - if (child.nodeName === 'rt' && ruby.length > 0) { + if (child.tagName === 'rp') { + continue; + } + if (child.tagName === 'rt' && ruby.length > 0) { const rt = getText(child); if (/\s|\[|\]/.test(rt)) { // If any space is included in rt, it is treated as a normal text @@ -217,7 +211,7 @@ export class MfmService { // block code (
)
 				case 'pre': {
-					if (node.childNodes.length === 1 && node.childNodes[0].nodeName === 'code') {
+					if (node.childNodes.length === 1 && isTag(node.childNodes[0]) && node.childNodes[0].tagName === 'code') {
 						text += '\n```\n';
 						text += getText(node.childNodes[0]);
 						text += '\n```\n';
@@ -302,17 +296,17 @@ export class MfmService {
 						let nonRtNodes = [];
 						// scan children, ignore `rp`, split on `rt`
 						for (const child of node.childNodes) {
-							if (treeAdapter.isTextNode(child)) {
+							if (isText(child)) {
 								nonRtNodes.push(child);
 								continue;
 							}
-							if (!treeAdapter.isElementNode(child)) {
+							if (!isTag(child)) {
 								continue;
 							}
-							if (child.nodeName === 'rp') {
+							if (child.tagName === 'rp') {
 								continue;
 							}
-							if (child.nodeName === 'rt') {
+							if (child.tagName === 'rt') {
 								// the only case in which we don't need a `$[group ]`
 								// is when both sides of the ruby are simple words
 								const needsGroup = nonRtNodes.length > 1 ||
@@ -350,45 +344,44 @@ export class MfmService {
 			return null;
 		}
 
-		const { happyDOM, window } = new Window();
+		const doc = new Document([]);
 
-		const doc = window.document;
+		const body = new Element('p', {});
+		doc.childNodes.push(body);
 
-		const body = doc.createElement('p');
-
-		function appendChildren(children: mfm.MfmNode[], targetElement: any): void {
-			if (children) {
-				for (const child of children.map(x => (handlers as any)[x.type](x))) targetElement.appendChild(child);
+		function appendChildren(children: mfm.MfmNode[], targetElement: ParentNode): void {
+			for (const child of children.map(x => handle(x))) {
+				targetElement.childNodes.push(child);
 			}
 		}
 
 		function fnDefault(node: mfm.MfmFn) {
-			const el = doc.createElement('i');
+			const el = new Element('i', {});
 			appendChildren(node.children, el);
 			return el;
 		}
 
-		const handlers: { [K in mfm.MfmNode['type']]: (node: mfm.NodeType) => any } = {
+		const handlers: { [K in mfm.MfmNode['type']]: (node: mfm.NodeType) => ChildNode } = {
 			bold: (node) => {
-				const el = doc.createElement('b');
+				const el = new Element('b', {});
 				appendChildren(node.children, el);
 				return el;
 			},
 
 			small: (node) => {
-				const el = doc.createElement('small');
+				const el = new Element('small', {});
 				appendChildren(node.children, el);
 				return el;
 			},
 
 			strike: (node) => {
-				const el = doc.createElement('del');
+				const el = new Element('del', {});
 				appendChildren(node.children, el);
 				return el;
 			},
 
 			italic: (node) => {
-				const el = doc.createElement('i');
+				const el = new Element('i', {});
 				appendChildren(node.children, el);
 				return el;
 			},
@@ -399,11 +392,12 @@ export class MfmService {
 						const text = node.children[0].type === 'text' ? node.children[0].props.text : '';
 						try {
 							const date = new Date(parseInt(text, 10) * 1000);
-							const el = doc.createElement('time');
-							el.setAttribute('datetime', date.toISOString());
-							el.textContent = date.toISOString();
+							const el = new Element('time', {
+								datetime: date.toISOString(),
+							});
+							el.childNodes.push(new Text(date.toISOString()));
 							return el;
-						} catch (err) {
+						} catch {
 							return fnDefault(node);
 						}
 					}
@@ -412,20 +406,20 @@ export class MfmService {
 						if (node.children.length === 1) {
 							const child = node.children[0];
 							const text = child.type === 'text' ? child.props.text : '';
-							const rubyEl = doc.createElement('ruby');
-							const rtEl = doc.createElement('rt');
+							const rubyEl = new Element('ruby', {});
+							const rtEl = new Element('rt', {});
 
 							// ruby未対応のHTMLサニタイザーを通したときにルビが「劉備（りゅうび）」となるようにする
-							const rpStartEl = doc.createElement('rp');
-							rpStartEl.appendChild(doc.createTextNode('('));
-							const rpEndEl = doc.createElement('rp');
-							rpEndEl.appendChild(doc.createTextNode(')'));
+							const rpStartEl = new Element('rp', {});
+							rpStartEl.childNodes.push(new Text('('));
+							const rpEndEl = new Element('rp', {});
+							rpEndEl.childNodes.push(new Text(')'));
 
-							rubyEl.appendChild(doc.createTextNode(text.split(' ')[0]));
-							rtEl.appendChild(doc.createTextNode(text.split(' ')[1]));
-							rubyEl.appendChild(rpStartEl);
-							rubyEl.appendChild(rtEl);
-							rubyEl.appendChild(rpEndEl);
+							rubyEl.childNodes.push(new Text(text.split(' ')[0]));
+							rtEl.childNodes.push(new Text(text.split(' ')[1]));
+							rubyEl.childNodes.push(rpStartEl);
+							rubyEl.childNodes.push(rtEl);
+							rubyEl.childNodes.push(rpEndEl);
 							return rubyEl;
 						} else {
 							const rt = node.children.at(-1);
@@ -435,20 +429,20 @@ export class MfmService {
 							}
 
 							const text = rt.type === 'text' ? rt.props.text : '';
-							const rubyEl = doc.createElement('ruby');
-							const rtEl = doc.createElement('rt');
+							const rubyEl = new Element('ruby', {});
+							const rtEl = new Element('rt', {});
 
 							// ruby未対応のHTMLサニタイザーを通したときにルビが「劉備（りゅうび）」となるようにする
-							const rpStartEl = doc.createElement('rp');
-							rpStartEl.appendChild(doc.createTextNode('('));
-							const rpEndEl = doc.createElement('rp');
-							rpEndEl.appendChild(doc.createTextNode(')'));
+							const rpStartEl = new Element('rp', {});
+							rpStartEl.childNodes.push(new Text('('));
+							const rpEndEl = new Element('rp', {});
+							rpEndEl.childNodes.push(new Text(')'));
 
 							appendChildren(node.children.slice(0, node.children.length - 1), rubyEl);
-							rtEl.appendChild(doc.createTextNode(text.trim()));
-							rubyEl.appendChild(rpStartEl);
-							rubyEl.appendChild(rtEl);
-							rubyEl.appendChild(rpEndEl);
+							rtEl.childNodes.push(new Text(text.trim()));
+							rubyEl.childNodes.push(rpStartEl);
+							rubyEl.childNodes.push(rtEl);
+							rubyEl.childNodes.push(rpEndEl);
 							return rubyEl;
 						}
 					}
@@ -456,7 +450,7 @@ export class MfmService {
 					// hack for ruby, should never be needed because we should
 					// never send this out to other instances
 					case 'group': {
-						const el = doc.createElement('span');
+						const el = new Element('span', {});
 						appendChildren(node.children, el);
 						return el;
 					}
@@ -468,125 +462,135 @@ export class MfmService {
 			},
 
 			blockCode: (node) => {
-				const pre = doc.createElement('pre');
-				const inner = doc.createElement('code');
-				inner.textContent = node.props.code;
-				pre.appendChild(inner);
+				const pre = new Element('pre', {});
+				const inner = new Element('code', {});
+				inner.childNodes.push(new Text(node.props.code));
+				pre.childNodes.push(inner);
 				return pre;
 			},
 
 			center: (node) => {
-				const el = doc.createElement('div');
+				const el = new Element('div', {});
 				appendChildren(node.children, el);
 				return el;
 			},
 
 			emojiCode: (node) => {
-				return doc.createTextNode(`\u200B:${node.props.name}:\u200B`);
+				return new Text(`\u200B:${node.props.name}:\u200B`);
 			},
 
 			unicodeEmoji: (node) => {
-				return doc.createTextNode(node.props.emoji);
+				return new Text(node.props.emoji);
 			},
 
 			hashtag: (node) => {
-				const a = doc.createElement('a');
-				a.setAttribute('href', `${this.config.url}/tags/${node.props.hashtag}`);
-				a.textContent = `#${node.props.hashtag}`;
-				a.setAttribute('rel', 'tag');
+				const a = new Element('a', {
+					href: `${this.config.url}/tags/${node.props.hashtag}`,
+					rel: 'tag',
+				});
+				a.childNodes.push(new Text(`#${node.props.hashtag}`));
 				return a;
 			},
 
 			inlineCode: (node) => {
-				const el = doc.createElement('code');
-				el.textContent = node.props.code;
+				const el = new Element('code', {});
+				el.childNodes.push(new Text(node.props.code));
 				return el;
 			},
 
 			mathInline: (node) => {
-				const el = doc.createElement('code');
-				el.textContent = node.props.formula;
+				const el = new Element('code', {});
+				el.childNodes.push(new Text(node.props.formula));
 				return el;
 			},
 
 			mathBlock: (node) => {
-				const el = doc.createElement('code');
-				el.textContent = node.props.formula;
+				const el = new Element('code', {});
+				el.childNodes.push(new Text(node.props.formula));
 				return el;
 			},
 
 			link: (node) => {
-				const a = doc.createElement('a');
-				a.setAttribute('href', node.props.url);
+				const a = new Element('a', {
+					href: node.props.url,
+				});
 				appendChildren(node.children, a);
 				return a;
 			},
 
 			mention: (node) => {
-				const a = doc.createElement('a');
 				const { username, host, acct } = node.props;
 				const remoteUserInfo = mentionedRemoteUsers.find(remoteUser => remoteUser.username.toLowerCase() === username.toLowerCase() && remoteUser.host?.toLowerCase() === host?.toLowerCase());
-				a.setAttribute('href', remoteUserInfo
-					? (remoteUserInfo.url ? remoteUserInfo.url : remoteUserInfo.uri)
-					: `${this.config.url}/${acct.endsWith(`@${this.config.url}`) ? acct.substring(0, acct.length - this.config.url.length - 1) : acct}`);
-				a.className = 'u-url mention';
-				a.textContent = acct;
+
+				const a = new Element('a', {
+					href: remoteUserInfo
+						? (remoteUserInfo.url ? remoteUserInfo.url : remoteUserInfo.uri)
+						: `${this.config.url}/${acct.endsWith(`@${this.config.url}`) ? acct.substring(0, acct.length - this.config.url.length - 1) : acct}`,
+					class: 'u-url mention',
+				});
+				a.childNodes.push(new Text(acct));
 				return a;
 			},
 
 			quote: (node) => {
-				const el = doc.createElement('blockquote');
+				const el = new Element('blockquote', {});
 				appendChildren(node.children, el);
 				return el;
 			},
 
 			text: (node) => {
 				if (!node.props.text.match(/[\r\n]/)) {
-					return doc.createTextNode(node.props.text);
+					return new Text(node.props.text);
 				}
 
-				const el = doc.createElement('span');
-				const nodes = node.props.text.split(/\r\n|\r|\n/).map(x => doc.createTextNode(x));
+				const el = new Element('span', {});
+				const nodes = node.props.text.split(/\r\n|\r|\n/).map(x => new Text(x));
 
 				for (const x of intersperse('br', nodes)) {
-					el.appendChild(x === 'br' ? doc.createElement('br') : x);
+					el.childNodes.push(x === 'br' ? new Element('br', {}) : x);
 				}
 
 				return el;
 			},
 
 			url: (node) => {
-				const a = doc.createElement('a');
-				a.setAttribute('href', node.props.url);
-				a.textContent = node.props.url;
+				const a = new Element('a', {
+					href: node.props.url,
+				});
+				a.childNodes.push(new Text(node.props.url));
 				return a;
 			},
 
 			search: (node) => {
-				const a = doc.createElement('a');
-				a.setAttribute('href', `https://www.google.com/search?q=${node.props.query}`);
-				a.textContent = node.props.content;
+				const a = new Element('a', {
+					href: `https://www.google.com/search?q=${node.props.query}`,
+				});
+				a.childNodes.push(new Text(node.props.content));
 				return a;
 			},
 
 			plain: (node) => {
-				const el = doc.createElement('span');
+				const el = new Element('span', {});
 				appendChildren(node.children, el);
 				return el;
 			},
 		};
 
+		// Utility function to make TypeScript behave
+		function handle(node: T): ChildNode {
+			const handler = handlers[node.type] as (node: T) => ChildNode;
+			return handler(node);
+		}
+
 		appendChildren(nodes, body);
 
 		for (const additionalAppender of additionalAppenders) {
 			additionalAppender(doc, body);
 		}
 
-		const serialized = body.outerHTML;
-
-		happyDOM.close().catch(err => {});
-
-		return serialized;
+		return domserializer.render(body, {
+			encodeEntities: 'utf8'
+		});
 	}
 
 	// the toMastoApiHtml function was taken from Iceshrimp and written by zotan and modified by marie to work with the current MK version
@@ -598,55 +602,55 @@ export class MfmService {
 			return null;
 		}
 
-		const { happyDOM, window } = new Window();
+		const doc = new Document([]);
 
-		const doc = window.document;
+		const body = new Element('p', {});
+		doc.childNodes.push(body);
 
-		const body = doc.createElement('p');
-
-		function appendChildren(children: mfm.MfmNode[], targetElement: any): void {
-			if (children) {
-				for (const child of children.map((x) => (handlers as any)[x.type](x))) targetElement.appendChild(child);
+		function appendChildren(children: mfm.MfmNode[], targetElement: ParentNode): void {
+			for (const child of children) {
+				const result = handle(child);
+				targetElement.childNodes.push(result);
 			}
 		}
 
 		const handlers: {
-			[K in mfm.MfmNode['type']]: (node: mfm.NodeType) => any;
+			[K in mfm.MfmNode['type']]: (node: mfm.NodeType) => ChildNode;
 		} = {
 			bold(node) {
-				const el = doc.createElement('span');
-				el.textContent = '**';
+				const el = new Element('span', {});
+				el.childNodes.push(new Text('**'));
 				appendChildren(node.children, el);
-				el.textContent += '**';
+				el.childNodes.push(new Text('**'));
 				return el;
 			},
 
 			small(node) {
-				const el = doc.createElement('small');
+				const el = new Element('small', {});
 				appendChildren(node.children, el);
 				return el;
 			},
 
 			strike(node) {
-				const el = doc.createElement('span');
-				el.textContent = '~~';
+				const el = new Element('span', {});
+				el.childNodes.push(new Text('~~'));
 				appendChildren(node.children, el);
-				el.textContent += '~~';
+				el.childNodes.push(new Text('~~'));
 				return el;
 			},
 
 			italic(node) {
-				const el = doc.createElement('span');
-				el.textContent = '*';
+				const el = new Element('span', {});
+				el.childNodes.push(new Text('*'));
 				appendChildren(node.children, el);
-				el.textContent += '*';
+				el.childNodes.push(new Text('*'));
 				return el;
 			},
 
 			fn(node) {
 				switch (node.props.name) {
 					case 'group': { // hack for ruby
-						const el = doc.createElement('span');
+						const el = new Element('span', {});
 						appendChildren(node.children, el);
 						return el;
 					}
@@ -654,119 +658,121 @@ export class MfmService {
 						if (node.children.length === 1) {
 							const child = node.children[0];
 							const text = child.type === 'text' ? child.props.text : '';
-							const rubyEl = doc.createElement('ruby');
-							const rtEl = doc.createElement('rt');
+							const rubyEl = new Element('ruby', {});
+							const rtEl = new Element('rt', {});
 
-							const rpStartEl = doc.createElement('rp');
-							rpStartEl.appendChild(doc.createTextNode('('));
-							const rpEndEl = doc.createElement('rp');
-							rpEndEl.appendChild(doc.createTextNode(')'));
+							const rpStartEl = new Element('rp', {});
+							rpStartEl.childNodes.push(new Text('('));
+							const rpEndEl = new Element('rp', {});
+							rpEndEl.childNodes.push(new Text(')'));
 
-							rubyEl.appendChild(doc.createTextNode(text.split(' ')[0]));
-							rtEl.appendChild(doc.createTextNode(text.split(' ')[1]));
-							rubyEl.appendChild(rpStartEl);
-							rubyEl.appendChild(rtEl);
-							rubyEl.appendChild(rpEndEl);
+							rubyEl.childNodes.push(new Text(text.split(' ')[0]));
+							rtEl.childNodes.push(new Text(text.split(' ')[1]));
+							rubyEl.childNodes.push(rpStartEl);
+							rubyEl.childNodes.push(rtEl);
+							rubyEl.childNodes.push(rpEndEl);
 							return rubyEl;
 						} else {
 							const rt = node.children.at(-1);
 
 							if (!rt) {
-								const el = doc.createElement('span');
+								const el = new Element('span', {});
 								appendChildren(node.children, el);
 								return el;
 							}
 
 							const text = rt.type === 'text' ? rt.props.text : '';
-							const rubyEl = doc.createElement('ruby');
-							const rtEl = doc.createElement('rt');
+							const rubyEl = new Element('ruby', {});
+							const rtEl = new Element('rt', {});
 
-							const rpStartEl = doc.createElement('rp');
-							rpStartEl.appendChild(doc.createTextNode('('));
-							const rpEndEl = doc.createElement('rp');
-							rpEndEl.appendChild(doc.createTextNode(')'));
+							const rpStartEl = new Element('rp', {});
+							rpStartEl.childNodes.push(new Text('('));
+							const rpEndEl = new Element('rp', {});
+							rpEndEl.childNodes.push(new Text(')'));
 
 							appendChildren(node.children.slice(0, node.children.length - 1), rubyEl);
-							rtEl.appendChild(doc.createTextNode(text.trim()));
-							rubyEl.appendChild(rpStartEl);
-							rubyEl.appendChild(rtEl);
-							rubyEl.appendChild(rpEndEl);
+							rtEl.childNodes.push(new Text(text.trim()));
+							rubyEl.childNodes.push(rpStartEl);
+							rubyEl.childNodes.push(rtEl);
+							rubyEl.childNodes.push(rpEndEl);
 							return rubyEl;
 						}
 					}
 
 					default: {
-						const el = doc.createElement('span');
-						el.textContent = '*';
+						const el = new Element('span', {});
+						el.childNodes.push(new Text('*'));
 						appendChildren(node.children, el);
-						el.textContent += '*';
+						el.childNodes.push(new Text('*'));
 						return el;
 					}
 				}
 			},
 
 			blockCode(node) {
-				const pre = doc.createElement('pre');
-				const inner = doc.createElement('code');
+				const pre = new Element('pre', {});
+				const inner = new Element('code', {});
 
 				const nodes = node.props.code
 					.split(/\r\n|\r|\n/)
-					.map((x) => doc.createTextNode(x));
+					.map((x) => new Text(x));
 
 				for (const x of intersperse('br', nodes)) {
-					inner.appendChild(x === 'br' ? doc.createElement('br') : x);
+					inner.childNodes.push(x === 'br' ? new Element('br', {}) : x);
 				}
 
-				pre.appendChild(inner);
+				pre.childNodes.push(inner);
 				return pre;
 			},
 
 			center(node) {
-				const el = doc.createElement('div');
+				const el = new Element('div', {});
 				appendChildren(node.children, el);
 				return el;
 			},
 
 			emojiCode(node) {
-				return doc.createTextNode(`\u200B:${node.props.name}:\u200B`);
+				return new Text(`\u200B:${node.props.name}:\u200B`);
 			},
 
 			unicodeEmoji(node) {
-				return doc.createTextNode(node.props.emoji);
+				return new Text(node.props.emoji);
 			},
 
 			hashtag: (node) => {
-				const a = doc.createElement('a');
-				a.setAttribute('href', `${this.config.url}/tags/${node.props.hashtag}`);
-				a.textContent = `#${node.props.hashtag}`;
-				a.setAttribute('rel', 'tag');
-				a.setAttribute('class', 'hashtag');
+				const a = new Element('a', {
+					href: `${this.config.url}/tags/${node.props.hashtag}`,
+					rel: 'tag',
+					class: 'hashtag',
+				});
+				a.childNodes.push(new Text(`#${node.props.hashtag}`));
 				return a;
 			},
 
 			inlineCode(node) {
-				const el = doc.createElement('code');
-				el.textContent = node.props.code;
+				const el = new Element('code', {});
+				el.childNodes.push(new Text(node.props.code));
 				return el;
 			},
 
 			mathInline(node) {
-				const el = doc.createElement('code');
-				el.textContent = node.props.formula;
+				const el = new Element('code', {});
+				el.childNodes.push(new Text(node.props.formula));
 				return el;
 			},
 
 			mathBlock(node) {
-				const el = doc.createElement('code');
-				el.textContent = node.props.formula;
+				const el = new Element('code', {});
+				el.childNodes.push(new Text(node.props.formula));
 				return el;
 			},
 
 			link(node) {
-				const a = doc.createElement('a');
-				a.setAttribute('rel', 'nofollow noopener noreferrer');
-				a.setAttribute('target', '_blank');
-				a.setAttribute('href', node.props.url);
+				const a = new Element('a', {
+					rel: 'nofollow noopener noreferrer',
+					target: '_blank',
+					href: node.props.url,
+				});
 				appendChildren(node.children, a);
 				return a;
 			},
@@ -775,92 +781,107 @@ export class MfmService {
 				const { username, host, acct } = node.props;
 				const resolved = mentionedRemoteUsers.find(remoteUser => remoteUser.username === username && remoteUser.host === host);
 
-				const el = doc.createElement('span');
+				const el = new Element('span', {});
 				if (!resolved) {
-					el.textContent = acct;
+					el.childNodes.push(new Text(acct));
 				} else {
-					el.setAttribute('class', 'h-card');
-					el.setAttribute('translate', 'no');
-					const a = doc.createElement('a');
-					a.setAttribute('href', resolved.url ? resolved.url : resolved.uri);
-					a.className = 'u-url mention';
-					const span = doc.createElement('span');
-					span.textContent = resolved.username || username;
-					a.textContent = '@';
-					a.appendChild(span);
-					el.appendChild(a);
+					el.attribs.class = 'h-card';
+					el.attribs.translate = 'no';
+					const a = new Element('a', {
+						href: resolved.url ? resolved.url : resolved.uri,
+						class: 'u-url mention',
+					});
+					const span = new Element('span', {});
+					span.childNodes.push(new Text(resolved.username || username));
+					a.childNodes.push(new Text('@'));
+					a.childNodes.push(span);
+					el.childNodes.push(a);
 				}
 
 				return el;
 			},
 
 			quote(node) {
-				const el = doc.createElement('blockquote');
+				const el = new Element('blockquote', {});
 				appendChildren(node.children, el);
 				return el;
 			},
 
 			text(node) {
-				const el = doc.createElement('span');
+				if (!node.props.text.match(/[\r\n]/)) {
+					return new Text(node.props.text);
+				}
+
+				const el = new Element('span', {});
 				const nodes = node.props.text
 					.split(/\r\n|\r|\n/)
-					.map((x) => doc.createTextNode(x));
+					.map((x) => new Text(x));
 
 				for (const x of intersperse('br', nodes)) {
-					el.appendChild(x === 'br' ? doc.createElement('br') : x);
+					el.childNodes.push(x === 'br' ? new Element('br', {}) : x);
 				}
 
 				return el;
 			},
 
 			url(node) {
-				const a = doc.createElement('a');
-				a.setAttribute('rel', 'nofollow noopener noreferrer');
-				a.setAttribute('target', '_blank');
-				a.setAttribute('href', node.props.url);
-				a.textContent = node.props.url.replace(/^https?:\/\//, '');
+				const a = new Element('a', {
+					rel: 'nofollow noopener noreferrer',
+					target: '_blank',
+					href: node.props.url,
+				});
+				a.childNodes.push(new Text(node.props.url.replace(/^https?:\/\//, '')));
 				return a;
 			},
 
 			search: (node) => {
-				const a = doc.createElement('a');
-				a.setAttribute('href', `https://www.google.com/search?q=${node.props.query}`);
-				a.textContent = node.props.content;
+				const a = new Element('a', {
+					href: `https://www.google.com/search?q=${node.props.query}`,
+				});
+				a.childNodes.push(new Text(node.props.content));
 				return a;
 			},
 
 			plain(node) {
-				const el = doc.createElement('span');
+				const el = new Element('span', {});
 				appendChildren(node.children, el);
 				return el;
 			},
 		};
 
+		// Utility function to make TypeScript behave
+		function handle(node: T): ChildNode {
+			const handler = handlers[node.type] as (node: T) => ChildNode;
+			return handler(node);
+		}
+
 		appendChildren(nodes, body);
 
 		if (quoteUri !== null) {
-			const a = doc.createElement('a');
-			a.setAttribute('href', quoteUri);
-			a.textContent = quoteUri.replace(/^https?:\/\//, '');
+			const a = new Element('a', {
+				href: quoteUri,
+			});
+			a.childNodes.push(new Text(quoteUri.replace(/^https?:\/\//, '')));
 
-			const quote = doc.createElement('span');
-			quote.setAttribute('class', 'quote-inline');
-			quote.appendChild(doc.createElement('br'));
-			quote.appendChild(doc.createElement('br'));
-			quote.innerHTML += 'RE: ';
-			quote.appendChild(a);
+			const quote = new Element('span', {
+				class: 'quote-inline',
+			});
+			quote.childNodes.push(new Element('br', {}));
+			quote.childNodes.push(new Element('br', {}));
+			quote.childNodes.push(new Text('RE: '));
+			quote.childNodes.push(a);
 
-			body.appendChild(quote);
+			body.childNodes.push(quote);
 		}
 
-		let result = body.outerHTML;
+		let result = domserializer.render(body, {
+			encodeEntities: 'utf8'
+		});
 
 		if (inline) {
 			result = result.replace(/^
/, '').replace(/<\/p>$/, '');
 		}
 
-		happyDOM.close().catch(() => {});
-
 		return result;
 	}
 }
diff --git a/packages/backend/src/core/WebfingerService.ts b/packages/backend/src/core/WebfingerService.ts
index 664963f3a3..bb9f0be4c6 100644
--- a/packages/backend/src/core/WebfingerService.ts
+++ b/packages/backend/src/core/WebfingerService.ts
@@ -5,7 +5,7 @@
 
 import { URL } from 'node:url';
 import { Injectable } from '@nestjs/common';
-import { XMLParser } from 'fast-xml-parser';
+import { load as cheerio } from 'cheerio/slim';
 import { HttpRequestService } from '@/core/HttpRequestService.js';
 import { bindThis } from '@/decorators.js';
 import type Logger from '@/logger.js';
@@ -101,14 +101,12 @@ export class WebfingerService {
 	private async fetchWebFingerTemplateFromHostMeta(url: string): Promise {
 		try {
 			const res = await this.httpRequestService.getHtml(url, 'application/xrd+xml');
-			const options = {
-				ignoreAttributes: false,
-				isArray: (_name: string, jpath: string) => jpath === 'XRD.Link',
-			};
-			const parser = new XMLParser(options);
-			const hostMeta = parser.parse(res);
-			const template = (hostMeta['XRD']['Link'] as Array).filter(p => p['@_rel'] === 'lrdd')[0]['@_template'];
-			return template.indexOf('{uri}') < 0 ? null : template;
+			const hostMeta = cheerio(res, {
+				xml: true,
+			});
+
+			const template = hostMeta('XRD > Link[rel="lrdd"][template*="{uri}"]').attr('template');
+			return template ?? null;
 		} catch (err) {
 			this.logger.error(`error while request host-meta for ${url}: ${renderInlineError(err)}`);
 			return null;
diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts
index 6068d707de..789611fd97 100644
--- a/packages/backend/src/core/activitypub/ApRendererService.ts
+++ b/packages/backend/src/core/activitypub/ApRendererService.ts
@@ -8,6 +8,7 @@ import { Inject, Injectable } from '@nestjs/common';
 import { In } from 'typeorm';
 import * as mfm from '@transfem-org/sfm-js';
 import { UnrecoverableError } from 'bullmq';
+import { Element, Text } from 'domhandler';
 import { DI } from '@/di-symbols.js';
 import type { Config } from '@/config.js';
 import type { MiPartialLocalUser, MiLocalUser, MiPartialRemoteUser, MiRemoteUser, MiUser } from '@/models/User.js';
@@ -475,16 +476,18 @@ export class ApRendererService {
 			// the claas name `quote-inline` is used in non-misskey clients for styling quote notes.
 			// For compatibility, the span part should be kept as possible.
 			apAppend.push((doc, body) => {
-				body.appendChild(doc.createElement('br'));
-				body.appendChild(doc.createElement('br'));
-				const span = doc.createElement('span');
-				span.className = 'quote-inline';
-				span.appendChild(doc.createTextNode('RE: '));
-				const link = doc.createElement('a');
-				link.setAttribute('href', quote);
-				link.textContent = quote;
-				span.appendChild(link);
-				body.appendChild(span);
+				body.childNodes.push(new Element('br', {}));
+				body.childNodes.push(new Element('br', {}));
+				const span = new Element('span', {
+					class: 'quote-inline',
+				});
+				span.childNodes.push(new Text('RE: '));
+				const link = new Element('a', {
+					href: quote,
+				});
+				link.childNodes.push(new Text(quote));
+				span.childNodes.push(link);
+				body.childNodes.push(span);
 			});
 		}
 
@@ -839,16 +842,18 @@ export class ApRendererService {
 			// the claas name `quote-inline` is used in non-misskey clients for styling quote notes.
 			// For compatibility, the span part should be kept as possible.
 			apAppend.push((doc, body) => {
-				body.appendChild(doc.createElement('br'));
-				body.appendChild(doc.createElement('br'));
-				const span = doc.createElement('span');
-				span.className = 'quote-inline';
-				span.appendChild(doc.createTextNode('RE: '));
-				const link = doc.createElement('a');
-				link.setAttribute('href', quote);
-				link.textContent = quote;
-				span.appendChild(link);
-				body.appendChild(span);
+				body.childNodes.push(new Element('br', {}));
+				body.childNodes.push(new Element('br', {}));
+				const span = new Element('span', {
+					class: 'quote-inline',
+				});
+				span.childNodes.push(new Text('RE: '));
+				const link = new Element('a', {
+					href: quote,
+				});
+				link.childNodes.push(new Text(quote));
+				span.childNodes.push(link);
+				body.childNodes.push(span);
 			});
 		}
 
diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts
index 4c7cac2169..e4db9b237c 100644
--- a/packages/backend/src/core/activitypub/ApRequestService.ts
+++ b/packages/backend/src/core/activitypub/ApRequestService.ts
@@ -6,7 +6,7 @@
 import * as crypto from 'node:crypto';
 import { URL } from 'node:url';
 import { Inject, Injectable } from '@nestjs/common';
-import { Window } from 'happy-dom';
+import { load as cheerio } from 'cheerio/slim';
 import { DI } from '@/di-symbols.js';
 import type { Config } from '@/config.js';
 import type { MiUser } from '@/models/User.js';
@@ -18,6 +18,8 @@ import { bindThis } from '@/decorators.js';
 import type Logger from '@/logger.js';
 import { validateContentTypeSetAsActivityPub } from '@/core/activitypub/misc/validator.js';
 import type { IObject, IObjectWithId } from './type.js';
+import type { Cheerio, CheerioAPI } from 'cheerio/slim';
+import type { AnyNode } from 'domhandler';
 
 type Request = {
 	url: string;
@@ -219,53 +221,33 @@ export class ApRequestService {
 			(contentType ?? '').split(';')[0].trimEnd().toLowerCase() === 'text/html' &&
 			_followAlternate === true
 		) {
-			const html = await res.text();
-			const { window, happyDOM } = new Window({
-				settings: {
-					disableJavaScriptEvaluation: true,
-					disableJavaScriptFileLoading: true,
-					disableCSSFileLoading: true,
-					disableComputedStyleRendering: true,
-					handleDisabledFileLoadingAsSuccess: true,
-					navigation: {
-						disableMainFrameNavigation: true,
-						disableChildFrameNavigation: true,
-						disableChildPageNavigation: true,
-						disableFallbackToSetURL: true,
-					},
-					timer: {
-						maxTimeout: 0,
-						maxIntervalTime: 0,
-						maxIntervalIterations: 0,
-					},
-				},
-			});
-			const document = window.document;
+			let alternate: Cheerio | null;
 			try {
-				document.documentElement.innerHTML = html;
+				const html = await res.text();
+				const document = cheerio(html);
 
 				// Search for any matching value in priority order:
 				// 1. Type=AP > Type=none > Type=anything
 				// 2. Alternate > Canonical
 				// 3. Page order (fallback)
-				const alternate =
-					document.querySelector('head > link[href][rel="alternate"][type="application/activity+json"]') ??
-					document.querySelector('head > link[href][rel="canonical"][type="application/activity+json"]') ??
-					document.querySelector('head > link[href][rel="alternate"]:not([type])') ??
-					document.querySelector('head > link[href][rel="canonical"]:not([type])') ??
-					document.querySelector('head > link[href][rel="alternate"]') ??
-					document.querySelector('head > link[href][rel="canonical"]');
-
-				if (alternate) {
-					const href = alternate.getAttribute('href');
-					if (href && this.apUtilityService.haveSameAuthority(url, href)) {
-						return await this.signedGet(href, user, allowAnonymous, false);
-					}
-				}
+				alternate = selectFirst(document, [
+					'head > link[href][rel="alternate"][type="application/activity+json"]',
+					'head > link[href][rel="canonical"][type="application/activity+json"]',
+					'head > link[href][rel="alternate"]:not([type])',
+					'head > link[href][rel="canonical"]:not([type])',
+					'head > link[href][rel="alternate"]',
+					'head > link[href][rel="canonical"]',
+				]);
 			} catch {
 				// something went wrong parsing the HTML, ignore the whole thing
-			} finally {
-				happyDOM.close().catch(err => {});
+				alternate = null;
+			}
+
+			if (alternate) {
+				const href = alternate.attr('href');
+				if (href && this.apUtilityService.haveSameAuthority(url, href)) {
+					return await this.signedGet(href, user, allowAnonymous, false);
+				}
 			}
 		}
 		//#endregion
@@ -285,3 +267,14 @@ export class ApRequestService {
 		return activity as IObjectWithId;
 	}
 }
+
+function selectFirst($: CheerioAPI, selectors: string[]): Cheerio | null {
+	for (const selector of selectors) {
+		const selection = $(selector);
+		if (selection.length > 0) {
+			return selection;
+		}
+	}
+
+	return null;
+}
diff --git a/packages/backend/src/misc/truncate.ts b/packages/backend/src/misc/truncate.ts
index 1c8a274609..a313ab7854 100644
--- a/packages/backend/src/misc/truncate.ts
+++ b/packages/backend/src/misc/truncate.ts
@@ -3,14 +3,12 @@
  * SPDX-License-Identifier: AGPL-3.0-only
  */
 
-import { substring } from 'stringz';
-
 export function truncate(input: string, size: number): string;
 export function truncate(input: string | undefined, size: number): string | undefined;
 export function truncate(input: string | undefined, size: number): string | undefined {
 	if (!input) {
 		return input;
 	} else {
-		return substring(input, 0, size);
+		return input.slice(0, size);
 	}
 }
diff --git a/packages/backend/src/misc/verify-field-link.ts b/packages/backend/src/misc/verify-field-link.ts
index 62542eaaa0..f9fc352806 100644
--- a/packages/backend/src/misc/verify-field-link.ts
+++ b/packages/backend/src/misc/verify-field-link.ts
@@ -3,7 +3,7 @@
 * SPDX-License-Identifier: AGPL-3.0-only
 */
 
-import { load as cheerio } from 'cheerio';
+import { load as cheerio } from 'cheerio/slim';
 import type { HttpRequestService } from '@/core/HttpRequestService.js';
 
 type Field = { name: string, value: string };
diff --git a/packages/backend/src/server/api/endpoints/fetch-rss.ts b/packages/backend/src/server/api/endpoints/fetch-rss.ts
index 03f35f16a5..11244b30f6 100644
--- a/packages/backend/src/server/api/endpoints/fetch-rss.ts
+++ b/packages/backend/src/server/api/endpoints/fetch-rss.ts
@@ -3,12 +3,12 @@
  * SPDX-License-Identifier: AGPL-3.0-only
  */
 
-import Parser from 'rss-parser';
 import { Injectable } from '@nestjs/common';
+import { parseFeed } from 'htmlparser2';
 import { Endpoint } from '@/server/api/endpoint-base.js';
 import { HttpRequestService } from '@/core/HttpRequestService.js';
-
-const rssParser = new Parser();
+import { ApiError } from '../error.js';
+import type { FeedItem } from 'domutils';
 
 export const meta = {
 	tags: ['meta'],
@@ -17,52 +17,32 @@ export const meta = {
 	allowGet: true,
 	cacheSec: 60 * 3,
 
+	errors: {
+		fetchFailed: {
+			id: '88f4356f-719d-4715-b4fc-703a10a812d2',
+			code: 'FETCH_FAILED',
+			message: 'Failed to fetch RSS feed',
+		},
+	},
+
 	res: {
 		type: 'object',
 		properties: {
-			image: {
-				type: 'object',
-				optional: true,
-				properties: {
-					link: {
-						type: 'string',
-						optional: true,
-					},
-					url: {
-						type: 'string',
-						optional: false,
-					},
-					title: {
-						type: 'string',
-						optional: true,
-					},
-				},
+			type: {
+				type: 'string',
+				optional: false,
 			},
-			paginationLinks: {
-				type: 'object',
+			id: {
+				type: 'string',
+				optional: true,
+			},
+			updated: {
+				type: 'string',
+				optional: true,
+			},
+			author: {
+				type: 'string',
 				optional: true,
-				properties: {
-					self: {
-						type: 'string',
-						optional: true,
-					},
-					first: {
-						type: 'string',
-						optional: true,
-					},
-					next: {
-						type: 'string',
-						optional: true,
-					},
-					last: {
-						type: 'string',
-						optional: true,
-					},
-					prev: {
-						type: 'string',
-						optional: true,
-					},
-				},
 			},
 			link: {
 				type: 'string',
@@ -94,113 +74,42 @@ export const meta = {
 							type: 'string',
 							optional: true,
 						},
-						creator: {
+						description: {
 							type: 'string',
 							optional: true,
 						},
-						summary: {
-							type: 'string',
-							optional: true,
-						},
-						content: {
-							type: 'string',
-							optional: true,
-						},
-						isoDate: {
-							type: 'string',
-							optional: true,
-						},
-						categories: {
+						media: {
 							type: 'array',
-							optional: true,
+							optional: false,
 							items: {
-								type: 'string',
-							},
-						},
-						contentSnippet: {
-							type: 'string',
-							optional: true,
-						},
-						enclosure: {
-							type: 'object',
-							optional: true,
-							properties: {
-								url: {
-									type: 'string',
-									optional: false,
-								},
-								length: {
-									type: 'number',
-									optional: true,
-								},
-								type: {
-									type: 'string',
-									optional: true,
+								type: 'object',
+								properties: {
+									medium: {
+										type: 'string',
+										optional: true,
+									},
+									url: {
+										type: 'string',
+										optional: true,
+									},
+									type: {
+										type: 'string',
+										optional: true,
+									},
+									lang: {
+										type: 'string',
+										optional: true,
+									},
 								},
 							},
 						},
 					},
 				},
 			},
-			feedUrl: {
-				type: 'string',
-				optional: true,
-			},
 			description: {
 				type: 'string',
 				optional: true,
 			},
-			itunes: {
-				type: 'object',
-				optional: true,
-				additionalProperties: true,
-				properties: {
-					image: {
-						type: 'string',
-						optional: true,
-					},
-					owner: {
-						type: 'object',
-						optional: true,
-						properties: {
-							name: {
-								type: 'string',
-								optional: true,
-							},
-							email: {
-								type: 'string',
-								optional: true,
-							},
-						},
-					},
-					author: {
-						type: 'string',
-						optional: true,
-					},
-					summary: {
-						type: 'string',
-						optional: true,
-					},
-					explicit: {
-						type: 'string',
-						optional: true,
-					},
-					categories: {
-						type: 'array',
-						optional: true,
-						items: {
-							type: 'string',
-						},
-					},
-					keywords: {
-						type: 'array',
-						optional: true,
-						items: {
-							type: 'string',
-						},
-					},
-				},
-			},
 		},
 	},
 
@@ -224,7 +133,7 @@ export default class extends Endpoint { // eslint-
 	constructor(
 		private httpRequestService: HttpRequestService,
 	) {
-		super(meta, paramDef, async (ps, me) => {
+		super(meta, paramDef, async (ps) => {
 			const res = await this.httpRequestService.send(ps.url, {
 				method: 'GET',
 				headers: {
@@ -234,8 +143,38 @@ export default class extends Endpoint { // eslint-
 			});
 
 			const text = await res.text();
+			const feed = parseFeed(text, {
+				xmlMode: true,
+			});
 
-			return rssParser.parseString(text);
+			if (!feed) {
+				throw new ApiError(meta.errors.fetchFailed);
+			}
+
+			return {
+				type: feed.type,
+				id: feed.id,
+				title: feed.title,
+				link: feed.link,
+				description: feed.description,
+				updated: feed.updated?.toISOString(),
+				author: feed.author,
+				items: feed.items
+					.filter((item): item is FeedItem & { link: string, title: string } => !!item.link && !!item.title)
+					.map(item => ({
+						guid: item.id,
+						title: item.title,
+						link: item.link,
+						description: item.description,
+						pubDate: item.pubDate?.toISOString(),
+						media: item.media.map(media => ({
+							medium: media.medium,
+							url: media.url,
+							type: media.type,
+							lang: media.lang,
+						})),
+					})),
+			};
 		});
 	}
 }
diff --git a/packages/backend/test/e2e/oauth.ts b/packages/backend/test/e2e/oauth.ts
index 47851e9474..1dc8d87593 100644
--- a/packages/backend/test/e2e/oauth.ts
+++ b/packages/backend/test/e2e/oauth.ts
@@ -19,7 +19,7 @@ import {
 	ResourceOwnerPassword,
 } from 'simple-oauth2';
 import pkceChallenge from 'pkce-challenge';
-import { load as cheerio } from 'cheerio';
+import { load as cheerio } from 'cheerio/slim';
 import Fastify, { type FastifyInstance, type FastifyReply } from 'fastify';
 import { api, port, sendEnvUpdateRequest, signup } from '../utils.js';
 import type * as misskey from 'misskey-js';
diff --git a/packages/backend/test/unit/MfmService.ts b/packages/backend/test/unit/MfmService.ts
index e54c006a4f..f96f3977d0 100644
--- a/packages/backend/test/unit/MfmService.ts
+++ b/packages/backend/test/unit/MfmService.ts
@@ -86,7 +86,7 @@ describe('MfmService', () => {
 
 		test('ruby', async () => {
 			const input = '$[ruby $[group *some* text] ignore me]';
-			const output = '
*some* text(ignore me)
';
+			const output = '
*some* text(ignore me)
';
 			assert.equal(await mfmService.toMastoApiHtml(mfm.parse(input)), output);
 		});
 	});
diff --git a/packages/backend/test/utils.ts b/packages/backend/test/utils.ts
index 7f2768488f..5da5353e09 100644
--- a/packages/backend/test/utils.ts
+++ b/packages/backend/test/utils.ts
@@ -11,12 +11,12 @@ import { inspect } from 'node:util';
 import WebSocket, { ClientOptions } from 'ws';
 import fetch, { File, RequestInit, type Headers } from 'node-fetch';
 import { DataSource } from 'typeorm';
-import { load as cheerio } from 'cheerio';
+import { load as cheerio } from 'cheerio/slim';
 import { type Response } from 'node-fetch';
 import Fastify from 'fastify';
 import { entities } from '../src/postgres.js';
 import { loadConfig } from '../src/config.js';
-import type { CheerioAPI } from 'cheerio';
+import type { CheerioAPI } from 'cheerio/slim';
 import type * as misskey from 'misskey-js';
 import { DEFAULT_POLICIES } from '@/core/RoleService.js';
 import { validateContentTypeSetAsActivityPub } from '@/core/activitypub/misc/validator.js';
diff --git a/packages/frontend-embed/package.json b/packages/frontend-embed/package.json
index 1a851df49b..5191fe6852 100644
--- a/packages/frontend-embed/package.json
+++ b/packages/frontend-embed/package.json
@@ -12,34 +12,24 @@
 	"dependencies": {
 		"@discordapp/twemoji": "15.1.0",
 		"@phosphor-icons/web": "^2.0.3",
-		"@rollup/plugin-json": "6.1.0",
-		"@rollup/plugin-replace": "6.0.2",
-		"@rollup/pluginutils": "5.1.4",
 		"@transfem-org/sfm-js": "0.24.5",
-		"@twemoji/parser": "15.1.1",
-		"@vitejs/plugin-vue": "5.2.3",
-		"@vue/compiler-sfc": "3.5.14",
-		"astring": "1.9.0",
 		"buraha": "0.0.1",
-		"estree-walker": "3.0.3",
 		"frontend-shared": "workspace:*",
 		"json5": "2.2.3",
 		"misskey-js": "workspace:*",
 		"punycode.js": "2.3.1",
-		"rollup": "4.40.0",
-		"sass": "1.87.0",
 		"shiki": "3.3.0",
 		"tinycolor2": "1.6.0",
-		"tsc-alias": "1.8.15",
-		"tsconfig-paths": "4.2.0",
-		"typescript": "5.8.3",
 		"uuid": "11.1.0",
-		"vite": "6.3.3",
 		"vue": "3.5.14"
 	},
 	"devDependencies": {
 		"@misskey-dev/summaly": "5.2.1",
+		"@rollup/plugin-json": "6.1.0",
+		"@rollup/plugin-replace": "6.0.2",
+		"@rollup/pluginutils": "5.1.4",
 		"@testing-library/vue": "8.1.0",
+		"@twemoji/parser": "15.1.1",
 		"@types/estree": "1.0.7",
 		"@types/micromatch": "4.0.9",
 		"@types/node": "22.15.2",
@@ -48,12 +38,16 @@
 		"@types/ws": "8.18.1",
 		"@typescript-eslint/eslint-plugin": "8.31.0",
 		"@typescript-eslint/parser": "8.31.0",
+		"@vitejs/plugin-vue": "5.2.3",
 		"@vitest/coverage-v8": "3.1.2",
+		"@vue/compiler-sfc": "3.5.14",
 		"@vue/runtime-core": "3.5.14",
 		"acorn": "8.14.1",
+		"astring": "1.9.0",
 		"cross-env": "7.0.3",
 		"eslint-plugin-import": "2.31.0",
 		"eslint-plugin-vue": "10.0.0",
+		"estree-walker": "3.0.3",
 		"fast-glob": "3.3.3",
 		"happy-dom": "17.4.4",
 		"intersection-observer": "0.12.2",
@@ -61,7 +55,13 @@
 		"msw": "2.7.5",
 		"nodemon": "3.1.10",
 		"prettier": "3.5.3",
+		"rollup": "4.40.0",
+		"sass": "1.87.0",
 		"start-server-and-test": "2.0.11",
+		"tsc-alias": "1.8.15",
+		"tsconfig-paths": "4.2.0",
+		"typescript": "5.8.3",
+		"vite": "6.3.3",
 		"vite-plugin-turbosnap": "1.0.3",
 		"vue-component-type-helpers": "2.2.10",
 		"vue-eslint-parser": "10.1.3",
diff --git a/packages/frontend-shared/package.json b/packages/frontend-shared/package.json
index f129121d19..b4a5dd89f5 100644
--- a/packages/frontend-shared/package.json
+++ b/packages/frontend-shared/package.json
@@ -35,7 +35,6 @@
 	],
 	"dependencies": {
 		"misskey-js": "workspace:*",
-		"nodemon": "3.1.7",
 		"vue": "3.5.13"
 	}
 }
diff --git a/packages/frontend/package.json b/packages/frontend/package.json
index 640ebe70d6..d04a04c78c 100644
--- a/packages/frontend/package.json
+++ b/packages/frontend/package.json
@@ -20,19 +20,11 @@
 		"@github/webauthn-json": "2.1.1",
 		"@mcaptcha/vanilla-glue": "0.1.0-alpha-3",
 		"@misskey-dev/browser-image-resizer": "2024.1.0",
-		"@phosphor-icons/web": "^2.0.3",
-		"@rollup/plugin-json": "6.1.0",
-		"@rollup/plugin-replace": "6.0.2",
-		"@rollup/pluginutils": "5.1.4",
+		"@phosphor-icons/web": "2.1.2",
 		"@ruffle-rs/ruffle": "0.1.0-nightly.2024.10.15",
 		"@sentry/vue": "9.14.0",
 		"@syuilo/aiscript": "0.19.0",
-		"@transfem-org/sfm-js": "0.24.6",
-		"@twemoji/parser": "15.1.1",
-		"@vitejs/plugin-vue": "5.2.3",
-		"@vue/compiler-sfc": "3.5.14",
 		"aiscript-vscode": "github:aiscript-dev/aiscript-vscode#v0.1.15",
-		"astring": "1.9.0",
 		"broadcast-channel": "7.1.0",
 		"buraha": "0.0.1",
 		"canvas-confetti": "1.9.3",
@@ -45,38 +37,30 @@
 		"compare-versions": "6.1.1",
 		"cropperjs": "2.0.0",
 		"date-fns": "4.1.0",
-		"estree-walker": "3.0.3",
 		"eventemitter3": "5.0.1",
 		"frontend-shared": "workspace:*",
 		"idb-keyval": "6.2.1",
 		"insert-text-at-cursor": "0.3.0",
 		"is-file-animated": "1.0.2",
 		"json5": "2.2.3",
-		"katex": "0.16.10",
-		"magic-string": "0.30.17",
+		"katex": "0.16.22",
 		"matter-js": "0.20.0",
 		"misskey-bubble-game": "workspace:*",
 		"misskey-js": "workspace:*",
 		"misskey-reversi": "workspace:*",
-		"moment": "^2.30.1",
+		"moment": "2.30.1",
 		"photoswipe": "5.4.4",
 		"promise-limit": "2.7.0",
 		"punycode.js": "2.3.1",
-		"rollup": "4.40.0",
 		"sanitize-html": "2.16.0",
-		"sass": "1.87.0",
 		"shiki": "3.3.0",
 		"strict-event-emitter-types": "2.0.0",
 		"textarea-caret": "3.1.0",
-		"three": "0.176.0",
 		"throttle-debounce": "5.0.2",
 		"tinycolor2": "1.6.0",
-		"tsc-alias": "1.8.15",
-		"tsconfig-paths": "4.2.0",
 		"typescript": "5.8.3",
 		"uuid": "11.1.0",
 		"v-code-diff": "1.13.1",
-		"vite": "6.3.3",
 		"vue": "3.5.14",
 		"vuedraggable": "next",
 		"wanakana": "5.3.1"
@@ -86,6 +70,9 @@
 	},
 	"devDependencies": {
 		"@misskey-dev/summaly": "5.2.1",
+		"@rollup/plugin-json": "6.1.0",
+		"@rollup/plugin-replace": "6.0.2",
+		"@rollup/pluginutils": "5.1.4",
 		"@storybook/addon-actions": "8.6.12",
 		"@storybook/addon-essentials": "8.6.12",
 		"@storybook/addon-interactions": "8.6.12",
@@ -105,6 +92,7 @@
 		"@storybook/vue3": "8.6.12",
 		"@storybook/vue3-vite": "8.6.12",
 		"@testing-library/vue": "8.1.0",
+		"@twemoji/parser": "15.1.1",
 		"@types/canvas-confetti": "1.9.0",
 		"@types/estree": "1.0.7",
 		"@types/katex": "^0.16.7",
@@ -119,16 +107,22 @@
 		"@types/ws": "8.18.1",
 		"@typescript-eslint/eslint-plugin": "8.31.0",
 		"@typescript-eslint/parser": "8.31.0",
+		"@vitejs/plugin-vue": "5.2.3",
 		"@vitest/coverage-v8": "3.1.2",
 		"@vue/compiler-core": "3.5.14",
+		"@vue/compiler-sfc": "3.5.14",
 		"@vue/runtime-core": "3.5.14",
+		"@transfem-org/sfm-js": "0.24.6",
 		"acorn": "8.14.1",
+		"astring": "1.9.0",
 		"cross-env": "7.0.3",
 		"eslint-plugin-import": "2.31.0",
 		"eslint-plugin-vue": "10.0.0",
+		"estree-walker": "3.0.3",
 		"fast-glob": "3.3.3",
 		"happy-dom": "17.4.4",
 		"intersection-observer": "0.12.2",
+		"magic-string": "0.30.17",
 		"micromatch": "4.0.8",
 		"minimatch": "10.0.1",
 		"msw": "2.7.5",
@@ -137,10 +131,16 @@
 		"prettier": "3.5.3",
 		"react": "19.1.0",
 		"react-dom": "19.1.0",
+		"rollup": "4.40.0",
+		"sass": "1.87.0",
 		"seedrandom": "3.0.5",
 		"start-server-and-test": "2.0.11",
 		"storybook": "8.6.12",
 		"storybook-addon-misskey-theme": "github:misskey-dev/storybook-addon-misskey-theme",
+		"three": "0.176.0",
+		"tsc-alias": "1.8.15",
+		"tsconfig-paths": "4.2.0",
+		"vite": "6.3.3",
 		"vite-plugin-turbosnap": "1.0.3",
 		"vitest": "3.1.2",
 		"vitest-fetch-mock": "0.4.5",
diff --git a/packages/misskey-js/src/autogen/types.ts b/packages/misskey-js/src/autogen/types.ts
index c09901c214..6cb52fcbea 100644
--- a/packages/misskey-js/src/autogen/types.ts
+++ b/packages/misskey-js/src/autogen/types.ts
@@ -19581,18 +19581,10 @@ export type operations = {
       200: {
         content: {
           'application/json': {
-            image?: {
-              link?: string;
-              url: string;
-              title?: string;
-            };
-            paginationLinks?: {
-              self?: string;
-              first?: string;
-              next?: string;
-              last?: string;
-              prev?: string;
-            };
+            type: string;
+            id?: string;
+            updated?: string;
+            author?: string;
             link?: string;
             title?: string;
             items: {
@@ -19600,33 +19592,15 @@ export type operations = {
                 guid?: string;
                 title?: string;
                 pubDate?: string;
-                creator?: string;
-                summary?: string;
-                content?: string;
-                isoDate?: string;
-                categories?: string[];
-                contentSnippet?: string;
-                enclosure?: {
-                  url: string;
-                  length?: number;
-                  type?: string;
-                };
+                description?: string;
+                media: {
+                    medium?: string;
+                    url?: string;
+                    type?: string;
+                    lang?: string;
+                  }[];
               }[];
-            feedUrl?: string;
             description?: string;
-            itunes?: {
-              image?: string;
-              owner?: {
-                name?: string;
-                email?: string;
-              };
-              author?: string;
-              summary?: string;
-              explicit?: string;
-              categories?: string[];
-              keywords?: string[];
-              [key: string]: unknown;
-            };
           };
         };
       };
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 880715e20e..bc055eb9f2 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -173,9 +173,6 @@ importers:
       argon2:
         specifier: ^0.40.1
         version: 0.40.1
-      async-mutex:
-        specifier: 0.5.0
-        version: 0.5.0
       axios:
         specifier: 1.7.4
         version: 1.7.4
@@ -185,9 +182,6 @@ importers:
       blurhash:
         specifier: 2.0.5
         version: 2.0.5
-      body-parser:
-        specifier: 1.20.3
-        version: 1.20.3
       bullmq:
         specifier: 5.51.1
         version: 5.51.1
@@ -209,9 +203,6 @@ importers:
       cheerio:
         specifier: 1.0.0
         version: 1.0.0
-      chokidar:
-        specifier: 4.0.3
-        version: 4.0.3
       cli-highlight:
         specifier: 2.1.11
         version: 2.1.11
@@ -227,9 +218,15 @@ importers:
       deep-email-validator:
         specifier: 0.1.21
         version: 0.1.21
-      fast-xml-parser:
-        specifier: 4.4.1
-        version: 4.4.1
+      dom-serializer:
+        specifier: 2.0.0
+        version: 2.0.0
+      domhandler:
+        specifier: 5.0.3
+        version: 5.0.3
+      domutils:
+        specifier: 3.2.2
+        version: 3.2.2
       fastify:
         specifier: 5.3.2
         version: 5.3.2
@@ -254,18 +251,15 @@ importers:
       got:
         specifier: 14.4.7
         version: 14.4.7
-      happy-dom:
-        specifier: 16.8.1
-        version: 16.8.1
       hpagent:
         specifier: 1.2.0
         version: 1.2.0
       htmlescape:
         specifier: 1.1.1
         version: 1.1.1
-      http-link-header:
-        specifier: 1.1.3
-        version: 1.1.3
+      htmlparser2:
+        specifier: 10.0.0
+        version: 10.0.0
       ioredis:
         specifier: 5.6.1
         version: 5.6.1
@@ -287,9 +281,6 @@ importers:
       jsonld:
         specifier: 8.3.3
         version: 8.3.3(web-streams-polyfill@4.0.0)
-      jsrsasign:
-        specifier: 11.1.0
-        version: 11.1.0
       juice:
         specifier: 11.0.1
         version: 11.0.1
@@ -299,9 +290,6 @@ importers:
       meilisearch:
         specifier: 0.50.0
         version: 0.50.0
-      microformats-parser:
-        specifier: 2.0.2
-        version: 2.0.2
       mime-types:
         specifier: 2.1.35
         version: 2.1.35
@@ -312,7 +300,7 @@ importers:
         specifier: workspace:*
         version: link:../misskey-reversi
       moment:
-        specifier: ^2.30.1
+        specifier: 2.30.1
         version: 2.30.1
       ms:
         specifier: 3.0.0-canary.1
@@ -329,24 +317,12 @@ importers:
       nodemailer:
         specifier: 6.10.1
         version: 6.10.1
-      oauth:
-        specifier: 0.10.2
-        version: 0.10.2
-      oauth2orize:
-        specifier: 1.12.0
-        version: 1.12.0
-      oauth2orize-pkce:
-        specifier: 0.1.2
-        version: 0.1.2
       os-utils:
         specifier: 0.0.14
         version: 0.0.14
       otpauth:
         specifier: 9.4.0
         version: 9.4.0
-      parse5:
-        specifier: 7.3.0
-        version: 7.3.0
       pg:
         specifier: 8.15.6
         version: 8.15.6
@@ -374,9 +350,6 @@ importers:
       random-seed:
         specifier: 0.3.0
         version: 0.3.0
-      ratelimiter:
-        specifier: 3.4.1
-        version: 3.4.1
       re2:
         specifier: 1.21.4
         version: 1.21.4
@@ -392,12 +365,6 @@ importers:
       rename:
         specifier: 1.0.4
         version: 1.0.4
-      rss-parser:
-        specifier: 3.13.0
-        version: 3.13.0
-      rxjs:
-        specifier: 7.8.2
-        version: 7.8.2
       sanitize-html:
         specifier: 2.16.0
         version: 2.16.0
@@ -413,9 +380,6 @@ importers:
       strict-event-emitter-types:
         specifier: 2.0.0
         version: 2.0.0
-      stringz:
-        specifier: 2.1.0
-        version: 2.1.0
       systeminformation:
         specifier: 5.25.11
         version: 5.25.11
@@ -562,9 +526,6 @@ importers:
       '@types/bcryptjs':
         specifier: 2.4.6
         version: 2.4.6
-      '@types/body-parser':
-        specifier: 1.19.5
-        version: 1.19.5
       '@types/color-convert':
         specifier: 2.0.4
         version: 2.0.4
@@ -577,9 +538,6 @@ importers:
       '@types/htmlescape':
         specifier: 1.1.3
         version: 1.1.3
-      '@types/http-link-header':
-        specifier: 1.0.7
-        version: 1.0.7
       '@types/jest':
         specifier: 29.5.14
         version: 29.5.14
@@ -631,9 +589,6 @@ importers:
       '@types/random-seed':
         specifier: 0.3.5
         version: 0.3.5
-      '@types/ratelimiter':
-        specifier: 3.4.6
-        version: 3.4.6
       '@types/redis-info':
         specifier: 3.0.3
         version: 3.0.3
@@ -728,44 +683,20 @@ importers:
         specifier: 2024.1.0
         version: 2024.1.0
       '@phosphor-icons/web':
-        specifier: ^2.0.3
-        version: 2.1.1
-      '@rollup/plugin-json':
-        specifier: 6.1.0
-        version: 6.1.0(rollup@4.40.0)
-      '@rollup/plugin-replace':
-        specifier: 6.0.2
-        version: 6.0.2(rollup@4.40.0)
-      '@rollup/pluginutils':
-        specifier: 5.1.4
-        version: 5.1.4(rollup@4.40.0)
+        specifier: 2.1.2
+        version: 2.1.2
       '@ruffle-rs/ruffle':
         specifier: 0.1.0-nightly.2024.10.15
         version: 0.1.0-nightly.2024.10.15
       '@sentry/vue':
         specifier: 9.14.0
-        version: 9.14.0(vue@3.5.14(typescript@5.8.3))
+        version: 9.14.0(vue@3.5.12(typescript@5.8.3))
       '@syuilo/aiscript':
         specifier: 0.19.0
         version: 0.19.0
-      '@transfem-org/sfm-js':
-        specifier: 0.24.6
-        version: 0.24.6
-      '@twemoji/parser':
-        specifier: 15.1.1
-        version: 15.1.1
-      '@vitejs/plugin-vue':
-        specifier: 5.2.3
-        version: 5.2.3(vite@6.3.3(@types/node@22.15.2)(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3))(vue@3.5.14(typescript@5.8.3))
-      '@vue/compiler-sfc':
-        specifier: 3.5.14
-        version: 3.5.14
       aiscript-vscode:
         specifier: github:aiscript-dev/aiscript-vscode#v0.1.15
         version: https://codeload.github.com/aiscript-dev/aiscript-vscode/tar.gz/c3cde89e79a41d93540cf8a48cd619c3f2dcb1b7
-      astring:
-        specifier: 1.9.0
-        version: 1.9.0
       broadcast-channel:
         specifier: 7.1.0
         version: 7.1.0
@@ -802,9 +733,6 @@ importers:
       date-fns:
         specifier: 4.1.0
         version: 4.1.0
-      estree-walker:
-        specifier: 3.0.3
-        version: 3.0.3
       eventemitter3:
         specifier: 5.0.1
         version: 5.0.1
@@ -824,11 +752,8 @@ importers:
         specifier: 2.2.3
         version: 2.2.3
       katex:
-        specifier: 0.16.10
-        version: 0.16.10
-      magic-string:
-        specifier: 0.30.17
-        version: 0.30.17
+        specifier: 0.16.22
+        version: 0.16.22
       matter-js:
         specifier: 0.20.0
         version: 0.20.0
@@ -842,26 +767,17 @@ importers:
         specifier: workspace:*
         version: link:../misskey-reversi
       moment:
-        specifier: ^2.30.1
+        specifier: 2.30.1
         version: 2.30.1
       photoswipe:
         specifier: 5.4.4
         version: 5.4.4
-      promise-limit:
-        specifier: 2.7.0
-        version: 2.7.0
       punycode.js:
         specifier: 2.3.1
         version: 2.3.1
-      rollup:
-        specifier: 4.40.0
-        version: 4.40.0
       sanitize-html:
         specifier: 2.16.0
         version: 2.16.0
-      sass:
-        specifier: 1.87.0
-        version: 1.87.0
       shiki:
         specifier: 3.3.0
         version: 3.3.0
@@ -871,21 +787,12 @@ importers:
       textarea-caret:
         specifier: 3.1.0
         version: 3.1.0
-      three:
-        specifier: 0.176.0
-        version: 0.176.0
       throttle-debounce:
         specifier: 5.0.2
         version: 5.0.2
       tinycolor2:
         specifier: 1.6.0
         version: 1.6.0
-      tsc-alias:
-        specifier: 1.8.15
-        version: 1.8.15
-      tsconfig-paths:
-        specifier: 4.2.0
-        version: 4.2.0
       typescript:
         specifier: 5.8.3
         version: 5.8.3
@@ -894,16 +801,13 @@ importers:
         version: 11.1.0
       v-code-diff:
         specifier: 1.13.1
-        version: 1.13.1(vue@3.5.14(typescript@5.8.3))
-      vite:
-        specifier: 6.3.3
-        version: 6.3.3(@types/node@22.15.2)(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3)
+        version: 1.13.1(vue@3.5.12(typescript@5.8.3))
       vue:
-        specifier: 3.5.14
-        version: 3.5.14(typescript@5.8.3)
+        specifier: 3.5.12
+        version: 3.5.12(typescript@5.8.3)
       vuedraggable:
         specifier: next
-        version: 4.1.0(vue@3.5.14(typescript@5.8.3))
+        version: 4.1.0(vue@3.5.12(typescript@5.8.3))
       wanakana:
         specifier: 5.3.1
         version: 5.3.1
@@ -915,6 +819,15 @@ importers:
       '@misskey-dev/summaly':
         specifier: 5.2.1
         version: 5.2.1
+      '@rollup/plugin-json':
+        specifier: 6.1.0
+        version: 6.1.0(rollup@4.40.0)
+      '@rollup/plugin-replace':
+        specifier: 6.0.2
+        version: 6.0.2(rollup@4.40.0)
+      '@rollup/pluginutils':
+        specifier: 5.1.4
+        version: 5.1.4(rollup@4.40.0)
       '@storybook/addon-actions':
         specifier: 8.6.12
         version: 8.6.12(storybook@8.6.12(bufferutil@4.0.9)(prettier@3.5.3)(utf-8-validate@6.0.5))
@@ -965,13 +878,19 @@ importers:
         version: 8.6.12(storybook@8.6.12(bufferutil@4.0.9)(prettier@3.5.3)(utf-8-validate@6.0.5))
       '@storybook/vue3':
         specifier: 8.6.12
-        version: 8.6.12(storybook@8.6.12(bufferutil@4.0.9)(prettier@3.5.3)(utf-8-validate@6.0.5))(vue@3.5.14(typescript@5.8.3))
+        version: 8.6.12(storybook@8.6.12(bufferutil@4.0.9)(prettier@3.5.3)(utf-8-validate@6.0.5))(vue@3.5.12(typescript@5.8.3))
       '@storybook/vue3-vite':
         specifier: 8.6.12
-        version: 8.6.12(storybook@8.6.12(bufferutil@4.0.9)(prettier@3.5.3)(utf-8-validate@6.0.5))(vite@6.3.3(@types/node@22.15.2)(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3))(vue@3.5.14(typescript@5.8.3))
+        version: 8.6.12(storybook@8.6.12(bufferutil@4.0.9)(prettier@3.5.3)(utf-8-validate@6.0.5))(vite@6.3.3(@types/node@22.15.2)(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3))(vue@3.5.12(typescript@5.8.3))
       '@testing-library/vue':
         specifier: 8.1.0
-        version: 8.1.0(@vue/compiler-sfc@3.5.14)(@vue/server-renderer@3.5.14(vue@3.5.14(typescript@5.8.3)))(vue@3.5.14(typescript@5.8.3))
+        version: 8.1.0(@vue/compiler-sfc@3.5.12)(@vue/server-renderer@3.5.14(vue@3.5.12(typescript@5.8.3)))(vue@3.5.12(typescript@5.8.3))
+      '@transfem-org/sfm-js':
+        specifier: 0.24.6
+        version: 0.24.6
+      '@twemoji/parser':
+        specifier: 15.1.1
+        version: 15.1.1
       '@types/canvas-confetti':
         specifier: 1.9.0
         version: 1.9.0
@@ -1014,18 +933,27 @@ importers:
       '@typescript-eslint/parser':
         specifier: 8.31.0
         version: 8.31.0(eslint@9.25.1)(typescript@5.8.3)
+      '@vitejs/plugin-vue':
+        specifier: 5.2.3
+        version: 5.2.3(vite@6.3.3(@types/node@22.15.2)(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3))(vue@3.5.12(typescript@5.8.3))
       '@vitest/coverage-v8':
         specifier: 3.1.2
         version: 3.1.2(vitest@3.1.2(@types/debug@4.1.12)(@types/node@22.15.2)(happy-dom@17.4.4)(jsdom@26.1.0(bufferutil@4.0.9)(utf-8-validate@6.0.5))(msw@2.7.5(@types/node@22.15.2)(typescript@5.8.3))(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3))
       '@vue/compiler-core':
-        specifier: 3.5.14
-        version: 3.5.14
+        specifier: 3.5.12
+        version: 3.5.12
+      '@vue/compiler-sfc':
+        specifier: 3.5.12
+        version: 3.5.12
       '@vue/runtime-core':
-        specifier: 3.5.14
-        version: 3.5.14
+        specifier: 3.5.12
+        version: 3.5.12
       acorn:
         specifier: 8.14.1
         version: 8.14.1
+      astring:
+        specifier: 1.9.0
+        version: 1.9.0
       cross-env:
         specifier: 7.0.3
         version: 7.0.3
@@ -1035,6 +963,9 @@ importers:
       eslint-plugin-vue:
         specifier: 10.0.0
         version: 10.0.0(eslint@9.25.1)(vue-eslint-parser@10.1.3(eslint@9.25.1))
+      estree-walker:
+        specifier: 3.0.3
+        version: 3.0.3
       fast-glob:
         specifier: 3.3.3
         version: 3.3.3
@@ -1044,6 +975,9 @@ importers:
       intersection-observer:
         specifier: 0.12.2
         version: 0.12.2
+      magic-string:
+        specifier: 0.30.17
+        version: 0.30.17
       micromatch:
         specifier: 4.0.8
         version: 4.0.8
@@ -1068,6 +1002,12 @@ importers:
       react-dom:
         specifier: 19.1.0
         version: 19.1.0(react@19.1.0)
+      rollup:
+        specifier: 4.40.0
+        version: 4.40.0
+      sass:
+        specifier: 1.87.0
+        version: 1.87.0
       seedrandom:
         specifier: 3.0.5
         version: 3.0.5
@@ -1080,6 +1020,18 @@ importers:
       storybook-addon-misskey-theme:
         specifier: github:misskey-dev/storybook-addon-misskey-theme
         version: https://codeload.github.com/misskey-dev/storybook-addon-misskey-theme/tar.gz/cf583db098365b2ccc81a82f63ca9c93bc32b640(@storybook/blocks@8.6.12(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(storybook@8.6.12(bufferutil@4.0.9)(prettier@3.5.3)(utf-8-validate@6.0.5)))(@storybook/components@8.6.12(storybook@8.6.12(bufferutil@4.0.9)(prettier@3.5.3)(utf-8-validate@6.0.5)))(@storybook/core-events@8.6.12(storybook@8.6.12(bufferutil@4.0.9)(prettier@3.5.3)(utf-8-validate@6.0.5)))(@storybook/manager-api@8.6.12(storybook@8.6.12(bufferutil@4.0.9)(prettier@3.5.3)(utf-8-validate@6.0.5)))(@storybook/preview-api@8.6.12(storybook@8.6.12(bufferutil@4.0.9)(prettier@3.5.3)(utf-8-validate@6.0.5)))(@storybook/theming@8.6.12(storybook@8.6.12(bufferutil@4.0.9)(prettier@3.5.3)(utf-8-validate@6.0.5)))(@storybook/types@8.6.12(storybook@8.6.12(bufferutil@4.0.9)(prettier@3.5.3)(utf-8-validate@6.0.5)))(react-dom@19.1.0(react@19.1.0))(react@19.1.0)
+      three:
+        specifier: 0.176.0
+        version: 0.176.0
+      tsc-alias:
+        specifier: 1.8.15
+        version: 1.8.15
+      tsconfig-paths:
+        specifier: 4.2.0
+        version: 4.2.0
+      vite:
+        specifier: 6.3.3
+        version: 6.3.3(@types/node@22.15.2)(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3)
       vite-plugin-turbosnap:
         specifier: 1.0.3
         version: 1.0.3
@@ -1107,36 +1059,12 @@ importers:
       '@phosphor-icons/web':
         specifier: ^2.0.3
         version: 2.1.1
-      '@rollup/plugin-json':
-        specifier: 6.1.0
-        version: 6.1.0(rollup@4.40.0)
-      '@rollup/plugin-replace':
-        specifier: 6.0.2
-        version: 6.0.2(rollup@4.40.0)
-      '@rollup/pluginutils':
-        specifier: 5.1.4
-        version: 5.1.4(rollup@4.40.0)
       '@transfem-org/sfm-js':
         specifier: 0.24.5
         version: 0.24.5
-      '@twemoji/parser':
-        specifier: 15.1.1
-        version: 15.1.1
-      '@vitejs/plugin-vue':
-        specifier: 5.2.3
-        version: 5.2.3(vite@6.3.3(@types/node@22.15.2)(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3))(vue@3.5.14(typescript@5.8.3))
-      '@vue/compiler-sfc':
-        specifier: 3.5.14
-        version: 3.5.14
-      astring:
-        specifier: 1.9.0
-        version: 1.9.0
       buraha:
         specifier: 0.0.1
         version: 0.0.1
-      estree-walker:
-        specifier: 3.0.3
-        version: 3.0.3
       frontend-shared:
         specifier: workspace:*
         version: link:../frontend-shared
@@ -1149,43 +1077,37 @@ importers:
       punycode.js:
         specifier: 2.3.1
         version: 2.3.1
-      rollup:
-        specifier: 4.40.0
-        version: 4.40.0
-      sass:
-        specifier: 1.87.0
-        version: 1.87.0
       shiki:
         specifier: 3.3.0
         version: 3.3.0
       tinycolor2:
         specifier: 1.6.0
         version: 1.6.0
-      tsc-alias:
-        specifier: 1.8.15
-        version: 1.8.15
-      tsconfig-paths:
-        specifier: 4.2.0
-        version: 4.2.0
-      typescript:
-        specifier: 5.8.3
-        version: 5.8.3
       uuid:
         specifier: 11.1.0
         version: 11.1.0
-      vite:
-        specifier: 6.3.3
-        version: 6.3.3(@types/node@22.15.2)(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3)
       vue:
-        specifier: 3.5.14
-        version: 3.5.14(typescript@5.8.3)
+        specifier: 3.5.12
+        version: 3.5.12(typescript@5.8.3)
     devDependencies:
       '@misskey-dev/summaly':
         specifier: 5.2.1
         version: 5.2.1
+      '@rollup/plugin-json':
+        specifier: 6.1.0
+        version: 6.1.0(rollup@4.40.0)
+      '@rollup/plugin-replace':
+        specifier: 6.0.2
+        version: 6.0.2(rollup@4.40.0)
+      '@rollup/pluginutils':
+        specifier: 5.1.4
+        version: 5.1.4(rollup@4.40.0)
       '@testing-library/vue':
         specifier: 8.1.0
-        version: 8.1.0(@vue/compiler-sfc@3.5.14)(@vue/server-renderer@3.5.14(vue@3.5.14(typescript@5.8.3)))(vue@3.5.14(typescript@5.8.3))
+        version: 8.1.0(@vue/compiler-sfc@3.5.12)(@vue/server-renderer@3.5.14(vue@3.5.12(typescript@5.8.3)))(vue@3.5.12(typescript@5.8.3))
+      '@twemoji/parser':
+        specifier: 15.1.1
+        version: 15.1.1
       '@types/estree':
         specifier: 1.0.7
         version: 1.0.7
@@ -1210,15 +1132,24 @@ importers:
       '@typescript-eslint/parser':
         specifier: 8.31.0
         version: 8.31.0(eslint@9.25.1)(typescript@5.8.3)
+      '@vitejs/plugin-vue':
+        specifier: 5.2.3
+        version: 5.2.3(vite@6.3.3(@types/node@22.15.2)(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3))(vue@3.5.12(typescript@5.8.3))
       '@vitest/coverage-v8':
         specifier: 3.1.2
         version: 3.1.2(vitest@3.1.2(@types/debug@4.1.12)(@types/node@22.15.2)(happy-dom@17.4.4)(jsdom@26.1.0(bufferutil@4.0.9)(utf-8-validate@6.0.5))(msw@2.7.5(@types/node@22.15.2)(typescript@5.8.3))(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3))
+      '@vue/compiler-sfc':
+        specifier: 3.5.12
+        version: 3.5.12
       '@vue/runtime-core':
-        specifier: 3.5.14
-        version: 3.5.14
+        specifier: 3.5.12
+        version: 3.5.12
       acorn:
         specifier: 8.14.1
         version: 8.14.1
+      astring:
+        specifier: 1.9.0
+        version: 1.9.0
       cross-env:
         specifier: 7.0.3
         version: 7.0.3
@@ -1228,6 +1159,9 @@ importers:
       eslint-plugin-vue:
         specifier: 10.0.0
         version: 10.0.0(eslint@9.25.1)(vue-eslint-parser@10.1.3(eslint@9.25.1))
+      estree-walker:
+        specifier: 3.0.3
+        version: 3.0.3
       fast-glob:
         specifier: 3.3.3
         version: 3.3.3
@@ -1249,9 +1183,27 @@ importers:
       prettier:
         specifier: 3.5.3
         version: 3.5.3
+      rollup:
+        specifier: 4.40.0
+        version: 4.40.0
+      sass:
+        specifier: 1.87.0
+        version: 1.87.0
       start-server-and-test:
         specifier: 2.0.11
         version: 2.0.11
+      tsc-alias:
+        specifier: 1.8.15
+        version: 1.8.15
+      tsconfig-paths:
+        specifier: 4.2.0
+        version: 4.2.0
+      typescript:
+        specifier: 5.8.3
+        version: 5.8.3
+      vite:
+        specifier: 6.3.3
+        version: 6.3.3(@types/node@22.15.2)(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3)
       vite-plugin-turbosnap:
         specifier: 1.0.3
         version: 1.0.3
@@ -1270,9 +1222,6 @@ importers:
       misskey-js:
         specifier: workspace:*
         version: link:../misskey-js
-      nodemon:
-        specifier: 3.1.7
-        version: 3.1.7
       vue:
         specifier: 3.5.13
         version: 3.5.13(typescript@5.8.3)
@@ -1292,6 +1241,9 @@ importers:
       eslint-plugin-vue:
         specifier: 10.0.0
         version: 10.0.0(eslint@9.25.1)(vue-eslint-parser@10.1.3(eslint@9.25.1))
+      nodemon:
+        specifier: 3.1.10
+        version: 3.1.10
       typescript:
         specifier: 5.8.3
         version: 5.8.3
@@ -1346,7 +1298,7 @@ importers:
         version: 3.5.3
       ts-jest:
         specifier: ^29.1.1
-        version: 29.1.2(@babel/core@7.24.7)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.24.7))(esbuild@0.25.3)(jest@29.7.0(@types/node@22.15.2))(typescript@5.8.3)
+        version: 29.1.2(@babel/core@7.23.5)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.5))(esbuild@0.25.3)(jest@29.7.0(@types/node@22.15.2))(typescript@5.8.3)
 
   packages/misskey-bubble-game:
     dependencies:
@@ -3152,6 +3104,9 @@ packages:
   '@phosphor-icons/web@2.1.1':
     resolution: {integrity: sha512-QjrfbItu5Rb2i37GzsKxmrRHfZPTVk3oXSPBnQ2+oACDbQRWGAeB0AsvZw263n1nFouQuff+khOCtRbrc6+k+A==}
 
+  '@phosphor-icons/web@2.1.2':
+    resolution: {integrity: sha512-rPAR9o/bEcp4Cw4DEeZHXf+nlGCMNGkNDRizYHM47NLxz9vvEHp/Tt6FMK1NcWadzw/pFDPnRBGi/ofRya958A==}
+
   '@pkgjs/parseargs@0.11.0':
     resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==}
     engines: {node: '>=14'}
@@ -3167,7 +3122,6 @@ packages:
 
   '@readme/json-schema-ref-parser@1.2.0':
     resolution: {integrity: sha512-Bt3QVovFSua4QmHa65EHUmh2xS0XJ3rgTEUPH998f4OW4VVJke3BuS16f+kM0ZLOGdvIrzrPRqwihuv5BAjtrA==}
-    deprecated: This package is no longer maintained. Please use `@apidevtools/json-schema-ref-parser` instead.
 
   '@readme/openapi-parser@2.7.0':
     resolution: {integrity: sha512-P8WSr8WTOxilnT89tcCRKWYsG/II4sAwt1a/DIWub8xTtkrG9cCBBy/IUcvc5X8oGWN82MwcTA3uEkDrXZd/7A==}
@@ -4199,9 +4153,6 @@ packages:
   '@types/http-cache-semantics@4.0.4':
     resolution: {integrity: sha512-1m0bIFVc7eJWyve9S0RnuRgcQqF/Xd5QsUZAZeQFr1Q3/p9JWoQQEqmVy+DPTNpGXwhgIetAoYF8JSc33q29QA==}
 
-  '@types/http-link-header@1.0.7':
-    resolution: {integrity: sha512-snm5oLckop0K3cTDAiBnZDy6ncx9DJ3mCRDvs42C884MbVYPP74Tiq2hFsSDRTyjK6RyDYDIulPiW23ge+g5Lw==}
-
   '@types/istanbul-lib-coverage@2.0.4':
     resolution: {integrity: sha512-z/QT1XN4K4KYuslS23k62yDIDLwLFkzxOuMplDtObz0+y7VqJCaO2o+SPwHCvLFZh7xazvvoor2tA/hPz9ee7g==}
 
@@ -4319,9 +4270,6 @@ packages:
   '@types/range-parser@1.2.4':
     resolution: {integrity: sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==}
 
-  '@types/ratelimiter@3.4.6':
-    resolution: {integrity: sha512-Bv6WLSXPGLVsBjkizXtn+ef78R92e36/DFQo2wXPTHtp1cYXF6rCULMqf9WcZPAtyMZMvQAtIPeYMA1xAyxghw==}
-
   '@types/react@18.0.28':
     resolution: {integrity: sha512-RD0ivG1kEztNBdoAK7lekI9M+azSnitIn85h4iOiaLjaTrMjzslhaqCGaI4IyCJ1RljWiLCEu4jyrLLgqxBTew==}
 
@@ -4555,24 +4503,36 @@ packages:
   '@volar/typescript@2.4.12':
     resolution: {integrity: sha512-HJB73OTJDgPc80K30wxi3if4fSsZZAOScbj2fcicMuOPoOkcf9NNAINb33o+DzhBdF9xTKC1gnPmIRDous5S0g==}
 
+  '@vue/compiler-core@3.5.12':
+    resolution: {integrity: sha512-ISyBTRMmMYagUxhcpyEH0hpXRd/KqDU4ymofPgl2XAkY9ZhQ+h0ovEZJIiPop13UmR/54oA2cgMDjgroRelaEw==}
+
   '@vue/compiler-core@3.5.13':
     resolution: {integrity: sha512-oOdAkwqUfW1WqpwSYJce06wvt6HljgY3fGeM9NcVA1HaYOij3mZG9Rkysn0OHuyUAGMbEbARIpsG+LPVlBJ5/Q==}
 
   '@vue/compiler-core@3.5.14':
     resolution: {integrity: sha512-k7qMHMbKvoCXIxPhquKQVw3Twid3Kg4s7+oYURxLGRd56LiuHJVrvFKI4fm2AM3c8apqODPfVJGoh8nePbXMRA==}
 
+  '@vue/compiler-dom@3.5.12':
+    resolution: {integrity: sha512-9G6PbJ03uwxLHKQ3P42cMTi85lDRvGLB2rSGOiQqtXELat6uI4n8cNz9yjfVHRPIu+MsK6TE418Giruvgptckg==}
+
   '@vue/compiler-dom@3.5.13':
     resolution: {integrity: sha512-ZOJ46sMOKUjO3e94wPdCzQ6P1Lx/vhp2RSvfaab88Ajexs0AHeV0uasYhi99WPaogmBlRHNRuly8xV75cNTMDA==}
 
   '@vue/compiler-dom@3.5.14':
     resolution: {integrity: sha512-1aOCSqxGOea5I80U2hQJvXYpPm/aXo95xL/m/mMhgyPUsKe9jhjwWpziNAw7tYRnbz1I61rd9Mld4W9KmmRoug==}
 
+  '@vue/compiler-sfc@3.5.12':
+    resolution: {integrity: sha512-2k973OGo2JuAa5+ZlekuQJtitI5CgLMOwgl94BzMCsKZCX/xiqzJYzapl4opFogKHqwJk34vfsaKpfEhd1k5nw==}
+
   '@vue/compiler-sfc@3.5.13':
     resolution: {integrity: sha512-6VdaljMpD82w6c2749Zhf5T9u5uLBWKnVue6XWxprDobftnletJ8+oel7sexFfM3qIxNmVE7LSFGTpv6obNyaQ==}
 
   '@vue/compiler-sfc@3.5.14':
     resolution: {integrity: sha512-9T6m/9mMr81Lj58JpzsiSIjBgv2LiVoWjIVa7kuXHICUi8LiDSIotMpPRXYJsXKqyARrzjT24NAwttrMnMaCXA==}
 
+  '@vue/compiler-ssr@3.5.12':
+    resolution: {integrity: sha512-eLwc7v6bfGBSM7wZOGPmRavSWzNFF6+PdRhE+VFJhNCgHiF8AM7ccoqcv5kBXA2eWUfigD7byekvf/JsOfKvPA==}
+
   '@vue/compiler-ssr@3.5.13':
     resolution: {integrity: sha512-wMH6vrYHxQl/IybKJagqbquvxpWCuVYpoUJfCqFZwa/JY1GdATAQ+TgVtgrwwMZ0D07QhA99rs/EAAWfvG6KpA==}
 
@@ -4598,24 +4558,38 @@ packages:
       typescript:
         optional: true
 
+  '@vue/reactivity@3.5.12':
+    resolution: {integrity: sha512-UzaN3Da7xnJXdz4Okb/BGbAaomRHc3RdoWqTzlvd9+WBR5m3J39J1fGcHes7U3za0ruYn/iYy/a1euhMEHvTAg==}
+
   '@vue/reactivity@3.5.13':
     resolution: {integrity: sha512-NaCwtw8o48B9I6L1zl2p41OHo/2Z4wqYGGIK1Khu5T7yxrn+ATOixn/Udn2m+6kZKB/J7cuT9DbWWhRxqixACg==}
 
   '@vue/reactivity@3.5.14':
     resolution: {integrity: sha512-7cK1Hp343Fu/SUCCO52vCabjvsYu7ZkOqyYu7bXV9P2yyfjUMUXHZafEbq244sP7gf+EZEz+77QixBTuEqkQQw==}
 
+  '@vue/runtime-core@3.5.12':
+    resolution: {integrity: sha512-hrMUYV6tpocr3TL3Ad8DqxOdpDe4zuQY4HPY3X/VRh+L2myQO8MFXPAMarIOSGNu0bFAjh1yBkMPXZBqCk62Uw==}
+
   '@vue/runtime-core@3.5.13':
     resolution: {integrity: sha512-Fj4YRQ3Az0WTZw1sFe+QDb0aXCerigEpw418pw1HBUKFtnQHWzwojaukAs2X/c9DQz4MQ4bsXTGlcpGxU/RCIw==}
 
   '@vue/runtime-core@3.5.14':
     resolution: {integrity: sha512-w9JWEANwHXNgieAhxPpEpJa+0V5G0hz3NmjAZwlOebtfKyp2hKxKF0+qSh0Xs6/PhfGihuSdqMprMVcQU/E6ag==}
 
+  '@vue/runtime-dom@3.5.12':
+    resolution: {integrity: sha512-q8VFxR9A2MRfBr6/55Q3umyoN7ya836FzRXajPB6/Vvuv0zOPL+qltd9rIMzG/DbRLAIlREmnLsplEF/kotXKA==}
+
   '@vue/runtime-dom@3.5.13':
     resolution: {integrity: sha512-dLaj94s93NYLqjLiyFzVs9X6dWhTdAlEAciC3Moq7gzAc13VJUdCnjjRurNM6uTLFATRHexHCTu/Xp3eW6yoog==}
 
   '@vue/runtime-dom@3.5.14':
     resolution: {integrity: sha512-lCfR++IakeI35TVR80QgOelsUIdcKjd65rWAMfdSlCYnaEY5t3hYwru7vvcWaqmrK+LpI7ZDDYiGU5V3xjMacw==}
 
+  '@vue/server-renderer@3.5.12':
+    resolution: {integrity: sha512-I3QoeDDeEPZm8yR28JtY+rk880Oqmj43hreIBVTicisFTx/Dl7JpG72g/X7YF8hnQD3IFhkky5i2bPonwrTVPg==}
+    peerDependencies:
+      vue: 3.5.12
+
   '@vue/server-renderer@3.5.13':
     resolution: {integrity: sha512-wAi4IRJV/2SAW3htkTlB+dHeRmpTiVIK1OGLWV1yeStVSebSQQOwGwIq0D3ZIoBj2C2qpgz5+vX9iEBkTdk5YA==}
     peerDependencies:
@@ -4626,6 +4600,9 @@ packages:
     peerDependencies:
       vue: 3.5.14
 
+  '@vue/shared@3.5.12':
+    resolution: {integrity: sha512-L2RPSAwUFbgZH20etwrXyVyCBu9OxRSi8T/38QsvnkJyvq2LufW2lDCOzm7t/U9C1mkhJGWYfCuFBCmIuNivrg==}
+
   '@vue/shared@3.5.13':
     resolution: {integrity: sha512-/hnE/qP5ZoGpol0a5mDi45bOd7t3tjYJBjsgCsivow7D48cJeV5l05RD82lPqi7gRiphZM37rnhW1l6ZoCNNnQ==}
 
@@ -4929,9 +4906,6 @@ packages:
     resolution: {integrity: sha512-LElXdjswlqjWrPpJFg1Fx4wpkOCxj1TDHlSV4PlaRxHGWko024xICaa97ZkMfs6DRKlCguiAI+rbXv5GWwXIkg==}
     hasBin: true
 
-  async-mutex@0.5.0:
-    resolution: {integrity: sha512-1A94B18jkJ3DYq284ohPxoXbfTA5HsQ7/Mf4DEhcyLx3Bz27Rh59iScbB6EPiP+B+joue6YCxcMXSbFC1tZKwA==}
-
   async@0.2.10:
     resolution: {integrity: sha512-eAkdoKxU6/LkKDBzLpT+t6Ff5EtfSF4wx1WfJiPEEV7WNLnDaRXk0oVysiEPm262roaachGexwUv94WhSgN5TQ==}
 
@@ -5901,11 +5875,8 @@ packages:
   domutils@2.8.0:
     resolution: {integrity: sha512-w96Cjofp72M5IIhpjgobBimYEfoPjx1Vx0BSX9P30WBdZW2WIKU0T1Bd0kz2eNZ9ikjKgHbEyKx8BB6H1L3h3A==}
 
-  domutils@3.0.1:
-    resolution: {integrity: sha512-z08c1l761iKhDFtfXO04C7kTdPBLi41zwOZl00WS8b5eiaebNpY00HKbztwBq+e3vyqWNwWF3mP9YLUeqIrF+Q==}
-
-  domutils@3.1.0:
-    resolution: {integrity: sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==}
+  domutils@3.2.2:
+    resolution: {integrity: sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==}
 
   dotenv@16.5.0:
     resolution: {integrity: sha512-m/C+AwOAr9/W1UOIZUo232ejMNnJAJtYQjUbHoNTBNTJSvqzzDh7vnrei3o3r3m9blf6ZoDkvcw0VmozNRFJxg==}
@@ -6630,12 +6601,10 @@ packages:
 
   glob@7.2.3:
     resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==}
-    deprecated: Glob versions prior to v9 are no longer supported
 
   glob@8.1.0:
     resolution: {integrity: sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==}
     engines: {node: '>=12'}
-    deprecated: Glob versions prior to v9 are no longer supported
 
   global-dirs@3.0.1:
     resolution: {integrity: sha512-NBcGGFbBA9s1VzD41QXDG+3++t9Mn5t1FpLdhESY6oKY4gYTFpX4wO3sqGUa0Srjtbfj3szX0RnemmrVRUdULA==}
@@ -6690,10 +6659,6 @@ packages:
     resolution: {integrity: sha512-tSQXBXS/MWQOn/RKckawJ61vvsDpCom87JgxiYdGwHdOa0ht0vzUWDlfioofFCRU0L+6NGDt6XzbgoJvZkMeRQ==}
     engines: {node: '>=0.8.0'}
 
-  happy-dom@16.8.1:
-    resolution: {integrity: sha512-n0QrmT9lD81rbpKsyhnlz3DgnMZlaOkJPpgi746doA+HvaMC79bdWkwjrNnGJRvDrWTI8iOcJiVTJ5CdT/AZRw==}
-    engines: {node: '>=18.0.0'}
-
   happy-dom@17.4.4:
     resolution: {integrity: sha512-/Pb0ctk3HTZ5xEL3BZ0hK1AqDSAUuRQitOmROPHhfUYEWpmTImwfD8vFDGADmMAX0JYgbcgxWoLFKtsWhcpuVA==}
     engines: {node: '>=18.0.0'}
@@ -6806,6 +6771,9 @@ packages:
     resolution: {integrity: sha512-eVcrzgbR4tim7c7soKQKtxa/kQM4TzjnlU83rcZ9bHU6t31ehfV7SktN6McWgwPWg+JYMA/O3qpGxBvFq1z2Jg==}
     engines: {node: '>=0.10'}
 
+  htmlparser2@10.0.0:
+    resolution: {integrity: sha512-TwAZM+zE5Tq3lrEHvOlvwgj1XLWQCtaaibSN11Q+gGBAS7Y1uZSWwXXRe4iF6OXnaq1riyQAPFOBtYc77Mxq0g==}
+
   htmlparser2@5.0.1:
     resolution: {integrity: sha512-vKZZra6CSe9qsJzh0BjBGXo8dvzNsq/oGvsjfRdOrrryfeD9UOBEEQdeoqCRmKZchF5h2zOBMQ6YuQ0uRUmdbQ==}
 
@@ -6822,10 +6790,6 @@ packages:
     resolution: {integrity: sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==}
     engines: {node: '>= 0.8'}
 
-  http-link-header@1.1.3:
-    resolution: {integrity: sha512-3cZ0SRL8fb9MUlU3mKM61FcQvPfXx2dBrZW3Vbg5CXa8jFlK8OaEpePenLe1oEXQduhz8b0QjsqfS59QP4AJDQ==}
-    engines: {node: '>=6.0.0'}
-
   http-proxy-agent@7.0.2:
     resolution: {integrity: sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==}
     engines: {node: '>= 14'}
@@ -6932,7 +6896,6 @@ packages:
 
   inflight@1.0.6:
     resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==}
-    deprecated: This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.
 
   inherits@2.0.4:
     resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==}
@@ -7497,9 +7460,6 @@ packages:
     resolution: {integrity: sha512-gqXddjPqQ6G40VdnI6T6yObEC+pDNvyP95wdQhkWkg7crHH3km5qP1FsOXEkzEQwnz6gz5qGTn1c2Y52wP3OyQ==}
     engines: {'0': node >=0.6.0}
 
-  jsrsasign@11.1.0:
-    resolution: {integrity: sha512-Ov74K9GihaK9/9WncTe1mPmvrO7Py665TUfUKvraXBpu+xcTWitrtuOwcjf4KMU9maPaYn0OuaWy0HOzy/GBXg==}
-
   jstransformer@1.0.0:
     resolution: {integrity: sha512-C9YK3Rf8q6VAPDCCU9fnqo3mAfOH6vUGnMcP4AQAYIEpWtfGLpwOTmZ+igtdK5y+VvI2n3CyYSzy4Qh34eq24A==}
 
@@ -7517,8 +7477,8 @@ packages:
   jws@4.0.0:
     resolution: {integrity: sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==}
 
-  katex@0.16.10:
-    resolution: {integrity: sha512-ZiqaC04tp2O5utMsl2TEZTXxa6WSC4yo0fv5ML++D3QZv/vx2Mct0mTlRx3O+uUkjfuAgOkzsCmq5MiUEsDDdA==}
+  katex@0.16.22:
+    resolution: {integrity: sha512-XCHRdUw4lf3SKBaJe4EvgqIuWwkPSo9XoeO8GjQW94Bp7TWv9hNhzZjZ+OH9yf1UmLygb7DIT5GSFQiyt16zYg==}
     hasBin: true
 
   keyv@4.5.4:
@@ -7598,7 +7558,6 @@ packages:
 
   lodash.get@4.4.2:
     resolution: {integrity: sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==}
-    deprecated: This package is deprecated. Use the optional chaining (?.) operator instead.
 
   lodash.isarguments@3.1.0:
     resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==}
@@ -7798,10 +7757,6 @@ packages:
     resolution: {integrity: sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==}
     engines: {node: '>= 0.6'}
 
-  microformats-parser@2.0.2:
-    resolution: {integrity: sha512-tUf9DmN4Jq/tGyp1YH2V6D/Cud+9Uc0WhjjUFirqVeHTRkkfLDacv6BQFT7h7HFsD0Z8wja5eKkRgzZU8bv0Fw==}
-    engines: {node: '>=18'}
-
   micromark-core-commonmark@2.0.0:
     resolution: {integrity: sha512-jThOz/pVmAYUtkroV3D5c1osFXAMv9e0ypGDOIZuCeAe91/sD6BoE2Sjzt30yuXtwOYUmySOhMas/PVyh02itA==}
 
@@ -8163,7 +8118,6 @@ packages:
   node-domexception@1.0.0:
     resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==}
     engines: {node: '>=10.5.0'}
-    deprecated: Use your platform's native DOMException instead
 
   node-fetch@2.7.0:
     resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==}
@@ -8209,11 +8163,6 @@ packages:
     engines: {node: '>=10'}
     hasBin: true
 
-  nodemon@3.1.7:
-    resolution: {integrity: sha512-hLj7fuMow6f0lbB0cD14Lz2xNjwsyruH251Pk4t/yIitCFJbmY1myuLlHm/q06aST4jg6EgAh74PIBBrRqpVAQ==}
-    engines: {node: '>=10'}
-    hasBin: true
-
   nofilter@3.1.0:
     resolution: {integrity: sha512-l2NNj07e9afPnhAhvgVrCD/oy2Ai1yfLpuo3EpiO1jFTsB4sFz6oIfAfSZyQzVpkZQ9xS8ZS5g1jCBgq4Hwo0g==}
     engines: {node: '>=12.19'}
@@ -8269,13 +8218,6 @@ packages:
   nwsapi@2.2.19:
     resolution: {integrity: sha512-94bcyI3RsqiZufXjkr3ltkI86iEl+I7uiHVDtcq9wJUTwYQJ5odHDeSzkkrRzi80jJ8MaeZgqKjH1bAWAFw9bA==}
 
-  oauth2orize-pkce@0.1.2:
-    resolution: {integrity: sha512-grto2UYhXHi9GLE3IBgBBbV87xci55+bCyjpVuxKyzol6I5Rg0K1MiTuXE+JZk54R86SG2wqXODMiZYHraPpxw==}
-
-  oauth2orize@1.12.0:
-    resolution: {integrity: sha512-j4XtFDQUBsvUHPjUmvmNDUDMYed2MphMIJBhyxVVe8hGCjkuYnjIsW+D9qk8c5ciXRdnk6x6tEbiO6PLeOZdCQ==}
-    engines: {node: '>= 0.4.0'}
-
   oauth@0.10.2:
     resolution: {integrity: sha512-JtFnB+8nxDEXgNyniwz573xxbKSOu3R8D40xQKqcjwJ2CDkYqUDI53o6IuzDJBx60Z8VKCm271+t8iFjakrl8Q==}
 
@@ -9078,9 +9020,6 @@ packages:
     resolution: {integrity: sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==}
     engines: {node: '>= 0.6'}
 
-  ratelimiter@3.4.1:
-    resolution: {integrity: sha512-5FJbRW/Jkkdk29ksedAfWFkQkhbUrMx3QJGwMKAypeIiQf4yrLW+gtPKZiaWt4zPrtw1uGufOjGO7UGM6VllsQ==}
-
   raw-body@2.5.2:
     resolution: {integrity: sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==}
     engines: {node: '>= 0.8'}
@@ -9301,9 +9240,6 @@ packages:
   rrweb-cssom@0.8.0:
     resolution: {integrity: sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw==}
 
-  rss-parser@3.13.0:
-    resolution: {integrity: sha512-7jWUBV5yGN3rqMMj7CZufl/291QAhvrrGpDNE4k/02ZchL0npisiYYqULF71jCEKoIiHvK/Q2e6IkDwPziT7+w==}
-
   run-parallel@1.2.0:
     resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==}
 
@@ -10300,9 +10236,6 @@ packages:
     engines: {node: '>=14.17'}
     hasBin: true
 
-  uid2@0.0.4:
-    resolution: {integrity: sha512-IevTus0SbGwQzYh3+fRsAMTVVPOoIVufzacXcHPmdlle1jUpq7BRL+mw3dgeLanvGZdwwbWhRV6XrcFNdBmjWA==}
-
   uid@2.0.2:
     resolution: {integrity: sha512-u3xV3X7uzvi5b1MncmZo3i2Aw222Zk1keqLA1YkHldREkAhAqi65wuPfe7lHx8H/Wzy+8CE7S7uS3jekIM5s8g==}
     engines: {node: '>=8'}
@@ -10632,6 +10565,14 @@ packages:
     peerDependencies:
       typescript: '>=5.0.0'
 
+  vue@3.5.12:
+    resolution: {integrity: sha512-CLVZtXtn2ItBIi/zHZ0Sg1Xkb7+PU32bJJ8Bmy7ts3jxXTcbfsEfBivFYYWz1Hur+lalqGAh65Coin0r+HRUfg==}
+    peerDependencies:
+      typescript: '*'
+    peerDependenciesMeta:
+      typescript:
+        optional: true
+
   vue@3.5.13:
     resolution: {integrity: sha512-wmeiSMxkZCSc+PM2w2VRsOYAZC8GdipNFRTsLSfodVqI9mbejKeXEGr8SckuLnrQPGe3oJN5c3K0vpoU9q/wCQ==}
     peerDependencies:
@@ -10809,14 +10750,6 @@ packages:
     resolution: {integrity: sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==}
     engines: {node: '>=18'}
 
-  xml2js@0.5.0:
-    resolution: {integrity: sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA==}
-    engines: {node: '>=4.0.0'}
-
-  xmlbuilder@11.0.1:
-    resolution: {integrity: sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==}
-    engines: {node: '>=4.0'}
-
   xmlchars@2.2.0:
     resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==}
 
@@ -11576,56 +11509,26 @@ snapshots:
       '@babel/core': 7.23.5
       '@babel/helper-plugin-utils': 7.22.5
 
-  '@babel/plugin-syntax-async-generators@7.8.4(@babel/core@7.24.7)':
-    dependencies:
-      '@babel/core': 7.24.7
-      '@babel/helper-plugin-utils': 7.22.5
-    optional: true
-
   '@babel/plugin-syntax-bigint@7.8.3(@babel/core@7.23.5)':
     dependencies:
       '@babel/core': 7.23.5
       '@babel/helper-plugin-utils': 7.22.5
 
-  '@babel/plugin-syntax-bigint@7.8.3(@babel/core@7.24.7)':
-    dependencies:
-      '@babel/core': 7.24.7
-      '@babel/helper-plugin-utils': 7.22.5
-    optional: true
-
   '@babel/plugin-syntax-class-properties@7.12.13(@babel/core@7.23.5)':
     dependencies:
       '@babel/core': 7.23.5
       '@babel/helper-plugin-utils': 7.22.5
 
-  '@babel/plugin-syntax-class-properties@7.12.13(@babel/core@7.24.7)':
-    dependencies:
-      '@babel/core': 7.24.7
-      '@babel/helper-plugin-utils': 7.22.5
-    optional: true
-
   '@babel/plugin-syntax-import-meta@7.10.4(@babel/core@7.23.5)':
     dependencies:
       '@babel/core': 7.23.5
       '@babel/helper-plugin-utils': 7.22.5
 
-  '@babel/plugin-syntax-import-meta@7.10.4(@babel/core@7.24.7)':
-    dependencies:
-      '@babel/core': 7.24.7
-      '@babel/helper-plugin-utils': 7.22.5
-    optional: true
-
   '@babel/plugin-syntax-json-strings@7.8.3(@babel/core@7.23.5)':
     dependencies:
       '@babel/core': 7.23.5
       '@babel/helper-plugin-utils': 7.22.5
 
-  '@babel/plugin-syntax-json-strings@7.8.3(@babel/core@7.24.7)':
-    dependencies:
-      '@babel/core': 7.24.7
-      '@babel/helper-plugin-utils': 7.22.5
-    optional: true
-
   '@babel/plugin-syntax-jsx@7.23.3(@babel/core@7.23.5)':
     dependencies:
       '@babel/core': 7.23.5
@@ -11636,78 +11539,36 @@ snapshots:
       '@babel/core': 7.23.5
       '@babel/helper-plugin-utils': 7.22.5
 
-  '@babel/plugin-syntax-logical-assignment-operators@7.10.4(@babel/core@7.24.7)':
-    dependencies:
-      '@babel/core': 7.24.7
-      '@babel/helper-plugin-utils': 7.22.5
-    optional: true
-
   '@babel/plugin-syntax-nullish-coalescing-operator@7.8.3(@babel/core@7.23.5)':
     dependencies:
       '@babel/core': 7.23.5
       '@babel/helper-plugin-utils': 7.22.5
 
-  '@babel/plugin-syntax-nullish-coalescing-operator@7.8.3(@babel/core@7.24.7)':
-    dependencies:
-      '@babel/core': 7.24.7
-      '@babel/helper-plugin-utils': 7.22.5
-    optional: true
-
   '@babel/plugin-syntax-numeric-separator@7.10.4(@babel/core@7.23.5)':
     dependencies:
       '@babel/core': 7.23.5
       '@babel/helper-plugin-utils': 7.22.5
 
-  '@babel/plugin-syntax-numeric-separator@7.10.4(@babel/core@7.24.7)':
-    dependencies:
-      '@babel/core': 7.24.7
-      '@babel/helper-plugin-utils': 7.22.5
-    optional: true
-
   '@babel/plugin-syntax-object-rest-spread@7.8.3(@babel/core@7.23.5)':
     dependencies:
       '@babel/core': 7.23.5
       '@babel/helper-plugin-utils': 7.22.5
 
-  '@babel/plugin-syntax-object-rest-spread@7.8.3(@babel/core@7.24.7)':
-    dependencies:
-      '@babel/core': 7.24.7
-      '@babel/helper-plugin-utils': 7.22.5
-    optional: true
-
   '@babel/plugin-syntax-optional-catch-binding@7.8.3(@babel/core@7.23.5)':
     dependencies:
       '@babel/core': 7.23.5
       '@babel/helper-plugin-utils': 7.22.5
 
-  '@babel/plugin-syntax-optional-catch-binding@7.8.3(@babel/core@7.24.7)':
-    dependencies:
-      '@babel/core': 7.24.7
-      '@babel/helper-plugin-utils': 7.22.5
-    optional: true
-
   '@babel/plugin-syntax-optional-chaining@7.8.3(@babel/core@7.23.5)':
     dependencies:
       '@babel/core': 7.23.5
       '@babel/helper-plugin-utils': 7.22.5
 
-  '@babel/plugin-syntax-optional-chaining@7.8.3(@babel/core@7.24.7)':
-    dependencies:
-      '@babel/core': 7.24.7
-      '@babel/helper-plugin-utils': 7.22.5
-    optional: true
-
   '@babel/plugin-syntax-top-level-await@7.14.5(@babel/core@7.23.5)':
     dependencies:
       '@babel/core': 7.23.5
       '@babel/helper-plugin-utils': 7.22.5
 
-  '@babel/plugin-syntax-top-level-await@7.14.5(@babel/core@7.24.7)':
-    dependencies:
-      '@babel/core': 7.24.7
-      '@babel/helper-plugin-utils': 7.22.5
-    optional: true
-
   '@babel/plugin-syntax-typescript@7.23.3(@babel/core@7.23.5)':
     dependencies:
       '@babel/core': 7.23.5
@@ -13138,6 +12999,8 @@ snapshots:
 
   '@phosphor-icons/web@2.1.1': {}
 
+  '@phosphor-icons/web@2.1.2': {}
+
   '@pkgjs/parseargs@0.11.0':
     optional: true
 
@@ -13390,6 +13253,12 @@ snapshots:
     transitivePeerDependencies:
       - supports-color
 
+  '@sentry/vue@9.14.0(vue@3.5.12(typescript@5.8.3))':
+    dependencies:
+      '@sentry/browser': 9.14.0
+      '@sentry/core': 9.14.0
+      vue: 3.5.12(typescript@5.8.3)
+
   '@sentry/vue@9.14.0(vue@3.5.14(typescript@5.8.3))':
     dependencies:
       '@sentry/browser': 9.14.0
@@ -14117,32 +13986,32 @@ snapshots:
     dependencies:
       storybook: 8.6.12(bufferutil@4.0.9)(prettier@3.5.3)(utf-8-validate@6.0.5)
 
-  '@storybook/vue3-vite@8.6.12(storybook@8.6.12(bufferutil@4.0.9)(prettier@3.5.3)(utf-8-validate@6.0.5))(vite@6.3.3(@types/node@22.15.2)(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3))(vue@3.5.14(typescript@5.8.3))':
+  '@storybook/vue3-vite@8.6.12(storybook@8.6.12(bufferutil@4.0.9)(prettier@3.5.3)(utf-8-validate@6.0.5))(vite@6.3.3(@types/node@22.15.2)(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3))(vue@3.5.12(typescript@5.8.3))':
     dependencies:
       '@storybook/builder-vite': 8.6.12(storybook@8.6.12(bufferutil@4.0.9)(prettier@3.5.3)(utf-8-validate@6.0.5))(vite@6.3.3(@types/node@22.15.2)(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3))
-      '@storybook/vue3': 8.6.12(storybook@8.6.12(bufferutil@4.0.9)(prettier@3.5.3)(utf-8-validate@6.0.5))(vue@3.5.14(typescript@5.8.3))
+      '@storybook/vue3': 8.6.12(storybook@8.6.12(bufferutil@4.0.9)(prettier@3.5.3)(utf-8-validate@6.0.5))(vue@3.5.12(typescript@5.8.3))
       find-package-json: 1.2.0
       magic-string: 0.30.17
       storybook: 8.6.12(bufferutil@4.0.9)(prettier@3.5.3)(utf-8-validate@6.0.5)
       typescript: 5.8.3
       vite: 6.3.3(@types/node@22.15.2)(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3)
       vue-component-meta: 2.0.16(typescript@5.8.3)
-      vue-docgen-api: 4.75.1(vue@3.5.14(typescript@5.8.3))
+      vue-docgen-api: 4.75.1(vue@3.5.12(typescript@5.8.3))
     transitivePeerDependencies:
       - vue
 
-  '@storybook/vue3@8.6.12(storybook@8.6.12(bufferutil@4.0.9)(prettier@3.5.3)(utf-8-validate@6.0.5))(vue@3.5.14(typescript@5.8.3))':
+  '@storybook/vue3@8.6.12(storybook@8.6.12(bufferutil@4.0.9)(prettier@3.5.3)(utf-8-validate@6.0.5))(vue@3.5.12(typescript@5.8.3))':
     dependencies:
       '@storybook/components': 8.6.12(storybook@8.6.12(bufferutil@4.0.9)(prettier@3.5.3)(utf-8-validate@6.0.5))
       '@storybook/global': 5.0.0
       '@storybook/manager-api': 8.6.12(storybook@8.6.12(bufferutil@4.0.9)(prettier@3.5.3)(utf-8-validate@6.0.5))
       '@storybook/preview-api': 8.6.12(storybook@8.6.12(bufferutil@4.0.9)(prettier@3.5.3)(utf-8-validate@6.0.5))
       '@storybook/theming': 8.6.12(storybook@8.6.12(bufferutil@4.0.9)(prettier@3.5.3)(utf-8-validate@6.0.5))
-      '@vue/compiler-core': 3.5.14
+      '@vue/compiler-core': 3.5.12
       storybook: 8.6.12(bufferutil@4.0.9)(prettier@3.5.3)(utf-8-validate@6.0.5)
       ts-dedent: 2.2.0
       type-fest: 2.19.0
-      vue: 3.5.14(typescript@5.8.3)
+      vue: 3.5.12(typescript@5.8.3)
       vue-component-type-helpers: 2.2.10
 
   '@stylistic/eslint-plugin@4.2.0(eslint@9.25.1)(typescript@5.8.3)':
@@ -14290,14 +14159,14 @@ snapshots:
     dependencies:
       '@testing-library/dom': 10.4.0
 
-  '@testing-library/vue@8.1.0(@vue/compiler-sfc@3.5.14)(@vue/server-renderer@3.5.14(vue@3.5.14(typescript@5.8.3)))(vue@3.5.14(typescript@5.8.3))':
+  '@testing-library/vue@8.1.0(@vue/compiler-sfc@3.5.12)(@vue/server-renderer@3.5.14(vue@3.5.12(typescript@5.8.3)))(vue@3.5.12(typescript@5.8.3))':
     dependencies:
       '@babel/runtime': 7.23.4
       '@testing-library/dom': 9.3.4
-      '@vue/test-utils': 2.4.1(@vue/server-renderer@3.5.14(vue@3.5.14(typescript@5.8.3)))(vue@3.5.14(typescript@5.8.3))
-      vue: 3.5.14(typescript@5.8.3)
+      '@vue/test-utils': 2.4.1(@vue/server-renderer@3.5.14(vue@3.5.12(typescript@5.8.3)))(vue@3.5.12(typescript@5.8.3))
+      vue: 3.5.12(typescript@5.8.3)
     optionalDependencies:
-      '@vue/compiler-sfc': 3.5.14
+      '@vue/compiler-sfc': 3.5.12
     transitivePeerDependencies:
       - '@vue/server-renderer'
 
@@ -14439,10 +14308,6 @@ snapshots:
 
   '@types/http-cache-semantics@4.0.4': {}
 
-  '@types/http-link-header@1.0.7':
-    dependencies:
-      '@types/node': 22.15.2
-
   '@types/istanbul-lib-coverage@2.0.4': {}
 
   '@types/istanbul-lib-report@3.0.0':
@@ -14562,8 +14427,6 @@ snapshots:
 
   '@types/range-parser@1.2.4': {}
 
-  '@types/ratelimiter@3.4.6': {}
-
   '@types/react@18.0.28':
     dependencies:
       '@types/prop-types': 15.7.14
@@ -14747,10 +14610,10 @@ snapshots:
 
   '@ungap/structured-clone@1.2.0': {}
 
-  '@vitejs/plugin-vue@5.2.3(vite@6.3.3(@types/node@22.15.2)(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3))(vue@3.5.14(typescript@5.8.3))':
+  '@vitejs/plugin-vue@5.2.3(vite@6.3.3(@types/node@22.15.2)(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3))(vue@3.5.12(typescript@5.8.3))':
     dependencies:
       vite: 6.3.3(@types/node@22.15.2)(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3)
-      vue: 3.5.14(typescript@5.8.3)
+      vue: 3.5.12(typescript@5.8.3)
 
   '@vitest/coverage-v8@3.1.2(vitest@3.1.2(@types/debug@4.1.12)(@types/node@22.15.2)(happy-dom@17.4.4)(jsdom@26.1.0(bufferutil@4.0.9)(utf-8-validate@6.0.5))(msw@2.7.5(@types/node@22.15.2)(typescript@5.8.3))(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3))':
     dependencies:
@@ -14868,6 +14731,14 @@ snapshots:
       path-browserify: 1.0.1
       vscode-uri: 3.0.8
 
+  '@vue/compiler-core@3.5.12':
+    dependencies:
+      '@babel/parser': 7.27.2
+      '@vue/shared': 3.5.12
+      entities: 4.5.0
+      estree-walker: 2.0.2
+      source-map-js: 1.2.1
+
   '@vue/compiler-core@3.5.13':
     dependencies:
       '@babel/parser': 7.27.2
@@ -14884,6 +14755,11 @@ snapshots:
       estree-walker: 2.0.2
       source-map-js: 1.2.1
 
+  '@vue/compiler-dom@3.5.12':
+    dependencies:
+      '@vue/compiler-core': 3.5.12
+      '@vue/shared': 3.5.12
+
   '@vue/compiler-dom@3.5.13':
     dependencies:
       '@vue/compiler-core': 3.5.13
@@ -14894,9 +14770,21 @@ snapshots:
       '@vue/compiler-core': 3.5.14
       '@vue/shared': 3.5.14
 
-  '@vue/compiler-sfc@3.5.13':
+  '@vue/compiler-sfc@3.5.12':
     dependencies:
       '@babel/parser': 7.27.2
+      '@vue/compiler-core': 3.5.12
+      '@vue/compiler-dom': 3.5.12
+      '@vue/compiler-ssr': 3.5.12
+      '@vue/shared': 3.5.12
+      estree-walker: 2.0.2
+      magic-string: 0.30.17
+      postcss: 8.5.3
+      source-map-js: 1.2.1
+
+  '@vue/compiler-sfc@3.5.13':
+    dependencies:
+      '@babel/parser': 7.25.7
       '@vue/compiler-core': 3.5.13
       '@vue/compiler-dom': 3.5.13
       '@vue/compiler-ssr': 3.5.13
@@ -14918,6 +14806,11 @@ snapshots:
       postcss: 8.5.3
       source-map-js: 1.2.1
 
+  '@vue/compiler-ssr@3.5.12':
+    dependencies:
+      '@vue/compiler-dom': 3.5.12
+      '@vue/shared': 3.5.12
+
   '@vue/compiler-ssr@3.5.13':
     dependencies:
       '@vue/compiler-dom': 3.5.13
@@ -14958,6 +14851,10 @@ snapshots:
     optionalDependencies:
       typescript: 5.8.3
 
+  '@vue/reactivity@3.5.12':
+    dependencies:
+      '@vue/shared': 3.5.12
+
   '@vue/reactivity@3.5.13':
     dependencies:
       '@vue/shared': 3.5.13
@@ -14966,6 +14863,11 @@ snapshots:
     dependencies:
       '@vue/shared': 3.5.14
 
+  '@vue/runtime-core@3.5.12':
+    dependencies:
+      '@vue/reactivity': 3.5.12
+      '@vue/shared': 3.5.12
+
   '@vue/runtime-core@3.5.13':
     dependencies:
       '@vue/reactivity': 3.5.13
@@ -14976,6 +14878,13 @@ snapshots:
       '@vue/reactivity': 3.5.14
       '@vue/shared': 3.5.14
 
+  '@vue/runtime-dom@3.5.12':
+    dependencies:
+      '@vue/reactivity': 3.5.12
+      '@vue/runtime-core': 3.5.12
+      '@vue/shared': 3.5.12
+      csstype: 3.1.3
+
   '@vue/runtime-dom@3.5.13':
     dependencies:
       '@vue/reactivity': 3.5.13
@@ -14990,29 +14899,44 @@ snapshots:
       '@vue/shared': 3.5.14
       csstype: 3.1.3
 
+  '@vue/server-renderer@3.5.12(vue@3.5.12(typescript@5.8.3))':
+    dependencies:
+      '@vue/compiler-ssr': 3.5.12
+      '@vue/shared': 3.5.12
+      vue: 3.5.12(typescript@5.8.3)
+
   '@vue/server-renderer@3.5.13(vue@3.5.13(typescript@5.8.3))':
     dependencies:
       '@vue/compiler-ssr': 3.5.13
       '@vue/shared': 3.5.13
       vue: 3.5.13(typescript@5.8.3)
 
+  '@vue/server-renderer@3.5.14(vue@3.5.12(typescript@5.8.3))':
+    dependencies:
+      '@vue/compiler-ssr': 3.5.14
+      '@vue/shared': 3.5.14
+      vue: 3.5.12(typescript@5.8.3)
+    optional: true
+
   '@vue/server-renderer@3.5.14(vue@3.5.14(typescript@5.8.3))':
     dependencies:
       '@vue/compiler-ssr': 3.5.14
       '@vue/shared': 3.5.14
       vue: 3.5.14(typescript@5.8.3)
 
+  '@vue/shared@3.5.12': {}
+
   '@vue/shared@3.5.13': {}
 
   '@vue/shared@3.5.14': {}
 
-  '@vue/test-utils@2.4.1(@vue/server-renderer@3.5.14(vue@3.5.14(typescript@5.8.3)))(vue@3.5.14(typescript@5.8.3))':
+  '@vue/test-utils@2.4.1(@vue/server-renderer@3.5.14(vue@3.5.12(typescript@5.8.3)))(vue@3.5.12(typescript@5.8.3))':
     dependencies:
       js-beautify: 1.14.9
-      vue: 3.5.14(typescript@5.8.3)
+      vue: 3.5.12(typescript@5.8.3)
       vue-component-type-helpers: 1.8.4
     optionalDependencies:
-      '@vue/server-renderer': 3.5.14(vue@3.5.14(typescript@5.8.3))
+      '@vue/server-renderer': 3.5.14(vue@3.5.12(typescript@5.8.3))
 
   '@xhmikosr/archive-type@7.0.0':
     dependencies:
@@ -15353,10 +15277,6 @@ snapshots:
 
   astring@1.9.0: {}
 
-  async-mutex@0.5.0:
-    dependencies:
-      tslib: 2.6.2
-
   async@0.2.10: {}
 
   async@3.2.4: {}
@@ -15428,20 +15348,6 @@ snapshots:
     transitivePeerDependencies:
       - supports-color
 
-  babel-jest@29.7.0(@babel/core@7.24.7):
-    dependencies:
-      '@babel/core': 7.24.7
-      '@jest/transform': 29.7.0
-      '@types/babel__core': 7.20.0
-      babel-plugin-istanbul: 6.1.1
-      babel-preset-jest: 29.6.3(@babel/core@7.24.7)
-      chalk: 4.1.2
-      graceful-fs: 4.2.11
-      slash: 3.0.0
-    transitivePeerDependencies:
-      - supports-color
-    optional: true
-
   babel-plugin-istanbul@6.1.1:
     dependencies:
       '@babel/helper-plugin-utils': 7.22.5
@@ -15475,36 +15381,12 @@ snapshots:
       '@babel/plugin-syntax-optional-chaining': 7.8.3(@babel/core@7.23.5)
       '@babel/plugin-syntax-top-level-await': 7.14.5(@babel/core@7.23.5)
 
-  babel-preset-current-node-syntax@1.0.1(@babel/core@7.24.7):
-    dependencies:
-      '@babel/core': 7.24.7
-      '@babel/plugin-syntax-async-generators': 7.8.4(@babel/core@7.24.7)
-      '@babel/plugin-syntax-bigint': 7.8.3(@babel/core@7.24.7)
-      '@babel/plugin-syntax-class-properties': 7.12.13(@babel/core@7.24.7)
-      '@babel/plugin-syntax-import-meta': 7.10.4(@babel/core@7.24.7)
-      '@babel/plugin-syntax-json-strings': 7.8.3(@babel/core@7.24.7)
-      '@babel/plugin-syntax-logical-assignment-operators': 7.10.4(@babel/core@7.24.7)
-      '@babel/plugin-syntax-nullish-coalescing-operator': 7.8.3(@babel/core@7.24.7)
-      '@babel/plugin-syntax-numeric-separator': 7.10.4(@babel/core@7.24.7)
-      '@babel/plugin-syntax-object-rest-spread': 7.8.3(@babel/core@7.24.7)
-      '@babel/plugin-syntax-optional-catch-binding': 7.8.3(@babel/core@7.24.7)
-      '@babel/plugin-syntax-optional-chaining': 7.8.3(@babel/core@7.24.7)
-      '@babel/plugin-syntax-top-level-await': 7.14.5(@babel/core@7.24.7)
-    optional: true
-
   babel-preset-jest@29.6.3(@babel/core@7.23.5):
     dependencies:
       '@babel/core': 7.23.5
       babel-plugin-jest-hoist: 29.6.3
       babel-preset-current-node-syntax: 1.0.1(@babel/core@7.23.5)
 
-  babel-preset-jest@29.6.3(@babel/core@7.24.7):
-    dependencies:
-      '@babel/core': 7.24.7
-      babel-plugin-jest-hoist: 29.6.3
-      babel-preset-current-node-syntax: 1.0.1(@babel/core@7.24.7)
-    optional: true
-
   babel-walk@3.0.0-canary-5:
     dependencies:
       '@babel/types': 7.27.1
@@ -15862,14 +15744,14 @@ snapshots:
       css-what: 6.1.0
       domelementtype: 2.3.0
       domhandler: 5.0.3
-      domutils: 3.1.0
+      domutils: 3.2.2
 
   cheerio@1.0.0:
     dependencies:
       cheerio-select: 2.1.0
       dom-serializer: 2.0.0
       domhandler: 5.0.3
-      domutils: 3.1.0
+      domutils: 3.2.2
       encoding-sniffer: 0.2.0
       htmlparser2: 9.1.0
       parse5: 7.3.0
@@ -16142,7 +16024,7 @@ snapshots:
       boolbase: 1.0.0
       css-what: 6.1.0
       domhandler: 5.0.3
-      domutils: 3.1.0
+      domutils: 3.2.2
       nth-check: 2.1.1
 
   css-tree@2.2.1:
@@ -16320,11 +16202,9 @@ snapshots:
     dependencies:
       ms: 2.1.2
 
-  debug@4.3.5(supports-color@5.5.0):
+  debug@4.3.5:
     dependencies:
       ms: 2.1.2
-    optionalDependencies:
-      supports-color: 5.5.0
 
   debug@4.4.0(supports-color@5.5.0):
     dependencies:
@@ -16515,13 +16395,7 @@ snapshots:
       domelementtype: 2.3.0
       domhandler: 4.3.1
 
-  domutils@3.0.1:
-    dependencies:
-      dom-serializer: 2.0.0
-      domelementtype: 2.3.0
-      domhandler: 5.0.3
-
-  domutils@3.1.0:
+  domutils@3.2.2:
     dependencies:
       dom-serializer: 2.0.0
       domelementtype: 2.3.0
@@ -17605,11 +17479,6 @@ snapshots:
 
   hammerjs@2.0.8: {}
 
-  happy-dom@16.8.1:
-    dependencies:
-      webidl-conversions: 7.0.0
-      whatwg-mimetype: 3.0.0
-
   happy-dom@17.4.4:
     dependencies:
       webidl-conversions: 7.0.0
@@ -17708,6 +17577,13 @@ snapshots:
 
   htmlescape@1.1.1: {}
 
+  htmlparser2@10.0.0:
+    dependencies:
+      domelementtype: 2.3.0
+      domhandler: 5.0.3
+      domutils: 3.2.2
+      entities: 6.0.0
+
   htmlparser2@5.0.1:
     dependencies:
       domelementtype: 2.3.0
@@ -17719,14 +17595,14 @@ snapshots:
     dependencies:
       domelementtype: 2.3.0
       domhandler: 5.0.3
-      domutils: 3.0.1
+      domutils: 3.2.2
       entities: 4.5.0
 
   htmlparser2@9.1.0:
     dependencies:
       domelementtype: 2.3.0
       domhandler: 5.0.3
-      domutils: 3.1.0
+      domutils: 3.2.2
       entities: 4.5.0
 
   http-cache-semantics@4.1.1: {}
@@ -17739,8 +17615,6 @@ snapshots:
       statuses: 2.0.1
       toidentifier: 1.0.1
 
-  http-link-header@1.1.3: {}
-
   http-proxy-agent@7.0.2:
     dependencies:
       agent-base: 7.1.3
@@ -18614,8 +18488,6 @@ snapshots:
       verror: 1.10.0
     optional: true
 
-  jsrsasign@11.1.0: {}
-
   jstransformer@1.0.0:
     dependencies:
       is-promise: 2.2.2
@@ -18643,7 +18515,7 @@ snapshots:
       jwa: 2.0.0
       safe-buffer: 5.2.1
 
-  katex@0.16.10:
+  katex@0.16.22:
     dependencies:
       commander: 8.3.0
 
@@ -18978,10 +18850,6 @@ snapshots:
 
   methods@1.1.2: {}
 
-  microformats-parser@2.0.2:
-    dependencies:
-      parse5: 7.3.0
-
   micromark-core-commonmark@2.0.0:
     dependencies:
       decode-named-character-reference: 1.0.2
@@ -19486,19 +19354,6 @@ snapshots:
       touch: 3.1.0
       undefsafe: 2.0.5
 
-  nodemon@3.1.7:
-    dependencies:
-      chokidar: 4.0.3
-      debug: 4.3.5(supports-color@5.5.0)
-      ignore-by-default: 1.0.1
-      minimatch: 3.1.2
-      pstree.remy: 1.1.8
-      semver: 7.6.0
-      simple-update-notifier: 2.0.0
-      supports-color: 5.5.0
-      touch: 3.1.0
-      undefsafe: 2.0.5
-
   nofilter@3.1.0: {}
 
   nopt@1.0.10:
@@ -19555,16 +19410,6 @@ snapshots:
   nwsapi@2.2.19:
     optional: true
 
-  oauth2orize-pkce@0.1.2: {}
-
-  oauth2orize@1.12.0:
-    dependencies:
-      debug: 2.6.9
-      uid2: 0.0.4
-      utils-merge: 1.0.1
-    transitivePeerDependencies:
-      - supports-color
-
   oauth@0.10.2: {}
 
   object-assign@4.1.1: {}
@@ -20348,8 +20193,6 @@ snapshots:
 
   range-parser@1.2.1: {}
 
-  ratelimiter@3.4.1: {}
-
   raw-body@2.5.2:
     dependencies:
       bytes: 3.1.2
@@ -20645,11 +20488,6 @@ snapshots:
   rrweb-cssom@0.8.0:
     optional: true
 
-  rss-parser@3.13.0:
-    dependencies:
-      entities: 2.2.0
-      xml2js: 0.5.0
-
   run-parallel@1.2.0:
     dependencies:
       queue-microtask: 1.2.3
@@ -20923,7 +20761,7 @@ snapshots:
     dependencies:
       '@hapi/hoek': 11.0.4
       '@hapi/wreck': 18.0.1
-      debug: 4.3.5(supports-color@5.5.0)
+      debug: 4.3.5
       joi: 17.11.0
     transitivePeerDependencies:
       - supports-color
@@ -20934,7 +20772,7 @@ snapshots:
 
   simple-update-notifier@2.0.0:
     dependencies:
-      semver: 7.6.3
+      semver: 7.7.1
 
   sinon@18.0.1:
     dependencies:
@@ -21553,7 +21391,7 @@ snapshots:
 
   ts-dedent@2.2.0: {}
 
-  ts-jest@29.1.2(@babel/core@7.24.7)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.24.7))(esbuild@0.25.3)(jest@29.7.0(@types/node@22.15.2))(typescript@5.8.3):
+  ts-jest@29.1.2(@babel/core@7.23.5)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.5))(esbuild@0.25.3)(jest@29.7.0(@types/node@22.15.2))(typescript@5.8.3):
     dependencies:
       bs-logger: 0.2.6
       fast-json-stable-stringify: 2.1.0
@@ -21566,9 +21404,9 @@ snapshots:
       typescript: 5.8.3
       yargs-parser: 21.1.1
     optionalDependencies:
-      '@babel/core': 7.24.7
+      '@babel/core': 7.23.5
       '@jest/types': 29.6.3
-      babel-jest: 29.7.0(@babel/core@7.24.7)
+      babel-jest: 29.7.0(@babel/core@7.23.5)
       esbuild: 0.25.3
 
   ts-map@1.0.3: {}
@@ -21714,8 +21552,6 @@ snapshots:
 
   typescript@5.8.3: {}
 
-  uid2@0.0.4: {}
-
   uid@2.0.2:
     dependencies:
       '@lukeed/csprng': 1.0.1
@@ -21854,13 +21690,13 @@ snapshots:
 
   uuid@9.0.1: {}
 
-  v-code-diff@1.13.1(vue@3.5.14(typescript@5.8.3)):
+  v-code-diff@1.13.1(vue@3.5.12(typescript@5.8.3)):
     dependencies:
       diff: 5.2.0
       diff-match-patch: 1.0.5
       highlight.js: 11.10.0
-      vue: 3.5.14(typescript@5.8.3)
-      vue-demi: 0.14.7(vue@3.5.14(typescript@5.8.3))
+      vue: 3.5.12(typescript@5.8.3)
+      vue-demi: 0.14.7(vue@3.5.12(typescript@5.8.3))
 
   v8-to-istanbul@9.2.0:
     dependencies:
@@ -22018,24 +21854,24 @@ snapshots:
 
   vue-component-type-helpers@2.2.10: {}
 
-  vue-demi@0.14.7(vue@3.5.14(typescript@5.8.3)):
+  vue-demi@0.14.7(vue@3.5.12(typescript@5.8.3)):
     dependencies:
-      vue: 3.5.14(typescript@5.8.3)
+      vue: 3.5.12(typescript@5.8.3)
 
-  vue-docgen-api@4.75.1(vue@3.5.14(typescript@5.8.3)):
+  vue-docgen-api@4.75.1(vue@3.5.12(typescript@5.8.3)):
     dependencies:
       '@babel/parser': 7.25.7
       '@babel/types': 7.25.7
       '@vue/compiler-dom': 3.5.14
-      '@vue/compiler-sfc': 3.5.14
+      '@vue/compiler-sfc': 3.5.12
       ast-types: 0.16.1
       hash-sum: 2.0.0
       lru-cache: 8.0.4
       pug: 3.0.3
       recast: 0.23.6
       ts-map: 1.0.3
-      vue: 3.5.14(typescript@5.8.3)
-      vue-inbrowser-compiler-independent-utils: 4.71.1(vue@3.5.14(typescript@5.8.3))
+      vue: 3.5.12(typescript@5.8.3)
+      vue-inbrowser-compiler-independent-utils: 4.71.1(vue@3.5.12(typescript@5.8.3))
 
   vue-eslint-parser@10.1.3(eslint@9.25.1):
     dependencies:
@@ -22050,9 +21886,9 @@ snapshots:
     transitivePeerDependencies:
       - supports-color
 
-  vue-inbrowser-compiler-independent-utils@4.71.1(vue@3.5.14(typescript@5.8.3)):
+  vue-inbrowser-compiler-independent-utils@4.71.1(vue@3.5.12(typescript@5.8.3)):
     dependencies:
-      vue: 3.5.14(typescript@5.8.3)
+      vue: 3.5.12(typescript@5.8.3)
 
   vue-template-compiler@2.7.14:
     dependencies:
@@ -22065,6 +21901,16 @@ snapshots:
       '@vue/language-core': 2.2.10(typescript@5.8.3)
       typescript: 5.8.3
 
+  vue@3.5.12(typescript@5.8.3):
+    dependencies:
+      '@vue/compiler-dom': 3.5.12
+      '@vue/compiler-sfc': 3.5.12
+      '@vue/runtime-dom': 3.5.12
+      '@vue/server-renderer': 3.5.12(vue@3.5.12(typescript@5.8.3))
+      '@vue/shared': 3.5.12
+    optionalDependencies:
+      typescript: 5.8.3
+
   vue@3.5.13(typescript@5.8.3):
     dependencies:
       '@vue/compiler-dom': 3.5.13
@@ -22085,10 +21931,10 @@ snapshots:
     optionalDependencies:
       typescript: 5.8.3
 
-  vuedraggable@4.1.0(vue@3.5.14(typescript@5.8.3)):
+  vuedraggable@4.1.0(vue@3.5.12(typescript@5.8.3)):
     dependencies:
       sortablejs: 1.14.0
-      vue: 3.5.14(typescript@5.8.3)
+      vue: 3.5.12(typescript@5.8.3)
 
   w3c-xmlserializer@5.0.0:
     dependencies:
@@ -22261,13 +22107,6 @@ snapshots:
   xml-name-validator@5.0.0:
     optional: true
 
-  xml2js@0.5.0:
-    dependencies:
-      sax: 1.2.4
-      xmlbuilder: 11.0.1
-
-  xmlbuilder@11.0.1: {}
-
   xmlchars@2.2.0:
     optional: true
 

From a348d7aa09d1c3624400e483fa590828a9f8e969 Mon Sep 17 00:00:00 2001
From: Hazelnoot 
Date: Fri, 23 May 2025 19:42:53 -0400
Subject: [PATCH 40/54] remove unused script from megalodon

---
 packages/megalodon/package.json | 16 +++++-----------
 1 file changed, 5 insertions(+), 11 deletions(-)

diff --git a/packages/megalodon/package.json b/packages/megalodon/package.json
index f10a9cf9dc..690b5d7f2a 100644
--- a/packages/megalodon/package.json
+++ b/packages/megalodon/package.json
@@ -6,7 +6,6 @@
   "typings": "./lib/src/index.d.ts",
   "scripts": {
     "build": "tsc -p ./",
-    "doc": "typedoc --out ../docs ./src",
     "test": "cross-env NODE_ENV=test jest -u --maxWorkers=3"
   },
   "engines": {
@@ -54,22 +53,17 @@
   },
   "homepage": "https://github.com/h3poteto/megalodon#readme",
   "dependencies": {
-    "@types/jest": "^29.5.10",
-    "@types/oauth": "^0.9.4",
-    "axios": "1.7.4",
-    "dayjs": "^1.11.10",
+    "axios": "1.9.0",
+    "dayjs": "1.11.13",
     "form-data": "4.0.2",
     "oauth": "0.10.2",
 		"typescript": "5.8.3"
   },
   "devDependencies": {
-		"@typescript-eslint/eslint-plugin": "8.31.0",
-		"@typescript-eslint/parser": "8.31.0",
-		"eslint": "9.25.1",
-    "eslint-config-prettier": "^9.0.0",
+		"@types/jest": "29.5.14",
+		"@types/oauth": "0.9.6",
     "jest": "29.7.0",
     "jest-worker": "29.7.0",
-    "prettier": "3.5.3",
-    "ts-jest": "^29.1.1"
+    "ts-jest": "29.3.4"
   }
 }

From 0a976720114492b52ae7d58078d6eccb95c20eed Mon Sep 17 00:00:00 2001
From: Hazelnoot 
Date: Fri, 23 May 2025 19:43:35 -0400
Subject: [PATCH 41/54] align megalodon tsconfig with the rest of the project

---
 packages/megalodon/tsconfig.json | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/packages/megalodon/tsconfig.json b/packages/megalodon/tsconfig.json
index 2b90738318..3cb31f67ba 100644
--- a/packages/megalodon/tsconfig.json
+++ b/packages/megalodon/tsconfig.json
@@ -1,14 +1,14 @@
 {
   "compilerOptions": {
     /* Basic Options */
-    "target": "ES2022",                          /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017','ES2018' or 'ESNEXT'. */
+    "target": "ES2022",                       /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017','ES2018' or 'ESNEXT'. */
     "module": "commonjs",                     /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */
-    "lib": ["ES2022", "dom"],                           /* Specify library files to be included in the compilation. */
+    "lib": ["ES2022", "dom"],                 /* Specify library files to be included in the compilation. */
     // "allowJs": true,                       /* Allow javascript files to be compiled. */
     // "checkJs": true,                       /* Report errors in .js files. */
     // "jsx": "preserve",                     /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */
     "declaration": true,                      /* Generates corresponding '.d.ts' file. */
-    // "declarationMap": true,                /* Generates a sourcemap for each corresponding '.d.ts' file. */
+    "declarationMap": true,                   /* Generates a sourcemap for each corresponding '.d.ts' file. */
     // "sourceMap": true,                     /* Generates corresponding '.map' file. */
     // "outFile": "./",                       /* Concatenate and emit output to single file. */
     "outDir": "./lib",                        /* Redirect output structure to the directory. */
@@ -17,8 +17,9 @@
     "removeComments": true,                   /* Do not emit comments to output. */
     // "noEmit": true,                        /* Do not emit outputs. */
     // "importHelpers": true,                 /* Import emit helpers from 'tslib'. */
-    "downlevelIteration": true,               /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
-    // "isolatedModules": true,               /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
+    "downlevelIteration": false,              /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
+    "isolatedModules": true,                  /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
+		"incremental": true,
 
     /* Strict Type-Checking Options */
     "strict": true,                           /* Enable all strict type-checking options. */
@@ -47,17 +48,17 @@
     // "typeRoots": [],                       /* List of folders to include type definitions from. */
     // "types": [],                           /* Type declaration files to be included in compilation. */
     // "allowSyntheticDefaultImports": true,  /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
-    "esModuleInterop": true                   /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */
+    "esModuleInterop": true,                  /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */
     // "preserveSymlinks": true,              /* Do not resolve the real path of symlinks. */
 
     /* Source Map Options */
     // "sourceRoot": "./",                    /* Specify the location where debugger should locate TypeScript files instead of source locations. */
     // "mapRoot": "./",                       /* Specify the location where debugger should locate map files instead of generated locations. */
-    // "inlineSourceMap": true,               /* Emit a single file with source maps instead of having a separate file. */
-    // "inlineSources": true,                 /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
+    "inlineSourceMap": false,                 /* Emit a single file with source maps instead of having a separate file. */
+    "inlineSources": false,                   /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
 
     /* Experimental Options */
-    // "experimentalDecorators": true,        /* Enables experimental support for ES7 decorators. */
+    "experimentalDecorators": true,           /* Enables experimental support for ES7 decorators. */
     // "emitDecoratorMetadata": true,         /* Enables experimental support for emitting type metadata for decorators. */
   },
   "include": ["./src", "./test"],

From 7eca06d672604fe03ce10591263bd04cf58e54c6 Mon Sep 17 00:00:00 2001
From: Hazelnoot 
Date: Fri, 23 May 2025 19:48:55 -0400
Subject: [PATCH 42/54] use pnpm package alias for @transfem-org/sfm-js

---
 packages/backend/package.json                                 | 2 +-
 packages/backend/src/core/MfmService.ts                       | 2 +-
 packages/backend/src/core/NoteCreateService.ts                | 2 +-
 packages/backend/src/core/NoteEditService.ts                  | 2 +-
 packages/backend/src/core/activitypub/ApMfmService.ts         | 2 +-
 packages/backend/src/core/activitypub/ApRendererService.ts    | 2 +-
 packages/backend/src/misc/extract-custom-emojis-from-mfm.ts   | 2 +-
 packages/backend/src/misc/extract-hashtags.ts                 | 2 +-
 packages/backend/src/misc/extract-mentions.ts                 | 2 +-
 packages/backend/src/server/api/endpoints/i/update.ts         | 2 +-
 .../backend/src/server/api/mastodon/MastodonConverters.ts     | 2 +-
 packages/backend/src/server/web/FeedService.ts                | 2 +-
 packages/backend/test/unit/MfmService.ts                      | 2 +-
 packages/backend/test/unit/extract-mentions.ts                | 2 +-
 packages/frontend-embed/package.json                          | 4 ++--
 packages/frontend-embed/src/components/EmMfm.ts               | 2 +-
 packages/frontend-embed/src/components/EmNote.vue             | 2 +-
 packages/frontend-embed/src/components/EmNoteDetailed.vue     | 2 +-
 packages/frontend/package.json                                | 2 +-
 packages/frontend/src/components/MkNote.vue                   | 2 +-
 packages/frontend/src/components/MkNoteDetailed.vue           | 2 +-
 packages/frontend/src/components/MkPostForm.vue               | 2 +-
 packages/frontend/src/components/MkSubNoteContent.vue         | 2 +-
 packages/frontend/src/components/SkNote.vue                   | 2 +-
 packages/frontend/src/components/SkNoteDetailed.vue           | 2 +-
 packages/frontend/src/components/SkOldNoteWindow.vue          | 2 +-
 packages/frontend/src/components/global/MkMfm.ts              | 2 +-
 packages/frontend/src/pages/chat/XMessage.vue                 | 2 +-
 packages/frontend/src/utility/check-animated-mfm.ts           | 2 +-
 packages/frontend/src/utility/extract-mentions.ts             | 2 +-
 packages/frontend/src/utility/extract-preview-urls.ts         | 2 +-
 packages/frontend/src/utility/extract-url-from-mfm.ts         | 2 +-
 32 files changed, 33 insertions(+), 33 deletions(-)

diff --git a/packages/backend/package.json b/packages/backend/package.json
index 69d38f3bfb..70432d1bd1 100644
--- a/packages/backend/package.json
+++ b/packages/backend/package.json
@@ -92,7 +92,7 @@
 		"@smithy/node-http-handler": "2.5.0",
 		"@swc/cli": "0.7.3",
 		"@swc/core": "1.11.24",
-		"@transfem-org/sfm-js": "0.24.6",
+		"mfm-js": "npm:@transfem-org/sfm-js@0.24.6",
 		"@twemoji/parser": "15.1.1",
 		"accepts": "1.3.8",
 		"ajv": "8.17.1",
diff --git a/packages/backend/src/core/MfmService.ts b/packages/backend/src/core/MfmService.ts
index d85ac7c807..551b25394a 100644
--- a/packages/backend/src/core/MfmService.ts
+++ b/packages/backend/src/core/MfmService.ts
@@ -15,7 +15,7 @@ import { intersperse } from '@/misc/prelude/array.js';
 import { normalizeForSearch } from '@/misc/normalize-for-search.js';
 import type { IMentionedRemoteUsers } from '@/models/Note.js';
 import { bindThis } from '@/decorators.js';
-import type * as mfm from '@transfem-org/sfm-js';
+import type * as mfm from 'mfm-js';
 
 const urlRegex = /^https?:\/\/[\w\/:%#@$&?!()\[\]~.,=+\-]+/;
 const urlRegexFull = /^https?:\/\/[\w\/:%#@$&?!()\[\]~.,=+\-]+$/;
diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts
index a9f4083446..f8584a4a48 100644
--- a/packages/backend/src/core/NoteCreateService.ts
+++ b/packages/backend/src/core/NoteCreateService.ts
@@ -4,7 +4,7 @@
  */
 
 import { setImmediate } from 'node:timers/promises';
-import * as mfm from '@transfem-org/sfm-js';
+import * as mfm from 'mfm-js';
 import { In, DataSource, IsNull, LessThan } from 'typeorm';
 import * as Redis from 'ioredis';
 import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
diff --git a/packages/backend/src/core/NoteEditService.ts b/packages/backend/src/core/NoteEditService.ts
index a359381573..d963bf1945 100644
--- a/packages/backend/src/core/NoteEditService.ts
+++ b/packages/backend/src/core/NoteEditService.ts
@@ -4,7 +4,7 @@
  */
 
 import { setImmediate } from 'node:timers/promises';
-import * as mfm from '@transfem-org/sfm-js';
+import * as mfm from 'mfm-js';
 import { DataSource, In, IsNull, LessThan } from 'typeorm';
 import * as Redis from 'ioredis';
 import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
diff --git a/packages/backend/src/core/activitypub/ApMfmService.ts b/packages/backend/src/core/activitypub/ApMfmService.ts
index c4a948429a..ddb6461746 100644
--- a/packages/backend/src/core/activitypub/ApMfmService.ts
+++ b/packages/backend/src/core/activitypub/ApMfmService.ts
@@ -4,7 +4,7 @@
  */
 
 import { Injectable } from '@nestjs/common';
-import * as mfm from '@transfem-org/sfm-js';
+import * as mfm from 'mfm-js';
 import { MfmService, Appender } from '@/core/MfmService.js';
 import type { MiNote } from '@/models/Note.js';
 import { bindThis } from '@/decorators.js';
diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts
index 789611fd97..08a8f30049 100644
--- a/packages/backend/src/core/activitypub/ApRendererService.ts
+++ b/packages/backend/src/core/activitypub/ApRendererService.ts
@@ -6,7 +6,7 @@
 import { createPublicKey, randomUUID } from 'node:crypto';
 import { Inject, Injectable } from '@nestjs/common';
 import { In } from 'typeorm';
-import * as mfm from '@transfem-org/sfm-js';
+import * as mfm from 'mfm-js';
 import { UnrecoverableError } from 'bullmq';
 import { Element, Text } from 'domhandler';
 import { DI } from '@/di-symbols.js';
diff --git a/packages/backend/src/misc/extract-custom-emojis-from-mfm.ts b/packages/backend/src/misc/extract-custom-emojis-from-mfm.ts
index 36a9b8e1f4..73ae9abb54 100644
--- a/packages/backend/src/misc/extract-custom-emojis-from-mfm.ts
+++ b/packages/backend/src/misc/extract-custom-emojis-from-mfm.ts
@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: AGPL-3.0-only
  */
 
-import * as mfm from '@transfem-org/sfm-js';
+import * as mfm from 'mfm-js';
 import { unique } from '@/misc/prelude/array.js';
 
 export function extractCustomEmojisFromMfm(nodes: mfm.MfmNode[]): string[] {
diff --git a/packages/backend/src/misc/extract-hashtags.ts b/packages/backend/src/misc/extract-hashtags.ts
index ed7606d995..d3d245d414 100644
--- a/packages/backend/src/misc/extract-hashtags.ts
+++ b/packages/backend/src/misc/extract-hashtags.ts
@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: AGPL-3.0-only
  */
 
-import * as mfm from '@transfem-org/sfm-js';
+import * as mfm from 'mfm-js';
 import { unique } from '@/misc/prelude/array.js';
 
 export function extractHashtags(nodes: mfm.MfmNode[]): string[] {
diff --git a/packages/backend/src/misc/extract-mentions.ts b/packages/backend/src/misc/extract-mentions.ts
index bb21c32ffb..2ec9349718 100644
--- a/packages/backend/src/misc/extract-mentions.ts
+++ b/packages/backend/src/misc/extract-mentions.ts
@@ -5,7 +5,7 @@
 
 // test is located in test/extract-mentions
 
-import * as mfm from '@transfem-org/sfm-js';
+import * as mfm from 'mfm-js';
 
 export function extractMentions(nodes: mfm.MfmNode[]): mfm.MfmMention['props'][] {
 	// TODO: 重複を削除
diff --git a/packages/backend/src/server/api/endpoints/i/update.ts b/packages/backend/src/server/api/endpoints/i/update.ts
index e632915f62..5767880531 100644
--- a/packages/backend/src/server/api/endpoints/i/update.ts
+++ b/packages/backend/src/server/api/endpoints/i/update.ts
@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: AGPL-3.0-only
  */
 
-import * as mfm from '@transfem-org/sfm-js';
+import * as mfm from 'mfm-js';
 import { Inject, Injectable } from '@nestjs/common';
 import ms from 'ms';
 import { extractCustomEmojisFromMfm } from '@/misc/extract-custom-emojis-from-mfm.js';
diff --git a/packages/backend/src/server/api/mastodon/MastodonConverters.ts b/packages/backend/src/server/api/mastodon/MastodonConverters.ts
index 02ce31c4f8..df8d68042a 100644
--- a/packages/backend/src/server/api/mastodon/MastodonConverters.ts
+++ b/packages/backend/src/server/api/mastodon/MastodonConverters.ts
@@ -5,7 +5,7 @@
 
 import { Inject, Injectable } from '@nestjs/common';
 import { Entity, MastodonEntity, MisskeyEntity } from 'megalodon';
-import mfm from '@transfem-org/sfm-js';
+import mfm from 'mfm-js';
 import { MastodonNotificationType } from 'megalodon/lib/src/mastodon/notification.js';
 import { NotificationType } from 'megalodon/lib/src/notification.js';
 import { DI } from '@/di-symbols.js';
diff --git a/packages/backend/src/server/web/FeedService.ts b/packages/backend/src/server/web/FeedService.ts
index dcd4d80303..a622ae7e34 100644
--- a/packages/backend/src/server/web/FeedService.ts
+++ b/packages/backend/src/server/web/FeedService.ts
@@ -15,7 +15,7 @@ import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.j
 import { bindThis } from '@/decorators.js';
 import { IdService } from '@/core/IdService.js';
 import { MfmService } from "@/core/MfmService.js";
-import { parse as mfmParse } from '@transfem-org/sfm-js';
+import { parse as mfmParse } from 'mfm-js';
 
 @Injectable()
 export class FeedService {
diff --git a/packages/backend/test/unit/MfmService.ts b/packages/backend/test/unit/MfmService.ts
index f96f3977d0..af1fc4e132 100644
--- a/packages/backend/test/unit/MfmService.ts
+++ b/packages/backend/test/unit/MfmService.ts
@@ -4,7 +4,7 @@
  */
 
 import * as assert from 'assert';
-import * as mfm from '@transfem-org/sfm-js';
+import * as mfm from 'mfm-js';
 import { Test } from '@nestjs/testing';
 
 import { CoreModule } from '@/core/CoreModule.js';
diff --git a/packages/backend/test/unit/extract-mentions.ts b/packages/backend/test/unit/extract-mentions.ts
index 2aad89d65b..3403387e30 100644
--- a/packages/backend/test/unit/extract-mentions.ts
+++ b/packages/backend/test/unit/extract-mentions.ts
@@ -5,7 +5,7 @@
 
 import * as assert from 'assert';
 
-import { parse } from '@transfem-org/sfm-js';
+import { parse } from 'mfm-js';
 import { extractMentions } from '@/misc/extract-mentions.js';
 
 describe('Extract mentions', () => {
diff --git a/packages/frontend-embed/package.json b/packages/frontend-embed/package.json
index 5191fe6852..5b4774c7af 100644
--- a/packages/frontend-embed/package.json
+++ b/packages/frontend-embed/package.json
@@ -11,8 +11,8 @@
 	},
 	"dependencies": {
 		"@discordapp/twemoji": "15.1.0",
-		"@phosphor-icons/web": "^2.0.3",
-		"@transfem-org/sfm-js": "0.24.5",
+		"@phosphor-icons/web": "2.1.2",
+		"mfm-js": "npm:@transfem-org/sfm-js@0.24.6",
 		"buraha": "0.0.1",
 		"frontend-shared": "workspace:*",
 		"json5": "2.2.3",
diff --git a/packages/frontend-embed/src/components/EmMfm.ts b/packages/frontend-embed/src/components/EmMfm.ts
index d377d492e0..74ae3373ef 100644
--- a/packages/frontend-embed/src/components/EmMfm.ts
+++ b/packages/frontend-embed/src/components/EmMfm.ts
@@ -5,7 +5,7 @@
 
 import { h, provide } from 'vue';
 import type { VNode, SetupContext } from 'vue';
-import * as mfm from '@transfem-org/sfm-js';
+import * as mfm from 'mfm-js';
 import * as Misskey from 'misskey-js';
 import { host } from '@@/js/config.js';
 import EmUrl from '@/components/EmUrl.vue';
diff --git a/packages/frontend-embed/src/components/EmNote.vue b/packages/frontend-embed/src/components/EmNote.vue
index 666cbde72d..0dc77d09a7 100644
--- a/packages/frontend-embed/src/components/EmNote.vue
+++ b/packages/frontend-embed/src/components/EmNote.vue
@@ -105,7 +105,7 @@ SPDX-License-Identifier: AGPL-3.0-only