diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4fa52b4094..ba2d2eacfb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -622,6 +622,35 @@ marginはそのコンポーネントを使う側が設定する ### indexというファイル名を使うな ESMではディレクトリインポートは廃止されているのと、ディレクトリインポートせずともファイル名が index だと何故か一部のライブラリ?でディレクトリインポートだと見做されてエラーになる +### Memory Caches + +Sharkey offers multiple memory cache implementations, each meant for a different use case. +The following table compares the available options: + +| Cache | Type | Consistency | Persistence | Data Source | Cardinality | Eviction | Description | +|---------------------|-----------|-------------|-------------|-------------|-------------|----------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `MemoryKVCache` | Key-Value | None | None | Caller | Single | Lifetime | Implements a basic in-memory Key-Value store. The implementation is entirely synchronous, except for user-provided data sources. | +| `MemorySingleCache` | Single | None | None | Caller | Single | Lifetime | Implements a basic in-memory Single Value store. The implementation is entirely synchronous, except for user-provided data sources. | +| `RedisKVCache` | Key-Value | Eventual | Redis | Callback | Single | Lifetime | Extends `MemoryKVCache` with Redis-backed persistence and a pre-defined callback data source. This provides eventual consistency guarantees based on the memory cache lifetime. | +| `RedisSingleCache` | Single | Eventual | Redis | Callback | Single | Lifetime | Extends `MemorySingleCache` with Redis-backed persistence and a pre-defined callback data source. This provides eventual consistency guarantees based on the memory cache lifetime. | +| `QuantumKVCache` | Key-Value | Immediate | None | Callback | Multiple | Lifetime | Combines `MemoryKVCache` with a pre-defined callback data source and immediate consistency via Redis sync events. The implementation offers multi-item batch overloads for efficient bulk operations. **This is the recommended cache implementation for most use cases.** | + +Key-Value caches store multiple entries per cache, while Single caches store a single value that can be accessed directly. +Consistency refers to the consistency of cached data between different processes in the instance cluster: "None" means no consistency guarantees, "Eventual" caches will gradually become consistent after some unknown time, and "Immediate" consistency ensures accurate data ASAP after the update. +Caches with persistence can retain their data after a reboot through an external service such as Redis. +If a data source is supported, then this allows the cache to directly load missing data in response to a fetch. +"Caller" data sources are passed into the fetch method(s) directly, while "Callback" sources are passed in as a function when the cache is first initialized. +The cardinality of a cache refers to the number of items that can be updated in a single operation, and eviction, finally, is the method that the cache uses to evict stale data. + +#### Selecting a cache implementation + +For most cache uses, `QuantumKVCache` should be considered first. +It offers strong consistency guarantees, multiple cardinality, and a cleaner API surface than the older caches. +An alternate cache implementation should be considered if any of the following apply: +* The data is particularly slow to calculate or difficult to access. In these cases, either `RedisKVCache` or `RedisSingleCache` should be considered. +* If stale data is acceptable, then consider `MemoryKVCache` or `MemorySingleCache`. These synchronous implementations have much less overhead than the other options. +* There is only one data item, or all data items must be fetched together. Using `MemorySingleCache` or `RedisSingleCache` could provide a cleaner implementation without resorting to hacks like a fixed key. + ## CSS Recipe ### Lighten CSS vars diff --git a/package.json b/package.json index 1db16bcd55..7b93e9416e 100644 --- a/package.json +++ b/package.json @@ -54,17 +54,7 @@ "lodash": "4.17.21" }, "dependencies": { - "cssnano": "7.0.6", - "esbuild": "0.25.3", - "execa": "9.5.2", - "fast-glob": "3.3.3", - "glob": "11.0.2", - "ignore-walk": "7.0.0", - "js-yaml": "4.1.0", - "postcss": "8.5.3", - "tar": "7.4.3", - "terser": "5.39.0", - "typescript": "5.8.3" + "js-yaml": "4.1.0" }, "optionalDependencies": { "cypress": "14.3.2" @@ -75,10 +65,20 @@ "@typescript-eslint/eslint-plugin": "8.31.0", "@typescript-eslint/parser": "8.31.0", "cross-env": "7.0.3", + "cssnano": "7.0.6", + "esbuild": "0.25.3", "eslint": "9.25.1", - "globals": "16.0.0", + "execa": "9.5.2", + "fast-glob": "3.3.3", + "glob": "11.0.2", + "globals": "16.1.0", "ncp": "2.0.0", - "pnpm": "10.10.0", - "start-server-and-test": "2.0.11" + "pnpm": "9.6.0", + "ignore-walk": "7.0.0", + "postcss": "8.5.3", + "start-server-and-test": "2.0.11", + "tar": "7.4.3", + "terser": "5.39.0", + "typescript": "5.8.3" } } diff --git a/packages/backend/package.json b/packages/backend/package.json index 8e92748884..331cddae78 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -80,7 +80,7 @@ "@fastify/static": "8.1.1", "@fastify/view": "10.0.2", "@misskey-dev/sharp-read-bmp": "1.3.0", - "@misskey-dev/summaly": "5.2.1", + "@misskey-dev/summaly": "npm:@transfem-org/summaly@5.2.2", "@nestjs/common": "11.1.0", "@nestjs/core": "11.1.0", "@nestjs/testing": "11.1.0", @@ -90,33 +90,30 @@ "@simplewebauthn/server": "12.0.0", "@sinonjs/fake-timers": "11.3.1", "@smithy/node-http-handler": "2.5.0", - "@swc/cli": "0.7.3", - "@swc/core": "1.11.24", - "@transfem-org/sfm-js": "0.24.6", + "mfm-js": "npm:@transfem-org/sfm-js@0.24.6", "@twemoji/parser": "15.1.1", "accepts": "1.3.8", "ajv": "8.17.1", "archiver": "7.0.1", - "argon2": "^0.40.1", + "argon2": "0.43.0", "axios": "1.7.4", - "async-mutex": "0.5.0", "bcryptjs": "2.4.3", "blurhash": "2.0.5", - "body-parser": "1.20.3", "bullmq": "5.51.1", "cacheable-lookup": "7.0.0", - "canvas": "^3.1.0", + "canvas": "3.1.0", "cbor": "9.0.2", "chalk": "5.4.1", "chalk-template": "1.1.0", "cheerio": "1.0.0", - "chokidar": "3.6.0", - "cli-highlight": "2.1.11", + "cli-highlight": "npm:@transfem-org/cli-highlight@2.1.12", "color-convert": "2.0.1", "content-disposition": "0.5.4", "date-fns": "2.30.0", "deep-email-validator": "0.1.21", - "fast-xml-parser": "4.4.1", + "dom-serializer": "2.0.0", + "domhandler": "5.0.3", + "domutils": "3.2.2", "fastify": "5.3.2", "fastify-raw-body": "5.0.0", "feed": "4.2.2", @@ -125,10 +122,9 @@ "form-data": "4.0.2", "glob": "11.0.0", "got": "14.4.7", - "happy-dom": "16.8.1", "hpagent": "1.2.0", "htmlescape": "1.1.1", - "http-link-header": "1.1.3", + "htmlparser2": "9.1.0", "ioredis": "5.6.1", "ip-cidr": "4.0.2", "ipaddr.js": "2.2.0", @@ -136,50 +132,40 @@ "js-yaml": "4.1.0", "json5": "2.2.3", "jsonld": "8.3.3", - "jsrsasign": "11.1.0", "juice": "11.0.1", "megalodon": "workspace:*", "meilisearch": "0.50.0", - "microformats-parser": "2.0.2", "mime-types": "2.1.35", "misskey-js": "workspace:*", "misskey-reversi": "workspace:*", - "moment": "^2.30.1", + "moment": "2.30.1", "ms": "3.0.0-canary.1", "nanoid": "5.1.5", "nested-property": "4.0.0", "node-fetch": "3.3.2", "nodemailer": "6.10.1", - "oauth": "0.10.2", - "oauth2orize": "1.12.0", - "oauth2orize-pkce": "0.1.2", "os-utils": "0.0.14", "otpauth": "9.4.0", - "parse5": "7.3.0", "pg": "8.15.6", "pkce-challenge": "4.1.0", "probe-image-size": "7.2.3", "promise-limit": "2.7.0", - "proxy-addr": "^2.0.7", - "psl": "^1.13.0", + "proxy-addr": "2.0.7", + "psl": "1.15.0", "pug": "3.0.3", "qrcode": "1.5.4", "random-seed": "0.3.0", - "ratelimiter": "3.4.1", "re2": "1.21.4", "redis-info": "3.1.0", "redis-lock": "0.1.4", "reflect-metadata": "0.2.2", "rename": "1.0.4", - "rss-parser": "3.13.0", - "rxjs": "7.8.2", "sanitize-html": "2.16.0", "secure-json-parse": "3.0.2", "sharp": "0.34.1", "semver": "7.7.1", "slacc": "0.0.10", "strict-event-emitter-types": "2.0.0", - "stringz": "2.1.0", "systeminformation": "5.25.11", "tinycolor2": "1.6.0", "tmp": "0.2.3", @@ -188,7 +174,7 @@ "typeorm": "0.3.22", "typescript": "5.8.3", "ulid": "2.4.0", - "uuid": "^9.0.1", + "uuid": "11.1.0", "vary": "1.1.2", "web-push": "3.6.7", "ws": "8.18.1", @@ -199,16 +185,16 @@ "@nestjs/platform-express": "11.1.0", "@sentry/vue": "9.14.0", "@simplewebauthn/types": "12.0.0", + "@swc/cli": "0.7.3", + "@swc/core": "1.11.24", "@swc/jest": "0.2.38", "@types/accepts": "1.3.7", "@types/archiver": "6.0.3", "@types/bcryptjs": "2.4.6", - "@types/body-parser": "1.19.5", "@types/color-convert": "2.0.4", "@types/content-disposition": "0.5.8", "@types/fluent-ffmpeg": "2.1.27", "@types/htmlescape": "1.1.3", - "@types/http-link-header": "1.0.7", "@types/jest": "29.5.14", "@types/js-yaml": "4.0.9", "@types/jsonld": "1.5.15", @@ -221,12 +207,11 @@ "@types/oauth2orize": "1.11.5", "@types/oauth2orize-pkce": "0.1.2", "@types/pg": "8.11.14", - "@types/proxy-addr": "^2.0.3", - "@types/psl": "^1.1.3", + "@types/proxy-addr": "2.0.3", + "@types/psl": "1.1.3", "@types/pug": "2.0.10", "@types/qrcode": "1.5.5", "@types/random-seed": "0.3.5", - "@types/ratelimiter": "3.4.6", "@types/redis-info": "3.0.3", "@types/rename": "1.0.7", "@types/sanitize-html": "2.15.0", @@ -236,7 +221,6 @@ "@types/supertest": "6.0.3", "@types/tinycolor2": "1.4.6", "@types/tmp": "0.2.6", - "@types/uuid": "^9.0.4", "@types/vary": "1.1.3", "@types/web-push": "3.6.4", "@types/ws": "8.18.1", @@ -245,7 +229,7 @@ "aws-sdk-client-mock": "4.1.0", "cross-env": "7.0.3", "eslint-plugin-import": "2.31.0", - "execa": "8.0.1", + "execa": "9.5.2", "fkill": "9.0.0", "jest": "29.7.0", "jest-mock": "29.7.0", diff --git a/packages/backend/src/boot/entry.ts b/packages/backend/src/boot/entry.ts index e52d77ab9b..afb48e526c 100644 --- a/packages/backend/src/boot/entry.ts +++ b/packages/backend/src/boot/entry.ts @@ -64,17 +64,35 @@ async function main() { } // Display detail of uncaught exception - process.on('uncaughtException', err => { + process.on('uncaughtExceptionMonitor', ((err, origin) => { try { - logger.error('Uncaught exception:', err); + logger.error(`Uncaught exception (${origin}):`, err); } catch { - console.error('Uncaught exception:', err); + console.error(`Uncaught exception (${origin}):`, err); } - }); + })); // Dying away... + process.on('disconnect', () => { + try { + logger.warn('IPC channel disconnected! The process may soon die.'); + } catch { + console.warn('IPC channel disconnected! The process may soon die.'); + } + }); + process.on('beforeExit', code => { + try { + logger.warn(`Event loop died! Process will exit with code ${code}.`); + } catch { + console.warn(`Event loop died! Process will exit with code ${code}.`); + } + }); process.on('exit', code => { - logger.info(`The process is going to exit with code ${code}`); + try { + logger.info(`The process is going to exit with code ${code}`); + } catch { + console.info(`The process is going to exit with code ${code}`); + } }); //#endregion diff --git a/packages/backend/src/core/AccountMoveService.ts b/packages/backend/src/core/AccountMoveService.ts index 738026f753..e107f02796 100644 --- a/packages/backend/src/core/AccountMoveService.ts +++ b/packages/backend/src/core/AccountMoveService.ts @@ -26,6 +26,7 @@ import PerUserFollowingChart from '@/core/chart/charts/per-user-following.js'; import { SystemAccountService } from '@/core/SystemAccountService.js'; import { RoleService } from '@/core/RoleService.js'; import { AntennaService } from '@/core/AntennaService.js'; +import { CacheService } from '@/core/CacheService.js'; @Injectable() export class AccountMoveService { @@ -68,6 +69,7 @@ export class AccountMoveService { private systemAccountService: SystemAccountService, private roleService: RoleService, private antennaService: AntennaService, + private readonly cacheService: CacheService, ) { } @@ -107,12 +109,10 @@ export class AccountMoveService { this.globalEventService.publishMainStream(src.id, 'meUpdated', iObj); // Unfollow after 24 hours - const followings = await this.followingsRepository.findBy({ - followerId: src.id, - }); - this.queueService.createDelayedUnfollowJob(followings.map(following => ({ + const followings = await this.cacheService.userFollowingsCache.fetch(src.id); + this.queueService.createDelayedUnfollowJob(Array.from(followings.keys()).map(followeeId => ({ from: { id: src.id }, - to: { id: following.followeeId }, + to: { id: followeeId }, })), process.env.NODE_ENV === 'test' ? 10000 : 1000 * 60 * 60 * 24); await this.postMoveProcess(src, dst); @@ -138,11 +138,9 @@ export class AccountMoveService { // follow the new account const proxy = await this.systemAccountService.fetch('proxy'); - const followings = await this.followingsRepository.findBy({ - followeeId: src.id, - followerHost: IsNull(), // follower is local - followerId: Not(proxy.id), - }); + const followings = await this.cacheService.userFollowersCache.fetch(src.id) + .then(fs => Array.from(fs.values()) + .filter(f => f.followerHost == null && f.followerId !== proxy.id)); const followJobs = followings.map(following => ({ from: { id: following.followerId }, to: { id: dst.id }, @@ -318,9 +316,9 @@ export class AccountMoveService { await this.usersRepository.decrement({ id: In(localFollowerIds) }, 'followingCount', 1); // Decrease follower counts of local followees by 1. - const oldFollowings = await this.followingsRepository.findBy({ followerId: oldAccount.id }); - if (oldFollowings.length > 0) { - await this.usersRepository.decrement({ id: In(oldFollowings.map(following => following.followeeId)) }, 'followersCount', 1); + const oldFollowings = await this.cacheService.userFollowingsCache.fetch(oldAccount.id); + if (oldFollowings.size > 0) { + await this.usersRepository.decrement({ id: In(Array.from(oldFollowings.keys())) }, 'followersCount', 1); } // Update instance stats by decreasing remote followers count by the number of local followers who were following the old account. diff --git a/packages/backend/src/core/AntennaService.ts b/packages/backend/src/core/AntennaService.ts index cf696e3599..667df57943 100644 --- a/packages/backend/src/core/AntennaService.ts +++ b/packages/backend/src/core/AntennaService.ts @@ -130,7 +130,8 @@ export class AntennaService implements OnApplicationShutdown { } if (note.visibility === 'followers') { - const isFollowing = Object.hasOwn(await this.cacheService.userFollowingsCache.fetch(antenna.userId), note.userId); + const followings = await this.cacheService.userFollowingsCache.fetch(antenna.userId); + const isFollowing = followings.has(note.userId); if (!isFollowing && antenna.userId !== note.userId) return false; } diff --git a/packages/backend/src/core/CacheService.ts b/packages/backend/src/core/CacheService.ts index 1cf63221f9..2d37cd6bab 100644 --- a/packages/backend/src/core/CacheService.ts +++ b/packages/backend/src/core/CacheService.ts @@ -5,14 +5,16 @@ import { Inject, Injectable } from '@nestjs/common'; import * as Redis from 'ioredis'; -import { IsNull } from 'typeorm'; -import type { BlockingsRepository, FollowingsRepository, MutingsRepository, RenoteMutingsRepository, MiUserProfile, UserProfilesRepository, UsersRepository, MiFollowing, MiNote } from '@/models/_.js'; +import { In, IsNull } from 'typeorm'; +import type { BlockingsRepository, FollowingsRepository, MutingsRepository, RenoteMutingsRepository, MiUserProfile, UserProfilesRepository, UsersRepository, MiNote, MiFollowing } from '@/models/_.js'; import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js'; +import { QuantumKVCache } from '@/misc/QuantumKVCache.js'; import type { MiLocalUser, MiUser } from '@/models/User.js'; import { DI } from '@/di-symbols.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { bindThis } from '@/decorators.js'; -import type { GlobalEvents } from '@/core/GlobalEventService.js'; +import type { InternalEventTypes } from '@/core/GlobalEventService.js'; +import { InternalEventService } from '@/core/InternalEventService.js'; import type { OnApplicationShutdown } from '@nestjs/common'; export interface FollowStats { @@ -27,7 +29,7 @@ export interface CachedTranslation { text: string | undefined; } -interface CachedTranslationEntity { +export interface CachedTranslationEntity { l?: string; t?: string; u?: number; @@ -39,14 +41,16 @@ export class CacheService implements OnApplicationShutdown { public localUserByNativeTokenCache: MemoryKVCache; public localUserByIdCache: MemoryKVCache; public uriPersonCache: MemoryKVCache; - public userProfileCache: RedisKVCache; - public userMutingsCache: RedisKVCache>; - public userBlockingCache: RedisKVCache>; - public userBlockedCache: RedisKVCache>; // NOTE: 「被」Blockキャッシュ - public renoteMutingsCache: RedisKVCache>; - public userFollowingsCache: RedisKVCache | undefined>>; - private readonly userFollowStatsCache = new MemoryKVCache(1000 * 60 * 10); // 10 minutes - private readonly translationsCache: RedisKVCache; + public userProfileCache: QuantumKVCache; + public userMutingsCache: QuantumKVCache>; + public userBlockingCache: QuantumKVCache>; + public userBlockedCache: QuantumKVCache>; // NOTE: 「被」Blockキャッシュ + public renoteMutingsCache: QuantumKVCache>; + public userFollowingsCache: QuantumKVCache>>; + public userFollowersCache: QuantumKVCache>>; + public hibernatedUserCache: QuantumKVCache; + protected userFollowStatsCache = new MemoryKVCache(1000 * 60 * 10); // 10 minutes + protected translationsCache: RedisKVCache; constructor( @Inject(DI.redis) @@ -74,6 +78,7 @@ export class CacheService implements OnApplicationShutdown { private followingsRepository: FollowingsRepository, private userEntityService: UserEntityService, + private readonly internalEventService: InternalEventService, ) { //this.onMessage = this.onMessage.bind(this); @@ -82,58 +87,148 @@ export class CacheService implements OnApplicationShutdown { this.localUserByIdCache = new MemoryKVCache(1000 * 60 * 5); // 5m this.uriPersonCache = new MemoryKVCache(1000 * 60 * 5); // 5m - this.userProfileCache = new RedisKVCache(this.redisClient, 'userProfile', { + this.userProfileCache = new QuantumKVCache(this.internalEventService, 'userProfile', { lifetime: 1000 * 60 * 30, // 30m - memoryCacheLifetime: 1000 * 60, // 1m fetcher: (key) => this.userProfilesRepository.findOneByOrFail({ userId: key }), - toRedisConverter: (value) => JSON.stringify(value), - fromRedisConverter: (value) => JSON.parse(value), // TODO: date型の考慮 + bulkFetcher: userIds => this.userProfilesRepository.findBy({ userId: In(userIds) }).then(ps => ps.map(p => [p.userId, p])), }); - this.userMutingsCache = new RedisKVCache>(this.redisClient, 'userMutings', { + this.userMutingsCache = new QuantumKVCache>(this.internalEventService, 'userMutings', { lifetime: 1000 * 60 * 30, // 30m - memoryCacheLifetime: 1000 * 60, // 1m fetcher: (key) => this.mutingsRepository.find({ where: { muterId: key }, select: ['muteeId'] }).then(xs => new Set(xs.map(x => x.muteeId))), - toRedisConverter: (value) => JSON.stringify(Array.from(value)), - fromRedisConverter: (value) => new Set(JSON.parse(value)), + bulkFetcher: muterIds => this.mutingsRepository + .createQueryBuilder('muting') + .select('"muting"."muterId"', 'muterId') + .addSelect('array_agg("muting"."muteeId")', 'muteeIds') + .where({ muterId: In(muterIds) }) + .groupBy('muting.muterId') + .getRawMany<{ muterId: string, muteeIds: string[] }>() + .then(ms => ms.map(m => [m.muterId, new Set(m.muteeIds)])), }); - this.userBlockingCache = new RedisKVCache>(this.redisClient, 'userBlocking', { + this.userBlockingCache = new QuantumKVCache>(this.internalEventService, 'userBlocking', { lifetime: 1000 * 60 * 30, // 30m - memoryCacheLifetime: 1000 * 60, // 1m fetcher: (key) => this.blockingsRepository.find({ where: { blockerId: key }, select: ['blockeeId'] }).then(xs => new Set(xs.map(x => x.blockeeId))), - toRedisConverter: (value) => JSON.stringify(Array.from(value)), - fromRedisConverter: (value) => new Set(JSON.parse(value)), + bulkFetcher: blockerIds => this.blockingsRepository + .createQueryBuilder('blocking') + .select('"blocking"."blockerId"', 'blockerId') + .addSelect('array_agg("blocking"."blockeeId")', 'blockeeIds') + .where({ blockerId: In(blockerIds) }) + .groupBy('blocking.blockerId') + .getRawMany<{ blockerId: string, blockeeIds: string[] }>() + .then(ms => ms.map(m => [m.blockerId, new Set(m.blockeeIds)])), }); - this.userBlockedCache = new RedisKVCache>(this.redisClient, 'userBlocked', { + this.userBlockedCache = new QuantumKVCache>(this.internalEventService, 'userBlocked', { lifetime: 1000 * 60 * 30, // 30m - memoryCacheLifetime: 1000 * 60, // 1m fetcher: (key) => this.blockingsRepository.find({ where: { blockeeId: key }, select: ['blockerId'] }).then(xs => new Set(xs.map(x => x.blockerId))), - toRedisConverter: (value) => JSON.stringify(Array.from(value)), - fromRedisConverter: (value) => new Set(JSON.parse(value)), + bulkFetcher: blockeeIds => this.blockingsRepository + .createQueryBuilder('blocking') + .select('"blocking"."blockeeId"', 'blockeeId') + .addSelect('array_agg("blocking"."blockeeId")', 'blockeeIds') + .where({ blockeeId: In(blockeeIds) }) + .groupBy('blocking.blockeeId') + .getRawMany<{ blockeeId: string, blockerIds: string[] }>() + .then(ms => ms.map(m => [m.blockeeId, new Set(m.blockerIds)])), }); - this.renoteMutingsCache = new RedisKVCache>(this.redisClient, 'renoteMutings', { + this.renoteMutingsCache = new QuantumKVCache>(this.internalEventService, 'renoteMutings', { lifetime: 1000 * 60 * 30, // 30m - memoryCacheLifetime: 1000 * 60, // 1m fetcher: (key) => this.renoteMutingsRepository.find({ where: { muterId: key }, select: ['muteeId'] }).then(xs => new Set(xs.map(x => x.muteeId))), - toRedisConverter: (value) => JSON.stringify(Array.from(value)), - fromRedisConverter: (value) => new Set(JSON.parse(value)), + bulkFetcher: muterIds => this.renoteMutingsRepository + .createQueryBuilder('muting') + .select('"muting"."muterId"', 'muterId') + .addSelect('array_agg("muting"."muteeId")', 'muteeIds') + .where({ muterId: In(muterIds) }) + .groupBy('muting.muterId') + .getRawMany<{ muterId: string, muteeIds: string[] }>() + .then(ms => ms.map(m => [m.muterId, new Set(m.muteeIds)])), }); - this.userFollowingsCache = new RedisKVCache | undefined>>(this.redisClient, 'userFollowings', { + this.userFollowingsCache = new QuantumKVCache>>(this.internalEventService, 'userFollowings', { lifetime: 1000 * 60 * 30, // 30m - memoryCacheLifetime: 1000 * 60, // 1m - fetcher: (key) => this.followingsRepository.find({ where: { followerId: key }, select: ['followeeId', 'withReplies'] }).then(xs => { - const obj: Record | undefined> = {}; - for (const x of xs) { - obj[x.followeeId] = { withReplies: x.withReplies }; + fetcher: (key) => this.followingsRepository.findBy({ followerId: key }).then(xs => new Map(xs.map(f => [f.followeeId, f]))), + bulkFetcher: followerIds => this.followingsRepository + .findBy({ followerId: In(followerIds) }) + .then(fs => fs + .reduce((groups, f) => { + let group = groups.get(f.followerId); + if (!group) { + group = new Map(); + groups.set(f.followerId, group); + } + group.set(f.followeeId, f); + return groups; + }, new Map>>)), + }); + + this.userFollowersCache = new QuantumKVCache>>(this.internalEventService, 'userFollowers', { + lifetime: 1000 * 60 * 30, // 30m + fetcher: followeeId => this.followingsRepository.findBy({ followeeId: followeeId }).then(xs => new Map(xs.map(x => [x.followerId, x]))), + bulkFetcher: followeeIds => this.followingsRepository + .findBy({ followeeId: In(followeeIds) }) + .then(fs => fs + .reduce((groups, f) => { + let group = groups.get(f.followeeId); + if (!group) { + group = new Map(); + groups.set(f.followeeId, group); + } + group.set(f.followerId, f); + return groups; + }, new Map>>)), + }); + + this.hibernatedUserCache = new QuantumKVCache(this.internalEventService, 'hibernatedUsers', { + lifetime: 1000 * 60 * 30, // 30m + fetcher: async userId => { + const { isHibernated } = await this.usersRepository.findOneOrFail({ + where: { id: userId }, + select: { isHibernated: true }, + }); + return isHibernated; + }, + bulkFetcher: async userIds => { + const results = await this.usersRepository.find({ + where: { id: In(userIds) }, + select: { id: true, isHibernated: true }, + }); + return results.map(({ id, isHibernated }) => [id, isHibernated]); + }, + onChanged: async userIds => { + // We only update local copies since each process will get this event, but we can have user objects in multiple different caches. + // Before doing anything else we must "find" all the objects to update. + const userObjects = new Map(); + const toUpdate: string[] = []; + for (const uid of userIds) { + const toAdd: MiUser[] = []; + + const localUserById = this.localUserByIdCache.get(uid); + if (localUserById) toAdd.push(localUserById); + + const userById = this.userByIdCache.get(uid); + if (userById) toAdd.push(userById); + + if (toAdd.length > 0) { + toUpdate.push(uid); + userObjects.set(uid, toAdd); + } } - return obj; - }), - toRedisConverter: (value) => JSON.stringify(value), - fromRedisConverter: (value) => JSON.parse(value), + + // In many cases, we won't have to do anything. + // Skipping the DB fetch ensures that this remains a single-step synchronous process. + if (toUpdate.length > 0) { + const hibernations = await this.usersRepository.find({ where: { id: In(toUpdate) }, select: { id: true, isHibernated: true } }); + for (const { id, isHibernated } of hibernations) { + const users = userObjects.get(id); + if (users) { + for (const u of users) { + u.isHibernated = isHibernated; + } + } + } + } + }, }); this.translationsCache = new RedisKVCache(this.redisClient, 'translations', { @@ -143,20 +238,21 @@ export class CacheService implements OnApplicationShutdown { // NOTE: チャンネルのフォロー状況キャッシュはChannelFollowingServiceで行っている - this.redisForSub.on('message', this.onMessage); + this.internalEventService.on('userChangeSuspendedState', this.onUserEvent); + this.internalEventService.on('userChangeDeletedState', this.onUserEvent); + this.internalEventService.on('remoteUserUpdated', this.onUserEvent); + this.internalEventService.on('localUserUpdated', this.onUserEvent); + this.internalEventService.on('userChangeSuspendedState', this.onUserEvent); + this.internalEventService.on('userTokenRegenerated', this.onTokenEvent); + this.internalEventService.on('follow', this.onFollowEvent); + this.internalEventService.on('unfollow', this.onFollowEvent); } @bindThis - private async onMessage(_: string, data: string): Promise { - const obj = JSON.parse(data); - - if (obj.channel === 'internal') { - const { type, body } = obj.message as GlobalEvents['internal']['payload']; - switch (type) { - case 'userChangeSuspendedState': - case 'userChangeDeletedState': - case 'remoteUserUpdated': - case 'localUserUpdated': { + private async onUserEvent(body: InternalEventTypes[E], _: E, isLocal: boolean): Promise { + { + { + { const user = await this.usersRepository.findOneBy({ id: body.id }); if (user == null) { this.userByIdCache.delete(body.id); @@ -166,6 +262,18 @@ export class CacheService implements OnApplicationShutdown { this.uriPersonCache.delete(k); } } + if (isLocal) { + await Promise.all([ + this.userProfileCache.delete(body.id), + this.userMutingsCache.delete(body.id), + this.userBlockingCache.delete(body.id), + this.userBlockedCache.delete(body.id), + this.renoteMutingsCache.delete(body.id), + this.userFollowingsCache.delete(body.id), + this.userFollowersCache.delete(body.id), + this.hibernatedUserCache.delete(body.id), + ]); + } } else { this.userByIdCache.set(user.id, user); for (const [k, v] of this.uriPersonCache.entries) { @@ -178,20 +286,37 @@ export class CacheService implements OnApplicationShutdown { this.localUserByIdCache.set(user.id, user); } } - break; } - case 'userTokenRegenerated': { + } + } + } + + @bindThis + private async onTokenEvent(body: InternalEventTypes[E]): Promise { + { + { + { const user = await this.usersRepository.findOneByOrFail({ id: body.id }) as MiLocalUser; this.localUserByNativeTokenCache.delete(body.oldToken); this.localUserByNativeTokenCache.set(body.newToken, user); - break; } + } + } + } + + @bindThis + private async onFollowEvent(body: InternalEventTypes[E], type: E): Promise { + { + switch (type) { case 'follow': { const follower = this.userByIdCache.get(body.followerId); if (follower) follower.followingCount++; const followee = this.userByIdCache.get(body.followeeId); if (followee) followee.followersCount++; - this.userFollowingsCache.delete(body.followerId); + await Promise.all([ + this.userFollowingsCache.delete(body.followerId), + this.userFollowersCache.delete(body.followeeId), + ]); this.userFollowStatsCache.delete(body.followerId); this.userFollowStatsCache.delete(body.followeeId); break; @@ -201,13 +326,14 @@ export class CacheService implements OnApplicationShutdown { if (follower) follower.followingCount--; const followee = this.userByIdCache.get(body.followeeId); if (followee) followee.followersCount--; - this.userFollowingsCache.delete(body.followerId); + await Promise.all([ + this.userFollowingsCache.delete(body.followerId), + this.userFollowersCache.delete(body.followeeId), + ]); this.userFollowStatsCache.delete(body.followerId); this.userFollowStatsCache.delete(body.followeeId); break; } - default: - break; } } } @@ -298,9 +424,115 @@ export class CacheService implements OnApplicationShutdown { }); } + @bindThis + public async getUsers(userIds: Iterable): Promise> { + const users = new Map; + + const toFetch: string[] = []; + for (const userId of userIds) { + const fromCache = this.userByIdCache.get(userId); + if (fromCache) { + users.set(userId, fromCache); + } else { + toFetch.push(userId); + } + } + + if (toFetch.length > 0) { + const fetched = await this.usersRepository.findBy({ + id: In(toFetch), + }); + + for (const user of fetched) { + users.set(user.id, user); + this.userByIdCache.set(user.id, user); + } + } + + return users; + } + + @bindThis + public async isFollowing(follower: string | { id: string }, followee: string | { id: string }): Promise { + const followerId = typeof(follower) === 'string' ? follower : follower.id; + const followeeId = typeof(followee) === 'string' ? followee : followee.id; + + // This lets us use whichever one is in memory, falling back to DB fetch via userFollowingsCache. + return this.userFollowersCache.get(followeeId)?.has(followerId) + ?? (await this.userFollowingsCache.fetch(followerId)).has(followeeId); + } + + /** + * Returns all hibernated followers. + */ + @bindThis + public async getHibernatedFollowers(followeeId: string): Promise { + const followers = await this.getFollowersWithHibernation(followeeId); + return followers.filter(f => f.isFollowerHibernated); + } + + /** + * Returns all non-hibernated followers. + */ + @bindThis + public async getNonHibernatedFollowers(followeeId: string): Promise { + const followers = await this.getFollowersWithHibernation(followeeId); + return followers.filter(f => !f.isFollowerHibernated); + } + + /** + * Returns follower relations with populated isFollowerHibernated. + * If you don't need this field, then please use userFollowersCache directly for reduced overhead. + */ + @bindThis + public async getFollowersWithHibernation(followeeId: string): Promise { + const followers = await this.userFollowersCache.fetch(followeeId); + const hibernations = await this.hibernatedUserCache.fetchMany(followers.keys()).then(fs => fs.reduce((map, f) => { + map.set(f[0], f[1]); + return map; + }, new Map)); + return Array.from(followers.values()).map(following => ({ + ...following, + isFollowerHibernated: hibernations.get(following.followerId) ?? false, + })); + } + + /** + * Refreshes follower and following relations for the given user. + */ + @bindThis + public async refreshFollowRelationsFor(userId: string): Promise { + const followings = await this.userFollowingsCache.refresh(userId); + const followees = Array.from(followings.values()).map(f => f.followeeId); + await this.userFollowersCache.deleteMany(followees); + } + + @bindThis + public clear(): void { + this.userByIdCache.clear(); + this.localUserByNativeTokenCache.clear(); + this.localUserByIdCache.clear(); + this.uriPersonCache.clear(); + this.userProfileCache.clear(); + this.userMutingsCache.clear(); + this.userBlockingCache.clear(); + this.userBlockedCache.clear(); + this.renoteMutingsCache.clear(); + this.userFollowingsCache.clear(); + this.userFollowStatsCache.clear(); + this.translationsCache.clear(); + } + @bindThis public dispose(): void { - this.redisForSub.off('message', this.onMessage); + this.internalEventService.off('userChangeSuspendedState', this.onUserEvent); + this.internalEventService.off('userChangeDeletedState', this.onUserEvent); + this.internalEventService.off('remoteUserUpdated', this.onUserEvent); + this.internalEventService.off('localUserUpdated', this.onUserEvent); + this.internalEventService.off('userChangeSuspendedState', this.onUserEvent); + this.internalEventService.off('userTokenRegenerated', this.onTokenEvent); + this.internalEventService.off('follow', this.onFollowEvent); + this.internalEventService.off('unfollow', this.onFollowEvent); this.userByIdCache.dispose(); this.localUserByNativeTokenCache.dispose(); this.localUserByIdCache.dispose(); diff --git a/packages/backend/src/core/ChannelFollowingService.ts b/packages/backend/src/core/ChannelFollowingService.ts index 12251595e2..430711fef1 100644 --- a/packages/backend/src/core/ChannelFollowingService.ts +++ b/packages/backend/src/core/ChannelFollowingService.ts @@ -9,14 +9,15 @@ import { DI } from '@/di-symbols.js'; import type { ChannelFollowingsRepository } from '@/models/_.js'; import { MiChannel } from '@/models/_.js'; import { IdService } from '@/core/IdService.js'; -import { GlobalEvents, GlobalEventService } from '@/core/GlobalEventService.js'; +import { GlobalEvents, GlobalEventService, InternalEventTypes } from '@/core/GlobalEventService.js'; import { bindThis } from '@/decorators.js'; import type { MiLocalUser } from '@/models/User.js'; -import { RedisKVCache } from '@/misc/cache.js'; +import { QuantumKVCache } from '@/misc/QuantumKVCache.js'; +import { InternalEventService } from './InternalEventService.js'; @Injectable() export class ChannelFollowingService implements OnModuleInit { - public userFollowingChannelsCache: RedisKVCache>; + public userFollowingChannelsCache: QuantumKVCache>; constructor( @Inject(DI.redis) @@ -27,19 +28,18 @@ export class ChannelFollowingService implements OnModuleInit { private channelFollowingsRepository: ChannelFollowingsRepository, private idService: IdService, private globalEventService: GlobalEventService, + private readonly internalEventService: InternalEventService, ) { - this.userFollowingChannelsCache = new RedisKVCache>(this.redisClient, 'userFollowingChannels', { + this.userFollowingChannelsCache = new QuantumKVCache>(this.internalEventService, 'userFollowingChannels', { lifetime: 1000 * 60 * 30, // 30m - memoryCacheLifetime: 1000 * 60, // 1m fetcher: (key) => this.channelFollowingsRepository.find({ where: { followerId: key }, select: ['followeeId'], }).then(xs => new Set(xs.map(x => x.followeeId))), - toRedisConverter: (value) => JSON.stringify(Array.from(value)), - fromRedisConverter: (value) => new Set(JSON.parse(value)), }); - this.redisForSub.on('message', this.onMessage); + this.internalEventService.on('followChannel', this.onMessage); + this.internalEventService.on('unfollowChannel', this.onMessage); } onModuleInit() { @@ -79,18 +79,15 @@ export class ChannelFollowingService implements OnModuleInit { } @bindThis - private async onMessage(_: string, data: string): Promise { - const obj = JSON.parse(data); - - if (obj.channel === 'internal') { - const { type, body } = obj.message as GlobalEvents['internal']['payload']; + private async onMessage(body: InternalEventTypes[E], type: E): Promise { + { switch (type) { case 'followChannel': { - this.userFollowingChannelsCache.refresh(body.userId); + await this.userFollowingChannelsCache.delete(body.userId); break; } case 'unfollowChannel': { - this.userFollowingChannelsCache.delete(body.userId); + await this.userFollowingChannelsCache.delete(body.userId); break; } } @@ -99,6 +96,8 @@ export class ChannelFollowingService implements OnModuleInit { @bindThis public dispose(): void { + this.internalEventService.off('followChannel', this.onMessage); + this.internalEventService.off('unfollowChannel', this.onMessage); this.userFollowingChannelsCache.dispose(); } diff --git a/packages/backend/src/core/CoreModule.ts b/packages/backend/src/core/CoreModule.ts index dd8e61d322..6839ba0159 100644 --- a/packages/backend/src/core/CoreModule.ts +++ b/packages/backend/src/core/CoreModule.ts @@ -41,6 +41,7 @@ import { HttpRequestService } from './HttpRequestService.js'; import { IdService } from './IdService.js'; import { ImageProcessingService } from './ImageProcessingService.js'; import { SystemAccountService } from './SystemAccountService.js'; +import { InternalEventService } from './InternalEventService.js'; import { InternalStorageService } from './InternalStorageService.js'; import { MetaService } from './MetaService.js'; import { MfmService } from './MfmService.js'; @@ -186,6 +187,7 @@ const $HashtagService: Provider = { provide: 'HashtagService', useExisting: Hash const $HttpRequestService: Provider = { provide: 'HttpRequestService', useExisting: HttpRequestService }; const $IdService: Provider = { provide: 'IdService', useExisting: IdService }; const $ImageProcessingService: Provider = { provide: 'ImageProcessingService', useExisting: ImageProcessingService }; +const $InternalEventService: Provider = { provide: 'InternalEventService', useExisting: InternalEventService }; const $InternalStorageService: Provider = { provide: 'InternalStorageService', useExisting: InternalStorageService }; const $MetaService: Provider = { provide: 'MetaService', useExisting: MetaService }; const $MfmService: Provider = { provide: 'MfmService', useExisting: MfmService }; @@ -345,6 +347,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp HttpRequestService, IdService, ImageProcessingService, + InternalEventService, InternalStorageService, MetaService, MfmService, @@ -500,6 +503,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp $HttpRequestService, $IdService, $ImageProcessingService, + $InternalEventService, $InternalStorageService, $MetaService, $MfmService, @@ -656,6 +660,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp HttpRequestService, IdService, ImageProcessingService, + InternalEventService, InternalStorageService, MetaService, MfmService, @@ -810,6 +815,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp $HttpRequestService, $IdService, $ImageProcessingService, + $InternalEventService, $InternalStorageService, $MetaService, $MfmService, diff --git a/packages/backend/src/core/FetchInstanceMetadataService.ts b/packages/backend/src/core/FetchInstanceMetadataService.ts index 9bfd7381f1..6fcfdfb596 100644 --- a/packages/backend/src/core/FetchInstanceMetadataService.ts +++ b/packages/backend/src/core/FetchInstanceMetadataService.ts @@ -7,7 +7,7 @@ import { URL } from 'node:url'; import { Inject, Injectable } from '@nestjs/common'; import tinycolor from 'tinycolor2'; import * as Redis from 'ioredis'; -import { load as cheerio } from 'cheerio'; +import { load as cheerio } from 'cheerio/slim'; import type { MiInstance } from '@/models/Instance.js'; import type Logger from '@/logger.js'; import { DI } from '@/di-symbols.js'; @@ -16,7 +16,7 @@ import { HttpRequestService } from '@/core/HttpRequestService.js'; import { bindThis } from '@/decorators.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; import { renderInlineError } from '@/misc/render-inline-error.js'; -import type { CheerioAPI } from 'cheerio'; +import type { CheerioAPI } from 'cheerio/slim'; type NodeInfo = { openRegistrations?: unknown; diff --git a/packages/backend/src/core/GlobalEventService.ts b/packages/backend/src/core/GlobalEventService.ts index c0027ae129..c146811331 100644 --- a/packages/backend/src/core/GlobalEventService.ts +++ b/packages/backend/src/core/GlobalEventService.ts @@ -265,6 +265,7 @@ export interface InternalEventTypes { unmute: { muterId: MiUser['id']; muteeId: MiUser['id']; }; userListMemberAdded: { userListId: MiUserList['id']; memberId: MiUser['id']; }; userListMemberRemoved: { userListId: MiUserList['id']; memberId: MiUser['id']; }; + quantumCacheUpdated: { name: string, keys: string[] }; } type EventTypesToEventPayload = EventUnionFromDictionary>>; @@ -353,12 +354,12 @@ export class GlobalEventService { } @bindThis - private publish(channel: StreamChannels, type: string | null, value?: any): void { + private async publish(channel: StreamChannels, type: string | null, value?: any): Promise { const message = type == null ? value : value == null ? { type: type, body: null } : { type: type, body: value }; - this.redisForPub.publish(this.config.host, JSON.stringify({ + await this.redisForPub.publish(this.config.host, JSON.stringify({ channel: channel, message: message, })); @@ -369,6 +370,11 @@ export class GlobalEventService { this.publish('internal', type, typeof value === 'undefined' ? null : value); } + @bindThis + public async publishInternalEventAsync(type: K, value?: InternalEventTypes[K]): Promise { + await this.publish('internal', type, typeof value === 'undefined' ? null : value); + } + @bindThis public publishBroadcastStream(type: K, value?: BroadcastTypes[K]): void { this.publish('broadcast', type, typeof value === 'undefined' ? null : value); diff --git a/packages/backend/src/core/InternalEventService.ts b/packages/backend/src/core/InternalEventService.ts new file mode 100644 index 0000000000..5b164b605e --- /dev/null +++ b/packages/backend/src/core/InternalEventService.ts @@ -0,0 +1,103 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; +import Redis from 'ioredis'; +import { DI } from '@/di-symbols.js'; +import { GlobalEventService } from '@/core/GlobalEventService.js'; +import type { GlobalEvents, InternalEventTypes } from '@/core/GlobalEventService.js'; +import { bindThis } from '@/decorators.js'; + +export type Listener = (value: InternalEventTypes[K], key: K, isLocal: boolean) => void | Promise; + +export interface ListenerProps { + ignoreLocal?: boolean, + ignoreRemote?: boolean, +} + +@Injectable() +export class InternalEventService implements OnApplicationShutdown { + private readonly listeners = new Map, ListenerProps>>(); + + constructor( + @Inject(DI.redisForSub) + private readonly redisForSub: Redis.Redis, + + private readonly globalEventService: GlobalEventService, + ) { + this.redisForSub.on('message', this.onMessage); + } + + @bindThis + public on(type: K, listener: Listener, props?: ListenerProps): void { + let set = this.listeners.get(type); + if (!set) { + set = new Map(); + this.listeners.set(type, set); + } + + // Functionally, this is just a set with metadata on the values. + set.set(listener as Listener, props ?? {}); + } + + @bindThis + public off(type: K, listener: Listener): void { + this.listeners.get(type)?.delete(listener as Listener); + } + + @bindThis + public async emit(type: K, value: InternalEventTypes[K]): Promise { + await this.emitInternal(type, value, true); + await this.globalEventService.publishInternalEventAsync(type, { ...value, _pid: process.pid }); + } + + @bindThis + private async emitInternal(type: K, value: InternalEventTypes[K], isLocal: boolean): Promise { + const listeners = this.listeners.get(type); + if (!listeners) { + return; + } + + const promises: Promise[] = []; + for (const [listener, props] of listeners) { + if ((isLocal && !props.ignoreLocal) || (!isLocal && !props.ignoreRemote)) { + const promise = Promise.resolve(listener(value, type, isLocal)); + promises.push(promise); + } + } + await Promise.all(promises); + } + + @bindThis + private async onMessage(_: string, data: string): Promise { + const obj = JSON.parse(data); + + if (obj.channel === 'internal') { + const { type, body } = obj.message as GlobalEvents['internal']['payload']; + if (!isLocalInternalEvent(body) || body._pid !== process.pid) { + await this.emitInternal(type, body as InternalEventTypes[keyof InternalEventTypes], false); + } + } + } + + @bindThis + public dispose(): void { + this.redisForSub.off('message', this.onMessage); + this.listeners.clear(); + } + + @bindThis + public onApplicationShutdown(): void { + this.dispose(); + } +} + +interface LocalInternalEvent { + _pid: number; +} + +function isLocalInternalEvent(body: object): body is LocalInternalEvent { + return '_pid' in body && typeof(body._pid) === 'number'; +} diff --git a/packages/backend/src/core/MfmService.ts b/packages/backend/src/core/MfmService.ts index 1ee3bd2275..551b25394a 100644 --- a/packages/backend/src/core/MfmService.ts +++ b/packages/backend/src/core/MfmService.ts @@ -5,25 +5,22 @@ import { URL } from 'node:url'; import { Inject, Injectable } from '@nestjs/common'; -import * as parse5 from 'parse5'; -import { type Document, type HTMLParagraphElement, Window } from 'happy-dom'; +import { isText, isTag, Text } from 'domhandler'; +import * as htmlparser2 from 'htmlparser2'; +import { Node, Document, ChildNode, Element, ParentNode } from 'domhandler'; +import * as domserializer from 'dom-serializer'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import { intersperse } from '@/misc/prelude/array.js'; import { normalizeForSearch } from '@/misc/normalize-for-search.js'; import type { IMentionedRemoteUsers } from '@/models/Note.js'; import { bindThis } from '@/decorators.js'; -import type { DefaultTreeAdapterMap } from 'parse5'; -import type * as mfm from '@transfem-org/sfm-js'; - -const treeAdapter = parse5.defaultTreeAdapter; -type Node = DefaultTreeAdapterMap['node']; -type ChildNode = DefaultTreeAdapterMap['childNode']; +import type * as mfm from 'mfm-js'; const urlRegex = /^https?:\/\/[\w\/:%#@$&?!()\[\]~.,=+\-]+/; const urlRegexFull = /^https?:\/\/[\w\/:%#@$&?!()\[\]~.,=+\-]+$/; -export type Appender = (document: Document, body: HTMLParagraphElement) => void; +export type Appender = (document: Document, body: Element) => void; @Injectable() export class MfmService { @@ -40,7 +37,7 @@ export class MfmService { const normalizedHashtagNames = hashtagNames == null ? undefined : new Set(hashtagNames.map(x => normalizeForSearch(x))); - const dom = parse5.parseFragment(html); + const dom = htmlparser2.parseDocument(html); let text = ''; @@ -51,37 +48,31 @@ export class MfmService { return text.trim(); function getText(node: Node): string { - if (treeAdapter.isTextNode(node)) return node.value; - if (!treeAdapter.isElementNode(node)) return ''; - if (node.nodeName === 'br') return '\n'; + if (isText(node)) return node.data; + if (!isTag(node)) return ''; + if (node.tagName === 'br') return '\n'; - if (node.childNodes) { - return node.childNodes.map(n => getText(n)).join(''); - } - - return ''; + return node.childNodes.map(n => getText(n)).join(''); } function appendChildren(childNodes: ChildNode[]): void { - if (childNodes) { - for (const n of childNodes) { - analyze(n); - } + for (const n of childNodes) { + analyze(n); } } function analyze(node: Node) { - if (treeAdapter.isTextNode(node)) { - text += node.value; + if (isText(node)) { + text += node.data; return; } // Skip comment or document type node - if (!treeAdapter.isElementNode(node)) { + if (!isTag(node)) { return; } - switch (node.nodeName) { + switch (node.tagName) { case 'br': { text += '\n'; break; @@ -89,19 +80,19 @@ export class MfmService { case 'a': { const txt = getText(node); - const rel = node.attrs.find(x => x.name === 'rel'); - const href = node.attrs.find(x => x.name === 'href'); + const rel = node.attribs.rel; + const href = node.attribs.href; // ハッシュタグ if (normalizedHashtagNames && href && normalizedHashtagNames.has(normalizeForSearch(txt))) { text += txt; // メンション - } else if (txt.startsWith('@') && !(rel && rel.value.startsWith('me '))) { + } else if (txt.startsWith('@') && !(rel && rel.startsWith('me '))) { const part = txt.split('@'); if (part.length === 2 && href) { //#region ホスト名部分が省略されているので復元する - const acct = `${txt}@${(new URL(href.value)).hostname}`; + const acct = `${txt}@${(new URL(href)).hostname}`; text += acct; //#endregion } else if (part.length === 3) { @@ -116,17 +107,17 @@ export class MfmService { if (!href) { return txt; } - if (!txt || txt === href.value) { // #6383: Missing text node - if (href.value.match(urlRegexFull)) { - return href.value; + if (!txt || txt === href) { // #6383: Missing text node + if (href.match(urlRegexFull)) { + return href; } else { - return `<${href.value}>`; + return `<${href}>`; } } - if (href.value.match(urlRegex) && !href.value.match(urlRegexFull)) { - return `[${txt}](<${href.value}>)`; // #6846 + if (href.match(urlRegex) && !href.match(urlRegexFull)) { + return `[${txt}](<${href}>)`; // #6846 } else { - return `[${txt}](${href.value})`; + return `[${txt}](${href})`; } }; @@ -185,14 +176,17 @@ export class MfmService { case 'ruby--': { let ruby: [string, string][] = []; for (const child of node.childNodes) { - if (child.nodeName === 'rp') { + if (isText(child) && !/\s|\[|\]/.test(child.data)) { + ruby.push([child.data, '']); continue; } - if (treeAdapter.isTextNode(child) && !/\s|\[|\]/.test(child.value)) { - ruby.push([child.value, '']); + if (!isTag(child)) { continue; } - if (child.nodeName === 'rt' && ruby.length > 0) { + if (child.tagName === 'rp') { + continue; + } + if (child.tagName === 'rt' && ruby.length > 0) { const rt = getText(child); if (/\s|\[|\]/.test(rt)) { // If any space is included in rt, it is treated as a normal text @@ -217,7 +211,7 @@ export class MfmService { // block code (
)
 				case 'pre': {
-					if (node.childNodes.length === 1 && node.childNodes[0].nodeName === 'code') {
+					if (node.childNodes.length === 1 && isTag(node.childNodes[0]) && node.childNodes[0].tagName === 'code') {
 						text += '\n```\n';
 						text += getText(node.childNodes[0]);
 						text += '\n```\n';
@@ -302,17 +296,17 @@ export class MfmService {
 						let nonRtNodes = [];
 						// scan children, ignore `rp`, split on `rt`
 						for (const child of node.childNodes) {
-							if (treeAdapter.isTextNode(child)) {
+							if (isText(child)) {
 								nonRtNodes.push(child);
 								continue;
 							}
-							if (!treeAdapter.isElementNode(child)) {
+							if (!isTag(child)) {
 								continue;
 							}
-							if (child.nodeName === 'rp') {
+							if (child.tagName === 'rp') {
 								continue;
 							}
-							if (child.nodeName === 'rt') {
+							if (child.tagName === 'rt') {
 								// the only case in which we don't need a `$[group ]`
 								// is when both sides of the ruby are simple words
 								const needsGroup = nonRtNodes.length > 1 ||
@@ -350,45 +344,44 @@ export class MfmService {
 			return null;
 		}
 
-		const { happyDOM, window } = new Window();
+		const doc = new Document([]);
 
-		const doc = window.document;
+		const body = new Element('p', {});
+		doc.childNodes.push(body);
 
-		const body = doc.createElement('p');
-
-		function appendChildren(children: mfm.MfmNode[], targetElement: any): void {
-			if (children) {
-				for (const child of children.map(x => (handlers as any)[x.type](x))) targetElement.appendChild(child);
+		function appendChildren(children: mfm.MfmNode[], targetElement: ParentNode): void {
+			for (const child of children.map(x => handle(x))) {
+				targetElement.childNodes.push(child);
 			}
 		}
 
 		function fnDefault(node: mfm.MfmFn) {
-			const el = doc.createElement('i');
+			const el = new Element('i', {});
 			appendChildren(node.children, el);
 			return el;
 		}
 
-		const handlers: { [K in mfm.MfmNode['type']]: (node: mfm.NodeType) => any } = {
+		const handlers: { [K in mfm.MfmNode['type']]: (node: mfm.NodeType) => ChildNode } = {
 			bold: (node) => {
-				const el = doc.createElement('b');
+				const el = new Element('b', {});
 				appendChildren(node.children, el);
 				return el;
 			},
 
 			small: (node) => {
-				const el = doc.createElement('small');
+				const el = new Element('small', {});
 				appendChildren(node.children, el);
 				return el;
 			},
 
 			strike: (node) => {
-				const el = doc.createElement('del');
+				const el = new Element('del', {});
 				appendChildren(node.children, el);
 				return el;
 			},
 
 			italic: (node) => {
-				const el = doc.createElement('i');
+				const el = new Element('i', {});
 				appendChildren(node.children, el);
 				return el;
 			},
@@ -399,11 +392,12 @@ export class MfmService {
 						const text = node.children[0].type === 'text' ? node.children[0].props.text : '';
 						try {
 							const date = new Date(parseInt(text, 10) * 1000);
-							const el = doc.createElement('time');
-							el.setAttribute('datetime', date.toISOString());
-							el.textContent = date.toISOString();
+							const el = new Element('time', {
+								datetime: date.toISOString(),
+							});
+							el.childNodes.push(new Text(date.toISOString()));
 							return el;
-						} catch (err) {
+						} catch {
 							return fnDefault(node);
 						}
 					}
@@ -412,20 +406,20 @@ export class MfmService {
 						if (node.children.length === 1) {
 							const child = node.children[0];
 							const text = child.type === 'text' ? child.props.text : '';
-							const rubyEl = doc.createElement('ruby');
-							const rtEl = doc.createElement('rt');
+							const rubyEl = new Element('ruby', {});
+							const rtEl = new Element('rt', {});
 
 							// ruby未対応のHTMLサニタイザーを通したときにルビが「劉備(りゅうび)」となるようにする
-							const rpStartEl = doc.createElement('rp');
-							rpStartEl.appendChild(doc.createTextNode('('));
-							const rpEndEl = doc.createElement('rp');
-							rpEndEl.appendChild(doc.createTextNode(')'));
+							const rpStartEl = new Element('rp', {});
+							rpStartEl.childNodes.push(new Text('('));
+							const rpEndEl = new Element('rp', {});
+							rpEndEl.childNodes.push(new Text(')'));
 
-							rubyEl.appendChild(doc.createTextNode(text.split(' ')[0]));
-							rtEl.appendChild(doc.createTextNode(text.split(' ')[1]));
-							rubyEl.appendChild(rpStartEl);
-							rubyEl.appendChild(rtEl);
-							rubyEl.appendChild(rpEndEl);
+							rubyEl.childNodes.push(new Text(text.split(' ')[0]));
+							rtEl.childNodes.push(new Text(text.split(' ')[1]));
+							rubyEl.childNodes.push(rpStartEl);
+							rubyEl.childNodes.push(rtEl);
+							rubyEl.childNodes.push(rpEndEl);
 							return rubyEl;
 						} else {
 							const rt = node.children.at(-1);
@@ -435,20 +429,20 @@ export class MfmService {
 							}
 
 							const text = rt.type === 'text' ? rt.props.text : '';
-							const rubyEl = doc.createElement('ruby');
-							const rtEl = doc.createElement('rt');
+							const rubyEl = new Element('ruby', {});
+							const rtEl = new Element('rt', {});
 
 							// ruby未対応のHTMLサニタイザーを通したときにルビが「劉備(りゅうび)」となるようにする
-							const rpStartEl = doc.createElement('rp');
-							rpStartEl.appendChild(doc.createTextNode('('));
-							const rpEndEl = doc.createElement('rp');
-							rpEndEl.appendChild(doc.createTextNode(')'));
+							const rpStartEl = new Element('rp', {});
+							rpStartEl.childNodes.push(new Text('('));
+							const rpEndEl = new Element('rp', {});
+							rpEndEl.childNodes.push(new Text(')'));
 
 							appendChildren(node.children.slice(0, node.children.length - 1), rubyEl);
-							rtEl.appendChild(doc.createTextNode(text.trim()));
-							rubyEl.appendChild(rpStartEl);
-							rubyEl.appendChild(rtEl);
-							rubyEl.appendChild(rpEndEl);
+							rtEl.childNodes.push(new Text(text.trim()));
+							rubyEl.childNodes.push(rpStartEl);
+							rubyEl.childNodes.push(rtEl);
+							rubyEl.childNodes.push(rpEndEl);
 							return rubyEl;
 						}
 					}
@@ -456,7 +450,7 @@ export class MfmService {
 					// hack for ruby, should never be needed because we should
 					// never send this out to other instances
 					case 'group': {
-						const el = doc.createElement('span');
+						const el = new Element('span', {});
 						appendChildren(node.children, el);
 						return el;
 					}
@@ -468,125 +462,135 @@ export class MfmService {
 			},
 
 			blockCode: (node) => {
-				const pre = doc.createElement('pre');
-				const inner = doc.createElement('code');
-				inner.textContent = node.props.code;
-				pre.appendChild(inner);
+				const pre = new Element('pre', {});
+				const inner = new Element('code', {});
+				inner.childNodes.push(new Text(node.props.code));
+				pre.childNodes.push(inner);
 				return pre;
 			},
 
 			center: (node) => {
-				const el = doc.createElement('div');
+				const el = new Element('div', {});
 				appendChildren(node.children, el);
 				return el;
 			},
 
 			emojiCode: (node) => {
-				return doc.createTextNode(`\u200B:${node.props.name}:\u200B`);
+				return new Text(`\u200B:${node.props.name}:\u200B`);
 			},
 
 			unicodeEmoji: (node) => {
-				return doc.createTextNode(node.props.emoji);
+				return new Text(node.props.emoji);
 			},
 
 			hashtag: (node) => {
-				const a = doc.createElement('a');
-				a.setAttribute('href', `${this.config.url}/tags/${node.props.hashtag}`);
-				a.textContent = `#${node.props.hashtag}`;
-				a.setAttribute('rel', 'tag');
+				const a = new Element('a', {
+					href: `${this.config.url}/tags/${node.props.hashtag}`,
+					rel: 'tag',
+				});
+				a.childNodes.push(new Text(`#${node.props.hashtag}`));
 				return a;
 			},
 
 			inlineCode: (node) => {
-				const el = doc.createElement('code');
-				el.textContent = node.props.code;
+				const el = new Element('code', {});
+				el.childNodes.push(new Text(node.props.code));
 				return el;
 			},
 
 			mathInline: (node) => {
-				const el = doc.createElement('code');
-				el.textContent = node.props.formula;
+				const el = new Element('code', {});
+				el.childNodes.push(new Text(node.props.formula));
 				return el;
 			},
 
 			mathBlock: (node) => {
-				const el = doc.createElement('code');
-				el.textContent = node.props.formula;
+				const el = new Element('code', {});
+				el.childNodes.push(new Text(node.props.formula));
 				return el;
 			},
 
 			link: (node) => {
-				const a = doc.createElement('a');
-				a.setAttribute('href', node.props.url);
+				const a = new Element('a', {
+					href: node.props.url,
+				});
 				appendChildren(node.children, a);
 				return a;
 			},
 
 			mention: (node) => {
-				const a = doc.createElement('a');
 				const { username, host, acct } = node.props;
 				const remoteUserInfo = mentionedRemoteUsers.find(remoteUser => remoteUser.username.toLowerCase() === username.toLowerCase() && remoteUser.host?.toLowerCase() === host?.toLowerCase());
-				a.setAttribute('href', remoteUserInfo
-					? (remoteUserInfo.url ? remoteUserInfo.url : remoteUserInfo.uri)
-					: `${this.config.url}/${acct.endsWith(`@${this.config.url}`) ? acct.substring(0, acct.length - this.config.url.length - 1) : acct}`);
-				a.className = 'u-url mention';
-				a.textContent = acct;
+
+				const a = new Element('a', {
+					href: remoteUserInfo
+						? (remoteUserInfo.url ? remoteUserInfo.url : remoteUserInfo.uri)
+						: `${this.config.url}/${acct.endsWith(`@${this.config.url}`) ? acct.substring(0, acct.length - this.config.url.length - 1) : acct}`,
+					class: 'u-url mention',
+				});
+				a.childNodes.push(new Text(acct));
 				return a;
 			},
 
 			quote: (node) => {
-				const el = doc.createElement('blockquote');
+				const el = new Element('blockquote', {});
 				appendChildren(node.children, el);
 				return el;
 			},
 
 			text: (node) => {
 				if (!node.props.text.match(/[\r\n]/)) {
-					return doc.createTextNode(node.props.text);
+					return new Text(node.props.text);
 				}
 
-				const el = doc.createElement('span');
-				const nodes = node.props.text.split(/\r\n|\r|\n/).map(x => doc.createTextNode(x));
+				const el = new Element('span', {});
+				const nodes = node.props.text.split(/\r\n|\r|\n/).map(x => new Text(x));
 
 				for (const x of intersperse('br', nodes)) {
-					el.appendChild(x === 'br' ? doc.createElement('br') : x);
+					el.childNodes.push(x === 'br' ? new Element('br', {}) : x);
 				}
 
 				return el;
 			},
 
 			url: (node) => {
-				const a = doc.createElement('a');
-				a.setAttribute('href', node.props.url);
-				a.textContent = node.props.url;
+				const a = new Element('a', {
+					href: node.props.url,
+				});
+				a.childNodes.push(new Text(node.props.url));
 				return a;
 			},
 
 			search: (node) => {
-				const a = doc.createElement('a');
-				a.setAttribute('href', `https://www.google.com/search?q=${node.props.query}`);
-				a.textContent = node.props.content;
+				const a = new Element('a', {
+					href: `https://www.google.com/search?q=${node.props.query}`,
+				});
+				a.childNodes.push(new Text(node.props.content));
 				return a;
 			},
 
 			plain: (node) => {
-				const el = doc.createElement('span');
+				const el = new Element('span', {});
 				appendChildren(node.children, el);
 				return el;
 			},
 		};
 
+		// Utility function to make TypeScript behave
+		function handle(node: T): ChildNode {
+			const handler = handlers[node.type] as (node: T) => ChildNode;
+			return handler(node);
+		}
+
 		appendChildren(nodes, body);
 
 		for (const additionalAppender of additionalAppenders) {
 			additionalAppender(doc, body);
 		}
 
-		const serialized = body.outerHTML;
-
-		happyDOM.close().catch(err => {});
-
-		return serialized;
+		return domserializer.render(body, {
+			encodeEntities: 'utf8'
+		});
 	}
 
 	// the toMastoApiHtml function was taken from Iceshrimp and written by zotan and modified by marie to work with the current MK version
@@ -598,55 +602,55 @@ export class MfmService {
 			return null;
 		}
 
-		const { happyDOM, window } = new Window();
+		const doc = new Document([]);
 
-		const doc = window.document;
+		const body = new Element('p', {});
+		doc.childNodes.push(body);
 
-		const body = doc.createElement('p');
-
-		function appendChildren(children: mfm.MfmNode[], targetElement: any): void {
-			if (children) {
-				for (const child of children.map((x) => (handlers as any)[x.type](x))) targetElement.appendChild(child);
+		function appendChildren(children: mfm.MfmNode[], targetElement: ParentNode): void {
+			for (const child of children) {
+				const result = handle(child);
+				targetElement.childNodes.push(result);
 			}
 		}
 
 		const handlers: {
-			[K in mfm.MfmNode['type']]: (node: mfm.NodeType) => any;
+			[K in mfm.MfmNode['type']]: (node: mfm.NodeType) => ChildNode;
 		} = {
 			bold(node) {
-				const el = doc.createElement('span');
-				el.textContent = '**';
+				const el = new Element('span', {});
+				el.childNodes.push(new Text('**'));
 				appendChildren(node.children, el);
-				el.textContent += '**';
+				el.childNodes.push(new Text('**'));
 				return el;
 			},
 
 			small(node) {
-				const el = doc.createElement('small');
+				const el = new Element('small', {});
 				appendChildren(node.children, el);
 				return el;
 			},
 
 			strike(node) {
-				const el = doc.createElement('span');
-				el.textContent = '~~';
+				const el = new Element('span', {});
+				el.childNodes.push(new Text('~~'));
 				appendChildren(node.children, el);
-				el.textContent += '~~';
+				el.childNodes.push(new Text('~~'));
 				return el;
 			},
 
 			italic(node) {
-				const el = doc.createElement('span');
-				el.textContent = '*';
+				const el = new Element('span', {});
+				el.childNodes.push(new Text('*'));
 				appendChildren(node.children, el);
-				el.textContent += '*';
+				el.childNodes.push(new Text('*'));
 				return el;
 			},
 
 			fn(node) {
 				switch (node.props.name) {
 					case 'group': { // hack for ruby
-						const el = doc.createElement('span');
+						const el = new Element('span', {});
 						appendChildren(node.children, el);
 						return el;
 					}
@@ -654,119 +658,121 @@ export class MfmService {
 						if (node.children.length === 1) {
 							const child = node.children[0];
 							const text = child.type === 'text' ? child.props.text : '';
-							const rubyEl = doc.createElement('ruby');
-							const rtEl = doc.createElement('rt');
+							const rubyEl = new Element('ruby', {});
+							const rtEl = new Element('rt', {});
 
-							const rpStartEl = doc.createElement('rp');
-							rpStartEl.appendChild(doc.createTextNode('('));
-							const rpEndEl = doc.createElement('rp');
-							rpEndEl.appendChild(doc.createTextNode(')'));
+							const rpStartEl = new Element('rp', {});
+							rpStartEl.childNodes.push(new Text('('));
+							const rpEndEl = new Element('rp', {});
+							rpEndEl.childNodes.push(new Text(')'));
 
-							rubyEl.appendChild(doc.createTextNode(text.split(' ')[0]));
-							rtEl.appendChild(doc.createTextNode(text.split(' ')[1]));
-							rubyEl.appendChild(rpStartEl);
-							rubyEl.appendChild(rtEl);
-							rubyEl.appendChild(rpEndEl);
+							rubyEl.childNodes.push(new Text(text.split(' ')[0]));
+							rtEl.childNodes.push(new Text(text.split(' ')[1]));
+							rubyEl.childNodes.push(rpStartEl);
+							rubyEl.childNodes.push(rtEl);
+							rubyEl.childNodes.push(rpEndEl);
 							return rubyEl;
 						} else {
 							const rt = node.children.at(-1);
 
 							if (!rt) {
-								const el = doc.createElement('span');
+								const el = new Element('span', {});
 								appendChildren(node.children, el);
 								return el;
 							}
 
 							const text = rt.type === 'text' ? rt.props.text : '';
-							const rubyEl = doc.createElement('ruby');
-							const rtEl = doc.createElement('rt');
+							const rubyEl = new Element('ruby', {});
+							const rtEl = new Element('rt', {});
 
-							const rpStartEl = doc.createElement('rp');
-							rpStartEl.appendChild(doc.createTextNode('('));
-							const rpEndEl = doc.createElement('rp');
-							rpEndEl.appendChild(doc.createTextNode(')'));
+							const rpStartEl = new Element('rp', {});
+							rpStartEl.childNodes.push(new Text('('));
+							const rpEndEl = new Element('rp', {});
+							rpEndEl.childNodes.push(new Text(')'));
 
 							appendChildren(node.children.slice(0, node.children.length - 1), rubyEl);
-							rtEl.appendChild(doc.createTextNode(text.trim()));
-							rubyEl.appendChild(rpStartEl);
-							rubyEl.appendChild(rtEl);
-							rubyEl.appendChild(rpEndEl);
+							rtEl.childNodes.push(new Text(text.trim()));
+							rubyEl.childNodes.push(rpStartEl);
+							rubyEl.childNodes.push(rtEl);
+							rubyEl.childNodes.push(rpEndEl);
 							return rubyEl;
 						}
 					}
 
 					default: {
-						const el = doc.createElement('span');
-						el.textContent = '*';
+						const el = new Element('span', {});
+						el.childNodes.push(new Text('*'));
 						appendChildren(node.children, el);
-						el.textContent += '*';
+						el.childNodes.push(new Text('*'));
 						return el;
 					}
 				}
 			},
 
 			blockCode(node) {
-				const pre = doc.createElement('pre');
-				const inner = doc.createElement('code');
+				const pre = new Element('pre', {});
+				const inner = new Element('code', {});
 
 				const nodes = node.props.code
 					.split(/\r\n|\r|\n/)
-					.map((x) => doc.createTextNode(x));
+					.map((x) => new Text(x));
 
 				for (const x of intersperse('br', nodes)) {
-					inner.appendChild(x === 'br' ? doc.createElement('br') : x);
+					inner.childNodes.push(x === 'br' ? new Element('br', {}) : x);
 				}
 
-				pre.appendChild(inner);
+				pre.childNodes.push(inner);
 				return pre;
 			},
 
 			center(node) {
-				const el = doc.createElement('div');
+				const el = new Element('div', {});
 				appendChildren(node.children, el);
 				return el;
 			},
 
 			emojiCode(node) {
-				return doc.createTextNode(`\u200B:${node.props.name}:\u200B`);
+				return new Text(`\u200B:${node.props.name}:\u200B`);
 			},
 
 			unicodeEmoji(node) {
-				return doc.createTextNode(node.props.emoji);
+				return new Text(node.props.emoji);
 			},
 
 			hashtag: (node) => {
-				const a = doc.createElement('a');
-				a.setAttribute('href', `${this.config.url}/tags/${node.props.hashtag}`);
-				a.textContent = `#${node.props.hashtag}`;
-				a.setAttribute('rel', 'tag');
-				a.setAttribute('class', 'hashtag');
+				const a = new Element('a', {
+					href: `${this.config.url}/tags/${node.props.hashtag}`,
+					rel: 'tag',
+					class: 'hashtag',
+				});
+				a.childNodes.push(new Text(`#${node.props.hashtag}`));
 				return a;
 			},
 
 			inlineCode(node) {
-				const el = doc.createElement('code');
-				el.textContent = node.props.code;
+				const el = new Element('code', {});
+				el.childNodes.push(new Text(node.props.code));
 				return el;
 			},
 
 			mathInline(node) {
-				const el = doc.createElement('code');
-				el.textContent = node.props.formula;
+				const el = new Element('code', {});
+				el.childNodes.push(new Text(node.props.formula));
 				return el;
 			},
 
 			mathBlock(node) {
-				const el = doc.createElement('code');
-				el.textContent = node.props.formula;
+				const el = new Element('code', {});
+				el.childNodes.push(new Text(node.props.formula));
 				return el;
 			},
 
 			link(node) {
-				const a = doc.createElement('a');
-				a.setAttribute('rel', 'nofollow noopener noreferrer');
-				a.setAttribute('target', '_blank');
-				a.setAttribute('href', node.props.url);
+				const a = new Element('a', {
+					rel: 'nofollow noopener noreferrer',
+					target: '_blank',
+					href: node.props.url,
+				});
 				appendChildren(node.children, a);
 				return a;
 			},
@@ -775,92 +781,107 @@ export class MfmService {
 				const { username, host, acct } = node.props;
 				const resolved = mentionedRemoteUsers.find(remoteUser => remoteUser.username === username && remoteUser.host === host);
 
-				const el = doc.createElement('span');
+				const el = new Element('span', {});
 				if (!resolved) {
-					el.textContent = acct;
+					el.childNodes.push(new Text(acct));
 				} else {
-					el.setAttribute('class', 'h-card');
-					el.setAttribute('translate', 'no');
-					const a = doc.createElement('a');
-					a.setAttribute('href', resolved.url ? resolved.url : resolved.uri);
-					a.className = 'u-url mention';
-					const span = doc.createElement('span');
-					span.textContent = resolved.username || username;
-					a.textContent = '@';
-					a.appendChild(span);
-					el.appendChild(a);
+					el.attribs.class = 'h-card';
+					el.attribs.translate = 'no';
+					const a = new Element('a', {
+						href: resolved.url ? resolved.url : resolved.uri,
+						class: 'u-url mention',
+					});
+					const span = new Element('span', {});
+					span.childNodes.push(new Text(resolved.username || username));
+					a.childNodes.push(new Text('@'));
+					a.childNodes.push(span);
+					el.childNodes.push(a);
 				}
 
 				return el;
 			},
 
 			quote(node) {
-				const el = doc.createElement('blockquote');
+				const el = new Element('blockquote', {});
 				appendChildren(node.children, el);
 				return el;
 			},
 
 			text(node) {
-				const el = doc.createElement('span');
+				if (!node.props.text.match(/[\r\n]/)) {
+					return new Text(node.props.text);
+				}
+
+				const el = new Element('span', {});
 				const nodes = node.props.text
 					.split(/\r\n|\r|\n/)
-					.map((x) => doc.createTextNode(x));
+					.map((x) => new Text(x));
 
 				for (const x of intersperse('br', nodes)) {
-					el.appendChild(x === 'br' ? doc.createElement('br') : x);
+					el.childNodes.push(x === 'br' ? new Element('br', {}) : x);
 				}
 
 				return el;
 			},
 
 			url(node) {
-				const a = doc.createElement('a');
-				a.setAttribute('rel', 'nofollow noopener noreferrer');
-				a.setAttribute('target', '_blank');
-				a.setAttribute('href', node.props.url);
-				a.textContent = node.props.url.replace(/^https?:\/\//, '');
+				const a = new Element('a', {
+					rel: 'nofollow noopener noreferrer',
+					target: '_blank',
+					href: node.props.url,
+				});
+				a.childNodes.push(new Text(node.props.url.replace(/^https?:\/\//, '')));
 				return a;
 			},
 
 			search: (node) => {
-				const a = doc.createElement('a');
-				a.setAttribute('href', `https://www.google.com/search?q=${node.props.query}`);
-				a.textContent = node.props.content;
+				const a = new Element('a', {
+					href: `https://www.google.com/search?q=${node.props.query}`,
+				});
+				a.childNodes.push(new Text(node.props.content));
 				return a;
 			},
 
 			plain(node) {
-				const el = doc.createElement('span');
+				const el = new Element('span', {});
 				appendChildren(node.children, el);
 				return el;
 			},
 		};
 
+		// Utility function to make TypeScript behave
+		function handle(node: T): ChildNode {
+			const handler = handlers[node.type] as (node: T) => ChildNode;
+			return handler(node);
+		}
+
 		appendChildren(nodes, body);
 
 		if (quoteUri !== null) {
-			const a = doc.createElement('a');
-			a.setAttribute('href', quoteUri);
-			a.textContent = quoteUri.replace(/^https?:\/\//, '');
+			const a = new Element('a', {
+				href: quoteUri,
+			});
+			a.childNodes.push(new Text(quoteUri.replace(/^https?:\/\//, '')));
 
-			const quote = doc.createElement('span');
-			quote.setAttribute('class', 'quote-inline');
-			quote.appendChild(doc.createElement('br'));
-			quote.appendChild(doc.createElement('br'));
-			quote.innerHTML += 'RE: ';
-			quote.appendChild(a);
+			const quote = new Element('span', {
+				class: 'quote-inline',
+			});
+			quote.childNodes.push(new Element('br', {}));
+			quote.childNodes.push(new Element('br', {}));
+			quote.childNodes.push(new Text('RE: '));
+			quote.childNodes.push(a);
 
-			body.appendChild(quote);
+			body.childNodes.push(quote);
 		}
 
-		let result = body.outerHTML;
+		let result = domserializer.render(body, {
+			encodeEntities: 'utf8'
+		});
 
 		if (inline) {
 			result = result.replace(/^

/, '').replace(/<\/p>$/, ''); } - happyDOM.close().catch(() => {}); - return result; } } diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 4dceb6e953..f8584a4a48 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -4,7 +4,7 @@ */ import { setImmediate } from 'node:timers/promises'; -import * as mfm from '@transfem-org/sfm-js'; +import * as mfm from 'mfm-js'; import { In, DataSource, IsNull, LessThan } from 'typeorm'; import * as Redis from 'ioredis'; import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; @@ -606,11 +606,11 @@ export class NoteCreateService implements OnApplicationShutdown { } if (data.reply == null) { - // TODO: キャッシュ - this.followingsRepository.findBy({ - followeeId: user.id, - notify: 'normal', - }).then(async followings => { + this.cacheService.userFollowersCache.fetch(user.id).then(async followingsMap => { + const followings = Array + .from(followingsMap.values()) + .filter(f => f.notify === 'normal'); + if (note.visibility !== 'specified') { const isPureRenote = this.isRenote(data) && !this.isQuote(data) ? true : false; for (const following of followings) { @@ -948,14 +948,7 @@ export class NoteCreateService implements OnApplicationShutdown { // TODO: キャッシュ? // eslint-disable-next-line prefer-const let [followings, userListMemberships] = await Promise.all([ - this.followingsRepository.find({ - where: { - followeeId: user.id, - followerHost: IsNull(), - isFollowerHibernated: false, - }, - select: ['followerId', 'withReplies'], - }), + this.cacheService.getNonHibernatedFollowers(user.id), this.userListMembershipsRepository.find({ where: { userId: user.id, @@ -1072,17 +1065,19 @@ export class NoteCreateService implements OnApplicationShutdown { }); if (hibernatedUsers.length > 0) { - this.usersRepository.update({ - id: In(hibernatedUsers.map(x => x.id)), - }, { - isHibernated: true, - }); - - this.followingsRepository.update({ - followerId: In(hibernatedUsers.map(x => x.id)), - }, { - isFollowerHibernated: true, - }); + await Promise.all([ + this.usersRepository.update({ + id: In(hibernatedUsers.map(x => x.id)), + }, { + isHibernated: true, + }), + this.followingsRepository.update({ + followerId: In(hibernatedUsers.map(x => x.id)), + }, { + isFollowerHibernated: true, + }), + this.cacheService.hibernatedUserCache.setMany(hibernatedUsers.map(x => [x.id, true])), + ]); } } diff --git a/packages/backend/src/core/NoteEditService.ts b/packages/backend/src/core/NoteEditService.ts index 34af1c76dd..d963bf1945 100644 --- a/packages/backend/src/core/NoteEditService.ts +++ b/packages/backend/src/core/NoteEditService.ts @@ -4,7 +4,7 @@ */ import { setImmediate } from 'node:timers/promises'; -import * as mfm from '@transfem-org/sfm-js'; +import * as mfm from 'mfm-js'; import { DataSource, In, IsNull, LessThan } from 'typeorm'; import * as Redis from 'ioredis'; import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; @@ -833,14 +833,7 @@ export class NoteEditService implements OnApplicationShutdown { // TODO: キャッシュ? // eslint-disable-next-line prefer-const let [followings, userListMemberships] = await Promise.all([ - this.followingsRepository.find({ - where: { - followeeId: user.id, - followerHost: IsNull(), - isFollowerHibernated: false, - }, - select: ['followerId', 'withReplies'], - }), + this.cacheService.getNonHibernatedFollowers(user.id), this.userListMembershipsRepository.find({ where: { userId: user.id, @@ -957,17 +950,19 @@ export class NoteEditService implements OnApplicationShutdown { }); if (hibernatedUsers.length > 0) { - this.usersRepository.update({ - id: In(hibernatedUsers.map(x => x.id)), - }, { - isHibernated: true, - }); - - this.followingsRepository.update({ - followerId: In(hibernatedUsers.map(x => x.id)), - }, { - isFollowerHibernated: true, - }); + await Promise.all([ + this.usersRepository.update({ + id: In(hibernatedUsers.map(x => x.id)), + }, { + isHibernated: true, + }), + this.followingsRepository.update({ + followerId: In(hibernatedUsers.map(x => x.id)), + }, { + isFollowerHibernated: true, + }), + this.cacheService.hibernatedUserCache.setMany(hibernatedUsers.map(x => [x.id, true])), + ]); } } diff --git a/packages/backend/src/core/NotificationService.ts b/packages/backend/src/core/NotificationService.ts index 0f05f5425d..2ce7bdb5a9 100644 --- a/packages/backend/src/core/NotificationService.ts +++ b/packages/backend/src/core/NotificationService.ts @@ -113,27 +113,27 @@ export class NotificationService implements OnApplicationShutdown { } if (recieveConfig?.type === 'following') { - const isFollowing = await this.cacheService.userFollowingsCache.fetch(notifieeId).then(followings => Object.hasOwn(followings, notifierId)); + const isFollowing = await this.cacheService.userFollowingsCache.fetch(notifieeId).then(followings => followings.has(notifierId)); if (!isFollowing) { return null; } } else if (recieveConfig?.type === 'follower') { - const isFollower = await this.cacheService.userFollowingsCache.fetch(notifierId).then(followings => Object.hasOwn(followings, notifieeId)); + const isFollower = await this.cacheService.userFollowingsCache.fetch(notifierId).then(followings => followings.has(notifieeId)); if (!isFollower) { return null; } } else if (recieveConfig?.type === 'mutualFollow') { const [isFollowing, isFollower] = await Promise.all([ - this.cacheService.userFollowingsCache.fetch(notifieeId).then(followings => Object.hasOwn(followings, notifierId)), - this.cacheService.userFollowingsCache.fetch(notifierId).then(followings => Object.hasOwn(followings, notifieeId)), + this.cacheService.userFollowingsCache.fetch(notifieeId).then(followings => followings.has(notifierId)), + this.cacheService.userFollowingsCache.fetch(notifierId).then(followings => followings.has(notifieeId)), ]); if (!(isFollowing && isFollower)) { return null; } } else if (recieveConfig?.type === 'followingOrFollower') { const [isFollowing, isFollower] = await Promise.all([ - this.cacheService.userFollowingsCache.fetch(notifieeId).then(followings => Object.hasOwn(followings, notifierId)), - this.cacheService.userFollowingsCache.fetch(notifierId).then(followings => Object.hasOwn(followings, notifieeId)), + this.cacheService.userFollowingsCache.fetch(notifieeId).then(followings => followings.has(notifierId)), + this.cacheService.userFollowingsCache.fetch(notifierId).then(followings => followings.has(notifieeId)), ]); if (!isFollowing && !isFollower) { return null; diff --git a/packages/backend/src/core/PushNotificationService.ts b/packages/backend/src/core/PushNotificationService.ts index 9333c1ebc5..e3f10d4504 100644 --- a/packages/backend/src/core/PushNotificationService.ts +++ b/packages/backend/src/core/PushNotificationService.ts @@ -12,7 +12,8 @@ import type { Packed } from '@/misc/json-schema.js'; import { getNoteSummary } from '@/misc/get-note-summary.js'; import type { MiMeta, MiSwSubscription, SwSubscriptionsRepository } from '@/models/_.js'; import { bindThis } from '@/decorators.js'; -import { RedisKVCache } from '@/misc/cache.js'; +import { QuantumKVCache } from '@/misc/QuantumKVCache.js'; +import { InternalEventService } from '@/core/InternalEventService.js'; // Defined also packages/sw/types.ts#L13 type PushNotificationsTypes = { @@ -48,7 +49,7 @@ function truncateBody(type: T, body: Pus @Injectable() export class PushNotificationService implements OnApplicationShutdown { - private subscriptionsCache: RedisKVCache; + private subscriptionsCache: QuantumKVCache; constructor( @Inject(DI.config) @@ -62,13 +63,11 @@ export class PushNotificationService implements OnApplicationShutdown { @Inject(DI.swSubscriptionsRepository) private swSubscriptionsRepository: SwSubscriptionsRepository, + private readonly internalEventService: InternalEventService, ) { - this.subscriptionsCache = new RedisKVCache(this.redisClient, 'userSwSubscriptions', { + this.subscriptionsCache = new QuantumKVCache(this.internalEventService, 'userSwSubscriptions', { lifetime: 1000 * 60 * 60 * 1, // 1h - memoryCacheLifetime: 1000 * 60 * 3, // 3m fetcher: (key) => this.swSubscriptionsRepository.findBy({ userId: key }), - toRedisConverter: (value) => JSON.stringify(value), - fromRedisConverter: (value) => JSON.parse(value), }); } @@ -114,8 +113,8 @@ export class PushNotificationService implements OnApplicationShutdown { endpoint: subscription.endpoint, auth: subscription.auth, publickey: subscription.publickey, - }).then(() => { - this.refreshCache(userId); + }).then(async () => { + await this.refreshCache(userId); }); } }); @@ -123,8 +122,8 @@ export class PushNotificationService implements OnApplicationShutdown { } @bindThis - public refreshCache(userId: string): void { - this.subscriptionsCache.refresh(userId); + public async refreshCache(userId: string): Promise { + await this.subscriptionsCache.refresh(userId); } @bindThis diff --git a/packages/backend/src/core/QueryService.ts b/packages/backend/src/core/QueryService.ts index 4e423e73f2..e1bfe8d3b9 100644 --- a/packages/backend/src/core/QueryService.ts +++ b/packages/backend/src/core/QueryService.ts @@ -164,17 +164,17 @@ export class QueryService { qb // My post .orWhere(':meId = note.userId') - // Reply to me - .orWhere(':meId = note.replyUserId') - // DM to me + // Visible to me .orWhere(':meIdAsList <@ note.visibleUserIds') - // Mentions me (and not a direct message) - .orWhere(new Brackets(qb => - qb.andWhere(':meIdAsList <@ note.mentions') - .andWhere('note.visibility != \'specified\''))) // Followers-only post - .orWhere(new Brackets(qb => this - .andFollowingUser(qb, ':meId', 'note.userId') + .orWhere(new Brackets(qb => qb + .andWhere(new Brackets(qbb => this + // Following author + .orFollowingUser(qbb, ':meId', 'note.userId') + // Mentions me + .orWhere(':meIdAsList <@ note.mentions') + // Reply to me + .orWhere(':meId = note.replyUserId'))) .andWhere('note.visibility = \'followers\''))); q.setParameters({ meId: me.id, meIdAsList: [me.id] }); diff --git a/packages/backend/src/core/ReactionService.ts b/packages/backend/src/core/ReactionService.ts index c23bb51178..8d2dc7d4e8 100644 --- a/packages/backend/src/core/ReactionService.ts +++ b/packages/backend/src/core/ReactionService.ts @@ -122,7 +122,7 @@ export class ReactionService { } // check visibility - if (!await this.noteEntityService.isVisibleForMe(note, user.id)) { + if (!await this.noteEntityService.isVisibleForMe(note, user.id, { me: user })) { throw new IdentifiableError('68e9d2d1-48bf-42c2-b90a-b20e09fd3d48', 'Note not accessible for you.'); } diff --git a/packages/backend/src/core/UserBlockingService.ts b/packages/backend/src/core/UserBlockingService.ts index 8da1bb2092..1a1e7c4778 100644 --- a/packages/backend/src/core/UserBlockingService.ts +++ b/packages/backend/src/core/UserBlockingService.ts @@ -77,8 +77,10 @@ export class UserBlockingService implements OnModuleInit { await this.blockingsRepository.insert(blocking); - this.cacheService.userBlockingCache.refresh(blocker.id); - this.cacheService.userBlockedCache.refresh(blockee.id); + await Promise.all([ + this.cacheService.userBlockingCache.delete(blocker.id), + this.cacheService.userBlockedCache.delete(blockee.id), + ]); this.globalEventService.publishInternalEvent('blockingCreated', { blockerId: blocker.id, @@ -168,8 +170,10 @@ export class UserBlockingService implements OnModuleInit { await this.blockingsRepository.delete(blocking.id); - this.cacheService.userBlockingCache.refresh(blocker.id); - this.cacheService.userBlockedCache.refresh(blockee.id); + await Promise.all([ + this.cacheService.userBlockingCache.delete(blocker.id), + this.cacheService.userBlockedCache.delete(blockee.id), + ]); this.globalEventService.publishInternalEvent('blockingDeleted', { blockerId: blocker.id, diff --git a/packages/backend/src/core/UserFollowingService.ts b/packages/backend/src/core/UserFollowingService.ts index 897b950022..8470872eac 100644 --- a/packages/backend/src/core/UserFollowingService.ts +++ b/packages/backend/src/core/UserFollowingService.ts @@ -29,6 +29,7 @@ import { AccountMoveService } from '@/core/AccountMoveService.js'; import { UtilityService } from '@/core/UtilityService.js'; import type { ThinUser } from '@/queue/types.js'; import { LoggerService } from '@/core/LoggerService.js'; +import { InternalEventService } from '@/core/InternalEventService.js'; import type Logger from '../logger.js'; type Local = MiLocalUser | { @@ -86,6 +87,7 @@ export class UserFollowingService implements OnModuleInit { private accountMoveService: AccountMoveService, private perUserFollowingChart: PerUserFollowingChart, private instanceChart: InstanceChart, + private readonly internalEventService: InternalEventService, loggerService: LoggerService, ) { @@ -145,12 +147,7 @@ export class UserFollowingService implements OnModuleInit { if (blocked) throw new IdentifiableError('3338392a-f764-498d-8855-db939dcf8c48', 'blocked'); } - if (await this.followingsRepository.exists({ - where: { - followerId: follower.id, - followeeId: followee.id, - }, - })) { + if (await this.cacheService.isFollowing(follower, followee)) { // すでにフォロー関係が存在している場合 if (this.userEntityService.isRemoteUser(follower) && this.userEntityService.isLocalUser(followee)) { // リモート → ローカル: acceptを送り返しておしまい @@ -178,24 +175,14 @@ export class UserFollowingService implements OnModuleInit { let autoAccept = false; // 鍵アカウントであっても、既にフォローされていた場合はスルー - const isFollowing = await this.followingsRepository.exists({ - where: { - followerId: follower.id, - followeeId: followee.id, - }, - }); + const isFollowing = await this.cacheService.isFollowing(follower, followee); if (isFollowing) { autoAccept = true; } // フォローしているユーザーは自動承認オプション if (!autoAccept && (this.userEntityService.isLocalUser(followee) && followeeProfile.autoAcceptFollowed)) { - const isFollowed = await this.followingsRepository.exists({ - where: { - followerId: followee.id, - followeeId: follower.id, - }, - }); + const isFollowed = await this.cacheService.isFollowing(followee, follower); // intentionally reversed parameters if (isFollowed) autoAccept = true; } @@ -204,12 +191,7 @@ export class UserFollowingService implements OnModuleInit { if (followee.isLocked && !autoAccept) { autoAccept = !!(await this.accountMoveService.validateAlsoKnownAs( follower, - (oldSrc, newSrc) => this.followingsRepository.exists({ - where: { - followeeId: followee.id, - followerId: newSrc.id, - }, - }), + (oldSrc, newSrc) => this.cacheService.isFollowing(newSrc, followee), true, )); } @@ -264,7 +246,8 @@ export class UserFollowingService implements OnModuleInit { } }); - this.cacheService.userFollowingsCache.refresh(follower.id); + // Handled by CacheService + //this.cacheService.userFollowingsCache.refresh(follower.id); const requestExist = await this.followRequestsRepository.exists({ where: { @@ -291,7 +274,7 @@ export class UserFollowingService implements OnModuleInit { }, followee.id); } - this.globalEventService.publishInternalEvent('follow', { followerId: follower.id, followeeId: followee.id }); + await this.internalEventService.emit('follow', { followerId: follower.id, followeeId: followee.id }); const [followeeUser, followerUser] = await Promise.all([ this.usersRepository.findOneByOrFail({ id: followee.id }), @@ -363,31 +346,29 @@ export class UserFollowingService implements OnModuleInit { }, silent = false, ): Promise { - const following = await this.followingsRepository.findOne({ - relations: { - follower: true, - followee: true, - }, - where: { - followerId: follower.id, - followeeId: followee.id, - }, - }); + const [ + followerUser, + followeeUser, + following, + ] = await Promise.all([ + this.cacheService.findUserById(follower.id), + this.cacheService.findUserById(followee.id), + this.cacheService.userFollowingsCache.fetch(follower.id).then(fs => fs.get(followee.id)), + ]); - if (following === null || !following.follower || !following.followee) { + if (following == null) { this.logger.warn('フォロー解除がリクエストされましたがフォローしていませんでした'); return; } await this.followingsRepository.delete(following.id); + await this.internalEventService.emit('unfollow', { followerId: follower.id, followeeId: followee.id }); - this.cacheService.userFollowingsCache.refresh(follower.id); - - this.decrementFollowing(following.follower, following.followee); + this.decrementFollowing(followerUser, followeeUser); if (!silent && this.userEntityService.isLocalUser(follower)) { // Publish unfollow event - this.userEntityService.pack(followee.id, follower, { + this.userEntityService.pack(followeeUser, follower, { schema: 'UserDetailedNotMe', }).then(async packed => { this.globalEventService.publishMainStream(follower.id, 'unfollow', packed); @@ -412,8 +393,6 @@ export class UserFollowingService implements OnModuleInit { follower: MiUser, followee: MiUser, ): Promise { - this.globalEventService.publishInternalEvent('unfollow', { followerId: follower.id, followeeId: followee.id }); - // Neither followee nor follower has moved. if (!follower.movedToUri && !followee.movedToUri) { //#region Decrement following / followers counts @@ -687,22 +666,22 @@ export class UserFollowingService implements OnModuleInit { */ @bindThis private async removeFollow(followee: Both, follower: Both): Promise { - const following = await this.followingsRepository.findOne({ - relations: { - followee: true, - follower: true, - }, - where: { - followeeId: followee.id, - followerId: follower.id, - }, - }); + const [ + followerUser, + followeeUser, + following, + ] = await Promise.all([ + this.cacheService.findUserById(follower.id), + this.cacheService.findUserById(followee.id), + this.cacheService.userFollowingsCache.fetch(follower.id).then(fs => fs.get(followee.id)), + ]); - if (!following || !following.followee || !following.follower) return; + if (!following) return; await this.followingsRepository.delete(following.id); + await this.internalEventService.emit('unfollow', { followerId: follower.id, followeeId: followee.id }); - this.decrementFollowing(following.follower, following.followee); + this.decrementFollowing(followerUser, followeeUser); } /** @@ -733,36 +712,26 @@ export class UserFollowingService implements OnModuleInit { } @bindThis - public getFollowees(userId: MiUser['id']) { - return this.followingsRepository.createQueryBuilder('following') - .select('following.followeeId') - .where('following.followerId = :followerId', { followerId: userId }) - .getMany(); + public async getFollowees(userId: MiUser['id']) { + const followings = await this.cacheService.userFollowingsCache.fetch(userId); + return Array.from(followings.values()); } @bindThis - public isFollowing(followerId: MiUser['id'], followeeId: MiUser['id']) { - return this.followingsRepository.exists({ - where: { - followerId, - followeeId, - }, - }); + public async isFollowing(followerId: MiUser['id'], followeeId: MiUser['id']) { + return this.cacheService.isFollowing(followerId, followeeId); } @bindThis public async isMutual(aUserId: MiUser['id'], bUserId: MiUser['id']) { - const count = await this.followingsRepository.createQueryBuilder('following') - .where(new Brackets(qb => { - qb.where('following.followerId = :aUserId', { aUserId }) - .andWhere('following.followeeId = :bUserId', { bUserId }); - })) - .orWhere(new Brackets(qb => { - qb.where('following.followerId = :bUserId', { bUserId }) - .andWhere('following.followeeId = :aUserId', { aUserId }); - })) - .getCount(); + const [ + isFollowing, + isFollowed, + ] = await Promise.all([ + this.isFollowing(aUserId, bUserId), + this.isFollowing(bUserId, aUserId), + ]); - return count === 2; + return isFollowing && isFollowed; } } diff --git a/packages/backend/src/core/UserKeypairService.ts b/packages/backend/src/core/UserKeypairService.ts index 92d61cd103..d8a67d273b 100644 --- a/packages/backend/src/core/UserKeypairService.ts +++ b/packages/backend/src/core/UserKeypairService.ts @@ -7,14 +7,14 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import * as Redis from 'ioredis'; import type { MiUser } from '@/models/User.js'; import type { UserKeypairsRepository } from '@/models/_.js'; -import { RedisKVCache } from '@/misc/cache.js'; +import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js'; import type { MiUserKeypair } from '@/models/UserKeypair.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; @Injectable() export class UserKeypairService implements OnApplicationShutdown { - private cache: RedisKVCache; + private cache: MemoryKVCache; constructor( @Inject(DI.redis) @@ -23,18 +23,12 @@ export class UserKeypairService implements OnApplicationShutdown { @Inject(DI.userKeypairsRepository) private userKeypairsRepository: UserKeypairsRepository, ) { - this.cache = new RedisKVCache(this.redisClient, 'userKeypair', { - lifetime: 1000 * 60 * 60 * 24, // 24h - memoryCacheLifetime: 1000 * 60 * 60, // 1h - fetcher: (key) => this.userKeypairsRepository.findOneByOrFail({ userId: key }), - toRedisConverter: (value) => JSON.stringify(value), - fromRedisConverter: (value) => JSON.parse(value), - }); + this.cache = new MemoryKVCache(1000 * 60 * 60 * 24); // 24h } @bindThis public async getUserKeypair(userId: MiUser['id']): Promise { - return await this.cache.fetch(userId); + return await this.cache.fetch(userId, () => this.userKeypairsRepository.findOneByOrFail({ userId })); } @bindThis diff --git a/packages/backend/src/core/UserListService.ts b/packages/backend/src/core/UserListService.ts index e7200ab1bf..b4486b9808 100644 --- a/packages/backend/src/core/UserListService.ts +++ b/packages/backend/src/core/UserListService.ts @@ -11,21 +11,22 @@ import type { MiUser } from '@/models/User.js'; import type { MiUserList } from '@/models/UserList.js'; import type { MiUserListMembership } from '@/models/UserListMembership.js'; import { IdService } from '@/core/IdService.js'; -import type { GlobalEvents } from '@/core/GlobalEventService.js'; +import type { GlobalEvents, InternalEventTypes } from '@/core/GlobalEventService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import { DI } from '@/di-symbols.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { bindThis } from '@/decorators.js'; import { QueueService } from '@/core/QueueService.js'; -import { RedisKVCache } from '@/misc/cache.js'; +import { QuantumKVCache } from '@/misc/QuantumKVCache.js'; import { RoleService } from '@/core/RoleService.js'; import { SystemAccountService } from '@/core/SystemAccountService.js'; +import { InternalEventService } from '@/core/InternalEventService.js'; @Injectable() export class UserListService implements OnApplicationShutdown, OnModuleInit { public static TooManyUsersError = class extends Error {}; - public membersCache: RedisKVCache>; + public membersCache: QuantumKVCache>; private roleService: RoleService; constructor( @@ -48,16 +49,15 @@ export class UserListService implements OnApplicationShutdown, OnModuleInit { private globalEventService: GlobalEventService, private queueService: QueueService, private systemAccountService: SystemAccountService, + private readonly internalEventService: InternalEventService, ) { - this.membersCache = new RedisKVCache>(this.redisClient, 'userListMembers', { + this.membersCache = new QuantumKVCache>(this.internalEventService, 'userListMembers', { lifetime: 1000 * 60 * 30, // 30m - memoryCacheLifetime: 1000 * 60, // 1m fetcher: (key) => this.userListMembershipsRepository.find({ where: { userListId: key }, select: ['userId'] }).then(xs => new Set(xs.map(x => x.userId))), - toRedisConverter: (value) => JSON.stringify(Array.from(value)), - fromRedisConverter: (value) => new Set(JSON.parse(value)), }); - this.redisForSub.on('message', this.onMessage); + this.internalEventService.on('userListMemberAdded', this.onMessage); + this.internalEventService.on('userListMemberRemoved', this.onMessage); } async onModuleInit() { @@ -65,15 +65,12 @@ export class UserListService implements OnApplicationShutdown, OnModuleInit { } @bindThis - private async onMessage(_: string, data: string): Promise { - const obj = JSON.parse(data); - - if (obj.channel === 'internal') { - const { type, body } = obj.message as GlobalEvents['internal']['payload']; + private async onMessage(body: InternalEventTypes[E], type: E): Promise { + { switch (type) { case 'userListMemberAdded': { const { userListId, memberId } = body; - const members = await this.membersCache.get(userListId); + const members = this.membersCache.get(userListId); if (members) { members.add(memberId); } @@ -81,7 +78,7 @@ export class UserListService implements OnApplicationShutdown, OnModuleInit { } case 'userListMemberRemoved': { const { userListId, memberId } = body; - const members = await this.membersCache.get(userListId); + const members = this.membersCache.get(userListId); if (members) { members.delete(memberId); } @@ -150,7 +147,8 @@ export class UserListService implements OnApplicationShutdown, OnModuleInit { @bindThis public dispose(): void { - this.redisForSub.off('message', this.onMessage); + this.internalEventService.off('userListMemberAdded', this.onMessage); + this.internalEventService.off('userListMemberRemoved', this.onMessage); this.membersCache.dispose(); } diff --git a/packages/backend/src/core/UserMutingService.ts b/packages/backend/src/core/UserMutingService.ts index 06643be5fb..c15a979d0f 100644 --- a/packages/backend/src/core/UserMutingService.ts +++ b/packages/backend/src/core/UserMutingService.ts @@ -32,7 +32,7 @@ export class UserMutingService { muteeId: target.id, }); - this.cacheService.userMutingsCache.refresh(user.id); + await this.cacheService.userMutingsCache.delete(user.id); } @bindThis @@ -43,9 +43,6 @@ export class UserMutingService { id: In(mutings.map(m => m.id)), }); - const muterIds = [...new Set(mutings.map(m => m.muterId))]; - for (const muterId of muterIds) { - this.cacheService.userMutingsCache.refresh(muterId); - } + await this.cacheService.userMutingsCache.deleteMany(mutings.map(m => m.muterId)); } } diff --git a/packages/backend/src/core/UserRenoteMutingService.ts b/packages/backend/src/core/UserRenoteMutingService.ts index bdc5e23f4b..7c0693f216 100644 --- a/packages/backend/src/core/UserRenoteMutingService.ts +++ b/packages/backend/src/core/UserRenoteMutingService.ts @@ -33,7 +33,7 @@ export class UserRenoteMutingService { muteeId: target.id, }); - await this.cacheService.renoteMutingsCache.refresh(user.id); + await this.cacheService.renoteMutingsCache.delete(user.id); } @bindThis @@ -44,9 +44,6 @@ export class UserRenoteMutingService { id: In(mutings.map(m => m.id)), }); - const muterIds = [...new Set(mutings.map(m => m.muterId))]; - for (const muterId of muterIds) { - await this.cacheService.renoteMutingsCache.refresh(muterId); - } + await this.cacheService.renoteMutingsCache.deleteMany(mutings.map(m => m.muterId)); } } diff --git a/packages/backend/src/core/UserService.ts b/packages/backend/src/core/UserService.ts index 1f471513f3..4a04910105 100644 --- a/packages/backend/src/core/UserService.ts +++ b/packages/backend/src/core/UserService.ts @@ -10,6 +10,7 @@ import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; import { SystemWebhookService } from '@/core/SystemWebhookService.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; +import { CacheService } from '@/core/CacheService.js'; @Injectable() export class UserService { @@ -20,6 +21,7 @@ export class UserService { private followingsRepository: FollowingsRepository, private systemWebhookService: SystemWebhookService, private userEntityService: UserEntityService, + private readonly cacheService: CacheService, ) { } @@ -38,14 +40,17 @@ export class UserService { }); const wokeUp = result.isHibernated; if (wokeUp) { - this.usersRepository.update(user.id, { - isHibernated: false, - }); - this.followingsRepository.update({ - followerId: user.id, - }, { - isFollowerHibernated: false, - }); + await Promise.all([ + this.usersRepository.update(user.id, { + isHibernated: false, + }), + this.followingsRepository.update({ + followerId: user.id, + }, { + isFollowerHibernated: false, + }), + this.cacheService.hibernatedUserCache.set(user.id, false), + ]); } } else { this.usersRepository.update(user.id, { diff --git a/packages/backend/src/core/UserSuspendService.ts b/packages/backend/src/core/UserSuspendService.ts index 30dcaa6f7d..f375dff862 100644 --- a/packages/backend/src/core/UserSuspendService.ts +++ b/packages/backend/src/core/UserSuspendService.ts @@ -16,6 +16,7 @@ import { bindThis } from '@/decorators.js'; import { RelationshipJobData } from '@/queue/types.js'; import { ModerationLogService } from '@/core/ModerationLogService.js'; import { isSystemAccount } from '@/misc/is-system-account.js'; +import { CacheService } from '@/core/CacheService.js'; @Injectable() export class UserSuspendService { @@ -34,6 +35,7 @@ export class UserSuspendService { private globalEventService: GlobalEventService, private apRendererService: ApRendererService, private moderationLogService: ModerationLogService, + private readonly cacheService: CacheService, ) { } @@ -143,12 +145,8 @@ export class UserSuspendService { @bindThis private async unFollowAll(follower: MiUser) { - const followings = await this.followingsRepository.find({ - where: { - followerId: follower.id, - followeeId: Not(IsNull()), - }, - }); + const followings = await this.cacheService.userFollowingsCache.fetch(follower.id) + .then(fs => Array.from(fs.values()).filter(f => f.followeeHost != null)); const jobs: RelationshipJobData[] = []; for (const following of followings) { diff --git a/packages/backend/src/core/WebfingerService.ts b/packages/backend/src/core/WebfingerService.ts index 664963f3a3..bb9f0be4c6 100644 --- a/packages/backend/src/core/WebfingerService.ts +++ b/packages/backend/src/core/WebfingerService.ts @@ -5,7 +5,7 @@ import { URL } from 'node:url'; import { Injectable } from '@nestjs/common'; -import { XMLParser } from 'fast-xml-parser'; +import { load as cheerio } from 'cheerio/slim'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { bindThis } from '@/decorators.js'; import type Logger from '@/logger.js'; @@ -101,14 +101,12 @@ export class WebfingerService { private async fetchWebFingerTemplateFromHostMeta(url: string): Promise { try { const res = await this.httpRequestService.getHtml(url, 'application/xrd+xml'); - const options = { - ignoreAttributes: false, - isArray: (_name: string, jpath: string) => jpath === 'XRD.Link', - }; - const parser = new XMLParser(options); - const hostMeta = parser.parse(res); - const template = (hostMeta['XRD']['Link'] as Array).filter(p => p['@_rel'] === 'lrdd')[0]['@_template']; - return template.indexOf('{uri}') < 0 ? null : template; + const hostMeta = cheerio(res, { + xml: true, + }); + + const template = hostMeta('XRD > Link[rel="lrdd"][template*="{uri}"]').attr('template'); + return template ?? null; } catch (err) { this.logger.error(`error while request host-meta for ${url}: ${renderInlineError(err)}`); return null; diff --git a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts index ab7f356f28..91f6f2d9fc 100644 --- a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts +++ b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts @@ -5,7 +5,6 @@ import { Inject, Injectable } from '@nestjs/common'; import { IsNull, Not } from 'typeorm'; -import { UnrecoverableError } from 'bullmq'; import { DI } from '@/di-symbols.js'; import type { FollowingsRepository } from '@/models/_.js'; import type { MiLocalUser, MiRemoteUser, MiUser } from '@/models/User.js'; @@ -14,6 +13,7 @@ import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { bindThis } from '@/decorators.js'; import type { IActivity } from '@/core/activitypub/type.js'; import { ThinUser } from '@/queue/types.js'; +import { CacheService } from '@/core/CacheService.js'; interface IRecipe { type: string; @@ -41,16 +41,14 @@ class DeliverManager { /** * Constructor - * @param userEntityService - * @param followingsRepository * @param queueService + * @param cacheService * @param actor Actor * @param activity Activity to deliver */ constructor( - private userEntityService: UserEntityService, - private followingsRepository: FollowingsRepository, private queueService: QueueService, + private readonly cacheService: CacheService, actor: { id: MiUser['id']; host: null; }, activity: IActivity | null, @@ -114,27 +112,23 @@ class DeliverManager { // Process follower recipes first to avoid duplication when processing direct recipes later. if (this.recipes.some(r => isFollowers(r))) { // followers deliver - // TODO: SELECT DISTINCT ON ("followerSharedInbox") "followerSharedInbox" みたいな問い合わせにすればよりパフォーマンス向上できそう // ただ、sharedInboxがnullなリモートユーザーも稀におり、その対応ができなさそう? - const followers = await this.followingsRepository.find({ - where: { - followeeId: this.actor.id, - followerHost: Not(IsNull()), - }, - select: { - followerSharedInbox: true, - followerInbox: true, - followerId: true, - }, - }); + const followers = await this.cacheService.userFollowersCache + .fetch(this.actor.id) + .then(f => Array + .from(f.values()) + .filter(f => f.followerHost != null) + .map(f => ({ + followerInbox: f.followerInbox, + followerSharedInbox: f.followerSharedInbox, + }))); for (const following of followers) { - const inbox = following.followerSharedInbox ?? following.followerInbox; - if (inbox === null) { - if (process.env.NODE_ENV === 'test') continue; - throw new UnrecoverableError(`deliver failed for ${this.actor.id}: follower ${following.followerId} inbox is null`); + if (following.followerSharedInbox) { + inboxes.set(following.followerSharedInbox, true); + } else if (following.followerInbox) { + inboxes.set(following.followerInbox, false); } - inboxes.set(inbox, following.followerSharedInbox != null); } } @@ -156,11 +150,8 @@ class DeliverManager { @Injectable() export class ApDeliverManagerService { constructor( - @Inject(DI.followingsRepository) - private followingsRepository: FollowingsRepository, - - private userEntityService: UserEntityService, private queueService: QueueService, + private readonly cacheService: CacheService, ) { } @@ -172,9 +163,8 @@ export class ApDeliverManagerService { @bindThis public async deliverToFollowers(actor: { id: MiLocalUser['id']; host: null; }, activity: IActivity): Promise { const manager = new DeliverManager( - this.userEntityService, - this.followingsRepository, this.queueService, + this.cacheService, actor, activity, ); @@ -191,9 +181,8 @@ export class ApDeliverManagerService { @bindThis public async deliverToUser(actor: { id: MiLocalUser['id']; host: null; }, activity: IActivity, to: MiRemoteUser): Promise { const manager = new DeliverManager( - this.userEntityService, - this.followingsRepository, this.queueService, + this.cacheService, actor, activity, ); @@ -210,9 +199,8 @@ export class ApDeliverManagerService { @bindThis public async deliverToUsers(actor: { id: MiLocalUser['id']; host: null; }, activity: IActivity, targets: MiRemoteUser[]): Promise { const manager = new DeliverManager( - this.userEntityService, - this.followingsRepository, this.queueService, + this.cacheService, actor, activity, ); @@ -223,9 +211,8 @@ export class ApDeliverManagerService { @bindThis public createDeliverManager(actor: { id: MiUser['id']; host: null; }, activity: IActivity | null): DeliverManager { return new DeliverManager( - this.userEntityService, - this.followingsRepository, this.queueService, + this.cacheService, actor, activity, diff --git a/packages/backend/src/core/activitypub/ApInboxService.ts b/packages/backend/src/core/activitypub/ApInboxService.ts index b384ec58c5..009d4cbd39 100644 --- a/packages/backend/src/core/activitypub/ApInboxService.ts +++ b/packages/backend/src/core/activitypub/ApInboxService.ts @@ -37,6 +37,7 @@ import InstanceChart from '@/core/chart/charts/instance.js'; import FederationChart from '@/core/chart/charts/federation.js'; import { FetchInstanceMetadataService } from '@/core/FetchInstanceMetadataService.js'; import { UpdateInstanceQueue } from '@/core/UpdateInstanceQueue.js'; +import { CacheService } from '@/core/CacheService.js'; import { getApHrefNullable, getApId, getApIds, getApType, getNullableApId, isAccept, isActor, isAdd, isAnnounce, isApObject, isBlock, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isDislike, isMove, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost, isActivity, IObjectWithId } from './type.js'; import { ApNoteService } from './models/ApNoteService.js'; import { ApLoggerService } from './ApLoggerService.js'; @@ -98,6 +99,7 @@ export class ApInboxService { private readonly instanceChart: InstanceChart, private readonly federationChart: FederationChart, private readonly updateInstanceQueue: UpdateInstanceQueue, + private readonly cacheService: CacheService, ) { this.logger = this.apLoggerService.logger; } @@ -365,7 +367,7 @@ export class ApInboxService { const renote = await this.apNoteService.resolveNote(target, { resolver, sentFrom: getApId(target) }); if (renote == null) return 'announce target is null'; - if (!await this.noteEntityService.isVisibleForMe(renote, actor.id)) { + if (!await this.noteEntityService.isVisibleForMe(renote, actor.id, { me: actor })) { return 'skip: invalid actor for this activity'; } @@ -766,12 +768,7 @@ export class ApInboxService { return 'skip: follower not found'; } - const isFollowing = await this.followingsRepository.exists({ - where: { - followerId: follower.id, - followeeId: actor.id, - }, - }); + const isFollowing = await this.cacheService.userFollowingsCache.fetch(follower.id).then(f => f.has(actor.id)); if (isFollowing) { await this.userFollowingService.unfollow(follower, actor); @@ -830,12 +827,7 @@ export class ApInboxService { }, }); - const isFollowing = await this.followingsRepository.exists({ - where: { - followerId: actor.id, - followeeId: followee.id, - }, - }); + const isFollowing = await this.cacheService.userFollowingsCache.fetch(actor.id).then(f => f.has(followee.id)); if (requestExist) { await this.userFollowingService.cancelFollowRequest(followee, actor); diff --git a/packages/backend/src/core/activitypub/ApMfmService.ts b/packages/backend/src/core/activitypub/ApMfmService.ts index c4a948429a..ddb6461746 100644 --- a/packages/backend/src/core/activitypub/ApMfmService.ts +++ b/packages/backend/src/core/activitypub/ApMfmService.ts @@ -4,7 +4,7 @@ */ import { Injectable } from '@nestjs/common'; -import * as mfm from '@transfem-org/sfm-js'; +import * as mfm from 'mfm-js'; import { MfmService, Appender } from '@/core/MfmService.js'; import type { MiNote } from '@/models/Note.js'; import { bindThis } from '@/decorators.js'; diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts index 46a78687f3..08a8f30049 100644 --- a/packages/backend/src/core/activitypub/ApRendererService.ts +++ b/packages/backend/src/core/activitypub/ApRendererService.ts @@ -6,8 +6,9 @@ import { createPublicKey, randomUUID } from 'node:crypto'; import { Inject, Injectable } from '@nestjs/common'; import { In } from 'typeorm'; -import * as mfm from '@transfem-org/sfm-js'; +import * as mfm from 'mfm-js'; import { UnrecoverableError } from 'bullmq'; +import { Element, Text } from 'domhandler'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import type { MiPartialLocalUser, MiLocalUser, MiPartialRemoteUser, MiRemoteUser, MiUser } from '@/models/User.js'; @@ -34,7 +35,7 @@ import { UtilityService } from '@/core/UtilityService.js'; import { JsonLdService } from './JsonLdService.js'; import { ApMfmService } from './ApMfmService.js'; import { CONTEXT } from './misc/contexts.js'; -import { getApId, IOrderedCollection, IOrderedCollectionPage } from './type.js'; +import { getApId, ILink, IOrderedCollection, IOrderedCollectionPage } from './type.js'; import type { IAccept, IActivity, IAdd, IAnnounce, IApDocument, IApEmoji, IApHashtag, IApImage, IApMention, IBlock, ICreate, IDelete, IFlag, IFollow, IKey, ILike, IMove, IObject, IPost, IQuestion, IReject, IRemove, ITombstone, IUndo, IUpdate } from './type.js'; @Injectable() @@ -419,7 +420,7 @@ export class ApRendererService { inReplyTo = null; } - let quote; + let quote: string | undefined = undefined; if (note.renoteId) { const renote = await this.notesRepository.findOneBy({ id: note.renoteId }); @@ -475,16 +476,18 @@ export class ApRendererService { // the claas name `quote-inline` is used in non-misskey clients for styling quote notes. // For compatibility, the span part should be kept as possible. apAppend.push((doc, body) => { - body.appendChild(doc.createElement('br')); - body.appendChild(doc.createElement('br')); - const span = doc.createElement('span'); - span.className = 'quote-inline'; - span.appendChild(doc.createTextNode('RE: ')); - const link = doc.createElement('a'); - link.setAttribute('href', quote); - link.textContent = quote; - span.appendChild(link); - body.appendChild(span); + body.childNodes.push(new Element('br', {})); + body.childNodes.push(new Element('br', {})); + const span = new Element('span', { + class: 'quote-inline', + }); + span.childNodes.push(new Text('RE: ')); + const link = new Element('a', { + href: quote, + }); + link.childNodes.push(new Text(quote)); + span.childNodes.push(link); + body.childNodes.push(span); }); } @@ -500,12 +503,22 @@ export class ApRendererService { const emojis = await this.getEmojis(note.emojis); const apemojis = emojis.filter(emoji => !emoji.localOnly).map(emoji => this.renderEmoji(emoji)); - const tag = [ + const tag: IObject[] = [ ...hashtagTags, ...mentionTags, ...apemojis, ]; + // https://codeberg.org/fediverse/fep/src/branch/main/fep/e232/fep-e232.md + if (quote) { + tag.push({ + type: 'Link', + mediaType: 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"', + rel: 'https://misskey-hub.net/ns#_misskey_quote', + href: quote, + } satisfies ILink); + } + const asPoll = poll ? { type: 'Question', [poll.expiresAt && poll.expiresAt < new Date() ? 'closed' : 'endTime']: poll.expiresAt, @@ -537,6 +550,8 @@ export class ApRendererService { _misskey_quote: quote, quoteUrl: quote, quoteUri: quote, + // https://codeberg.org/fediverse/fep/src/branch/main/fep/044f/fep-044f.md + quote: quote, published: this.idService.parse(note.id).date.toISOString(), to, cc, @@ -774,7 +789,7 @@ export class ApRendererService { inReplyTo = null; } - let quote; + let quote: string | undefined = undefined; if (note.renoteId) { const renote = await this.notesRepository.findOneBy({ id: note.renoteId }); @@ -827,16 +842,18 @@ export class ApRendererService { // the claas name `quote-inline` is used in non-misskey clients for styling quote notes. // For compatibility, the span part should be kept as possible. apAppend.push((doc, body) => { - body.appendChild(doc.createElement('br')); - body.appendChild(doc.createElement('br')); - const span = doc.createElement('span'); - span.className = 'quote-inline'; - span.appendChild(doc.createTextNode('RE: ')); - const link = doc.createElement('a'); - link.setAttribute('href', quote); - link.textContent = quote; - span.appendChild(link); - body.appendChild(span); + body.childNodes.push(new Element('br', {})); + body.childNodes.push(new Element('br', {})); + const span = new Element('span', { + class: 'quote-inline', + }); + span.childNodes.push(new Text('RE: ')); + const link = new Element('a', { + href: quote, + }); + link.childNodes.push(new Text(quote)); + span.childNodes.push(link); + body.childNodes.push(span); }); } @@ -852,12 +869,22 @@ export class ApRendererService { const emojis = await this.getEmojis(note.emojis); const apemojis = emojis.filter(emoji => !emoji.localOnly).map(emoji => this.renderEmoji(emoji)); - const tag = [ + const tag: IObject[] = [ ...hashtagTags, ...mentionTags, ...apemojis, ]; + // https://codeberg.org/fediverse/fep/src/branch/main/fep/e232/fep-e232.md + if (quote) { + tag.push({ + type: 'Link', + mediaType: 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"', + rel: 'https://misskey-hub.net/ns#_misskey_quote', + href: quote, + } satisfies ILink); + } + const asPoll = poll ? { type: 'Question', [poll.expiresAt && poll.expiresAt < new Date() ? 'closed' : 'endTime']: poll.expiresAt, @@ -886,6 +913,8 @@ export class ApRendererService { _misskey_quote: quote, quoteUrl: quote, quoteUri: quote, + // https://codeberg.org/fediverse/fep/src/branch/main/fep/044f/fep-044f.md + quote: quote, published: this.idService.parse(note.id).date.toISOString(), to, cc, @@ -936,9 +965,7 @@ export class ApRendererService { const keypair = await this.userKeypairService.getUserKeypair(user.id); - const jsonLd = this.jsonLdService.use(); - jsonLd.debug = false; - activity = await jsonLd.signRsaSignature2017(activity, keypair.privateKey, `${this.config.url}/users/${user.id}#main-key`); + activity = await this.jsonLdService.signRsaSignature2017(activity, keypair.privateKey, `${this.config.url}/users/${user.id}#main-key`); return activity; } diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index 4c7cac2169..e4db9b237c 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -6,7 +6,7 @@ import * as crypto from 'node:crypto'; import { URL } from 'node:url'; import { Inject, Injectable } from '@nestjs/common'; -import { Window } from 'happy-dom'; +import { load as cheerio } from 'cheerio/slim'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import type { MiUser } from '@/models/User.js'; @@ -18,6 +18,8 @@ import { bindThis } from '@/decorators.js'; import type Logger from '@/logger.js'; import { validateContentTypeSetAsActivityPub } from '@/core/activitypub/misc/validator.js'; import type { IObject, IObjectWithId } from './type.js'; +import type { Cheerio, CheerioAPI } from 'cheerio/slim'; +import type { AnyNode } from 'domhandler'; type Request = { url: string; @@ -219,53 +221,33 @@ export class ApRequestService { (contentType ?? '').split(';')[0].trimEnd().toLowerCase() === 'text/html' && _followAlternate === true ) { - const html = await res.text(); - const { window, happyDOM } = new Window({ - settings: { - disableJavaScriptEvaluation: true, - disableJavaScriptFileLoading: true, - disableCSSFileLoading: true, - disableComputedStyleRendering: true, - handleDisabledFileLoadingAsSuccess: true, - navigation: { - disableMainFrameNavigation: true, - disableChildFrameNavigation: true, - disableChildPageNavigation: true, - disableFallbackToSetURL: true, - }, - timer: { - maxTimeout: 0, - maxIntervalTime: 0, - maxIntervalIterations: 0, - }, - }, - }); - const document = window.document; + let alternate: Cheerio | null; try { - document.documentElement.innerHTML = html; + const html = await res.text(); + const document = cheerio(html); // Search for any matching value in priority order: // 1. Type=AP > Type=none > Type=anything // 2. Alternate > Canonical // 3. Page order (fallback) - const alternate = - document.querySelector('head > link[href][rel="alternate"][type="application/activity+json"]') ?? - document.querySelector('head > link[href][rel="canonical"][type="application/activity+json"]') ?? - document.querySelector('head > link[href][rel="alternate"]:not([type])') ?? - document.querySelector('head > link[href][rel="canonical"]:not([type])') ?? - document.querySelector('head > link[href][rel="alternate"]') ?? - document.querySelector('head > link[href][rel="canonical"]'); - - if (alternate) { - const href = alternate.getAttribute('href'); - if (href && this.apUtilityService.haveSameAuthority(url, href)) { - return await this.signedGet(href, user, allowAnonymous, false); - } - } + alternate = selectFirst(document, [ + 'head > link[href][rel="alternate"][type="application/activity+json"]', + 'head > link[href][rel="canonical"][type="application/activity+json"]', + 'head > link[href][rel="alternate"]:not([type])', + 'head > link[href][rel="canonical"]:not([type])', + 'head > link[href][rel="alternate"]', + 'head > link[href][rel="canonical"]', + ]); } catch { // something went wrong parsing the HTML, ignore the whole thing - } finally { - happyDOM.close().catch(err => {}); + alternate = null; + } + + if (alternate) { + const href = alternate.attr('href'); + if (href && this.apUtilityService.haveSameAuthority(url, href)) { + return await this.signedGet(href, user, allowAnonymous, false); + } } } //#endregion @@ -285,3 +267,14 @@ export class ApRequestService { return activity as IObjectWithId; } } + +function selectFirst($: CheerioAPI, selectors: string[]): Cheerio | null { + for (const selector of selectors) { + const selection = $(selector); + if (selection.length > 0) { + return selection; + } + } + + return null; +} diff --git a/packages/backend/src/core/activitypub/JsonLdService.ts b/packages/backend/src/core/activitypub/JsonLdService.ts index 1db5df6ad9..8f150ab201 100644 --- a/packages/backend/src/core/activitypub/JsonLdService.ts +++ b/packages/backend/src/core/activitypub/JsonLdService.ts @@ -8,26 +8,61 @@ import { Injectable } from '@nestjs/common'; import { UnrecoverableError } from 'bullmq'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { bindThis } from '@/decorators.js'; +import Logger from '@/logger.js'; +import { LoggerService } from '@/core/LoggerService.js'; import { StatusError } from '@/misc/status-error.js'; import { CONTEXT, PRELOADED_CONTEXTS } from './misc/contexts.js'; import { validateContentTypeSetAsJsonLD } from './misc/validator.js'; -import type { JsonLdDocument } from 'jsonld'; +import type { ContextDefinition, JsonLdDocument } from 'jsonld'; import type { JsonLd as JsonLdObject, RemoteDocument } from 'jsonld/jsonld-spec.js'; +// https://stackoverflow.com/a/66252656 +type RemoveIndex = { + [ K in keyof T as string extends K + ? never + : number extends K + ? never + : symbol extends K + ? never + : K + ] : T[K]; +}; + +export type Document = RemoveIndex; + +export type Signature = { + id?: string; + type: string; + creator: string; + domain?: string; + nonce: string; + created: string; + signatureValue: string; +}; + +export type Signed = T & { + signature: Signature; +}; + +export function isSigned(doc: T): doc is Signed { + return 'signature' in doc && typeof(doc.signature) === 'object'; +} + // RsaSignature2017 implementation is based on https://github.com/transmute-industries/RsaSignature2017 -class JsonLd { - public debug = false; - public preLoad = true; - public loderTimeout = 5000; +@Injectable() +export class JsonLdService { + private readonly logger: Logger; constructor( private httpRequestService: HttpRequestService, + loggerService: LoggerService, ) { + this.logger = loggerService.getLogger('json-ld'); } @bindThis - public async signRsaSignature2017(data: any, privateKey: string, creator: string, domain?: string, created?: Date): Promise { + public async signRsaSignature2017(data: T, privateKey: string, creator: string, domain?: string, created?: Date): Promise> { const options: { type: string; creator: string; @@ -63,7 +98,7 @@ class JsonLd { } @bindThis - public async verifyRsaSignature2017(data: any, publicKey: string): Promise { + public async verifyRsaSignature2017(data: Signed, publicKey: string): Promise { const toBeSigned = await this.createVerifyData(data, data.signature); const verifier = crypto.createVerify('sha256'); verifier.update(toBeSigned); @@ -71,7 +106,7 @@ class JsonLd { } @bindThis - public async createVerifyData(data: any, options: any): Promise { + public async createVerifyData(data: T, options: Partial): Promise { const transformedOptions = { ...options, '@context': 'https://w3id.org/identity/v1', @@ -81,17 +116,18 @@ class JsonLd { delete transformedOptions['signatureValue']; const canonizedOptions = await this.normalize(transformedOptions); const optionsHash = this.sha256(canonizedOptions.toString()); - const transformedData = { ...data }; + const transformedData = { ...data } as T & { signature?: unknown }; delete transformedData['signature']; const cannonidedData = await this.normalize(transformedData); - if (this.debug) console.debug(`cannonidedData: ${cannonidedData}`); + this.logger.debug('cannonidedData', cannonidedData); const documentHash = this.sha256(cannonidedData.toString()); const verifyData = `${optionsHash}${documentHash}`; return verifyData; } @bindThis - public async compact(data: any, context: any = CONTEXT): Promise { + // TODO our default CONTEXT isn't valid for the library, is this a bug? + public async compact(data: Document, context: ContextDefinition = CONTEXT as unknown as ContextDefinition): Promise { const customLoader = this.getLoader(); // XXX: Importing jsonld dynamically since Jest frequently fails to import it statically // https://github.com/misskey-dev/misskey/pull/9894#discussion_r1103753595 @@ -101,7 +137,7 @@ class JsonLd { } @bindThis - public async normalize(data: JsonLdDocument): Promise { + public async normalize(data: Document): Promise { const customLoader = this.getLoader(); return (await import('jsonld')).default.normalize(data, { documentLoader: customLoader, @@ -113,9 +149,9 @@ class JsonLd { return async (url: string): Promise => { if (!/^https?:\/\//.test(url)) throw new UnrecoverableError(`Invalid URL: ${url}`); - if (this.preLoad) { + { if (url in PRELOADED_CONTEXTS) { - if (this.debug) console.debug(`HIT: ${url}`); + this.logger.debug(`Preload HIT: ${url}`); return { contextUrl: undefined, document: PRELOADED_CONTEXTS[url], @@ -124,7 +160,7 @@ class JsonLd { } } - if (this.debug) console.debug(`MISS: ${url}`); + this.logger.debug(`Preload MISS: ${url}`); const document = await this.fetchDocument(url); return { contextUrl: undefined, @@ -142,7 +178,6 @@ class JsonLd { headers: { Accept: 'application/ld+json, application/json', }, - timeout: this.loderTimeout, }, { throwErrorWhenResponseNotOk: false, @@ -166,16 +201,3 @@ class JsonLd { return hash.digest('hex'); } } - -@Injectable() -export class JsonLdService { - constructor( - private httpRequestService: HttpRequestService, - ) { - } - - @bindThis - public use(): JsonLd { - return new JsonLd(this.httpRequestService); - } -} diff --git a/packages/backend/src/core/activitypub/misc/contexts.ts b/packages/backend/src/core/activitypub/misc/contexts.ts index cedd1d8dd5..fa003b1791 100644 --- a/packages/backend/src/core/activitypub/misc/contexts.ts +++ b/packages/backend/src/core/activitypub/misc/contexts.ts @@ -540,6 +540,10 @@ const extension_context_definition = { quoteUrl: 'as:quoteUrl', fedibird: 'http://fedibird.com/ns#', quoteUri: 'fedibird:quoteUri', + quote: { + '@id': 'https://w3id.org/fep/044f#quote', + '@type': '@id', + }, // Mastodon toot: 'http://joinmastodon.org/ns#', Emoji: 'toot:Emoji', diff --git a/packages/backend/src/core/activitypub/models/ApNoteService.ts b/packages/backend/src/core/activitypub/models/ApNoteService.ts index 57d4303982..2a28405121 100644 --- a/packages/backend/src/core/activitypub/models/ApNoteService.ts +++ b/packages/backend/src/core/activitypub/models/ApNoteService.ts @@ -27,7 +27,7 @@ import { checkHttps } from '@/misc/check-https.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; import { isRetryableError } from '@/misc/is-retryable-error.js'; import { renderInlineError } from '@/misc/render-inline-error.js'; -import { getOneApId, getApId, validPost, isEmoji, getApType, isApObject, isDocument, IApDocument } from '../type.js'; +import { getOneApId, getApId, validPost, isEmoji, getApType, isApObject, isDocument, IApDocument, isLink } from '../type.js'; import { ApLoggerService } from '../ApLoggerService.js'; import { ApMfmService } from '../ApMfmService.js'; import { ApDbResolverService } from '../ApDbResolverService.js'; @@ -657,9 +657,29 @@ export class ApNoteService { */ private async getQuote(note: IPost, entryUri: string, resolver: Resolver): Promise { const quoteUris = new Set(); - if (note._misskey_quote) quoteUris.add(note._misskey_quote); - if (note.quoteUrl) quoteUris.add(note.quoteUrl); - if (note.quoteUri) quoteUris.add(note.quoteUri); + if (note._misskey_quote && typeof(note._misskey_quote as unknown) === 'string') quoteUris.add(note._misskey_quote); + if (note.quoteUrl && typeof(note.quoteUrl as unknown) === 'string') quoteUris.add(note.quoteUrl); + if (note.quoteUri && typeof(note.quoteUri as unknown) === 'string') quoteUris.add(note.quoteUri); + + // https://codeberg.org/fediverse/fep/src/branch/main/fep/044f/fep-044f.md + if (note.quote && typeof(note.quote as unknown) === 'string') quoteUris.add(note.quote); + + // https://codeberg.org/fediverse/fep/src/branch/main/fep/e232/fep-e232.md + const tags = toArray(note.tag).filter(tag => typeof(tag) === 'object' && isLink(tag)); + for (const tag of tags) { + if (!tag.href || typeof (tag.href as unknown) !== 'string') continue; + + const mediaTypes = toArray(tag.mediaType); + if ( + !mediaTypes.includes('application/ld+json; profile="https://www.w3.org/ns/activitystreams"') && + !mediaTypes.includes('application/activity+json') + ) continue; + + const rels = toArray(tag.rel); + if (!rels.includes('https://misskey-hub.net/ns#_misskey_quote')) continue; + + quoteUris.add(tag.href); + } // No quote, return undefined if (quoteUris.size < 1) return undefined; diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index b7aa036068..29f7459219 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -741,10 +741,17 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { this.hashtagService.updateUsertags(exist, tags); // 該当ユーザーが既にフォロワーになっていた場合はFollowingもアップデートする - await this.followingsRepository.update( - { followerId: exist.id }, - { followerSharedInbox: person.sharedInbox ?? person.endpoints?.sharedInbox ?? null }, - ); + if (exist.inbox !== person.inbox || exist.sharedInbox !== (person.sharedInbox ?? person.endpoints?.sharedInbox)) { + await this.followingsRepository.update( + { followerId: exist.id }, + { + followerInbox: person.inbox, + followerSharedInbox: person.sharedInbox ?? person.endpoints?.sharedInbox ?? null, + }, + ); + + await this.cacheService.refreshFollowRelationsFor(exist.id); + } await this.updateFeatured(exist.id, resolver).catch(err => { // Permanent error implies hidden or inaccessible, which is a normal thing. diff --git a/packages/backend/src/core/activitypub/type.ts b/packages/backend/src/core/activitypub/type.ts index cc7599d394..554420d670 100644 --- a/packages/backend/src/core/activitypub/type.ts +++ b/packages/backend/src/core/activitypub/type.ts @@ -35,6 +35,7 @@ export interface IObject { mediaType?: string; url?: ApObject | string; href?: string; + rel?: string | string[]; tag?: IObject | IObject[]; sensitive?: boolean; } @@ -55,6 +56,16 @@ export function isAnonymousObject(object: IObject): object is IAnonymousObject { return object.id === undefined; } +export interface ILink extends IObject { + '@context'?: string | string[] | Obj | Obj[]; + type: 'Link' | 'Mention'; + href: string; +} + +export const isLink = (object: IObject): object is ILink => + (getApType(object) === 'Link' || getApType(object) === 'Link') && + typeof object.href === 'string'; + /** * Get array of ActivityStreams Objects id */ @@ -204,6 +215,7 @@ export interface IPost extends IObject { _misskey_content?: string; quoteUrl?: string; quoteUri?: string; + quote?: string; updated?: string; } @@ -306,9 +318,8 @@ export const isPropertyValue = (object: IObject): object is IApPropertyValue => 'value' in object && typeof object.value === 'string'; -export interface IApMention extends IObject { +export interface IApMention extends ILink { type: 'Mention'; - href: string; name: string; } diff --git a/packages/backend/src/core/chart/charts/federation.ts b/packages/backend/src/core/chart/charts/federation.ts index b6db6f5454..4bbb5437cc 100644 --- a/packages/backend/src/core/chart/charts/federation.ts +++ b/packages/backend/src/core/chart/charts/federation.ts @@ -44,6 +44,7 @@ export default class FederationChart extends Chart { // eslint-di } protected async tickMinor(): Promise>> { + // TODO optimization: replace these with exists() const pubsubSubQuery = this.followingsRepository.createQueryBuilder('f') .select('f.followerHost') .where('f.followerHost IS NOT NULL'); diff --git a/packages/backend/src/core/chart/charts/per-user-following.ts b/packages/backend/src/core/chart/charts/per-user-following.ts index 588ac638de..8d75a30e9a 100644 --- a/packages/backend/src/core/chart/charts/per-user-following.ts +++ b/packages/backend/src/core/chart/charts/per-user-following.ts @@ -15,6 +15,7 @@ import Chart from '../core.js'; import { ChartLoggerService } from '../ChartLoggerService.js'; import { name, schema } from './entities/per-user-following.js'; import type { KVs } from '../core.js'; +import { CacheService } from '@/core/CacheService.js'; /** * ユーザーごとのフォローに関するチャート @@ -31,23 +32,25 @@ export default class PerUserFollowingChart extends Chart { // esl private appLockService: AppLockService, private userEntityService: UserEntityService, private chartLoggerService: ChartLoggerService, + private readonly cacheService: CacheService, ) { super(db, (k) => appLockService.getChartInsertLock(k), chartLoggerService.logger, name, schema, true); } protected async tickMajor(group: string): Promise>> { const [ - localFollowingsCount, - localFollowersCount, - remoteFollowingsCount, - remoteFollowersCount, + followees, + followers, ] = await Promise.all([ - this.followingsRepository.countBy({ followerId: group, followeeHost: IsNull() }), - this.followingsRepository.countBy({ followeeId: group, followerHost: IsNull() }), - this.followingsRepository.countBy({ followerId: group, followeeHost: Not(IsNull()) }), - this.followingsRepository.countBy({ followeeId: group, followerHost: Not(IsNull()) }), + this.cacheService.userFollowingsCache.fetch(group).then(fs => Array.from(fs.values())), + this.cacheService.userFollowersCache.fetch(group).then(fs => Array.from(fs.values())), ]); + const localFollowingsCount = followees.reduce((sum, f) => sum + (f.followeeHost == null ? 1 : 0), 0); + const localFollowersCount = followers.reduce((sum, f) => sum + (f.followerHost == null ? 1 : 0), 0); + const remoteFollowingsCount = followees.reduce((sum, f) => sum + (f.followeeHost == null ? 0 : 1), 0); + const remoteFollowersCount = followers.reduce((sum, f) => sum + (f.followerHost == null ? 0 : 1), 0); + return { 'local.followings.total': localFollowingsCount, 'local.followers.total': localFollowersCount, diff --git a/packages/backend/src/core/entities/NoteEntityService.ts b/packages/backend/src/core/entities/NoteEntityService.ts index d2373a70a2..4248fde77f 100644 --- a/packages/backend/src/core/entities/NoteEntityService.ts +++ b/packages/backend/src/core/entities/NoteEntityService.ts @@ -11,7 +11,7 @@ import type { Packed } from '@/misc/json-schema.js'; import { awaitAll } from '@/misc/prelude/await-all.js'; import type { MiUser } from '@/models/User.js'; import type { MiNote } from '@/models/Note.js'; -import type { UsersRepository, NotesRepository, FollowingsRepository, PollsRepository, PollVotesRepository, NoteReactionsRepository, ChannelsRepository, MiMeta } from '@/models/_.js'; +import type { UsersRepository, NotesRepository, FollowingsRepository, PollsRepository, PollVotesRepository, NoteReactionsRepository, ChannelsRepository, MiMeta, MiPollVote, MiPoll, MiChannel, MiFollowing } from '@/models/_.js'; import { bindThis } from '@/decorators.js'; import { DebounceLoader } from '@/misc/loader.js'; import { IdService } from '@/core/IdService.js'; @@ -26,13 +26,13 @@ import type { UserEntityService } from './UserEntityService.js'; import type { DriveFileEntityService } from './DriveFileEntityService.js'; // is-renote.tsとよしなにリンク -function isPureRenote(note: MiNote): note is MiNote & { renoteId: MiNote['id']; renote: MiNote } { +function isPureRenote(note: MiNote): note is MiNote & { renoteId: MiNote['id'] } { return ( - note.renote != null && - note.reply == null && + note.renoteId != null && + note.replyId == null && note.text == null && note.cw == null && - (note.fileIds == null || note.fileIds.length === 0) && + note.fileIds.length === 0 && !note.hasPoll ); } @@ -132,7 +132,10 @@ export class NoteEntityService implements OnModuleInit { } @bindThis - public async hideNote(packedNote: Packed<'Note'>, meId: MiUser['id'] | null): Promise { + public async hideNote(packedNote: Packed<'Note'>, meId: MiUser['id'] | null, hint?: { + myFollowing?: ReadonlyMap, + myBlockers?: ReadonlySet, + }): Promise { if (meId === packedNote.userId) return; // TODO: isVisibleForMe を使うようにしても良さそう(型違うけど) @@ -188,14 +191,9 @@ export class NoteEntityService implements OnModuleInit { } else if (packedNote.renote && (meId === packedNote.renote.userId)) { hide = false; } else { - // フォロワーかどうか - // TODO: 当関数呼び出しごとにクエリが走るのは重そうだからなんとかする - const isFollowing = await this.followingsRepository.exists({ - where: { - followeeId: packedNote.userId, - followerId: meId, - }, - }); + const isFollowing = hint?.myFollowing + ? hint.myFollowing.has(packedNote.userId) + : (await this.cacheService.userFollowingsCache.fetch(meId)).has(packedNote.userId); hide = !isFollowing; } @@ -211,7 +209,8 @@ export class NoteEntityService implements OnModuleInit { } if (!hide && meId && packedNote.userId !== meId) { - const isBlocked = (await this.cacheService.userBlockedCache.fetch(meId)).has(packedNote.userId); + const blockers = hint?.myBlockers ?? await this.cacheService.userBlockedCache.fetch(meId); + const isBlocked = blockers.has(packedNote.userId); if (isBlocked) hide = true; } @@ -235,8 +234,11 @@ export class NoteEntityService implements OnModuleInit { } @bindThis - private async populatePoll(note: MiNote, meId: MiUser['id'] | null) { - const poll = await this.pollsRepository.findOneByOrFail({ noteId: note.id }); + private async populatePoll(note: MiNote, meId: MiUser['id'] | null, hint?: { + poll?: MiPoll, + myVotes?: MiPollVote[], + }) { + const poll = hint?.poll ?? await this.pollsRepository.findOneByOrFail({ noteId: note.id }); const choices = poll.choices.map(c => ({ text: c, votes: poll.votes[poll.choices.indexOf(c)], @@ -245,7 +247,7 @@ export class NoteEntityService implements OnModuleInit { if (meId) { if (poll.multiple) { - const votes = await this.pollVotesRepository.findBy({ + const votes = hint?.myVotes ?? await this.pollVotesRepository.findBy({ userId: meId, noteId: note.id, }); @@ -255,7 +257,7 @@ export class NoteEntityService implements OnModuleInit { choices[myChoice].isVoted = true; } } else { - const vote = await this.pollVotesRepository.findOneBy({ + const vote = hint?.myVotes ? hint.myVotes[0] : await this.pollVotesRepository.findOneBy({ userId: meId, noteId: note.id, }); @@ -317,7 +319,12 @@ export class NoteEntityService implements OnModuleInit { } @bindThis - public async isVisibleForMe(note: MiNote, meId: MiUser['id'] | null): Promise { + public async isVisibleForMe(note: MiNote, meId: MiUser['id'] | null, hint?: { + myFollowing?: ReadonlySet, + myBlocking?: ReadonlySet, + myBlockers?: ReadonlySet, + me?: Pick | null, + }): Promise { // This code must always be synchronized with the checks in generateVisibilityQuery. // visibility が specified かつ自分が指定されていなかったら非表示 if (note.visibility === 'specified') { @@ -345,16 +352,16 @@ export class NoteEntityService implements OnModuleInit { return true; } else { // フォロワーかどうか - const [blocked, following, user] = await Promise.all([ - this.cacheService.userBlockingCache.fetch(meId).then((ids) => ids.has(note.userId)), - this.followingsRepository.count({ - where: { - followeeId: note.userId, - followerId: meId, - }, - take: 1, - }), - this.usersRepository.findOneByOrFail({ id: meId }), + const [blocked, following, userHost] = await Promise.all([ + hint?.myBlocking + ? hint.myBlocking.has(note.userId) + : this.cacheService.userBlockingCache.fetch(meId).then((ids) => ids.has(note.userId)), + hint?.myFollowing + ? hint.myFollowing.has(note.userId) + : this.cacheService.userFollowingsCache.fetch(meId).then(ids => ids.has(note.userId)), + hint?.me !== undefined + ? (hint.me?.host ?? null) + : this.cacheService.findUserById(meId).then(me => me.host), ]); if (blocked) return false; @@ -366,12 +373,13 @@ export class NoteEntityService implements OnModuleInit { in which case we can never know the following. Instead we have to assume that the users are following each other. */ - return following > 0 || (note.userHost != null && user.host != null); + return following || (note.userHost != null && userHost != null); } } if (meId != null) { - const isBlocked = (await this.cacheService.userBlockedCache.fetch(meId)).has(note.userId); + const blockers = hint?.myBlockers ?? await this.cacheService.userBlockedCache.fetch(meId); + const isBlocked = blockers.has(note.userId); if (isBlocked) return false; } @@ -408,6 +416,12 @@ export class NoteEntityService implements OnModuleInit { packedFiles: Map | null>; packedUsers: Map>; mentionHandles: Record; + userFollowings: Map>>; + userBlockers: Map>; + polls: Map; + pollVotes: Map>; + channels: Map; + notes: Map; }; }, ): Promise> { @@ -437,9 +451,7 @@ export class NoteEntityService implements OnModuleInit { } const channel = note.channelId - ? note.channel - ? note.channel - : await this.channelsRepository.findOneBy({ id: note.channelId }) + ? (opts._hint_?.channels.get(note.channelId) ?? note.channel ?? await this.channelsRepository.findOneBy({ id: note.channelId })) : null; const reactionEmojiNames = Object.keys(reactions) @@ -485,7 +497,10 @@ export class NoteEntityService implements OnModuleInit { mentionHandles: note.mentions.length > 0 ? this.getUserHandles(note.mentions, options?._hint_?.mentionHandles) : undefined, uri: note.uri ?? undefined, url: note.url ?? undefined, - poll: note.hasPoll ? this.populatePoll(note, meId) : undefined, + poll: note.hasPoll ? this.populatePoll(note, meId, { + poll: opts._hint_?.polls.get(note.id), + myVotes: opts._hint_?.pollVotes.get(note.id)?.get(note.userId), + }) : undefined, ...(meId && Object.keys(reactions).length > 0 ? { myReaction: this.populateMyReaction({ @@ -499,14 +514,14 @@ export class NoteEntityService implements OnModuleInit { clippedCount: note.clippedCount, processErrors: note.processErrors, - reply: note.replyId ? this.pack(note.reply ?? note.replyId, me, { + reply: note.replyId ? this.pack(note.reply ?? opts._hint_?.notes.get(note.replyId) ?? note.replyId, me, { detail: false, skipHide: opts.skipHide, withReactionAndUserPairCache: opts.withReactionAndUserPairCache, _hint_: options?._hint_, }) : undefined, - renote: note.renoteId ? this.pack(note.renote ?? note.renoteId, me, { + renote: note.renoteId ? this.pack(note.renote ?? opts._hint_?.notes.get(note.renoteId) ?? note.renoteId, me, { detail: true, skipHide: opts.skipHide, withReactionAndUserPairCache: opts.withReactionAndUserPairCache, @@ -518,7 +533,10 @@ export class NoteEntityService implements OnModuleInit { this.treatVisibility(packed); if (!opts.skipHide) { - await this.hideNote(packed, meId); + await this.hideNote(packed, meId, meId == null ? undefined : { + myFollowing: opts._hint_?.userFollowings.get(meId), + myBlockers: opts._hint_?.userBlockers.get(meId), + }); } return packed; @@ -535,79 +553,139 @@ export class NoteEntityService implements OnModuleInit { ) { if (notes.length === 0) return []; - const targetNotes: MiNote[] = []; + const targetNotesMap = new Map(); + const targetNotesToFetch : string[] = []; for (const note of notes) { if (isPureRenote(note)) { // we may need to fetch 'my reaction' for renote target. - targetNotes.push(note.renote); - if (note.renote.reply) { - // idem if the renote is also a reply. - targetNotes.push(note.renote.reply); + if (note.renote) { + targetNotesMap.set(note.renote.id, note.renote); + if (note.renote.reply) { + // idem if the renote is also a reply. + targetNotesMap.set(note.renote.reply.id, note.renote.reply); + } + } else if (options?.detail) { + targetNotesToFetch.push(note.renoteId); } } else { if (note.reply) { // idem for OP of a regular reply. - targetNotes.push(note.reply); + targetNotesMap.set(note.reply.id, note.reply); + } else if (note.replyId && options?.detail) { + targetNotesToFetch.push(note.replyId); } - targetNotes.push(note); + targetNotesMap.set(note.id, note); } } - const bufferedReactions = this.meta.enableReactionsBuffering ? await this.reactionsBufferingService.getMany([...getAppearNoteIds(notes)]) : null; + // Don't fetch notes that were added by ID and then found inline in another note. + for (let i = targetNotesToFetch.length - 1; i >= 0; i--) { + if (targetNotesMap.has(targetNotesToFetch[i])) { + targetNotesToFetch.splice(i, 1); + } + } - const meId = me ? me.id : null; - const myReactionsMap = new Map(); - if (meId) { - const idsNeedFetchMyReaction = new Set(); + // Populate any relations that weren't included in the source + if (targetNotesToFetch.length > 0) { + const newNotes = await this.notesRepository.find({ + where: { + id: In(targetNotesToFetch), + }, + relations: { + user: { + userProfile: true, + }, + reply: { + user: { + userProfile: true, + }, + }, + renote: { + user: { + userProfile: true, + }, + reply: { + user: { + userProfile: true, + }, + }, + }, + channel: true, + }, + }); - for (const note of targetNotes) { - const reactionsCount = Object.values(this.reactionsBufferingService.mergeReactions(note.reactions, bufferedReactions?.get(note.id)?.deltas ?? {})).reduce((a, b) => a + b, 0); - if (reactionsCount === 0) { - myReactionsMap.set(note.id, null); - } else if (reactionsCount <= note.reactionAndUserPairCache.length + (bufferedReactions?.get(note.id)?.pairs.length ?? 0)) { - const pairInBuffer = bufferedReactions?.get(note.id)?.pairs.find(p => p[0] === meId); - if (pairInBuffer) { - myReactionsMap.set(note.id, pairInBuffer[1]); - } else { - const pair = note.reactionAndUserPairCache.find(p => p.startsWith(meId)); - myReactionsMap.set(note.id, pair ? pair.split('/')[1] : null); + for (const note of newNotes) { + targetNotesMap.set(note.id, note); + } + } + + const targetNotes = Array.from(targetNotesMap.values()); + const noteIds = Array.from(targetNotesMap.keys()); + + const usersMap = new Map(); + const allUsers = notes.flatMap(note => [ + note.user ?? note.userId, + note.reply?.user ?? note.replyUserId, + note.renote?.user ?? note.renoteUserId, + ]); + + for (const user of allUsers) { + if (!user) continue; + + if (typeof(user) === 'object') { + // ID -> Entity + usersMap.set(user.id, user); + } else if (!usersMap.has(user)) { + // ID -> ID + usersMap.set(user, user); + } + } + + const users = Array.from(usersMap.values()); + const userIds = Array.from(usersMap.keys()); + + const fileIds = new Set(targetNotes.flatMap(n => n.fileIds)); + const mentionedUsers = new Set(targetNotes.flatMap(note => note.mentions)); + + const [{ bufferedReactions, myReactionsMap }, packedFiles, packedUsers, mentionHandles, userFollowings, userBlockers, polls, pollVotes, channels] = await Promise.all([ + // bufferedReactions & myReactionsMap + this.getReactions(targetNotes, me), + // packedFiles + this.driveFileEntityService.packManyByIdsMap(Array.from(fileIds)), + // packedUsers + this.userEntityService.packMany(users, me) + .then(users => new Map(users.map(u => [u.id, u]))), + // mentionHandles + this.getUserHandles(Array.from(mentionedUsers)), + // userFollowings + this.cacheService.userFollowingsCache.fetchMany(userIds).then(fs => new Map(fs)), + // userBlockers + this.cacheService.userBlockedCache.fetchMany(userIds).then(bs => new Map(bs)), + // polls + this.pollsRepository.findBy({ noteId: In(noteIds) }) + .then(polls => new Map(polls.map(p => [p.noteId, p]))), + // pollVotes + this.pollVotesRepository.findBy({ noteId: In(noteIds), userId: In(userIds) }) + .then(votes => votes.reduce((noteMap, vote) => { + let userMap = noteMap.get(vote.noteId); + if (!userMap) { + userMap = new Map(); + noteMap.set(vote.noteId, userMap); } - } else { - idsNeedFetchMyReaction.add(note.id); - } - } - - const myReactions = idsNeedFetchMyReaction.size > 0 ? await this.noteReactionsRepository.findBy({ - userId: meId, - noteId: In(Array.from(idsNeedFetchMyReaction)), - }) : []; - - for (const id of idsNeedFetchMyReaction) { - myReactionsMap.set(id, myReactions.find(reaction => reaction.noteId === id)?.reaction ?? null); - } - } - - await this.customEmojiService.prefetchEmojis(this.aggregateNoteEmojis(notes)); - // TODO: 本当は renote とか reply がないのに renoteId とか replyId があったらここで解決しておく - const fileIds = notes.map(n => [n.fileIds, n.renote?.fileIds, n.reply?.fileIds]).flat(2).filter(x => x != null); - const packedFiles = fileIds.length > 0 ? await this.driveFileEntityService.packManyByIdsMap(fileIds) : new Map(); - const users = [ - ...notes.map(({ user, userId }) => user ?? userId), - ...notes.map(({ replyUserId }) => replyUserId).filter(x => x != null), - ...notes.map(({ renoteUserId }) => renoteUserId).filter(x => x != null), - ]; - const packedUsers = await this.userEntityService.packMany(users, me) - .then(users => new Map(users.map(u => [u.id, u]))); - - // Recursively add all mentioned users from all notes + replies + renotes - const allMentionedUsers = targetNotes.reduce((users, note) => { - for (const user of note.mentions) { - users.add(user); - } - return users; - }, new Set()); - const mentionHandles = await this.getUserHandles(Array.from(allMentionedUsers)); + let voteList = userMap.get(vote.userId); + if (!voteList) { + voteList = []; + userMap.set(vote.userId, voteList); + } + voteList.push(vote); + return noteMap; + }, new Map>)), + // channels + this.getChannels(targetNotes), + // (not returned) + this.customEmojiService.prefetchEmojis(this.aggregateNoteEmojis(notes)), + ]); return await Promise.all(notes.map(n => this.pack(n, me, { ...options, @@ -617,6 +695,12 @@ export class NoteEntityService implements OnModuleInit { packedFiles, packedUsers, mentionHandles, + userFollowings, + userBlockers, + polls, + pollVotes, + channels, + notes: new Map(targetNotes.map(n => [n.id, n])), }, }))); } @@ -685,6 +769,68 @@ export class NoteEntityService implements OnModuleInit { }, {} as Record); } + private async getChannels(notes: MiNote[]): Promise> { + const channels = new Map(); + const channelsToFetch = new Set(); + + for (const note of notes) { + if (note.channel) { + channels.set(note.channel.id, note.channel); + } else if (note.channelId) { + channelsToFetch.add(note.channelId); + } + } + + if (channelsToFetch.size > 0) { + const newChannels = await this.channelsRepository.findBy({ + id: In(Array.from(channelsToFetch)), + }); + for (const channel of newChannels) { + channels.set(channel.id, channel); + } + } + + return channels; + } + + private async getReactions(notes: MiNote[], me: { id: string } | null | undefined) { + const bufferedReactions = this.meta.enableReactionsBuffering ? await this.reactionsBufferingService.getMany([...getAppearNoteIds(notes)]) : null; + + const meId = me ? me.id : null; + const myReactionsMap = new Map(); + if (meId) { + const idsNeedFetchMyReaction = new Set(); + + for (const note of notes) { + const reactionsCount = Object.values(this.reactionsBufferingService.mergeReactions(note.reactions, bufferedReactions?.get(note.id)?.deltas ?? {})).reduce((a, b) => a + b, 0); + if (reactionsCount === 0) { + myReactionsMap.set(note.id, null); + } else if (reactionsCount <= note.reactionAndUserPairCache.length + (bufferedReactions?.get(note.id)?.pairs.length ?? 0)) { + const pairInBuffer = bufferedReactions?.get(note.id)?.pairs.find(p => p[0] === meId); + if (pairInBuffer) { + myReactionsMap.set(note.id, pairInBuffer[1]); + } else { + const pair = note.reactionAndUserPairCache.find(p => p.startsWith(meId)); + myReactionsMap.set(note.id, pair ? pair.split('/')[1] : null); + } + } else { + idsNeedFetchMyReaction.add(note.id); + } + } + + const myReactions = idsNeedFetchMyReaction.size > 0 ? await this.noteReactionsRepository.findBy({ + userId: meId, + noteId: In(Array.from(idsNeedFetchMyReaction)), + }) : []; + + for (const id of idsNeedFetchMyReaction) { + myReactionsMap.set(id, myReactions.find(reaction => reaction.noteId === id)?.reaction ?? null); + } + } + + return { bufferedReactions, myReactionsMap }; + } + @bindThis public genLocalNoteUri(noteId: string): string { return `${this.config.url}/notes/${noteId}`; diff --git a/packages/backend/src/core/entities/UserEntityService.ts b/packages/backend/src/core/entities/UserEntityService.ts index 326baaefd4..91bf258ff4 100644 --- a/packages/backend/src/core/entities/UserEntityService.ts +++ b/packages/backend/src/core/entities/UserEntityService.ts @@ -30,6 +30,7 @@ import type { FollowingsRepository, FollowRequestsRepository, MiFollowing, + MiInstance, MiMeta, MiUserNotePining, MiUserProfile, @@ -42,7 +43,7 @@ import type { UsersRepository, } from '@/models/_.js'; import { bindThis } from '@/decorators.js'; -import { RoleService } from '@/core/RoleService.js'; +import { RolePolicies, RoleService } from '@/core/RoleService.js'; import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; import { IdService } from '@/core/IdService.js'; @@ -52,6 +53,7 @@ import { AvatarDecorationService } from '@/core/AvatarDecorationService.js'; import { ChatService } from '@/core/ChatService.js'; import { isSystemAccount } from '@/misc/is-system-account.js'; import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js'; +import type { CacheService } from '@/core/CacheService.js'; import type { OnModuleInit } from '@nestjs/common'; import type { NoteEntityService } from './NoteEntityService.js'; import type { PageEntityService } from './PageEntityService.js'; @@ -77,7 +79,7 @@ function isRemoteUser(user: MiUser | { host: MiUser['host'] }): boolean { export type UserRelation = { id: MiUser['id'] - following: MiFollowing | null, + following: Omit | null, isFollowing: boolean isFollowed: boolean hasPendingFollowRequestFromYou: boolean @@ -103,6 +105,7 @@ export class UserEntityService implements OnModuleInit { private idService: IdService; private avatarDecorationService: AvatarDecorationService; private chatService: ChatService; + private cacheService: CacheService; constructor( private moduleRef: ModuleRef, @@ -163,6 +166,7 @@ export class UserEntityService implements OnModuleInit { this.idService = this.moduleRef.get('IdService'); this.avatarDecorationService = this.moduleRef.get('AvatarDecorationService'); this.chatService = this.moduleRef.get('ChatService'); + this.cacheService = this.moduleRef.get('CacheService'); } //#region Validators @@ -193,16 +197,8 @@ export class UserEntityService implements OnModuleInit { memo, mutedInstances, ] = await Promise.all([ - this.followingsRepository.findOneBy({ - followerId: me, - followeeId: target, - }), - this.followingsRepository.exists({ - where: { - followerId: target, - followeeId: me, - }, - }), + this.cacheService.userFollowingsCache.fetch(me).then(f => f.get(target) ?? null), + this.cacheService.userFollowingsCache.fetch(target).then(f => f.has(me)), this.followRequestsRepository.exists({ where: { followerId: me, @@ -215,45 +211,22 @@ export class UserEntityService implements OnModuleInit { followeeId: me, }, }), - this.blockingsRepository.exists({ - where: { - blockerId: me, - blockeeId: target, - }, - }), - this.blockingsRepository.exists({ - where: { - blockerId: target, - blockeeId: me, - }, - }), - this.mutingsRepository.exists({ - where: { - muterId: me, - muteeId: target, - }, - }), - this.renoteMutingsRepository.exists({ - where: { - muterId: me, - muteeId: target, - }, - }), - this.usersRepository.createQueryBuilder('u') - .select('u.host') - .where({ id: target }) - .getRawOne<{ u_host: string }>() - .then(it => it?.u_host ?? null), + this.cacheService.userBlockingCache.fetch(me) + .then(blockees => blockees.has(target)), + this.cacheService.userBlockedCache.fetch(me) + .then(blockers => blockers.has(target)), + this.cacheService.userMutingsCache.fetch(me) + .then(mutings => mutings.has(target)), + this.cacheService.renoteMutingsCache.fetch(me) + .then(mutings => mutings.has(target)), + this.cacheService.findUserById(target).then(u => u.host), this.userMemosRepository.createQueryBuilder('m') .select('m.memo') .where({ userId: me, targetUserId: target }) .getRawOne<{ m_memo: string | null }>() .then(it => it?.m_memo ?? null), - this.userProfilesRepository.createQueryBuilder('p') - .select('p.mutedInstances') - .where({ userId: me }) - .getRawOne<{ p_mutedInstances: string[] }>() - .then(it => it?.p_mutedInstances ?? []), + this.cacheService.userProfileCache.fetch(me) + .then(profile => profile.mutedInstances), ]); const isInstanceMuted = !!host && mutedInstances.includes(host); @@ -277,8 +250,8 @@ export class UserEntityService implements OnModuleInit { @bindThis public async getRelations(me: MiUser['id'], targets: MiUser['id'][]): Promise> { const [ - followers, - followees, + myFollowing, + myFollowers, followersRequests, followeesRequests, blockers, @@ -289,13 +262,8 @@ export class UserEntityService implements OnModuleInit { memos, mutedInstances, ] = await Promise.all([ - this.followingsRepository.findBy({ followerId: me }) - .then(f => new Map(f.map(it => [it.followeeId, it]))), - this.followingsRepository.createQueryBuilder('f') - .select('f.followerId') - .where('f.followeeId = :me', { me }) - .getRawMany<{ f_followerId: string }>() - .then(it => it.map(it => it.f_followerId)), + this.cacheService.userFollowingsCache.fetch(me), + this.cacheService.userFollowersCache.fetch(me), this.followRequestsRepository.createQueryBuilder('f') .select('f.followeeId') .where('f.followerId = :me', { me }) @@ -306,34 +274,18 @@ export class UserEntityService implements OnModuleInit { .where('f.followeeId = :me', { me }) .getRawMany<{ f_followerId: string }>() .then(it => it.map(it => it.f_followerId)), - this.blockingsRepository.createQueryBuilder('b') - .select('b.blockeeId') - .where('b.blockerId = :me', { me }) - .getRawMany<{ b_blockeeId: string }>() - .then(it => it.map(it => it.b_blockeeId)), - this.blockingsRepository.createQueryBuilder('b') - .select('b.blockerId') - .where('b.blockeeId = :me', { me }) - .getRawMany<{ b_blockerId: string }>() - .then(it => it.map(it => it.b_blockerId)), - this.mutingsRepository.createQueryBuilder('m') - .select('m.muteeId') - .where('m.muterId = :me', { me }) - .getRawMany<{ m_muteeId: string }>() - .then(it => it.map(it => it.m_muteeId)), - this.renoteMutingsRepository.createQueryBuilder('m') - .select('m.muteeId') - .where('m.muterId = :me', { me }) - .getRawMany<{ m_muteeId: string }>() - .then(it => it.map(it => it.m_muteeId)), - this.usersRepository.createQueryBuilder('u') - .select(['u.id', 'u.host']) - .where({ id: In(targets) } ) - .getRawMany<{ m_id: string, m_host: string }>() - .then(it => it.reduce((map, it) => { - map[it.m_id] = it.m_host; - return map; - }, {} as Record)), + this.cacheService.userBlockedCache.fetch(me), + this.cacheService.userBlockingCache.fetch(me), + this.cacheService.userMutingsCache.fetch(me), + this.cacheService.renoteMutingsCache.fetch(me), + this.cacheService.getUsers(targets) + .then(users => { + const record: Record = {}; + for (const [id, user] of users) { + record[id] = user.host; + } + return record; + }), this.userMemosRepository.createQueryBuilder('m') .select(['m.targetUserId', 'm.memo']) .where({ userId: me, targetUserId: In(targets) }) @@ -342,16 +294,13 @@ export class UserEntityService implements OnModuleInit { map[it.m_targetUserId] = it.m_memo; return map; }, {} as Record)), - this.userProfilesRepository.createQueryBuilder('p') - .select('p.mutedInstances') - .where({ userId: me }) - .getRawOne<{ p_mutedInstances: string[] }>() - .then(it => it?.p_mutedInstances ?? []), + this.cacheService.userProfileCache.fetch(me) + .then(p => p.mutedInstances), ]); return new Map( targets.map(target => { - const following = followers.get(target) ?? null; + const following = myFollowing.get(target) ?? null; return [ target, @@ -359,14 +308,14 @@ export class UserEntityService implements OnModuleInit { id: target, following: following, isFollowing: following != null, - isFollowed: followees.includes(target), + isFollowed: myFollowers.has(target), hasPendingFollowRequestFromYou: followersRequests.includes(target), hasPendingFollowRequestToYou: followeesRequests.includes(target), - isBlocking: blockers.includes(target), - isBlocked: blockees.includes(target), - isMuted: muters.includes(target), - isRenoteMuted: renoteMuters.includes(target), - isInstanceMuted: mutedInstances.includes(hosts[target]), + isBlocking: blockees.has(target), + isBlocked: blockers.has(target), + isMuted: muters.has(target), + isRenoteMuted: renoteMuters.has(target), + isInstanceMuted: hosts[target] != null && mutedInstances.includes(hosts[target]), memo: memos[target] ?? null, }, ]; @@ -391,6 +340,7 @@ export class UserEntityService implements OnModuleInit { return false; // TODO } + // TODO optimization: make redis calls in MULTI @bindThis public async getNotificationsInfo(userId: MiUser['id']): Promise<{ hasUnread: boolean; @@ -424,16 +374,14 @@ export class UserEntityService implements OnModuleInit { @bindThis public async getHasPendingReceivedFollowRequest(userId: MiUser['id']): Promise { - const count = await this.followRequestsRepository.countBy({ + return await this.followRequestsRepository.existsBy({ followeeId: userId, }); - - return count > 0; } @bindThis public async getHasPendingSentFollowRequest(userId: MiUser['id']): Promise { - return this.followRequestsRepository.existsBy({ + return await this.followRequestsRepository.existsBy({ followerId: userId, }); } @@ -480,6 +428,12 @@ export class UserEntityService implements OnModuleInit { userRelations?: Map, userMemos?: Map, pinNotes?: Map, + iAmModerator?: boolean, + userIdsByUri?: Map, + instances?: Map, + securityKeyCounts?: Map, + pendingReceivedFollows?: Set, + pendingSentFollows?: Set, }, ): Promise> { const opts = Object.assign({ @@ -521,7 +475,7 @@ export class UserEntityService implements OnModuleInit { const isDetailed = opts.schema !== 'UserLite'; const meId = me ? me.id : null; const isMe = meId === user.id; - const iAmModerator = me ? await this.roleService.isModerator(me as MiUser) : false; + const iAmModerator = opts.iAmModerator ?? (me ? await this.roleService.isModerator(me as MiUser) : false); const profile = isDetailed ? (opts.userProfile ?? user.userProfile ?? await this.userProfilesRepository.findOneByOrFail({ userId: user.id })) @@ -582,6 +536,9 @@ export class UserEntityService implements OnModuleInit { const checkHost = user.host == null ? this.config.host : user.host; const notificationsInfo = isMe && isDetailed ? await this.getNotificationsInfo(user.id) : null; + let fetchPoliciesPromise: Promise | null = null; + const fetchPolicies = () => fetchPoliciesPromise ??= this.roleService.getUserPolicies(user); + const packed = { id: user.id, name: user.name, @@ -607,13 +564,13 @@ export class UserEntityService implements OnModuleInit { mandatoryCW: user.mandatoryCW, rejectQuotes: user.rejectQuotes, attributionDomains: user.attributionDomains, - isSilenced: user.isSilenced || this.roleService.getUserPolicies(user.id).then(r => !r.canPublicNote), + isSilenced: user.isSilenced || fetchPolicies().then(r => !r.canPublicNote), speakAsCat: user.speakAsCat ?? false, approved: user.approved, requireSigninToViewContents: user.requireSigninToViewContents === false ? undefined : true, makeNotesFollowersOnlyBefore: user.makeNotesFollowersOnlyBefore ?? undefined, makeNotesHiddenBefore: user.makeNotesHiddenBefore ?? undefined, - instance: user.host ? this.federatedInstanceService.fetch(user.host).then(instance => instance ? { + instance: user.host ? Promise.resolve(opts.instances?.has(user.host) ? opts.instances.get(user.host) : this.federatedInstanceService.fetch(user.host)).then(instance => instance ? { name: instance.name, softwareName: instance.softwareName, softwareVersion: instance.softwareVersion, @@ -628,7 +585,7 @@ export class UserEntityService implements OnModuleInit { emojis: this.customEmojiService.populateEmojis(user.emojis, checkHost), onlineStatus: this.getOnlineStatus(user), // パフォーマンス上の理由でローカルユーザーのみ - badgeRoles: user.host == null ? this.roleService.getUserBadgeRoles(user.id).then((rs) => rs + badgeRoles: user.host == null ? this.roleService.getUserBadgeRoles(user).then((rs) => rs .filter((r) => r.isPublic || iAmModerator) .sort((a, b) => b.displayOrder - a.displayOrder) .map((r) => ({ @@ -641,9 +598,9 @@ export class UserEntityService implements OnModuleInit { ...(isDetailed ? { url: profile!.url, uri: user.uri, - movedTo: user.movedToUri ? this.apPersonService.resolvePerson(user.movedToUri).then(user => user.id).catch(() => null) : null, + movedTo: user.movedToUri ? Promise.resolve(opts.userIdsByUri?.get(user.movedToUri) ?? this.apPersonService.resolvePerson(user.movedToUri).then(user => user.id).catch(() => null)) : null, alsoKnownAs: user.alsoKnownAs - ? Promise.all(user.alsoKnownAs.map(uri => this.apPersonService.fetchPerson(uri).then(user => user?.id).catch(() => null))) + ? Promise.all(user.alsoKnownAs.map(uri => Promise.resolve(opts.userIdsByUri?.get(uri) ?? this.apPersonService.fetchPerson(uri).then(user => user?.id).catch(() => null)))) .then(xs => xs.length === 0 ? null : xs.filter(x => x != null)) : null, updatedAt: user.updatedAt ? user.updatedAt.toISOString() : null, @@ -670,8 +627,8 @@ export class UserEntityService implements OnModuleInit { followersVisibility: profile!.followersVisibility, followingVisibility: profile!.followingVisibility, chatScope: user.chatScope, - canChat: this.roleService.getUserPolicies(user.id).then(r => r.chatAvailability === 'available'), - roles: this.roleService.getUserRoles(user.id).then(roles => roles.filter(role => role.isPublic).sort((a, b) => b.displayOrder - a.displayOrder).map(role => ({ + canChat: fetchPolicies().then(r => r.chatAvailability === 'available'), + roles: this.roleService.getUserRoles(user).then(roles => roles.filter(role => role.isPublic).sort((a, b) => b.displayOrder - a.displayOrder).map(role => ({ id: role.id, name: role.name, color: role.color, @@ -689,7 +646,7 @@ export class UserEntityService implements OnModuleInit { twoFactorEnabled: profile!.twoFactorEnabled, usePasswordLessLogin: profile!.usePasswordLessLogin, securityKeys: profile!.twoFactorEnabled - ? this.userSecurityKeysRepository.countBy({ userId: user.id }).then(result => result >= 1) + ? Promise.resolve(opts.securityKeyCounts?.get(user.id) ?? this.userSecurityKeysRepository.countBy({ userId: user.id })).then(result => result >= 1) : false, } : {}), @@ -722,8 +679,8 @@ export class UserEntityService implements OnModuleInit { hasUnreadAntenna: this.getHasUnreadAntenna(user.id), hasUnreadChannel: false, // 後方互換性のため hasUnreadNotification: notificationsInfo?.hasUnread, // 後方互換性のため - hasPendingReceivedFollowRequest: this.getHasPendingReceivedFollowRequest(user.id), - hasPendingSentFollowRequest: this.getHasPendingSentFollowRequest(user.id), + hasPendingReceivedFollowRequest: opts.pendingReceivedFollows?.has(user.id) ?? this.getHasPendingReceivedFollowRequest(user.id), + hasPendingSentFollowRequest: opts.pendingSentFollows?.has(user.id) ?? this.getHasPendingSentFollowRequest(user.id), unreadNotificationsCount: notificationsInfo?.unreadCount, mutedWords: profile!.mutedWords, hardMutedWords: profile!.hardMutedWords, @@ -733,7 +690,7 @@ export class UserEntityService implements OnModuleInit { emailNotificationTypes: profile!.emailNotificationTypes, achievements: profile!.achievements, loggedInDays: profile!.loggedInDates.length, - policies: this.roleService.getUserPolicies(user.id), + policies: fetchPolicies(), defaultCW: profile!.defaultCW, defaultCWPriority: profile!.defaultCWPriority, allowUnsignedFetch: user.allowUnsignedFetch, @@ -783,6 +740,8 @@ export class UserEntityService implements OnModuleInit { includeSecrets?: boolean, }, ): Promise[]> { + if (users.length === 0) return []; + // -- IDのみの要素を補完して完全なエンティティ一覧を作る const _users = users.filter((user): user is MiUser => typeof user !== 'string'); @@ -800,57 +759,105 @@ export class UserEntityService implements OnModuleInit { } const _userIds = _users.map(u => u.id); - // -- 実行者の有無や指定スキーマの種別によって要否が異なる値群を取得 + const iAmModerator = await this.roleService.isModerator(me as MiUser); + const meId = me ? me.id : null; + const isMe = meId && _userIds.includes(meId); + const isDetailed = options && options.schema !== 'UserLite'; + const isDetailedAndMe = isDetailed && isMe; + const isDetailedAndMeOrMod = isDetailed && (isMe || iAmModerator); + const isDetailedAndNotMe = isDetailed && !isMe; - let profilesMap: Map = new Map(); - let userRelations: Map = new Map(); - let userMemos: Map = new Map(); - let pinNotes: Map = new Map(); + const userUris = new Set(_users + .flatMap(user => [user.uri, user.movedToUri]) + .filter((uri): uri is string => uri != null)); - if (options?.schema !== 'UserLite') { - const _profiles: MiUserProfile[] = []; - const _profilesToFetch: string[] = []; - for (const user of _users) { - if (user.userProfile) { - _profiles.push(user.userProfile); - } else { - _profilesToFetch.push(user.id); - } - } - if (_profilesToFetch.length > 0) { - const fetched = await this.userProfilesRepository.findBy({ userId: In(_profilesToFetch) }); - _profiles.push(...fetched); - } - profilesMap = new Map(_profiles.map(p => [p.userId, p])); + const userHosts = new Set(_users + .map(user => user.host) + .filter((host): host is string => host != null)); - const meId = me ? me.id : null; - if (meId) { - userMemos = await this.userMemosRepository.findBy({ userId: meId }) - .then(memos => new Map(memos.map(memo => [memo.targetUserId, memo.memo]))); - - if (_userIds.length > 0) { - userRelations = await this.getRelations(meId, _userIds); - pinNotes = await this.userNotePiningsRepository.createQueryBuilder('pin') - .where('pin.userId IN (:...userIds)', { userIds: _userIds }) - .innerJoinAndSelect('pin.note', 'note') - .getMany() - .then(pinsNotes => { - const map = new Map(); - for (const note of pinsNotes) { - const notes = map.get(note.userId) ?? []; - notes.push(note); - map.set(note.userId, notes); - } - for (const [, notes] of map.entries()) { - // pack側ではDESCで取得しているので、それに合わせて降順に並び替えておく - notes.sort((a, b) => b.id.localeCompare(a.id)); - } - return map; - }); - } + const _profilesFromUsers: [string, MiUserProfile][] = []; + const _profilesToFetch: string[] = []; + for (const user of _users) { + if (user.userProfile) { + _profilesFromUsers.push([user.id, user.userProfile]); + } else { + _profilesToFetch.push(user.id); } } + // -- 実行者の有無や指定スキーマの種別によって要否が異なる値群を取得 + + const [profilesMap, userMemos, userRelations, pinNotes, userIdsByUri, instances, securityKeyCounts, pendingReceivedFollows, pendingSentFollows] = await Promise.all([ + // profilesMap + this.cacheService.userProfileCache.fetchMany(_profilesToFetch).then(profiles => new Map(profiles.concat(_profilesFromUsers))), + // userMemos + isDetailed && meId ? this.userMemosRepository.findBy({ userId: meId }) + .then(memos => new Map(memos.map(memo => [memo.targetUserId, memo.memo]))) : new Map(), + // userRelations + isDetailedAndNotMe && meId ? this.getRelations(meId, _userIds) : new Map(), + // pinNotes + isDetailed ? this.userNotePiningsRepository.createQueryBuilder('pin') + .where('pin.userId IN (:...userIds)', { userIds: _userIds }) + .innerJoinAndSelect('pin.note', 'note') + .getMany() + .then(pinsNotes => { + const map = new Map(); + for (const note of pinsNotes) { + const notes = map.get(note.userId) ?? []; + notes.push(note); + map.set(note.userId, notes); + } + for (const [, notes] of map.entries()) { + // pack側ではDESCで取得しているので、それに合わせて降順に並び替えておく + notes.sort((a, b) => b.id.localeCompare(a.id)); + } + return map; + }) : new Map(), + // userIdsByUrl + isDetailed ? this.usersRepository.createQueryBuilder('user') + .select([ + 'user.id', + 'user.uri', + ]) + .where({ + uri: In(Array.from(userUris)), + }) + .getRawMany<{ user_uri: string, user_id: string }>() + .then(users => new Map(users.map(u => [u.user_uri, u.user_id]))) : new Map(), + // instances + Promise.all(Array.from(userHosts).map(async host => [host, await this.federatedInstanceService.fetch(host)] as const)) + .then(hosts => new Map(hosts)), + // securityKeyCounts + isDetailedAndMeOrMod ? this.userSecurityKeysRepository.createQueryBuilder('key') + .select('key.userId', 'userId') + .addSelect('count(key.id)', 'userCount') + .where({ + userId: In(_userIds), + }) + .groupBy('key.userId') + .getRawMany<{ userId: string, userCount: number }>() + .then(counts => new Map(counts.map(c => [c.userId, c.userCount]))) : new Map(), + // TODO optimization: cache follow requests + // pendingReceivedFollows + isDetailedAndMe ? this.followRequestsRepository.createQueryBuilder('req') + .select('req.followeeId', 'followeeId') + .where({ + followeeId: In(_userIds), + }) + .groupBy('req.followeeId') + .getRawMany<{ followeeId: string }>() + .then(reqs => new Set(reqs.map(r => r.followeeId))) : new Set(), + // pendingSentFollows + isDetailedAndMe ? this.followRequestsRepository.createQueryBuilder('req') + .select('req.followerId', 'followerId') + .where({ + followerId: In(_userIds), + }) + .groupBy('req.followerId') + .getRawMany<{ followerId: string }>() + .then(reqs => new Set(reqs.map(r => r.followerId))) : new Set(), + ]); + return Promise.all( _users.map(u => this.pack( u, @@ -861,6 +868,12 @@ export class UserEntityService implements OnModuleInit { userRelations: userRelations, userMemos: userMemos, pinNotes: pinNotes, + iAmModerator, + userIdsByUri, + instances, + securityKeyCounts, + pendingReceivedFollows, + pendingSentFollows, }, )), ); diff --git a/packages/backend/src/logger.ts b/packages/backend/src/logger.ts index ca9b494ff2..4bf45fc76b 100644 --- a/packages/backend/src/logger.ts +++ b/packages/backend/src/logger.ts @@ -23,6 +23,14 @@ export type DataElement = DataObject | Error | string | null; // https://stackoverflow.com/questions/61148466/typescript-type-that-matches-any-object-but-not-arrays export type DataObject = Record | (object & { length?: never; }); +const levelFuncs = { + error: 'error', + warning: 'warn', + success: 'info', + info: 'log', + debug: 'debug', +} as const satisfies Record; + // eslint-disable-next-line import/no-default-export export default class Logger { private context: Context; @@ -86,7 +94,7 @@ export default class Logger { } else if (data != null) { args.push(data); } - console.log(...args); + console[levelFuncs[level]](...args); } @bindThis diff --git a/packages/backend/src/misc/QuantumKVCache.ts b/packages/backend/src/misc/QuantumKVCache.ts new file mode 100644 index 0000000000..b96937d6f2 --- /dev/null +++ b/packages/backend/src/misc/QuantumKVCache.ts @@ -0,0 +1,385 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { InternalEventService } from '@/core/InternalEventService.js'; +import { bindThis } from '@/decorators.js'; +import { InternalEventTypes } from '@/core/GlobalEventService.js'; +import { MemoryKVCache } from '@/misc/cache.js'; + +export interface QuantumKVOpts { + /** + * Memory cache lifetime in milliseconds. + */ + lifetime: number; + + /** + * Callback to fetch the value for a key that wasn't found in the cache. + * May be synchronous or async. + */ + fetcher: (key: string, cache: QuantumKVCache) => T | Promise; + + /** + * Optional callback to fetch the value for multiple keys that weren't found in the cache. + * May be synchronous or async. + * If not provided, then the implementation will fall back on repeated calls to fetcher(). + */ + bulkFetcher?: (keys: string[], cache: QuantumKVCache) => Iterable<[key: string, value: T]> | Promise>; + + /** + * Optional callback when one or more values are changed (created, updated, or deleted) in the cache, either locally or elsewhere in the cluster. + * This is called *after* the cache state is updated. + * Implementations may be synchronous or async. + */ + onChanged?: (keys: string[], cache: QuantumKVCache) => void | Promise; +} + +/** + * QuantumKVCache is a lifetime-bounded memory cache (like MemoryKVCache) with automatic cross-cluster synchronization via Redis. + * All nodes in the cluster are guaranteed to have a *subset* view of the current accurate state, though individual processes may have different items in their local cache. + * This ensures that a call to get() will never return stale data. + */ +export class QuantumKVCache implements Iterable<[key: string, value: T]> { + private readonly memoryCache: MemoryKVCache; + + public readonly fetcher: QuantumKVOpts['fetcher']; + public readonly bulkFetcher: QuantumKVOpts['bulkFetcher']; + public readonly onChanged: QuantumKVOpts['onChanged']; + + /** + * @param internalEventService Service bus to synchronize events. + * @param name Unique name of the cache - must be the same in all processes. + * @param opts Cache options + */ + constructor( + private readonly internalEventService: InternalEventService, + private readonly name: string, + opts: QuantumKVOpts, + ) { + this.memoryCache = new MemoryKVCache(opts.lifetime); + this.fetcher = opts.fetcher; + this.bulkFetcher = opts.bulkFetcher; + this.onChanged = opts.onChanged; + + this.internalEventService.on('quantumCacheUpdated', this.onQuantumCacheUpdated, { + // Ignore our own events, otherwise we'll immediately erase any set value. + ignoreLocal: true, + }); + } + + /** + * The number of items currently in memory. + * This applies to the local subset view, not the cross-cluster cache state. + */ + public get size() { + return this.memoryCache.size; + } + + /** + * Iterates all [key, value] pairs in memory. + * This applies to the local subset view, not the cross-cluster cache state. + */ + @bindThis + public *entries(): Generator<[key: string, value: T]> { + for (const entry of this.memoryCache.entries) { + yield [entry[0], entry[1].value]; + } + } + + /** + * Iterates all keys in memory. + * This applies to the local subset view, not the cross-cluster cache state. + */ + @bindThis + public *keys() { + for (const entry of this.memoryCache.entries) { + yield entry[0]; + } + } + + /** + * Iterates all values pairs in memory. + * This applies to the local subset view, not the cross-cluster cache state. + */ + @bindThis + public *values() { + for (const entry of this.memoryCache.entries) { + yield entry[1].value; + } + } + + /** + * Creates or updates a value in the cache, and erases any stale caches across the cluster. + * Fires an onSet event after the cache has been updated in all processes. + * Skips if the value is unchanged. + */ + @bindThis + public async set(key: string, value: T): Promise { + if (this.memoryCache.get(key) === value) { + return; + } + + this.memoryCache.set(key, value); + + await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, keys: [key] }); + + if (this.onChanged) { + await this.onChanged([key], this); + } + } + + /** + * Creates or updates multiple value in the cache, and erases any stale caches across the cluster. + * Fires an onSet for each changed item event after the cache has been updated in all processes. + * Skips if all values are unchanged. + */ + @bindThis + public async setMany(items: Iterable<[key: string, value: T]>): Promise { + const changedKeys: string[] = []; + + for (const item of items) { + if (this.memoryCache.get(item[0]) !== item[1]) { + changedKeys.push(item[0]); + this.memoryCache.set(item[0], item[1]); + } + } + + if (changedKeys.length > 0) { + await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, keys: changedKeys }); + + if (this.onChanged) { + await this.onChanged(changedKeys, this); + } + } + } + + /** + * Adds a value to the local memory cache without notifying other process. + * Neither a Redis event nor onSet callback will be fired, as the value has not actually changed. + * This should only be used when the value is known to be current, like after fetching from the database. + */ + @bindThis + public add(key: string, value: T): void { + this.memoryCache.set(key, value); + } + + /** + * Adds multiple values to the local memory cache without notifying other process. + * Neither a Redis event nor onSet callback will be fired, as the value has not actually changed. + * This should only be used when the value is known to be current, like after fetching from the database. + */ + @bindThis + public addMany(items: Iterable<[key: string, value: T]>): void { + for (const [key, value] of items) { + this.memoryCache.set(key, value); + } + } + + /** + * Gets a value from the local memory cache, or returns undefined if not found. + * Returns cached data only - does not make any fetches. + */ + @bindThis + public get(key: string): T | undefined { + return this.memoryCache.get(key); + } + + /** + * Gets multiple values from the local memory cache; returning undefined for any missing keys. + * Returns cached data only - does not make any fetches. + */ + @bindThis + public getMany(keys: Iterable): [key: string, value: T | undefined][] { + const results: [key: string, value: T | undefined][] = []; + for (const key of keys) { + results.push([key, this.get(key)]); + } + return results; + } + + /** + * Gets or fetches a value from the cache. + * Fires an onSet event, but does not emit an update event to other processes. + */ + @bindThis + public async fetch(key: string): Promise { + let value = this.memoryCache.get(key); + if (value === undefined) { + value = await this.fetcher(key, this); + this.memoryCache.set(key, value); + + if (this.onChanged) { + await this.onChanged([key], this); + } + } + return value; + } + + /** + * Gets or fetches multiple values from the cache. + * Fires onSet events, but does not emit any update events to other processes. + */ + @bindThis + public async fetchMany(keys: Iterable): Promise<[key: string, value: T][]> { + const results: [key: string, value: T][] = []; + const toFetch: string[] = []; + + // Spliterate into cached results / uncached keys. + for (const key of keys) { + const fromCache = this.get(key); + if (fromCache) { + results.push([key, fromCache]); + } else { + toFetch.push(key); + } + } + + // Fetch any uncached keys + if (toFetch.length > 0) { + const fetched = await this.bulkFetch(toFetch); + + // Add to cache and return set + this.addMany(fetched); + results.push(...fetched); + + // Emit event + if (this.onChanged) { + await this.onChanged(toFetch, this); + } + } + + return results; + } + + /** + * Returns true is a key exists in memory. + * This applies to the local subset view, not the cross-cluster cache state. + */ + @bindThis + public has(key: string): boolean { + return this.memoryCache.get(key) !== undefined; + } + + /** + * Deletes a value from the cache, and erases any stale caches across the cluster. + * Fires an onDelete event after the cache has been updated in all processes. + */ + @bindThis + public async delete(key: string): Promise { + this.memoryCache.delete(key); + + await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, keys: [key] }); + + if (this.onChanged) { + await this.onChanged([key], this); + } + } + /** + * Deletes multiple values from the cache, and erases any stale caches across the cluster. + * Fires an onDelete event for each key after the cache has been updated in all processes. + * Skips if the input is empty. + */ + @bindThis + public async deleteMany(keys: Iterable): Promise { + const deleted: string[] = []; + + for (const key of keys) { + this.memoryCache.delete(key); + deleted.push(key); + } + + if (deleted.length === 0) { + return; + } + + await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, keys: deleted }); + + if (this.onChanged) { + await this.onChanged(deleted, this); + } + } + + /** + * Refreshes the value of a key from the fetcher, and erases any stale caches across the cluster. + * Fires an onSet event after the cache has been updated in all processes. + */ + @bindThis + public async refresh(key: string): Promise { + const value = await this.fetcher(key, this); + await this.set(key, value); + return value; + } + + @bindThis + public async refreshMany(keys: Iterable): Promise<[key: string, value: T][]> { + const values = await this.bulkFetch(keys); + await this.setMany(values); + return values; + } + + /** + * Erases all entries from the local memory cache. + * Does not send any events or update other processes. + */ + @bindThis + public clear() { + this.memoryCache.clear(); + } + + /** + * Removes expired cache entries from the local view. + * Does not send any events or update other processes. + */ + @bindThis + public gc() { + this.memoryCache.gc(); + } + + /** + * Erases all data and disconnects from the cluster. + * This *must* be called when shutting down to prevent memory leaks! + */ + @bindThis + public dispose() { + this.internalEventService.off('quantumCacheUpdated', this.onQuantumCacheUpdated); + + this.memoryCache.dispose(); + } + + @bindThis + private async bulkFetch(keys: Iterable): Promise<[key: string, value: T][]> { + if (this.bulkFetcher) { + const results = await this.bulkFetcher(Array.from(keys), this); + return Array.from(results); + } + + const results: [key: string, value: T][] = []; + for (const key of keys) { + const value = await this.fetcher(key, this); + results.push([key, value]); + } + return results; + } + + @bindThis + private async onQuantumCacheUpdated(data: InternalEventTypes['quantumCacheUpdated']): Promise { + if (data.name === this.name) { + for (const key of data.keys) { + this.memoryCache.delete(key); + } + + if (this.onChanged) { + await this.onChanged(data.keys, this); + } + } + } + + /** + * Iterates all [key, value] pairs in memory. + * This applies to the local subset view, not the cross-cluster cache state. + */ + [Symbol.iterator](): Iterator<[key: string, value: T]> { + return this.entries(); + } +} diff --git a/packages/backend/src/misc/cache.ts b/packages/backend/src/misc/cache.ts index a6ab96c189..666e684c1c 100644 --- a/packages/backend/src/misc/cache.ts +++ b/packages/backend/src/misc/cache.ts @@ -9,9 +9,9 @@ import { bindThis } from '@/decorators.js'; export class RedisKVCache { private readonly lifetime: number; private readonly memoryCache: MemoryKVCache; - private readonly fetcher: (key: string) => Promise; - private readonly toRedisConverter: (value: T) => string; - private readonly fromRedisConverter: (value: string) => T | undefined; + public readonly fetcher: (key: string) => Promise; + public readonly toRedisConverter: (value: T) => string; + public readonly fromRedisConverter: (value: string) => T | undefined; constructor( private redisClient: Redis.Redis, @@ -99,6 +99,11 @@ export class RedisKVCache { // TODO: イベント発行して他プロセスのメモリキャッシュも更新できるようにする } + @bindThis + public clear() { + this.memoryCache.clear(); + } + @bindThis public gc() { this.memoryCache.gc(); @@ -113,9 +118,9 @@ export class RedisKVCache { export class RedisSingleCache { private readonly lifetime: number; private readonly memoryCache: MemorySingleCache; - private readonly fetcher: () => Promise; - private readonly toRedisConverter: (value: T) => string; - private readonly fromRedisConverter: (value: string) => T | undefined; + public readonly fetcher: () => Promise; + public readonly toRedisConverter: (value: T) => string; + public readonly fromRedisConverter: (value: string) => T | undefined; constructor( private redisClient: Redis.Redis, @@ -123,16 +128,17 @@ export class RedisSingleCache { opts: { lifetime: number; memoryCacheLifetime: number; - fetcher: RedisSingleCache['fetcher']; - toRedisConverter: RedisSingleCache['toRedisConverter']; - fromRedisConverter: RedisSingleCache['fromRedisConverter']; + fetcher?: RedisSingleCache['fetcher']; + toRedisConverter?: RedisSingleCache['toRedisConverter']; + fromRedisConverter?: RedisSingleCache['fromRedisConverter']; }, ) { this.lifetime = opts.lifetime; this.memoryCache = new MemorySingleCache(opts.memoryCacheLifetime); - this.fetcher = opts.fetcher; - this.toRedisConverter = opts.toRedisConverter; - this.fromRedisConverter = opts.fromRedisConverter; + + this.fetcher = opts.fetcher ?? (() => { throw new Error('fetch not supported - use get/set directly'); }); + this.toRedisConverter = opts.toRedisConverter ?? ((value) => JSON.stringify(value)); + this.fromRedisConverter = opts.fromRedisConverter ?? ((value) => JSON.parse(value)); } @bindThis @@ -237,6 +243,16 @@ export class MemoryKVCache { return cached.value; } + public has(key: string): boolean { + const cached = this.cache.get(key); + if (cached == null) return false; + if ((Date.now() - cached.date) > this.lifetime) { + this.cache.delete(key); + return false; + } + return true; + } + @bindThis public delete(key: string): void { this.cache.delete(key); @@ -322,6 +338,10 @@ export class MemoryKVCache { clearInterval(this.gcIntervalHandle); } + public get size() { + return this.cache.size; + } + public get entries() { return this.cache.entries(); } diff --git a/packages/backend/src/misc/extract-custom-emojis-from-mfm.ts b/packages/backend/src/misc/extract-custom-emojis-from-mfm.ts index 36a9b8e1f4..73ae9abb54 100644 --- a/packages/backend/src/misc/extract-custom-emojis-from-mfm.ts +++ b/packages/backend/src/misc/extract-custom-emojis-from-mfm.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import * as mfm from '@transfem-org/sfm-js'; +import * as mfm from 'mfm-js'; import { unique } from '@/misc/prelude/array.js'; export function extractCustomEmojisFromMfm(nodes: mfm.MfmNode[]): string[] { diff --git a/packages/backend/src/misc/extract-hashtags.ts b/packages/backend/src/misc/extract-hashtags.ts index ed7606d995..d3d245d414 100644 --- a/packages/backend/src/misc/extract-hashtags.ts +++ b/packages/backend/src/misc/extract-hashtags.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import * as mfm from '@transfem-org/sfm-js'; +import * as mfm from 'mfm-js'; import { unique } from '@/misc/prelude/array.js'; export function extractHashtags(nodes: mfm.MfmNode[]): string[] { diff --git a/packages/backend/src/misc/extract-mentions.ts b/packages/backend/src/misc/extract-mentions.ts index bb21c32ffb..2ec9349718 100644 --- a/packages/backend/src/misc/extract-mentions.ts +++ b/packages/backend/src/misc/extract-mentions.ts @@ -5,7 +5,7 @@ // test is located in test/extract-mentions -import * as mfm from '@transfem-org/sfm-js'; +import * as mfm from 'mfm-js'; export function extractMentions(nodes: mfm.MfmNode[]): mfm.MfmMention['props'][] { // TODO: 重複を削除 diff --git a/packages/backend/src/misc/truncate.ts b/packages/backend/src/misc/truncate.ts index 1c8a274609..a313ab7854 100644 --- a/packages/backend/src/misc/truncate.ts +++ b/packages/backend/src/misc/truncate.ts @@ -3,14 +3,12 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { substring } from 'stringz'; - export function truncate(input: string, size: number): string; export function truncate(input: string | undefined, size: number): string | undefined; export function truncate(input: string | undefined, size: number): string | undefined { if (!input) { return input; } else { - return substring(input, 0, size); + return input.slice(0, size); } } diff --git a/packages/backend/src/misc/verify-field-link.ts b/packages/backend/src/misc/verify-field-link.ts index 62542eaaa0..f9fc352806 100644 --- a/packages/backend/src/misc/verify-field-link.ts +++ b/packages/backend/src/misc/verify-field-link.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { load as cheerio } from 'cheerio'; +import { load as cheerio } from 'cheerio/slim'; import type { HttpRequestService } from '@/core/HttpRequestService.js'; type Field = { name: string, value: string }; diff --git a/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts b/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts index 6a1a8bcc66..5bf64e4f04 100644 --- a/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts +++ b/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts @@ -18,6 +18,7 @@ import { SearchService } from '@/core/SearchService.js'; import { ApLogService } from '@/core/ApLogService.js'; import { ReactionService } from '@/core/ReactionService.js'; import { QueueService } from '@/core/QueueService.js'; +import { CacheService } from '@/core/CacheService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; import type { DbUserDeleteJobData } from '../types.js'; @@ -94,6 +95,7 @@ export class DeleteAccountProcessorService { private searchService: SearchService, private reactionService: ReactionService, private readonly apLogService: ApLogService, + private readonly cacheService: CacheService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('delete-account'); } @@ -140,6 +142,22 @@ export class DeleteAccountProcessorService { } { // Delete user relations + await this.cacheService.refreshFollowRelationsFor(user.id); + await this.cacheService.userFollowingsCache.delete(user.id); + await this.cacheService.userFollowingsCache.delete(user.id); + await this.cacheService.userBlockingCache.delete(user.id); + await this.cacheService.userBlockedCache.delete(user.id); + await this.cacheService.userMutingsCache.delete(user.id); + await this.cacheService.userMutingsCache.delete(user.id); + await this.cacheService.hibernatedUserCache.delete(user.id); + await this.cacheService.renoteMutingsCache.delete(user.id); + await this.cacheService.userProfileCache.delete(user.id); + this.cacheService.userByIdCache.delete(user.id); + this.cacheService.localUserByIdCache.delete(user.id); + if (user.token) { + this.cacheService.localUserByNativeTokenCache.delete(user.token); + } + await this.followingsRepository.delete({ followerId: user.id, }); diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index 612b16dfbf..5f82d558b3 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -21,7 +21,7 @@ import { ApDbResolverService } from '@/core/activitypub/ApDbResolverService.js'; import { StatusError } from '@/misc/status-error.js'; import { UtilityService } from '@/core/UtilityService.js'; import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js'; -import { JsonLdService } from '@/core/activitypub/JsonLdService.js'; +import { isSigned, JsonLdService } from '@/core/activitypub/JsonLdService.js'; import { ApInboxService } from '@/core/activitypub/ApInboxService.js'; import { bindThis } from '@/decorators.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; @@ -179,8 +179,8 @@ export class InboxProcessorService implements OnApplicationShutdown { // また、signatureのsignerは、activity.actorと一致する必要がある if (!httpSignatureValidated || authUser.user.uri !== actorId) { // 一致しなくても、でもLD-Signatureがありそうならそっちも見る - const ldSignature = activity.signature; - if (ldSignature) { + if (isSigned(activity)) { + const ldSignature = activity.signature; if (ldSignature.type !== 'RsaSignature2017') { throw new Bull.UnrecoverableError(`skip: unsupported LD-signature type ${ldSignature.type}`); } @@ -202,24 +202,21 @@ export class InboxProcessorService implements OnApplicationShutdown { throw new Bull.UnrecoverableError('skip: LD-SignatureのユーザーはpublicKeyを持っていませんでした'); } - const jsonLd = this.jsonLdService.use(); - // LD-Signature検証 - const verified = await jsonLd.verifyRsaSignature2017(activity, authUser.key.keyPem).catch(() => false); + const verified = await this.jsonLdService.verifyRsaSignature2017(activity, authUser.key.keyPem).catch(() => false); if (!verified) { throw new Bull.UnrecoverableError('skip: LD-Signatureの検証に失敗しました'); } // アクティビティを正規化 - delete activity.signature; + const copy = { ...activity, signature: undefined }; try { - activity = await jsonLd.compact(activity) as IActivity; + activity = await this.jsonLdService.compact(copy) as IActivity; } catch (e) { throw new Bull.UnrecoverableError(`skip: failed to compact activity: ${e}`); } // TODO: 元のアクティビティと非互換な形に正規化される場合は転送をスキップする // https://github.com/mastodon/mastodon/blob/664b0ca/app/services/activitypub/process_collection_service.rb#L24-L29 - activity.signature = ldSignature; // もう一度actorチェック if (authUser.user.uri !== actorId) { diff --git a/packages/backend/src/server/api/endpoints/admin/nsfw-user.ts b/packages/backend/src/server/api/endpoints/admin/nsfw-user.ts index 194e793eda..f6c4f0b635 100644 --- a/packages/backend/src/server/api/endpoints/admin/nsfw-user.ts +++ b/packages/backend/src/server/api/endpoints/admin/nsfw-user.ts @@ -47,7 +47,7 @@ export default class extends Endpoint { // eslint- alwaysMarkNsfw: true, }); - await this.cacheService.userProfileCache.refresh(ps.userId); + await this.cacheService.userProfileCache.delete(ps.userId); }); } } diff --git a/packages/backend/src/server/api/endpoints/fetch-rss.ts b/packages/backend/src/server/api/endpoints/fetch-rss.ts index 03f35f16a5..11244b30f6 100644 --- a/packages/backend/src/server/api/endpoints/fetch-rss.ts +++ b/packages/backend/src/server/api/endpoints/fetch-rss.ts @@ -3,12 +3,12 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import Parser from 'rss-parser'; import { Injectable } from '@nestjs/common'; +import { parseFeed } from 'htmlparser2'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { HttpRequestService } from '@/core/HttpRequestService.js'; - -const rssParser = new Parser(); +import { ApiError } from '../error.js'; +import type { FeedItem } from 'domutils'; export const meta = { tags: ['meta'], @@ -17,52 +17,32 @@ export const meta = { allowGet: true, cacheSec: 60 * 3, + errors: { + fetchFailed: { + id: '88f4356f-719d-4715-b4fc-703a10a812d2', + code: 'FETCH_FAILED', + message: 'Failed to fetch RSS feed', + }, + }, + res: { type: 'object', properties: { - image: { - type: 'object', - optional: true, - properties: { - link: { - type: 'string', - optional: true, - }, - url: { - type: 'string', - optional: false, - }, - title: { - type: 'string', - optional: true, - }, - }, + type: { + type: 'string', + optional: false, }, - paginationLinks: { - type: 'object', + id: { + type: 'string', + optional: true, + }, + updated: { + type: 'string', + optional: true, + }, + author: { + type: 'string', optional: true, - properties: { - self: { - type: 'string', - optional: true, - }, - first: { - type: 'string', - optional: true, - }, - next: { - type: 'string', - optional: true, - }, - last: { - type: 'string', - optional: true, - }, - prev: { - type: 'string', - optional: true, - }, - }, }, link: { type: 'string', @@ -94,113 +74,42 @@ export const meta = { type: 'string', optional: true, }, - creator: { + description: { type: 'string', optional: true, }, - summary: { - type: 'string', - optional: true, - }, - content: { - type: 'string', - optional: true, - }, - isoDate: { - type: 'string', - optional: true, - }, - categories: { + media: { type: 'array', - optional: true, + optional: false, items: { - type: 'string', - }, - }, - contentSnippet: { - type: 'string', - optional: true, - }, - enclosure: { - type: 'object', - optional: true, - properties: { - url: { - type: 'string', - optional: false, - }, - length: { - type: 'number', - optional: true, - }, - type: { - type: 'string', - optional: true, + type: 'object', + properties: { + medium: { + type: 'string', + optional: true, + }, + url: { + type: 'string', + optional: true, + }, + type: { + type: 'string', + optional: true, + }, + lang: { + type: 'string', + optional: true, + }, }, }, }, }, }, }, - feedUrl: { - type: 'string', - optional: true, - }, description: { type: 'string', optional: true, }, - itunes: { - type: 'object', - optional: true, - additionalProperties: true, - properties: { - image: { - type: 'string', - optional: true, - }, - owner: { - type: 'object', - optional: true, - properties: { - name: { - type: 'string', - optional: true, - }, - email: { - type: 'string', - optional: true, - }, - }, - }, - author: { - type: 'string', - optional: true, - }, - summary: { - type: 'string', - optional: true, - }, - explicit: { - type: 'string', - optional: true, - }, - categories: { - type: 'array', - optional: true, - items: { - type: 'string', - }, - }, - keywords: { - type: 'array', - optional: true, - items: { - type: 'string', - }, - }, - }, - }, }, }, @@ -224,7 +133,7 @@ export default class extends Endpoint { // eslint- constructor( private httpRequestService: HttpRequestService, ) { - super(meta, paramDef, async (ps, me) => { + super(meta, paramDef, async (ps) => { const res = await this.httpRequestService.send(ps.url, { method: 'GET', headers: { @@ -234,8 +143,38 @@ export default class extends Endpoint { // eslint- }); const text = await res.text(); + const feed = parseFeed(text, { + xmlMode: true, + }); - return rssParser.parseString(text); + if (!feed) { + throw new ApiError(meta.errors.fetchFailed); + } + + return { + type: feed.type, + id: feed.id, + title: feed.title, + link: feed.link, + description: feed.description, + updated: feed.updated?.toISOString(), + author: feed.author, + items: feed.items + .filter((item): item is FeedItem & { link: string, title: string } => !!item.link && !!item.title) + .map(item => ({ + guid: item.id, + title: item.title, + link: item.link, + description: item.description, + pubDate: item.pubDate?.toISOString(), + media: item.media.map(media => ({ + medium: media.medium, + url: media.url, + type: media.type, + lang: media.lang, + })), + })), + }; }); } } diff --git a/packages/backend/src/server/api/endpoints/following/delete.ts b/packages/backend/src/server/api/endpoints/following/delete.ts index ba146b6703..442352a4d2 100644 --- a/packages/backend/src/server/api/endpoints/following/delete.ts +++ b/packages/backend/src/server/api/endpoints/following/delete.ts @@ -12,6 +12,7 @@ import { UserFollowingService } from '@/core/UserFollowingService.js'; import { DI } from '@/di-symbols.js'; import { GetterService } from '@/server/api/GetterService.js'; import { ApiError } from '../../error.js'; +import { CacheService } from '@/core/CacheService.js'; export const meta = { tags: ['following', 'users'], @@ -69,6 +70,7 @@ export default class extends Endpoint { // eslint- private userEntityService: UserEntityService, private getterService: GetterService, private userFollowingService: UserFollowingService, + private readonly cacheService: CacheService, ) { super(meta, paramDef, async (ps, me) => { const follower = me; @@ -85,12 +87,7 @@ export default class extends Endpoint { // eslint- }); // Check not following - const exist = await this.followingsRepository.exists({ - where: { - followerId: follower.id, - followeeId: followee.id, - }, - }); + const exist = await this.cacheService.userFollowingsCache.fetch(follower.id).then(f => f.has(followee.id)); if (!exist) { throw new ApiError(meta.errors.notFollowing); diff --git a/packages/backend/src/server/api/endpoints/following/invalidate.ts b/packages/backend/src/server/api/endpoints/following/invalidate.ts index b45d21410b..3809bf29b0 100644 --- a/packages/backend/src/server/api/endpoints/following/invalidate.ts +++ b/packages/backend/src/server/api/endpoints/following/invalidate.ts @@ -11,6 +11,7 @@ import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { UserFollowingService } from '@/core/UserFollowingService.js'; import { DI } from '@/di-symbols.js'; import { GetterService } from '@/server/api/GetterService.js'; +import { CacheService } from '@/core/CacheService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -69,6 +70,7 @@ export default class extends Endpoint { // eslint- private userEntityService: UserEntityService, private getterService: GetterService, private userFollowingService: UserFollowingService, + private readonly cacheService: CacheService, ) { super(meta, paramDef, async (ps, me) => { const followee = me; @@ -85,12 +87,9 @@ export default class extends Endpoint { // eslint- }); // Check not following - const exist = await this.followingsRepository.findOneBy({ - followerId: follower.id, - followeeId: followee.id, - }); + const isFollowing = await this.cacheService.userFollowingsCache.fetch(follower.id).then(f => f.has(followee.id)); - if (exist == null) { + if (!isFollowing) { throw new ApiError(meta.errors.notFollowing); } diff --git a/packages/backend/src/server/api/endpoints/following/update-all.ts b/packages/backend/src/server/api/endpoints/following/update-all.ts index c953feb393..a02b51cc79 100644 --- a/packages/backend/src/server/api/endpoints/following/update-all.ts +++ b/packages/backend/src/server/api/endpoints/following/update-all.ts @@ -12,6 +12,7 @@ import { UserFollowingService } from '@/core/UserFollowingService.js'; import { DI } from '@/di-symbols.js'; import { GetterService } from '@/server/api/GetterService.js'; import { ApiError } from '../../error.js'; +import { CacheService } from '@/core/CacheService.js'; export const meta = { tags: ['following', 'users'], @@ -39,6 +40,7 @@ export default class extends Endpoint { // eslint- constructor( @Inject(DI.followingsRepository) private followingsRepository: FollowingsRepository, + private readonly cacheService: CacheService, ) { super(meta, paramDef, async (ps, me) => { await this.followingsRepository.update({ @@ -48,6 +50,8 @@ export default class extends Endpoint { // eslint- withReplies: ps.withReplies != null ? ps.withReplies : undefined, }); + await this.cacheService.refreshFollowRelationsFor(me.id); + return; }); } diff --git a/packages/backend/src/server/api/endpoints/following/update.ts b/packages/backend/src/server/api/endpoints/following/update.ts index d62cf210ed..f4ca21856f 100644 --- a/packages/backend/src/server/api/endpoints/following/update.ts +++ b/packages/backend/src/server/api/endpoints/following/update.ts @@ -11,6 +11,7 @@ import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { UserFollowingService } from '@/core/UserFollowingService.js'; import { DI } from '@/di-symbols.js'; import { GetterService } from '@/server/api/GetterService.js'; +import { CacheService } from '@/core/CacheService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -71,6 +72,7 @@ export default class extends Endpoint { // eslint- private userEntityService: UserEntityService, private getterService: GetterService, private userFollowingService: UserFollowingService, + private readonly cacheService: CacheService, ) { super(meta, paramDef, async (ps, me) => { const follower = me; @@ -87,10 +89,7 @@ export default class extends Endpoint { // eslint- }); // Check not following - const exist = await this.followingsRepository.findOneBy({ - followerId: follower.id, - followeeId: followee.id, - }); + const exist = await this.cacheService.userFollowingsCache.fetch(follower.id).then(f => f.get(followee.id)); if (exist == null) { throw new ApiError(meta.errors.notFollowing); @@ -103,6 +102,8 @@ export default class extends Endpoint { // eslint- withReplies: ps.withReplies != null ? ps.withReplies : undefined, }); + await this.cacheService.refreshFollowRelationsFor(follower.id); + return await this.userEntityService.pack(follower.id, me); }); } diff --git a/packages/backend/src/server/api/endpoints/i/update.ts b/packages/backend/src/server/api/endpoints/i/update.ts index dda42ce0e4..5767880531 100644 --- a/packages/backend/src/server/api/endpoints/i/update.ts +++ b/packages/backend/src/server/api/endpoints/i/update.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import * as mfm from '@transfem-org/sfm-js'; +import * as mfm from 'mfm-js'; import { Inject, Injectable } from '@nestjs/common'; import ms from 'ms'; import { extractCustomEmojisFromMfm } from '@/misc/extract-custom-emojis-from-mfm.js'; @@ -617,7 +617,7 @@ export default class extends Endpoint { // eslint- const updatedProfile = await this.userProfilesRepository.findOneByOrFail({ userId: user.id }); - this.cacheService.userProfileCache.set(user.id, updatedProfile); + await this.cacheService.userProfileCache.set(user.id, updatedProfile); // Publish meUpdated event this.globalEventService.publishMainStream(user.id, 'meUpdated', iObj); diff --git a/packages/backend/src/server/api/endpoints/notes/create.ts b/packages/backend/src/server/api/endpoints/notes/create.ts index ac1c9bfad9..97ff5a2ea3 100644 --- a/packages/backend/src/server/api/endpoints/notes/create.ts +++ b/packages/backend/src/server/api/endpoints/notes/create.ts @@ -350,7 +350,7 @@ export default class extends Endpoint { // eslint- throw new ApiError(meta.errors.noSuchReplyTarget); } else if (isRenote(reply) && !isQuote(reply)) { throw new ApiError(meta.errors.cannotReplyToPureRenote); - } else if (!await this.noteEntityService.isVisibleForMe(reply, me.id)) { + } else if (!await this.noteEntityService.isVisibleForMe(reply, me.id, { me })) { throw new ApiError(meta.errors.cannotReplyToInvisibleNote); } else if (reply.visibility === 'specified' && ps.visibility !== 'specified') { throw new ApiError(meta.errors.cannotReplyToSpecifiedVisibilityNoteWithExtendedVisibility); diff --git a/packages/backend/src/server/api/endpoints/notes/edit.ts b/packages/backend/src/server/api/endpoints/notes/edit.ts index 2c01b26584..bd70cb7835 100644 --- a/packages/backend/src/server/api/endpoints/notes/edit.ts +++ b/packages/backend/src/server/api/endpoints/notes/edit.ts @@ -402,7 +402,7 @@ export default class extends Endpoint { // eslint- throw new ApiError(meta.errors.noSuchReplyTarget); } else if (isRenote(reply) && !isQuote(reply)) { throw new ApiError(meta.errors.cannotReplyToPureRenote); - } else if (!await this.noteEntityService.isVisibleForMe(reply, me.id)) { + } else if (!await this.noteEntityService.isVisibleForMe(reply, me.id, { me })) { throw new ApiError(meta.errors.cannotReplyToInvisibleNote); } else if (reply.visibility === 'specified' && ps.visibility !== 'specified') { throw new ApiError(meta.errors.cannotReplyToSpecifiedVisibilityNoteWithExtendedVisibility); diff --git a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts index 613cf7b9b4..fa4ee44c0f 100644 --- a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts @@ -164,7 +164,7 @@ export default class extends Endpoint { // eslint- excludeBots: !ps.withBots, noteFilter: note => { if (note.reply && note.reply.visibility === 'followers') { - if (!Object.hasOwn(followings, note.reply.userId) && note.reply.userId !== me.id) return false; + if (!followings.has(note.reply.userId) && note.reply.userId !== me.id) return false; } return true; diff --git a/packages/backend/src/server/api/endpoints/notes/timeline.ts b/packages/backend/src/server/api/endpoints/notes/timeline.ts index 17065bdd9a..47dc9bb9ba 100644 --- a/packages/backend/src/server/api/endpoints/notes/timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/timeline.ts @@ -115,7 +115,7 @@ export default class extends Endpoint { // eslint- excludePureRenotes: !ps.withRenotes, noteFilter: note => { if (note.reply && note.reply.visibility === 'followers') { - if (!Object.hasOwn(followings, note.reply.userId) && note.reply.userId !== me.id) return false; + if (!followings.has(note.reply.userId) && note.reply.userId !== me.id) return false; } if (!ps.withBots && note.user?.isBot) return false; diff --git a/packages/backend/src/server/api/endpoints/notes/translate.ts b/packages/backend/src/server/api/endpoints/notes/translate.ts index e55168e296..5ebd5ef362 100644 --- a/packages/backend/src/server/api/endpoints/notes/translate.ts +++ b/packages/backend/src/server/api/endpoints/notes/translate.ts @@ -91,7 +91,7 @@ export default class extends Endpoint { // eslint- throw err; }); - if (!(await this.noteEntityService.isVisibleForMe(note, me?.id ?? null))) { + if (!(await this.noteEntityService.isVisibleForMe(note, me?.id ?? null, { me }))) { throw new ApiError(meta.errors.cannotTranslateInvisibleNote); } diff --git a/packages/backend/src/server/api/endpoints/sw/register.ts b/packages/backend/src/server/api/endpoints/sw/register.ts index f447b5598b..2f72e6ce1d 100644 --- a/packages/backend/src/server/api/endpoints/sw/register.ts +++ b/packages/backend/src/server/api/endpoints/sw/register.ts @@ -104,7 +104,7 @@ export default class extends Endpoint { // eslint- sendReadMessage: ps.sendReadMessage, }); - this.pushNotificationService.refreshCache(me.id); + await this.pushNotificationService.refreshCache(me.id); return { state: 'subscribed' as const, diff --git a/packages/backend/src/server/api/endpoints/sw/unregister.ts b/packages/backend/src/server/api/endpoints/sw/unregister.ts index aa7e03dceb..f43a2cce28 100644 --- a/packages/backend/src/server/api/endpoints/sw/unregister.ts +++ b/packages/backend/src/server/api/endpoints/sw/unregister.ts @@ -46,7 +46,7 @@ export default class extends Endpoint { // eslint- }); if (me) { - this.pushNotificationService.refreshCache(me.id); + await this.pushNotificationService.refreshCache(me.id); } }); } diff --git a/packages/backend/src/server/api/endpoints/sw/update-registration.ts b/packages/backend/src/server/api/endpoints/sw/update-registration.ts index 78b9323b7b..0cbed273e8 100644 --- a/packages/backend/src/server/api/endpoints/sw/update-registration.ts +++ b/packages/backend/src/server/api/endpoints/sw/update-registration.ts @@ -86,7 +86,7 @@ export default class extends Endpoint { // eslint- sendReadMessage: swSubscription.sendReadMessage, }); - this.pushNotificationService.refreshCache(me.id); + await this.pushNotificationService.refreshCache(me.id); return { userId: swSubscription.userId, diff --git a/packages/backend/src/server/api/endpoints/users/followers.ts b/packages/backend/src/server/api/endpoints/users/followers.ts index c1617e14e5..82ce282bfc 100644 --- a/packages/backend/src/server/api/endpoints/users/followers.ts +++ b/packages/backend/src/server/api/endpoints/users/followers.ts @@ -12,6 +12,7 @@ import { FollowingEntityService } from '@/core/entities/FollowingEntityService.j import { UtilityService } from '@/core/UtilityService.js'; import { DI } from '@/di-symbols.js'; import { RoleService } from '@/core/RoleService.js'; +import { CacheService } from '@/core/CacheService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -89,6 +90,7 @@ export default class extends Endpoint { // eslint- private followingEntityService: FollowingEntityService, private queryService: QueryService, private roleService: RoleService, + private readonly cacheService: CacheService, ) { super(meta, paramDef, async (ps, me) => { const user = await this.usersRepository.findOneBy(ps.userId != null @@ -110,12 +112,7 @@ export default class extends Endpoint { // eslint- if (me == null) { throw new ApiError(meta.errors.forbidden); } else if (me.id !== user.id) { - const isFollowing = await this.followingsRepository.exists({ - where: { - followeeId: user.id, - followerId: me.id, - }, - }); + const isFollowing = await this.cacheService.userFollowingsCache.fetch(me.id).then(f => f.has(user.id)); if (!isFollowing) { throw new ApiError(meta.errors.forbidden); } diff --git a/packages/backend/src/server/api/endpoints/users/following.ts b/packages/backend/src/server/api/endpoints/users/following.ts index c292c6d6a3..80f0b0c484 100644 --- a/packages/backend/src/server/api/endpoints/users/following.ts +++ b/packages/backend/src/server/api/endpoints/users/following.ts @@ -13,6 +13,7 @@ import { FollowingEntityService } from '@/core/entities/FollowingEntityService.j import { UtilityService } from '@/core/UtilityService.js'; import { DI } from '@/di-symbols.js'; import { RoleService } from '@/core/RoleService.js'; +import { CacheService } from '@/core/CacheService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -98,6 +99,7 @@ export default class extends Endpoint { // eslint- private followingEntityService: FollowingEntityService, private queryService: QueryService, private roleService: RoleService, + private readonly cacheService: CacheService, ) { super(meta, paramDef, async (ps, me) => { const user = await this.usersRepository.findOneBy(ps.userId != null @@ -119,12 +121,7 @@ export default class extends Endpoint { // eslint- if (me == null) { throw new ApiError(meta.errors.forbidden); } else if (me.id !== user.id) { - const isFollowing = await this.followingsRepository.exists({ - where: { - followeeId: user.id, - followerId: me.id, - }, - }); + const isFollowing = await this.cacheService.userFollowingsCache.fetch(me.id).then(f => f.has(user.id)); if (!isFollowing) { throw new ApiError(meta.errors.forbidden); } diff --git a/packages/backend/src/server/api/endpoints/users/notes.ts b/packages/backend/src/server/api/endpoints/users/notes.ts index cd5ef53915..5b16ef32a4 100644 --- a/packages/backend/src/server/api/endpoints/users/notes.ts +++ b/packages/backend/src/server/api/endpoints/users/notes.ts @@ -134,7 +134,7 @@ export default class extends Endpoint { // eslint- if (ps.withReplies) redisTimelines.push(`userTimelineWithReplies:${ps.userId}`); if (ps.withChannelNotes) redisTimelines.push(`userTimelineWithChannel:${ps.userId}`); - const isFollowing = me && Object.hasOwn(await this.cacheService.userFollowingsCache.fetch(me.id), ps.userId); + const isFollowing = me && (await this.cacheService.userFollowingsCache.fetch(me.id)).has(ps.userId); const timeline = await this.fanoutTimelineEndpointService.timeline({ untilId, diff --git a/packages/backend/src/server/api/endpoints/users/recommendation.ts b/packages/backend/src/server/api/endpoints/users/recommendation.ts index 642d788459..52dd2197b2 100644 --- a/packages/backend/src/server/api/endpoints/users/recommendation.ts +++ b/packages/backend/src/server/api/endpoints/users/recommendation.ts @@ -71,6 +71,7 @@ export default class extends Endpoint { // eslint- this.queryService.generateBlockQueryForUsers(query, me); this.queryService.generateBlockedUserQueryForNotes(query, me); + // TODO optimization: replace with exists() const followingQuery = this.followingsRepository.createQueryBuilder('following') .select('following.followeeId') .where('following.followerId = :followerId', { followerId: me.id }); diff --git a/packages/backend/src/server/api/mastodon/MastodonConverters.ts b/packages/backend/src/server/api/mastodon/MastodonConverters.ts index 02ce31c4f8..df8d68042a 100644 --- a/packages/backend/src/server/api/mastodon/MastodonConverters.ts +++ b/packages/backend/src/server/api/mastodon/MastodonConverters.ts @@ -5,7 +5,7 @@ import { Inject, Injectable } from '@nestjs/common'; import { Entity, MastodonEntity, MisskeyEntity } from 'megalodon'; -import mfm from '@transfem-org/sfm-js'; +import mfm from 'mfm-js'; import { MastodonNotificationType } from 'megalodon/lib/src/mastodon/notification.js'; import { NotificationType } from 'megalodon/lib/src/notification.js'; import { DI } from '@/di-symbols.js'; diff --git a/packages/backend/src/server/api/mastodon/MastodonDataService.ts b/packages/backend/src/server/api/mastodon/MastodonDataService.ts index 73cd553b9a..e080cb10bd 100644 --- a/packages/backend/src/server/api/mastodon/MastodonDataService.ts +++ b/packages/backend/src/server/api/mastodon/MastodonDataService.ts @@ -57,19 +57,19 @@ export class MastodonDataService { if (relations.reply) { query.leftJoinAndSelect('note.reply', 'reply'); if (typeof(relations.reply) === 'object') { - if (relations.reply.reply) query.leftJoinAndSelect('note.reply.reply', 'replyReply'); - if (relations.reply.renote) query.leftJoinAndSelect('note.reply.renote', 'replyRenote'); - if (relations.reply.user) query.innerJoinAndSelect('note.reply.user', 'replyUser'); - if (relations.reply.channel) query.leftJoinAndSelect('note.reply.channel', 'replyChannel'); + if (relations.reply.reply) query.leftJoinAndSelect('reply.reply', 'replyReply'); + if (relations.reply.renote) query.leftJoinAndSelect('reply.renote', 'replyRenote'); + if (relations.reply.user) query.innerJoinAndSelect('reply.user', 'replyUser'); + if (relations.reply.channel) query.leftJoinAndSelect('reply.channel', 'replyChannel'); } } if (relations.renote) { query.leftJoinAndSelect('note.renote', 'renote'); if (typeof(relations.renote) === 'object') { - if (relations.renote.reply) query.leftJoinAndSelect('note.renote.reply', 'renoteReply'); - if (relations.renote.renote) query.leftJoinAndSelect('note.renote.renote', 'renoteRenote'); - if (relations.renote.user) query.innerJoinAndSelect('note.renote.user', 'renoteUser'); - if (relations.renote.channel) query.leftJoinAndSelect('note.renote.channel', 'renoteChannel'); + if (relations.renote.reply) query.leftJoinAndSelect('renote.reply', 'renoteReply'); + if (relations.renote.renote) query.leftJoinAndSelect('renote.renote', 'renoteRenote'); + if (relations.renote.user) query.innerJoinAndSelect('renote.user', 'renoteUser'); + if (relations.renote.channel) query.leftJoinAndSelect('renote.channel', 'renoteChannel'); } } if (relations.user) { diff --git a/packages/backend/src/server/api/stream/Connection.ts b/packages/backend/src/server/api/stream/Connection.ts index e0535a2f14..0ee7078eb2 100644 --- a/packages/backend/src/server/api/stream/Connection.ts +++ b/packages/backend/src/server/api/stream/Connection.ts @@ -36,7 +36,7 @@ export default class Connection { private channels = new Map(); private subscribingNotes = new Map(); public userProfile: MiUserProfile | null = null; - public following: Record | undefined> = {}; + public following: Map> = new Map(); public followingChannels: Set = new Set(); public userIdsWhoMeMuting: Set = new Set(); public userIdsWhoBlockingMe: Set = new Set(); diff --git a/packages/backend/src/server/api/stream/channel.ts b/packages/backend/src/server/api/stream/channel.ts index 3a82865577..40ad454adb 100644 --- a/packages/backend/src/server/api/stream/channel.ts +++ b/packages/backend/src/server/api/stream/channel.ts @@ -70,7 +70,7 @@ export default abstract class Channel { if (!this.user) return false; if (this.user.id === note.userId) return true; if (note.visibility === 'followers') { - return this.following[note.userId] != null; + return this.following.has(note.userId); } if (!note.visibleUserIds) return false; return note.visibleUserIds.includes(this.user.id); @@ -84,7 +84,7 @@ export default abstract class Channel { if (note.user.requireSigninToViewContents && !this.user) return true; // 流れてきたNoteがインスタンスミュートしたインスタンスが関わる - if (isInstanceMuted(note, this.userMutedInstances) && !this.following[note.userId]) return true; + if (isInstanceMuted(note, this.userMutedInstances) && !this.following.has(note.userId)) return true; // 流れてきたNoteがミュートしているユーザーが関わる if (isUserRelated(note, this.userIdsWhoMeMuting)) return true; @@ -101,7 +101,7 @@ export default abstract class Channel { if (note.user.isSilenced || note.user.instance?.isSilenced) { if (this.user == null) return true; if (this.user.id === note.userId) return false; - if (this.following[note.userId] == null) return true; + if (!this.following.has(note.userId)) return true; } // TODO muted threads diff --git a/packages/backend/src/server/api/stream/channels/bubble-timeline.ts b/packages/backend/src/server/api/stream/channels/bubble-timeline.ts index 393fe3883c..72f719b411 100644 --- a/packages/backend/src/server/api/stream/channels/bubble-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/bubble-timeline.ts @@ -62,7 +62,7 @@ class BubbleTimelineChannel extends Channel { const reply = note.reply; // 自分のフォローしていないユーザーの visibility: followers な投稿への返信は弾く if (!this.isNoteVisibleToMe(reply)) return; - if (!this.following[note.userId]?.withReplies) { + if (!this.following.get(note.userId)?.withReplies) { // 「チャンネル接続主への返信」でもなければ、「チャンネル接続主が行った返信」でもなければ、「投稿者の投稿者自身への返信」でもない場合 if (reply.userId !== this.user?.id && !isMe && reply.userId !== note.userId) return; } diff --git a/packages/backend/src/server/api/stream/channels/global-timeline.ts b/packages/backend/src/server/api/stream/channels/global-timeline.ts index bac0277538..5c73f637c7 100644 --- a/packages/backend/src/server/api/stream/channels/global-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/global-timeline.ts @@ -63,7 +63,7 @@ class GlobalTimelineChannel extends Channel { const reply = note.reply; // 自分のフォローしていないユーザーの visibility: followers な投稿への返信は弾く if (!this.isNoteVisibleToMe(reply)) return; - if (!this.following[note.userId]?.withReplies) { + if (!this.following.get(note.userId)?.withReplies) { // 「チャンネル接続主への返信」でもなければ、「チャンネル接続主が行った返信」でもなければ、「投稿者の投稿者自身への返信」でもない場合 if (reply.userId !== this.user?.id && !isMe && reply.userId !== note.userId) return; } diff --git a/packages/backend/src/server/api/stream/channels/home-timeline.ts b/packages/backend/src/server/api/stream/channels/home-timeline.ts index d1dcbd07e5..c7062c0394 100644 --- a/packages/backend/src/server/api/stream/channels/home-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/home-timeline.ts @@ -47,7 +47,7 @@ class HomeTimelineChannel extends Channel { if (!this.followingChannels.has(note.channelId)) return; } else { // その投稿のユーザーをフォローしていなかったら弾く - if (!isMe && !Object.hasOwn(this.following, note.userId)) return; + if (!isMe && !this.following.has(note.userId)) return; } if (this.isNoteMutedOrBlocked(note)) return; @@ -57,7 +57,7 @@ class HomeTimelineChannel extends Channel { const reply = note.reply; // 自分のフォローしていないユーザーの visibility: followers な投稿への返信は弾く if (!this.isNoteVisibleToMe(reply)) return; - if (!this.following[note.userId]?.withReplies) { + if (!this.following.get(note.userId)?.withReplies) { // 「チャンネル接続主への返信」でもなければ、「チャンネル接続主が行った返信」でもなければ、「投稿者の投稿者自身への返信」でもない場合 if (reply.userId !== this.user!.id && !isMe && reply.userId !== note.userId) return; } diff --git a/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts b/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts index d923167e04..7cb64c9f89 100644 --- a/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts @@ -62,7 +62,7 @@ class HybridTimelineChannel extends Channel { // フォローしているチャンネルの投稿 の場合だけ if (!( (note.channelId == null && isMe) || - (note.channelId == null && Object.hasOwn(this.following, note.userId)) || + (note.channelId == null && this.following.has(note.userId)) || (note.channelId == null && (note.user.host == null && note.visibility === 'public')) || (note.channelId != null && this.followingChannels.has(note.channelId)) )) return; @@ -74,7 +74,7 @@ class HybridTimelineChannel extends Channel { const reply = note.reply; // 自分のフォローしていないユーザーの visibility: followers な投稿への返信は弾く if (!this.isNoteVisibleToMe(reply)) return; - if (!this.following[note.userId]?.withReplies && !this.withReplies) { + if (!this.following.get(note.userId)?.withReplies && !this.withReplies) { // 「チャンネル接続主への返信」でもなければ、「チャンネル接続主が行った返信」でもなければ、「投稿者の投稿者自身への返信」でもない場合 if (reply.userId !== this.user!.id && !isMe && reply.userId !== note.userId) return; } diff --git a/packages/backend/src/server/api/stream/channels/local-timeline.ts b/packages/backend/src/server/api/stream/channels/local-timeline.ts index 2eb3460efa..4869d871d6 100644 --- a/packages/backend/src/server/api/stream/channels/local-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/local-timeline.ts @@ -67,7 +67,7 @@ class LocalTimelineChannel extends Channel { const reply = note.reply; // 自分のフォローしていないユーザーの visibility: followers な投稿への返信は弾く if (!this.isNoteVisibleToMe(reply)) return; - if (!this.following[note.userId]?.withReplies) { + if (!this.following.get(note.userId)?.withReplies) { // 「チャンネル接続主への返信」でもなければ、「チャンネル接続主が行った返信」でもなければ、「投稿者の投稿者自身への返信」でもない場合 if (reply.userId !== this.user?.id && !isMe && reply.userId !== note.userId) return; } diff --git a/packages/backend/src/server/api/stream/channels/role-timeline.ts b/packages/backend/src/server/api/stream/channels/role-timeline.ts index f5984b5ae9..a3886618f1 100644 --- a/packages/backend/src/server/api/stream/channels/role-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/role-timeline.ts @@ -55,7 +55,7 @@ class RoleTimelineChannel extends Channel { const reply = note.reply; // 自分のフォローしていないユーザーの visibility: followers な投稿への返信は弾く if (!this.isNoteVisibleToMe(reply)) return; - if (!this.following[note.userId]?.withReplies) { + if (!this.following.get(note.userId)?.withReplies) { // 「チャンネル接続主への返信」でもなければ、「チャンネル接続主が行った返信」でもなければ、「投稿者の投稿者自身への返信」でもない場合 if (reply.userId !== this.user?.id && !isMe && reply.userId !== note.userId) return; } diff --git a/packages/backend/src/server/api/stream/channels/user-list.ts b/packages/backend/src/server/api/stream/channels/user-list.ts index 3f1a5a8f8f..4dae24a696 100644 --- a/packages/backend/src/server/api/stream/channels/user-list.ts +++ b/packages/backend/src/server/api/stream/channels/user-list.ts @@ -98,7 +98,7 @@ class UserListChannel extends Channel { const reply = note.reply; // 自分のフォローしていないユーザーの visibility: followers な投稿への返信は弾く if (!this.isNoteVisibleToMe(reply)) return; - if (!this.following[note.userId]?.withReplies) { + if (!this.following.get(note.userId)?.withReplies) { // 「チャンネル接続主への返信」でもなければ、「チャンネル接続主が行った返信」でもなければ、「投稿者の投稿者自身への返信」でもない場合 if (reply.userId !== this.user!.id && !isMe && reply.userId !== note.userId) return; } diff --git a/packages/backend/src/server/web/FeedService.ts b/packages/backend/src/server/web/FeedService.ts index dcd4d80303..a622ae7e34 100644 --- a/packages/backend/src/server/web/FeedService.ts +++ b/packages/backend/src/server/web/FeedService.ts @@ -15,7 +15,7 @@ import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.j import { bindThis } from '@/decorators.js'; import { IdService } from '@/core/IdService.js'; import { MfmService } from "@/core/MfmService.js"; -import { parse as mfmParse } from '@transfem-org/sfm-js'; +import { parse as mfmParse } from 'mfm-js'; @Injectable() export class FeedService { diff --git a/packages/backend/test/e2e/oauth.ts b/packages/backend/test/e2e/oauth.ts index aba7a15d28..97830bc987 100644 --- a/packages/backend/test/e2e/oauth.ts +++ b/packages/backend/test/e2e/oauth.ts @@ -19,7 +19,7 @@ import { ResourceOwnerPassword, } from 'simple-oauth2'; import pkceChallenge from 'pkce-challenge'; -import { load as cheerio } from 'cheerio'; +import { load as cheerio } from 'cheerio/slim'; import Fastify, { type FastifyInstance, type FastifyReply } from 'fastify'; import { api, port, sendEnvUpdateRequest, signup } from '../utils.js'; import type * as misskey from 'misskey-js'; diff --git a/packages/backend/test/misc/FakeInternalEventService.ts b/packages/backend/test/misc/FakeInternalEventService.ts new file mode 100644 index 0000000000..d18a080eaf --- /dev/null +++ b/packages/backend/test/misc/FakeInternalEventService.ts @@ -0,0 +1,92 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import type { Listener, ListenerProps } from '@/core/InternalEventService.js'; +import type Redis from 'ioredis'; +import type { GlobalEventService, InternalEventTypes } from '@/core/GlobalEventService.js'; +import { InternalEventService } from '@/core/InternalEventService.js'; +import { bindThis } from '@/decorators.js'; + +type FakeCall = [K, Parameters]; +type FakeListener = [K, Listener, ListenerProps]; + +/** + * Minimal implementation of InternalEventService meant for use in unit tests. + * There is no redis connection, and metadata is tracked in the public _calls and _listeners arrays. + * The on/off/emit methods are fully functional and can be called in tests to invoke any registered listeners. + */ +export class FakeInternalEventService extends InternalEventService { + /** + * List of calls to public methods, in chronological order. + */ + public _calls: FakeCall[] = []; + + /** + * List of currently registered listeners. + */ + public _listeners: FakeListener[] = []; + + /** + * Resets the mock. + * Clears all listeners and tracked calls. + */ + public _reset() { + this._calls = []; + this._listeners = []; + } + + /** + * Simulates a remote event sent from another process in the cluster via redis. + */ + @bindThis + public async _emitRedis(type: K, value: InternalEventTypes[K]): Promise { + await this.emit(type, value, false); + } + + constructor() { + super( + { on: () => {} } as unknown as Redis.Redis, + {} as unknown as GlobalEventService, + ); + } + + @bindThis + public on(type: K, listener: Listener, props?: ListenerProps): void { + if (!this._listeners.some(l => l[0] === type && l[1] === listener)) { + this._listeners.push([type, listener as Listener, props ?? {}]); + } + this._calls.push(['on', [type, listener as Listener, props]]); + } + + @bindThis + public off(type: K, listener: Listener): void { + this._listeners = this._listeners.filter(l => l[0] !== type || l[1] !== listener); + this._calls.push(['off', [type, listener as Listener]]); + } + + @bindThis + public async emit(type: K, value: InternalEventTypes[K], isLocal = true): Promise { + for (const listener of this._listeners) { + if (listener[0] === type) { + if ((isLocal && !listener[2].ignoreLocal) || (!isLocal && !listener[2].ignoreRemote)) { + await listener[1](value, type, isLocal); + } + } + } + this._calls.push(['emit', [type, value]]); + } + + @bindThis + public dispose(): void { + this._listeners = []; + this._calls.push(['dispose', []]); + } + + @bindThis + public onApplicationShutdown(): void { + this._calls.push(['onApplicationShutdown', []]); + } +} + diff --git a/packages/backend/test/misc/noOpCaches.ts b/packages/backend/test/misc/noOpCaches.ts new file mode 100644 index 0000000000..f3cc1e2ba2 --- /dev/null +++ b/packages/backend/test/misc/noOpCaches.ts @@ -0,0 +1,187 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import * as Redis from 'ioredis'; +import { Inject } from '@nestjs/common'; +import { FakeInternalEventService } from './FakeInternalEventService.js'; +import type { BlockingsRepository, FollowingsRepository, MiUser, MutingsRepository, RenoteMutingsRepository, UserProfilesRepository, UsersRepository } from '@/models/_.js'; +import type { MiLocalUser } from '@/models/User.js'; +import { MemoryKVCache, MemorySingleCache, RedisKVCache, RedisSingleCache } from '@/misc/cache.js'; +import { QuantumKVCache, QuantumKVOpts } from '@/misc/QuantumKVCache.js'; +import { CacheService, FollowStats } from '@/core/CacheService.js'; +import { DI } from '@/di-symbols.js'; +import { UserEntityService } from '@/core/entities/UserEntityService.js'; +import { InternalEventService } from '@/core/InternalEventService.js'; + +export function noOpRedis() { + return { + set: () => Promise.resolve(), + get: () => Promise.resolve(null), + del: () => Promise.resolve(), + on: () => {}, + off: () => {}, + } as unknown as Redis.Redis; +} + +export class NoOpCacheService extends CacheService { + public readonly fakeRedis: { + [K in keyof Redis.Redis]: Redis.Redis[K]; + }; + public readonly fakeInternalEventService: FakeInternalEventService; + + constructor( + @Inject(DI.usersRepository) + usersRepository: UsersRepository, + + @Inject(DI.userProfilesRepository) + userProfilesRepository: UserProfilesRepository, + + @Inject(DI.mutingsRepository) + mutingsRepository: MutingsRepository, + + @Inject(DI.blockingsRepository) + blockingsRepository: BlockingsRepository, + + @Inject(DI.renoteMutingsRepository) + renoteMutingsRepository: RenoteMutingsRepository, + + @Inject(DI.followingsRepository) + followingsRepository: FollowingsRepository, + + @Inject(UserEntityService) + userEntityService: UserEntityService, + ) { + const fakeRedis = noOpRedis(); + const fakeInternalEventService = new FakeInternalEventService(); + + super( + fakeRedis, + fakeRedis, + usersRepository, + userProfilesRepository, + mutingsRepository, + blockingsRepository, + renoteMutingsRepository, + followingsRepository, + userEntityService, + fakeInternalEventService, + ); + + this.fakeRedis = fakeRedis; + this.fakeInternalEventService = fakeInternalEventService; + + // Override caches + this.userByIdCache = new NoOpMemoryKVCache(); + this.localUserByNativeTokenCache = new NoOpMemoryKVCache(); + this.localUserByIdCache = new NoOpMemoryKVCache(); + this.uriPersonCache = new NoOpMemoryKVCache(); + this.userProfileCache = NoOpQuantumKVCache.copy(this.userProfileCache, fakeInternalEventService); + this.userMutingsCache = NoOpQuantumKVCache.copy(this.userMutingsCache, fakeInternalEventService); + this.userBlockingCache = NoOpQuantumKVCache.copy(this.userBlockingCache, fakeInternalEventService); + this.userBlockedCache = NoOpQuantumKVCache.copy(this.userBlockedCache, fakeInternalEventService); + this.renoteMutingsCache = NoOpQuantumKVCache.copy(this.renoteMutingsCache, fakeInternalEventService); + this.userFollowingsCache = NoOpQuantumKVCache.copy(this.userFollowingsCache, fakeInternalEventService); + this.userFollowersCache = NoOpQuantumKVCache.copy(this.userFollowersCache, fakeInternalEventService); + this.hibernatedUserCache = NoOpQuantumKVCache.copy(this.hibernatedUserCache, fakeInternalEventService); + this.userFollowStatsCache = new NoOpMemoryKVCache(); + this.translationsCache = NoOpRedisKVCache.copy(this.translationsCache, fakeRedis); + } +} + +export class NoOpMemoryKVCache extends MemoryKVCache { + constructor() { + super(-1); + } +} + +export class NoOpMemorySingleCache extends MemorySingleCache { + constructor() { + super(-1); + } +} + +export class NoOpRedisKVCache extends RedisKVCache { + constructor(opts?: { + redis?: Redis.Redis; + fetcher?: RedisKVCache['fetcher']; + toRedisConverter?: RedisKVCache['toRedisConverter']; + fromRedisConverter?: RedisKVCache['fromRedisConverter']; + }) { + super( + opts?.redis ?? noOpRedis(), + 'no-op', + { + lifetime: -1, + memoryCacheLifetime: -1, + fetcher: opts?.fetcher, + toRedisConverter: opts?.toRedisConverter, + fromRedisConverter: opts?.fromRedisConverter, + }, + ); + } + + public static copy(cache: RedisKVCache, redis?: Redis.Redis): NoOpRedisKVCache { + return new NoOpRedisKVCache({ + redis, + fetcher: cache.fetcher, + toRedisConverter: cache.toRedisConverter, + fromRedisConverter: cache.fromRedisConverter, + }); + } +} + +export class NoOpRedisSingleCache extends RedisSingleCache { + constructor(opts?: { + redis?: Redis.Redis; + fetcher?: RedisSingleCache['fetcher']; + toRedisConverter?: RedisSingleCache['toRedisConverter']; + fromRedisConverter?: RedisSingleCache['fromRedisConverter']; + }) { + super( + opts?.redis ?? noOpRedis(), + 'no-op', + { + lifetime: -1, + memoryCacheLifetime: -1, + fetcher: opts?.fetcher, + toRedisConverter: opts?.toRedisConverter, + fromRedisConverter: opts?.fromRedisConverter, + }, + ); + } + + public static copy(cache: RedisSingleCache, redis?: Redis.Redis): NoOpRedisSingleCache { + return new NoOpRedisSingleCache({ + redis, + fetcher: cache.fetcher, + toRedisConverter: cache.toRedisConverter, + fromRedisConverter: cache.fromRedisConverter, + }); + } +} + +export class NoOpQuantumKVCache extends QuantumKVCache { + constructor(opts: Omit, 'lifetime'> & { + internalEventService?: InternalEventService, + }) { + super( + opts.internalEventService ?? new FakeInternalEventService(), + 'no-op', + { + ...opts, + lifetime: -1, + }, + ); + } + + public static copy(cache: QuantumKVCache, internalEventService?: InternalEventService): NoOpQuantumKVCache { + return new NoOpQuantumKVCache({ + internalEventService, + fetcher: cache.fetcher, + bulkFetcher: cache.bulkFetcher, + onChanged: cache.onChanged, + }); + } +} diff --git a/packages/backend/test/unit/AnnouncementService.ts b/packages/backend/test/unit/AnnouncementService.ts index a79655c9aa..32d7df05bf 100644 --- a/packages/backend/test/unit/AnnouncementService.ts +++ b/packages/backend/test/unit/AnnouncementService.ts @@ -8,9 +8,12 @@ process.env.NODE_ENV = 'test'; import { jest } from '@jest/globals'; import { ModuleMocker } from 'jest-mock'; import { Test } from '@nestjs/testing'; +import { NoOpCacheService } from '../misc/noOpCaches.js'; +import { FakeInternalEventService } from '../misc/FakeInternalEventService.js'; import { GlobalModule } from '@/GlobalModule.js'; import { AnnouncementService } from '@/core/AnnouncementService.js'; import { AnnouncementEntityService } from '@/core/entities/AnnouncementEntityService.js'; +import { InternalEventService } from '@/core/InternalEventService.js'; import type { AnnouncementReadsRepository, AnnouncementsRepository, @@ -71,24 +74,27 @@ describe('AnnouncementService', () => { AnnouncementEntityService, CacheService, IdService, + InternalEventService, + GlobalEventService, + ModerationLogService, ], }) .useMocker((token) => { - if (token === GlobalEventService) { - return { - publishMainStream: jest.fn(), - publishBroadcastStream: jest.fn(), - }; - } else if (token === ModerationLogService) { - return { - log: jest.fn(), - }; - } else if (typeof token === 'function') { + if (typeof token === 'function') { const mockMetadata = moduleMocker.getMetadata(token) as MockFunctionMetadata; const Mock = moduleMocker.generateFromMetadata(mockMetadata); return new Mock(); } }) + .overrideProvider(GlobalEventService).useValue({ + publishMainStream: jest.fn(), + publishBroadcastStream: jest.fn(), + } as unknown as GlobalEventService) + .overrideProvider(ModerationLogService).useValue({ + log: jest.fn(), + }) + .overrideProvider(InternalEventService).useClass(FakeInternalEventService) + .overrideProvider(CacheService).useClass(NoOpCacheService) .compile(); app.enableShutdownHooks(); diff --git a/packages/backend/test/unit/MfmService.ts b/packages/backend/test/unit/MfmService.ts index e54c006a4f..af1fc4e132 100644 --- a/packages/backend/test/unit/MfmService.ts +++ b/packages/backend/test/unit/MfmService.ts @@ -4,7 +4,7 @@ */ import * as assert from 'assert'; -import * as mfm from '@transfem-org/sfm-js'; +import * as mfm from 'mfm-js'; import { Test } from '@nestjs/testing'; import { CoreModule } from '@/core/CoreModule.js'; @@ -86,7 +86,7 @@ describe('MfmService', () => { test('ruby', async () => { const input = '$[ruby $[group *some* text] ignore me]'; - const output = '

*some* text(ignore me)

'; + const output = '

*some* text(ignore me)

'; assert.equal(await mfmService.toMastoApiHtml(mfm.parse(input)), output); }); }); diff --git a/packages/backend/test/unit/RoleService.ts b/packages/backend/test/unit/RoleService.ts index 839402418e..2afe22618d 100644 --- a/packages/backend/test/unit/RoleService.ts +++ b/packages/backend/test/unit/RoleService.ts @@ -10,12 +10,15 @@ import { jest } from '@jest/globals'; import { ModuleMocker } from 'jest-mock'; import { Test } from '@nestjs/testing'; import * as lolex from '@sinonjs/fake-timers'; +import { NoOpCacheService } from '../misc/noOpCaches.js'; +import { FakeInternalEventService } from '../misc/FakeInternalEventService.js'; import type { TestingModule } from '@nestjs/testing'; import type { MockFunctionMetadata } from 'jest-mock'; import { GlobalModule } from '@/GlobalModule.js'; import { RoleService } from '@/core/RoleService.js'; import { InstancesRepository, + MetasRepository, MiMeta, MiRole, MiRoleAssignment, @@ -34,6 +37,7 @@ import { secureRndstr } from '@/misc/secure-rndstr.js'; import { NotificationService } from '@/core/NotificationService.js'; import { RoleCondFormulaValue } from '@/models/Role.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; +import { InternalEventService } from '@/core/InternalEventService.js'; const moduleMocker = new ModuleMocker(global); @@ -45,6 +49,7 @@ describe('RoleService', () => { let rolesRepository: RolesRepository; let roleAssignmentsRepository: RoleAssignmentsRepository; let meta: jest.Mocked; + let metasRepository: MetasRepository; let notificationService: jest.Mocked; let clock: lolex.InstalledClock; @@ -143,18 +148,20 @@ describe('RoleService', () => { provide: NotificationService.name, useExisting: NotificationService, }, + MetaService, + InternalEventService, ], }) .useMocker((token) => { - if (token === MetaService) { - return { fetch: jest.fn() }; - } if (typeof token === 'function') { const mockMetadata = moduleMocker.getMetadata(token) as MockFunctionMetadata; const Mock = moduleMocker.generateFromMetadata(mockMetadata); return new Mock(); } }) + .overrideProvider(MetaService).useValue({ fetch: jest.fn() }) + .overrideProvider(InternalEventService).useClass(FakeInternalEventService) + .overrideProvider(CacheService).useClass(NoOpCacheService) .compile(); app.enableShutdownHooks(); @@ -164,6 +171,7 @@ describe('RoleService', () => { usersRepository = app.get(DI.usersRepository); rolesRepository = app.get(DI.rolesRepository); roleAssignmentsRepository = app.get(DI.roleAssignmentsRepository); + metasRepository = app.get(DI.metasRepository); meta = app.get(DI.meta) as jest.Mocked; notificationService = app.get(NotificationService) as jest.Mocked; @@ -175,7 +183,7 @@ describe('RoleService', () => { clock.uninstall(); await Promise.all([ - app.get(DI.metasRepository).delete({}), + metasRepository.delete({}), usersRepository.delete({}), rolesRepository.delete({}), roleAssignmentsRepository.delete({}), diff --git a/packages/backend/test/unit/activitypub.ts b/packages/backend/test/unit/activitypub.ts index 94dec16401..3ad9cdc4d5 100644 --- a/packages/backend/test/unit/activitypub.ts +++ b/packages/backend/test/unit/activitypub.ts @@ -9,8 +9,12 @@ import { generateKeyPair } from 'crypto'; import { Test } from '@nestjs/testing'; import { jest } from '@jest/globals'; +import { NoOpCacheService } from '../misc/noOpCaches.js'; +import { FakeInternalEventService } from '../misc/FakeInternalEventService.js'; import type { Config } from '@/config.js'; import type { MiLocalUser, MiRemoteUser } from '@/models/User.js'; +import { InternalEventService } from '@/core/InternalEventService.js'; +import { CacheService } from '@/core/CacheService.js'; import { ApImageService } from '@/core/activitypub/models/ApImageService.js'; import { ApNoteService } from '@/core/activitypub/models/ApNoteService.js'; import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js'; @@ -30,7 +34,7 @@ import { genAidx } from '@/misc/id/aidx.js'; import { IdService } from '@/core/IdService.js'; import { MockResolver } from '../misc/mock-resolver.js'; import { UserKeypairService } from '@/core/UserKeypairService.js'; -import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js'; +import { MemoryKVCache } from '@/misc/cache.js'; const host = 'https://host1.test'; @@ -154,6 +158,8 @@ describe('ActivityPub', () => { }, }) .overrideProvider(DI.meta).useFactory({ factory: () => meta }) + .overrideProvider(CacheService).useClass(NoOpCacheService) + .overrideProvider(InternalEventService).useClass(FakeInternalEventService) .compile(); await app.init(); @@ -473,8 +479,6 @@ describe('ActivityPub', () => { describe('JSON-LD', () => { test('Compaction', async () => { - const jsonLd = jsonLdService.use(); - const object = { '@context': [ 'https://www.w3.org/ns/activitystreams', @@ -493,7 +497,7 @@ describe('ActivityPub', () => { unknown: 'test test bar', undefined: 'test test baz', }; - const compacted = await jsonLd.compact(object); + const compacted = await jsonLdService.compact(object); assert.deepStrictEqual(compacted, { '@context': CONTEXT, @@ -556,7 +560,7 @@ describe('ActivityPub', () => { publicKey, privateKey, }); - ((userKeypairService as unknown as { cache: RedisKVCache }).cache as unknown as { memoryCache: MemoryKVCache }).memoryCache.set(author.id, keypair); + (userKeypairService as unknown as { cache: MemoryKVCache }).cache.set(author.id, keypair); note = new MiNote({ id: idService.gen(), diff --git a/packages/backend/test/unit/entities/UserEntityService.ts b/packages/backend/test/unit/entities/UserEntityService.ts index ca6a639be8..3eda3eee01 100644 --- a/packages/backend/test/unit/entities/UserEntityService.ts +++ b/packages/backend/test/unit/entities/UserEntityService.ts @@ -4,6 +4,8 @@ */ import { Test, TestingModule } from '@nestjs/testing'; +import { FakeInternalEventService } from '../../misc/FakeInternalEventService.js'; +import { NoOpCacheService } from '../../misc/noOpCaches.js'; import type { MiUser } from '@/models/User.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { GlobalModule } from '@/GlobalModule.js'; @@ -51,6 +53,7 @@ import { ReactionService } from '@/core/ReactionService.js'; import { NotificationService } from '@/core/NotificationService.js'; import { ReactionsBufferingService } from '@/core/ReactionsBufferingService.js'; import { ChatService } from '@/core/ChatService.js'; +import { InternalEventService } from '@/core/InternalEventService.js'; process.env.NODE_ENV = 'test'; @@ -174,6 +177,7 @@ describe('UserEntityService', () => { ReactionsBufferingService, NotificationService, ChatService, + InternalEventService, ]; app = await Test.createTestingModule({ @@ -182,7 +186,10 @@ describe('UserEntityService', () => { ...services, ...services.map(x => ({ provide: x.name, useExisting: x })), ], - }).compile(); + }) + .overrideProvider(InternalEventService).useClass(FakeInternalEventService) + .overrideProvider(CacheService).useClass(NoOpCacheService) + .compile(); await app.init(); app.enableShutdownHooks(); diff --git a/packages/backend/test/unit/extract-mentions.ts b/packages/backend/test/unit/extract-mentions.ts index 2aad89d65b..3403387e30 100644 --- a/packages/backend/test/unit/extract-mentions.ts +++ b/packages/backend/test/unit/extract-mentions.ts @@ -5,7 +5,7 @@ import * as assert from 'assert'; -import { parse } from '@transfem-org/sfm-js'; +import { parse } from 'mfm-js'; import { extractMentions } from '@/misc/extract-mentions.js'; describe('Extract mentions', () => { diff --git a/packages/backend/test/unit/misc/QuantumKVCache.ts b/packages/backend/test/unit/misc/QuantumKVCache.ts new file mode 100644 index 0000000000..92792171be --- /dev/null +++ b/packages/backend/test/unit/misc/QuantumKVCache.ts @@ -0,0 +1,799 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { jest } from '@jest/globals'; +import { FakeInternalEventService } from '../../misc/FakeInternalEventService.js'; +import { QuantumKVCache, QuantumKVOpts } from '@/misc/QuantumKVCache.js'; + +describe(QuantumKVCache, () => { + let fakeInternalEventService: FakeInternalEventService; + let madeCaches: { dispose: () => void }[]; + + function makeCache(opts?: Partial> & { name?: string }): QuantumKVCache { + const _opts = { + name: 'test', + lifetime: Infinity, + fetcher: () => { throw new Error('not implemented'); }, + } satisfies QuantumKVOpts & { name: string }; + + if (opts) { + Object.assign(_opts, opts); + } + + const cache = new QuantumKVCache(fakeInternalEventService, _opts.name, _opts); + madeCaches.push(cache); + return cache; + } + + beforeEach(() => { + madeCaches = []; + fakeInternalEventService = new FakeInternalEventService(); + }); + + afterEach(() => { + madeCaches.forEach(cache => { + cache.dispose(); + }); + }); + + it('should connect on construct', () => { + makeCache(); + + expect(fakeInternalEventService._calls).toContainEqual(['on', ['quantumCacheUpdated', expect.anything(), { ignoreLocal: true }]]); + }); + + it('should disconnect on dispose', () => { + const cache = makeCache(); + + cache.dispose(); + + const callback = fakeInternalEventService._calls + .find(c => c[0] === 'on' && c[1][0] === 'quantumCacheUpdated') + ?.[1][1]; + expect(fakeInternalEventService._calls).toContainEqual(['off', ['quantumCacheUpdated', callback]]); + }); + + it('should store in memory cache', async () => { + const cache = makeCache(); + + await cache.set('foo', 'bar'); + await cache.set('alpha', 'omega'); + + const result1 = await cache.get('foo'); + const result2 = await cache.get('alpha'); + + expect(result1).toBe('bar'); + expect(result2).toBe('omega'); + }); + + it('should emit event when storing', async () => { + const cache = makeCache({ name: 'fake' }); + + await cache.set('foo', 'bar'); + + expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo'] }]]); + }); + + it('should call onChanged when storing', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + name: 'fake', + onChanged: fakeOnChanged, + }); + + await cache.set('foo', 'bar'); + + expect(fakeOnChanged).toHaveBeenCalledWith(['foo'], cache); + }); + + it('should not emit event when storing unchanged value', async () => { + const cache = makeCache({ name: 'fake' }); + + await cache.set('foo', 'bar'); + await cache.set('foo', 'bar'); + + expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(1); + }); + + it('should not call onChanged when storing unchanged value', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + name: 'fake', + onChanged: fakeOnChanged, + }); + + await cache.set('foo', 'bar'); + await cache.set('foo', 'bar'); + + expect(fakeOnChanged).toHaveBeenCalledTimes(1); + }); + + it('should fetch an unknown value', async () => { + const cache = makeCache({ + name: 'fake', + fetcher: key => `value#${key}`, + }); + + const result = await cache.fetch('foo'); + + expect(result).toBe('value#foo'); + }); + + it('should store fetched value in memory cache', async () => { + const cache = makeCache({ + name: 'fake', + fetcher: key => `value#${key}`, + }); + + await cache.fetch('foo'); + + const result = cache.has('foo'); + expect(result).toBe(true); + }); + + it('should call onChanged when fetching', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + name: 'fake', + fetcher: key => `value#${key}`, + onChanged: fakeOnChanged, + }); + + await cache.fetch('foo'); + + expect(fakeOnChanged).toHaveBeenCalledWith(['foo'], cache); + }); + + it('should not emit event when fetching', async () => { + const cache = makeCache({ + name: 'fake', + fetcher: key => `value#${key}`, + }); + + await cache.fetch('foo'); + + expect(fakeInternalEventService._calls).not.toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo'] }]]); + }); + + it('should delete from memory cache', async () => { + const cache = makeCache(); + + await cache.set('foo', 'bar'); + await cache.delete('foo'); + + const result = cache.has('foo'); + expect(result).toBe(false); + }); + + it('should call onChanged when deleting', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + name: 'fake', + onChanged: fakeOnChanged, + }); + + await cache.set('foo', 'bar'); + await cache.delete('foo'); + + expect(fakeOnChanged).toHaveBeenCalledWith(['foo'], cache); + }); + + it('should emit event when deleting', async () => { + const cache = makeCache({ name: 'fake' }); + + await cache.set('foo', 'bar'); + await cache.delete('foo'); + + expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo'] }]]); + }); + + it('should delete when receiving set event', async () => { + const cache = makeCache({ name: 'fake' }); + await cache.set('foo', 'bar'); + + await fakeInternalEventService._emitRedis('quantumCacheUpdated', { name: 'fake', keys: ['foo'] }); + + const result = cache.has('foo'); + expect(result).toBe(false); + }); + + it('should call onChanged when receiving set event', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + name: 'fake', + onChanged: fakeOnChanged, + }); + + await fakeInternalEventService._emitRedis('quantumCacheUpdated', { name: 'fake', keys: ['foo'] }); + + expect(fakeOnChanged).toHaveBeenCalledWith(['foo'], cache); + }); + + it('should delete when receiving delete event', async () => { + const cache = makeCache({ name: 'fake' }); + await cache.set('foo', 'bar'); + + await fakeInternalEventService._emitRedis('quantumCacheUpdated', { name: 'fake', keys: ['foo'] }); + + const result = cache.has('foo'); + expect(result).toBe(false); + }); + + it('should call onChanged when receiving delete event', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + name: 'fake', + onChanged: fakeOnChanged, + }); + await cache.set('foo', 'bar'); + + await fakeInternalEventService._emitRedis('quantumCacheUpdated', { name: 'fake', keys: ['foo'] }); + + expect(fakeOnChanged).toHaveBeenCalledWith(['foo'], cache); + }); + + describe('get', () => { + it('should return value if present', async () => { + const cache = makeCache(); + await cache.set('foo', 'bar'); + + const result = cache.get('foo'); + + expect(result).toBe('bar'); + }); + it('should return undefined if missing', () => { + const cache = makeCache(); + + const result = cache.get('foo'); + + expect(result).toBe(undefined); + }); + }); + + describe('setMany', () => { + it('should populate all values', async () => { + const cache = makeCache(); + + await cache.setMany([['foo', 'bar'], ['alpha', 'omega']]); + + expect(cache.has('foo')).toBe(true); + expect(cache.has('alpha')).toBe(true); + }); + + it('should emit one event', async () => { + const cache = makeCache({ + name: 'fake', + }); + + await cache.setMany([['foo', 'bar'], ['alpha', 'omega']]); + + expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo', 'alpha'] }]]); + expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(1); + }); + + it('should call onChanged once with all items', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + name: 'fake', + onChanged: fakeOnChanged, + }); + + await cache.setMany([['foo', 'bar'], ['alpha', 'omega']]); + + expect(fakeOnChanged).toHaveBeenCalledWith(['foo', 'alpha'], cache); + expect(fakeOnChanged).toHaveBeenCalledTimes(1); + }); + + it('should emit events only for changed items', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + name: 'fake', + onChanged: fakeOnChanged, + }); + + await cache.set('foo', 'bar'); + fakeOnChanged.mockClear(); + fakeInternalEventService._reset(); + + await cache.setMany([['foo', 'bar'], ['alpha', 'omega']]); + + expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['alpha'] }]]); + expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(1); + expect(fakeOnChanged).toHaveBeenCalledWith(['alpha'], cache); + expect(fakeOnChanged).toHaveBeenCalledTimes(1); + }); + }); + + describe('getMany', () => { + it('should return empty for empty input', () => { + const cache = makeCache(); + const result = cache.getMany([]); + expect(result).toEqual([]); + }); + + it('should return the value for all keys', () => { + const cache = makeCache(); + cache.add('foo', 'bar'); + cache.add('alpha', 'omega'); + + const result = cache.getMany(['foo', 'alpha']); + + expect(result).toEqual([['foo', 'bar'], ['alpha', 'omega']]); + }); + + it('should return undefined for missing keys', () => { + const cache = makeCache(); + cache.add('foo', 'bar'); + + const result = cache.getMany(['foo', 'alpha']); + + expect(result).toEqual([['foo', 'bar'], ['alpha', undefined]]); + }); + }); + + describe('fetchMany', () => { + it('should do nothing for empty input', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + onChanged: fakeOnChanged, + }); + + await cache.fetchMany([]); + + expect(fakeOnChanged).not.toHaveBeenCalled(); + expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0); + }); + + it('should return existing items', async () => { + const cache = makeCache(); + cache.add('foo', 'bar'); + cache.add('alpha', 'omega'); + + const result = await cache.fetchMany(['foo', 'alpha']); + + expect(result).toEqual([['foo', 'bar'], ['alpha', 'omega']]); + }); + + it('should return existing items without events', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + onChanged: fakeOnChanged, + }); + cache.add('foo', 'bar'); + cache.add('alpha', 'omega'); + + await cache.fetchMany(['foo', 'alpha']); + + expect(fakeOnChanged).not.toHaveBeenCalled(); + expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0); + }); + + it('should call bulkFetcher for missing items', async () => { + const cache = makeCache({ + bulkFetcher: keys => keys.map(k => [k, `${k}#many`]), + fetcher: key => `${key}#single`, + }); + + const results = await cache.fetchMany(['foo', 'alpha']); + + expect(results).toEqual([['foo', 'foo#many'], ['alpha', 'alpha#many']]); + }); + + it('should call bulkFetcher only once', async () => { + const mockBulkFetcher = jest.fn((keys: string[]) => keys.map(k => [k, `${k}#value`] as [string, string])); + const cache = makeCache({ + bulkFetcher: mockBulkFetcher, + }); + + await cache.fetchMany(['foo', 'bar']); + + expect(mockBulkFetcher).toHaveBeenCalledTimes(1); + }); + + it('should call fetcher when fetchMany is undefined', async () => { + const cache = makeCache({ + fetcher: key => `${key}#single`, + }); + + const results = await cache.fetchMany(['foo', 'alpha']); + + expect(results).toEqual([['foo', 'foo#single'], ['alpha', 'alpha#single']]); + }); + + it('should call onChanged', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + onChanged: fakeOnChanged, + fetcher: k => k, + }); + + await cache.fetchMany(['foo', 'alpha']); + + expect(fakeOnChanged).toHaveBeenCalledWith(['foo', 'alpha'], cache); + expect(fakeOnChanged).toHaveBeenCalledTimes(1); + }); + + it('should call onChanged only for changed', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + onChanged: fakeOnChanged, + fetcher: k => k, + }); + cache.add('foo', 'bar'); + + await cache.fetchMany(['foo', 'alpha']); + + expect(fakeOnChanged).toHaveBeenCalledWith(['alpha'], cache); + expect(fakeOnChanged).toHaveBeenCalledTimes(1); + }); + + it('should not emit event', async () => { + const cache = makeCache({ + fetcher: k => k, + }); + + await cache.fetchMany(['foo', 'alpha']); + + expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0); + }); + }); + + describe('refreshMany', () => { + it('should do nothing for empty input', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + onChanged: fakeOnChanged, + }); + + const result = await cache.refreshMany([]); + + expect(result).toEqual([]); + expect(fakeOnChanged).not.toHaveBeenCalled(); + expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0); + }); + + it('should call bulkFetcher for all keys', async () => { + const mockBulkFetcher = jest.fn((keys: string[]) => keys.map(k => [k, `${k}#value`] as [string, string])); + const cache = makeCache({ + bulkFetcher: mockBulkFetcher, + }); + + const result = await cache.refreshMany(['foo', 'alpha']); + + expect(result).toEqual([['foo', 'foo#value'], ['alpha', 'alpha#value']]); + expect(mockBulkFetcher).toHaveBeenCalledWith(['foo', 'alpha'], cache); + expect(mockBulkFetcher).toHaveBeenCalledTimes(1); + }); + + it('should replace any existing keys', async () => { + const mockBulkFetcher = jest.fn((keys: string[]) => keys.map(k => [k, `${k}#value`] as [string, string])); + const cache = makeCache({ + bulkFetcher: mockBulkFetcher, + }); + cache.add('foo', 'bar'); + + const result = await cache.refreshMany(['foo', 'alpha']); + + expect(result).toEqual([['foo', 'foo#value'], ['alpha', 'alpha#value']]); + expect(mockBulkFetcher).toHaveBeenCalledWith(['foo', 'alpha'], cache); + expect(mockBulkFetcher).toHaveBeenCalledTimes(1); + }); + + it('should call onChanged for all keys', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + bulkFetcher: keys => keys.map(k => [k, `${k}#value`]), + onChanged: fakeOnChanged, + }); + cache.add('foo', 'bar'); + + await cache.refreshMany(['foo', 'alpha']); + + expect(fakeOnChanged).toHaveBeenCalledWith(['foo', 'alpha'], cache); + expect(fakeOnChanged).toHaveBeenCalledTimes(1); + }); + + it('should emit event for all keys', async () => { + const cache = makeCache({ + name: 'fake', + bulkFetcher: keys => keys.map(k => [k, `${k}#value`]), + }); + cache.add('foo', 'bar'); + + await cache.refreshMany(['foo', 'alpha']); + + expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo', 'alpha'] }]]); + expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(1); + }); + }); + + describe('deleteMany', () => { + it('should remove keys from memory cache', async () => { + const cache = makeCache(); + + await cache.set('foo', 'bar'); + await cache.set('alpha', 'omega'); + await cache.deleteMany(['foo', 'alpha']); + + expect(cache.has('foo')).toBe(false); + expect(cache.has('alpha')).toBe(false); + }); + + it('should emit only one event', async () => { + const cache = makeCache({ + name: 'fake', + }); + + await cache.deleteMany(['foo', 'alpha']); + + expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo', 'alpha'] }]]); + expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(1); + }); + + it('should call onChanged once with all items', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + name: 'fake', + onChanged: fakeOnChanged, + }); + + await cache.deleteMany(['foo', 'alpha']); + + expect(fakeOnChanged).toHaveBeenCalledWith(['foo', 'alpha'], cache); + expect(fakeOnChanged).toHaveBeenCalledTimes(1); + }); + + it('should do nothing if no keys are provided', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + name: 'fake', + onChanged: fakeOnChanged, + }); + + await cache.deleteMany([]); + + expect(fakeOnChanged).not.toHaveBeenCalled(); + expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0); + }); + }); + + describe('refresh', () => { + it('should populate the value', async () => { + const cache = makeCache({ + name: 'fake', + fetcher: key => `value#${key}`, + }); + + await cache.refresh('foo'); + + const result = cache.has('foo'); + expect(result).toBe(true); + }); + + it('should return the value', async () => { + const cache = makeCache({ + name: 'fake', + fetcher: key => `value#${key}`, + }); + + const result = await cache.refresh('foo'); + + expect(result).toBe('value#foo'); + }); + + it('should replace the value if it exists', async () => { + const cache = makeCache({ + name: 'fake', + fetcher: key => `value#${key}`, + }); + + await cache.set('foo', 'bar'); + const result = await cache.refresh('foo'); + + expect(result).toBe('value#foo'); + }); + + it('should call onChanged', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + name: 'fake', + fetcher: key => `value#${key}`, + onChanged: fakeOnChanged, + }); + + await cache.refresh('foo'); + + expect(fakeOnChanged).toHaveBeenCalledWith(['foo'], cache); + }); + + it('should emit event', async () => { + const cache = makeCache({ + name: 'fake', + fetcher: key => `value#${key}`, + }); + + await cache.refresh('foo'); + + expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo'] }]]); + }); + }); + + describe('add', () => { + it('should add the item', () => { + const cache = makeCache(); + cache.add('foo', 'bar'); + expect(cache.has('foo')).toBe(true); + }); + + it('should not emit event', () => { + const cache = makeCache({ + name: 'fake', + }); + + cache.add('foo', 'bar'); + + expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0); + }); + + it('should not call onChanged', () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + onChanged: fakeOnChanged, + }); + + cache.add('foo', 'bar'); + + expect(fakeOnChanged).not.toHaveBeenCalled(); + }); + }); + + describe('addMany', () => { + it('should add all items', () => { + const cache = makeCache(); + + cache.addMany([['foo', 'bar'], ['alpha', 'omega']]); + + expect(cache.has('foo')).toBe(true); + expect(cache.has('alpha')).toBe(true); + }); + + it('should not emit event', () => { + const cache = makeCache({ + name: 'fake', + }); + + cache.addMany([['foo', 'bar'], ['alpha', 'omega']]); + + expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0); + }); + + it('should not call onChanged', () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + onChanged: fakeOnChanged, + }); + + cache.addMany([['foo', 'bar'], ['alpha', 'omega']]); + + expect(fakeOnChanged).not.toHaveBeenCalled(); + }); + }); + + describe('has', () => { + it('should return false when empty', () => { + const cache = makeCache(); + const result = cache.has('foo'); + expect(result).toBe(false); + }); + + it('should return false when value is not in memory', async () => { + const cache = makeCache(); + await cache.set('foo', 'bar'); + + const result = cache.has('alpha'); + + expect(result).toBe(false); + }); + + it('should return true when value is in memory', async () => { + const cache = makeCache(); + await cache.set('foo', 'bar'); + + const result = cache.has('foo'); + + expect(result).toBe(true); + }); + }); + + describe('size', () => { + it('should return 0 when empty', () => { + const cache = makeCache(); + expect(cache.size).toBe(0); + }); + + it('should return correct size when populated', async () => { + const cache = makeCache(); + await cache.set('foo', 'bar'); + + expect(cache.size).toBe(1); + }); + }); + + describe('entries', () => { + it('should return empty when empty', () => { + const cache = makeCache(); + + const result = Array.from(cache.entries()); + + expect(result).toHaveLength(0); + }); + + it('should return all entries when populated', async () => { + const cache = makeCache(); + await cache.set('foo', 'bar'); + + const result = Array.from(cache.entries()); + + expect(result).toEqual([['foo', 'bar']]); + }); + }); + + describe('keys', () => { + it('should return empty when empty', () => { + const cache = makeCache(); + + const result = Array.from(cache.keys()); + + expect(result).toHaveLength(0); + }); + + it('should return all keys when populated', async () => { + const cache = makeCache(); + await cache.set('foo', 'bar'); + + const result = Array.from(cache.keys()); + + expect(result).toEqual(['foo']); + }); + }); + + describe('values', () => { + it('should return empty when empty', () => { + const cache = makeCache(); + + const result = Array.from(cache.values()); + + expect(result).toHaveLength(0); + }); + + it('should return all values when populated', async () => { + const cache = makeCache(); + await cache.set('foo', 'bar'); + + const result = Array.from(cache.values()); + + expect(result).toEqual(['bar']); + }); + }); + + describe('[Symbol.iterator]', () => { + it('should return empty when empty', () => { + const cache = makeCache(); + + const result = Array.from(cache); + + expect(result).toHaveLength(0); + }); + + it('should return all entries when populated', async () => { + const cache = makeCache(); + await cache.set('foo', 'bar'); + + const result = Array.from(cache); + + expect(result).toEqual([['foo', 'bar']]); + }); + }); +}); diff --git a/packages/backend/test/utils.ts b/packages/backend/test/utils.ts index 0e7a0b591f..96332a92e4 100644 --- a/packages/backend/test/utils.ts +++ b/packages/backend/test/utils.ts @@ -11,12 +11,12 @@ import { inspect } from 'node:util'; import WebSocket, { ClientOptions } from 'ws'; import fetch, { File, RequestInit, type Headers } from 'node-fetch'; import { DataSource } from 'typeorm'; -import { load as cheerio } from 'cheerio'; +import { load as cheerio } from 'cheerio/slim'; import { type Response } from 'node-fetch'; import Fastify from 'fastify'; import { entities } from '../src/postgres.js'; import { loadConfig } from '../src/config.js'; -import type { CheerioAPI } from 'cheerio'; +import type { CheerioAPI } from 'cheerio/slim'; import type * as misskey from 'misskey-js'; import { DEFAULT_POLICIES } from '@/core/RoleService.js'; import { validateContentTypeSetAsActivityPub } from '@/core/activitypub/misc/validator.js'; diff --git a/packages/frontend-embed/package.json b/packages/frontend-embed/package.json index fa54842914..b6ed56b2e2 100644 --- a/packages/frontend-embed/package.json +++ b/packages/frontend-embed/package.json @@ -11,35 +11,25 @@ }, "dependencies": { "@discordapp/twemoji": "15.1.0", - "@phosphor-icons/web": "^2.0.3", - "@rollup/plugin-json": "6.1.0", - "@rollup/plugin-replace": "6.0.2", - "@rollup/pluginutils": "5.1.4", - "@transfem-org/sfm-js": "0.24.5", - "@twemoji/parser": "15.1.1", - "@vitejs/plugin-vue": "5.2.3", - "@vue/compiler-sfc": "3.5.14", - "astring": "1.9.0", + "@phosphor-icons/web": "2.1.2", + "mfm-js": "npm:@transfem-org/sfm-js@0.24.6", "buraha": "0.0.1", - "estree-walker": "3.0.3", "frontend-shared": "workspace:*", "json5": "2.2.3", "misskey-js": "workspace:*", "punycode.js": "2.3.1", - "rollup": "4.40.0", - "sass": "1.87.0", "shiki": "3.3.0", "tinycolor2": "1.6.0", - "tsc-alias": "1.8.15", - "tsconfig-paths": "4.2.0", - "typescript": "5.8.3", "uuid": "11.1.0", - "vite": "6.3.4", "vue": "3.5.14" }, "devDependencies": { - "@misskey-dev/summaly": "5.2.1", + "@misskey-dev/summaly": "npm:@transfem-org/summaly@5.2.2", + "@rollup/plugin-json": "6.1.0", + "@rollup/plugin-replace": "6.0.2", + "@rollup/pluginutils": "5.1.4", "@testing-library/vue": "8.1.0", + "@twemoji/parser": "15.1.1", "@types/estree": "1.0.7", "@types/micromatch": "4.0.9", "@types/node": "22.15.2", @@ -48,12 +38,16 @@ "@types/ws": "8.18.1", "@typescript-eslint/eslint-plugin": "8.31.0", "@typescript-eslint/parser": "8.31.0", + "@vitejs/plugin-vue": "5.2.3", "@vitest/coverage-v8": "3.1.2", + "@vue/compiler-sfc": "3.5.14", "@vue/runtime-core": "3.5.14", "acorn": "8.14.1", + "astring": "1.9.0", "cross-env": "7.0.3", "eslint-plugin-import": "2.31.0", "eslint-plugin-vue": "10.0.0", + "estree-walker": "3.0.3", "fast-glob": "3.3.3", "happy-dom": "17.4.4", "intersection-observer": "0.12.2", @@ -61,7 +55,13 @@ "msw": "2.7.5", "nodemon": "3.1.10", "prettier": "3.5.3", + "rollup": "4.40.0", + "sass": "1.87.0", "start-server-and-test": "2.0.11", + "tsc-alias": "1.8.15", + "tsconfig-paths": "4.2.0", + "typescript": "5.8.3", + "vite": "6.3.4", "vite-plugin-turbosnap": "1.0.3", "vue-component-type-helpers": "2.2.10", "vue-eslint-parser": "10.1.3", diff --git a/packages/frontend-embed/src/components/EmMfm.ts b/packages/frontend-embed/src/components/EmMfm.ts index d377d492e0..74ae3373ef 100644 --- a/packages/frontend-embed/src/components/EmMfm.ts +++ b/packages/frontend-embed/src/components/EmMfm.ts @@ -5,7 +5,7 @@ import { h, provide } from 'vue'; import type { VNode, SetupContext } from 'vue'; -import * as mfm from '@transfem-org/sfm-js'; +import * as mfm from 'mfm-js'; import * as Misskey from 'misskey-js'; import { host } from '@@/js/config.js'; import EmUrl from '@/components/EmUrl.vue'; diff --git a/packages/frontend-embed/src/components/EmNote.vue b/packages/frontend-embed/src/components/EmNote.vue index 666cbde72d..0dc77d09a7 100644 --- a/packages/frontend-embed/src/components/EmNote.vue +++ b/packages/frontend-embed/src/components/EmNote.vue @@ -105,7 +105,7 @@ SPDX-License-Identifier: AGPL-3.0-only