misty.cafe/mistykey
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Merge branch 'develop' into upstream/2025.5.0

Browse source
This commit is contained in:
dakkar 2025-06-13 08:43:17 +01:00
commit 33aee38a59
125 changed files with 3926 additions and 2148 deletions
Download patch file Download diff file Expand all files Collapse all files

29
CONTRIBUTING.md
View file

@ -622,6 +622,35 @@ marginはそのコンポーネントを使う側が設定する
### indexというファイル名を使うな
ESMではディレクトリインポートは廃止されているのと、ディレクトリインポートせずともファイル名が index だと何故か一部のライブラリ？でディレクトリインポートだと見做されてエラーになる
### Memory Caches
Sharkey offers multiple memory cache implementations, each meant for a different use case.
The following table compares the available options:
| Cache | Type | Consistency | Persistence | Data Source | Cardinality | Eviction | Description |
|---------------------|-----------|-------------|-------------|-------------|-------------|----------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `MemoryKVCache` | Key-Value | None | None | Caller | Single | Lifetime | Implements a basic in-memory Key-Value store. The implementation is entirely synchronous, except for user-provided data sources. |
| `MemorySingleCache` | Single | None | None | Caller | Single | Lifetime | Implements a basic in-memory Single Value store. The implementation is entirely synchronous, except for user-provided data sources. |
| `RedisKVCache` | Key-Value | Eventual | Redis | Callback | Single | Lifetime | Extends `MemoryKVCache` with Redis-backed persistence and a pre-defined callback data source. This provides eventual consistency guarantees based on the memory cache lifetime. |
| `RedisSingleCache` | Single | Eventual | Redis | Callback | Single | Lifetime | Extends `MemorySingleCache` with Redis-backed persistence and a pre-defined callback data source. This provides eventual consistency guarantees based on the memory cache lifetime. |
| `QuantumKVCache` | Key-Value | Immediate | None | Callback | Multiple | Lifetime | Combines `MemoryKVCache` with a pre-defined callback data source and immediate consistency via Redis sync events. The implementation offers multi-item batch overloads for efficient bulk operations. **This is the recommended cache implementation for most use cases.** |
Key-Value caches store multiple entries per cache, while Single caches store a single value that can be accessed directly.
Consistency refers to the consistency of cached data between different processes in the instance cluster: "None" means no consistency guarantees, "Eventual" caches will gradually become consistent after some unknown time, and "Immediate" consistency ensures accurate data ASAP after the update.
Caches with persistence can retain their data after a reboot through an external service such as Redis.
If a data source is supported, then this allows the cache to directly load missing data in response to a fetch.
"Caller" data sources are passed into the fetch method(s) directly, while "Callback" sources are passed in as a function when the cache is first initialized.
The cardinality of a cache refers to the number of items that can be updated in a single operation, and eviction, finally, is the method that the cache uses to evict stale data.
#### Selecting a cache implementation
For most cache uses, `QuantumKVCache` should be considered first.
It offers strong consistency guarantees, multiple cardinality, and a cleaner API surface than the older caches.
An alternate cache implementation should be considered if any of the following apply:
* The data is particularly slow to calculate or difficult to access. In these cases, either `RedisKVCache` or `RedisSingleCache` should be considered.
* If stale data is acceptable, then consider `MemoryKVCache` or `MemorySingleCache`. These synchronous implementations have much less overhead than the other options.
* There is only one data item, or all data items must be fetched together. Using `MemorySingleCache` or `RedisSingleCache` could provide a cleaner implementation without resorting to hacks like a fixed key.
## CSS Recipe
### Lighten CSS vars

28
package.json
View file

@ -54,17 +54,7 @@
"lodash": "4.17.21"
},
"dependencies": {
"cssnano": "7.0.6",
"esbuild": "0.25.3",
"execa": "9.5.2",
"fast-glob": "3.3.3",
"glob": "11.0.2",
"ignore-walk": "7.0.0",
"js-yaml": "4.1.0",
"postcss": "8.5.3",
"tar": "7.4.3",
"terser": "5.39.0",
"typescript": "5.8.3"
"js-yaml": "4.1.0"
},
"optionalDependencies": {
"cypress": "14.3.2"
@ -75,10 +65,20 @@
"@typescript-eslint/eslint-plugin": "8.31.0",
"@typescript-eslint/parser": "8.31.0",
"cross-env": "7.0.3",
"cssnano": "7.0.6",
"esbuild": "0.25.3",
"eslint": "9.25.1",
"globals": "16.0.0",
"execa": "9.5.2",
"fast-glob": "3.3.3",
"glob": "11.0.2",
"globals": "16.1.0",
"ncp": "2.0.0",
"pnpm": "10.10.0",
"start-server-and-test": "2.0.11"
"pnpm": "9.6.0",
"ignore-walk": "7.0.0",
"postcss": "8.5.3",
"start-server-and-test": "2.0.11",
"tar": "7.4.3",
"terser": "5.39.0",
"typescript": "5.8.3"
}
}

52
packages/backend/package.json
View file

@ -80,7 +80,7 @@
"@fastify/static": "8.1.1",
"@fastify/view": "10.0.2",
"@misskey-dev/sharp-read-bmp": "1.3.0",
"@misskey-dev/summaly": "5.2.1",
"@misskey-dev/summaly": "npm:@transfem-org/summaly@5.2.2",
"@nestjs/common": "11.1.0",
"@nestjs/core": "11.1.0",
"@nestjs/testing": "11.1.0",
@ -90,33 +90,30 @@
"@simplewebauthn/server": "12.0.0",
"@sinonjs/fake-timers": "11.3.1",
"@smithy/node-http-handler": "2.5.0",
"@swc/cli": "0.7.3",
"@swc/core": "1.11.24",
"@transfem-org/sfm-js": "0.24.6",
"mfm-js": "npm:@transfem-org/sfm-js@0.24.6",
"@twemoji/parser": "15.1.1",
"accepts": "1.3.8",
"ajv": "8.17.1",
"archiver": "7.0.1",
"argon2": "^0.40.1",
"argon2": "0.43.0",
"axios": "1.7.4",
"async-mutex": "0.5.0",
"bcryptjs": "2.4.3",
"blurhash": "2.0.5",
"body-parser": "1.20.3",
"bullmq": "5.51.1",
"cacheable-lookup": "7.0.0",
"canvas": "^3.1.0",
"canvas": "3.1.0",
"cbor": "9.0.2",
"chalk": "5.4.1",
"chalk-template": "1.1.0",
"cheerio": "1.0.0",
"chokidar": "3.6.0",
"cli-highlight": "2.1.11",
"cli-highlight": "npm:@transfem-org/cli-highlight@2.1.12",
"color-convert": "2.0.1",
"content-disposition": "0.5.4",
"date-fns": "2.30.0",
"deep-email-validator": "0.1.21",
"fast-xml-parser": "4.4.1",
"dom-serializer": "2.0.0",
"domhandler": "5.0.3",
"domutils": "3.2.2",
"fastify": "5.3.2",
"fastify-raw-body": "5.0.0",
"feed": "4.2.2",
@ -125,10 +122,9 @@
"form-data": "4.0.2",
"glob": "11.0.0",
"got": "14.4.7",
"happy-dom": "16.8.1",
"hpagent": "1.2.0",
"htmlescape": "1.1.1",
"http-link-header": "1.1.3",
"htmlparser2": "9.1.0",
"ioredis": "5.6.1",
"ip-cidr": "4.0.2",
"ipaddr.js": "2.2.0",
@ -136,50 +132,40 @@
"js-yaml": "4.1.0",
"json5": "2.2.3",
"jsonld": "8.3.3",
"jsrsasign": "11.1.0",
"juice": "11.0.1",
"megalodon": "workspace:*",
"meilisearch": "0.50.0",
"microformats-parser": "2.0.2",
"mime-types": "2.1.35",
"misskey-js": "workspace:*",
"misskey-reversi": "workspace:*",
"moment": "^2.30.1",
"moment": "2.30.1",
"ms": "3.0.0-canary.1",
"nanoid": "5.1.5",
"nested-property": "4.0.0",
"node-fetch": "3.3.2",
"nodemailer": "6.10.1",
"oauth": "0.10.2",
"oauth2orize": "1.12.0",
"oauth2orize-pkce": "0.1.2",
"os-utils": "0.0.14",
"otpauth": "9.4.0",
"parse5": "7.3.0",
"pg": "8.15.6",
"pkce-challenge": "4.1.0",
"probe-image-size": "7.2.3",
"promise-limit": "2.7.0",
"proxy-addr": "^2.0.7",
"psl": "^1.13.0",
"proxy-addr": "2.0.7",
"psl": "1.15.0",
"pug": "3.0.3",
"qrcode": "1.5.4",
"random-seed": "0.3.0",
"ratelimiter": "3.4.1",
"re2": "1.21.4",
"redis-info": "3.1.0",
"redis-lock": "0.1.4",
"reflect-metadata": "0.2.2",
"rename": "1.0.4",
"rss-parser": "3.13.0",
"rxjs": "7.8.2",
"sanitize-html": "2.16.0",
"secure-json-parse": "3.0.2",
"sharp": "0.34.1",
"semver": "7.7.1",
"slacc": "0.0.10",
"strict-event-emitter-types": "2.0.0",
"stringz": "2.1.0",
"systeminformation": "5.25.11",
"tinycolor2": "1.6.0",
"tmp": "0.2.3",
@ -188,7 +174,7 @@
"typeorm": "0.3.22",
"typescript": "5.8.3",
"ulid": "2.4.0",
"uuid": "^9.0.1",
"uuid": "11.1.0",
"vary": "1.1.2",
"web-push": "3.6.7",
"ws": "8.18.1",
@ -199,16 +185,16 @@
"@nestjs/platform-express": "11.1.0",
"@sentry/vue": "9.14.0",
"@simplewebauthn/types": "12.0.0",
"@swc/cli": "0.7.3",
"@swc/core": "1.11.24",
"@swc/jest": "0.2.38",
"@types/accepts": "1.3.7",
"@types/archiver": "6.0.3",
"@types/bcryptjs": "2.4.6",
"@types/body-parser": "1.19.5",
"@types/color-convert": "2.0.4",
"@types/content-disposition": "0.5.8",
"@types/fluent-ffmpeg": "2.1.27",
"@types/htmlescape": "1.1.3",
"@types/http-link-header": "1.0.7",
"@types/jest": "29.5.14",
"@types/js-yaml": "4.0.9",
"@types/jsonld": "1.5.15",
@ -221,12 +207,11 @@
"@types/oauth2orize": "1.11.5",
"@types/oauth2orize-pkce": "0.1.2",
"@types/pg": "8.11.14",
"@types/proxy-addr": "^2.0.3",
"@types/psl": "^1.1.3",
"@types/proxy-addr": "2.0.3",
"@types/psl": "1.1.3",
"@types/pug": "2.0.10",
"@types/qrcode": "1.5.5",
"@types/random-seed": "0.3.5",
"@types/ratelimiter": "3.4.6",
"@types/redis-info": "3.0.3",
"@types/rename": "1.0.7",
"@types/sanitize-html": "2.15.0",
@ -236,7 +221,6 @@
"@types/supertest": "6.0.3",
"@types/tinycolor2": "1.4.6",
"@types/tmp": "0.2.6",
"@types/uuid": "^9.0.4",
"@types/vary": "1.1.3",
"@types/web-push": "3.6.4",
"@types/ws": "8.18.1",
@ -245,7 +229,7 @@
"aws-sdk-client-mock": "4.1.0",
"cross-env": "7.0.3",
"eslint-plugin-import": "2.31.0",
"execa": "8.0.1",
"execa": "9.5.2",
"fkill": "9.0.0",
"jest": "29.7.0",
"jest-mock": "29.7.0",

28
packages/backend/src/boot/entry.ts
View file

@ -64,17 +64,35 @@ async function main() {
}
// Display detail of uncaught exception
process.on('uncaughtException', err => {
process.on('uncaughtExceptionMonitor', ((err, origin) => {
try {
logger.error('Uncaught exception:', err);
logger.error(`Uncaught exception (${origin}):`, err);
} catch {
console.error('Uncaught exception:', err);
console.error(`Uncaught exception (${origin}):`, err);
}
});
}));
// Dying away...
process.on('disconnect', () => {
try {
logger.warn('IPC channel disconnected! The process may soon die.');
} catch {
console.warn('IPC channel disconnected! The process may soon die.');
}
});
process.on('beforeExit', code => {
try {
logger.warn(`Event loop died! Process will exit with code ${code}.`);
} catch {
console.warn(`Event loop died! Process will exit with code ${code}.`);
}
});
process.on('exit', code => {
logger.info(`The process is going to exit with code ${code}`);
try {
logger.info(`The process is going to exit with code ${code}`);
} catch {
console.info(`The process is going to exit with code ${code}`);
}
});
//#endregion

24
packages/backend/src/core/AccountMoveService.ts
View file

@ -26,6 +26,7 @@ import PerUserFollowingChart from '@/core/chart/charts/per-user-following.js';
import { SystemAccountService } from '@/core/SystemAccountService.js';
import { RoleService } from '@/core/RoleService.js';
import { AntennaService } from '@/core/AntennaService.js';
import { CacheService } from '@/core/CacheService.js';
@Injectable()
export class AccountMoveService {
@ -68,6 +69,7 @@ export class AccountMoveService {
private systemAccountService: SystemAccountService,
private roleService: RoleService,
private antennaService: AntennaService,
private readonly cacheService: CacheService,
) {
}
@ -107,12 +109,10 @@ export class AccountMoveService {
this.globalEventService.publishMainStream(src.id, 'meUpdated', iObj);
// Unfollow after 24 hours
const followings = await this.followingsRepository.findBy({
followerId: src.id,
});
this.queueService.createDelayedUnfollowJob(followings.map(following => ({
const followings = await this.cacheService.userFollowingsCache.fetch(src.id);
this.queueService.createDelayedUnfollowJob(Array.from(followings.keys()).map(followeeId => ({
from: { id: src.id },
to: { id: following.followeeId },
to: { id: followeeId },
})), process.env.NODE_ENV === 'test' ? 10000 : 1000 * 60 * 60 * 24);
await this.postMoveProcess(src, dst);
@ -138,11 +138,9 @@ export class AccountMoveService {
// follow the new account
const proxy = await this.systemAccountService.fetch('proxy');
const followings = await this.followingsRepository.findBy({
followeeId: src.id,
followerHost: IsNull(), // follower is local
followerId: Not(proxy.id),
});
const followings = await this.cacheService.userFollowersCache.fetch(src.id)
.then(fs => Array.from(fs.values())
.filter(f => f.followerHost == null && f.followerId !== proxy.id));
const followJobs = followings.map(following => ({
from: { id: following.followerId },
to: { id: dst.id },
@ -318,9 +316,9 @@ export class AccountMoveService {
await this.usersRepository.decrement({ id: In(localFollowerIds) }, 'followingCount', 1);
// Decrease follower counts of local followees by 1.
const oldFollowings = await this.followingsRepository.findBy({ followerId: oldAccount.id });
if (oldFollowings.length > 0) {
await this.usersRepository.decrement({ id: In(oldFollowings.map(following => following.followeeId)) }, 'followersCount', 1);
const oldFollowings = await this.cacheService.userFollowingsCache.fetch(oldAccount.id);
if (oldFollowings.size > 0) {
await this.usersRepository.decrement({ id: In(Array.from(oldFollowings.keys())) }, 'followersCount', 1);
}
// Update instance stats by decreasing remote followers count by the number of local followers who were following the old account.

3
packages/backend/src/core/AntennaService.ts
View file

@ -130,7 +130,8 @@ export class AntennaService implements OnApplicationShutdown {
}
if (note.visibility === 'followers') {
const isFollowing = Object.hasOwn(await this.cacheService.userFollowingsCache.fetch(antenna.userId), note.userId);
const followings = await this.cacheService.userFollowingsCache.fetch(antenna.userId);
const isFollowing = followings.has(note.userId);
if (!isFollowing && antenna.userId !== note.userId) return false;
}

354
packages/backend/src/core/CacheService.ts
View file

@ -5,14 +5,16 @@
import { Inject, Injectable } from '@nestjs/common';
import * as Redis from 'ioredis';
import { IsNull } from 'typeorm';
import type { BlockingsRepository, FollowingsRepository, MutingsRepository, RenoteMutingsRepository, MiUserProfile, UserProfilesRepository, UsersRepository, MiFollowing, MiNote } from '@/models/_.js';
import { In, IsNull } from 'typeorm';
import type { BlockingsRepository, FollowingsRepository, MutingsRepository, RenoteMutingsRepository, MiUserProfile, UserProfilesRepository, UsersRepository, MiNote, MiFollowing } from '@/models/_.js';
import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js';
import { QuantumKVCache } from '@/misc/QuantumKVCache.js';
import type { MiLocalUser, MiUser } from '@/models/User.js';
import { DI } from '@/di-symbols.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { bindThis } from '@/decorators.js';
import type { GlobalEvents } from '@/core/GlobalEventService.js';
import type { InternalEventTypes } from '@/core/GlobalEventService.js';
import { InternalEventService } from '@/core/InternalEventService.js';
import type { OnApplicationShutdown } from '@nestjs/common';
export interface FollowStats {
@ -27,7 +29,7 @@ export interface CachedTranslation {
text: string | undefined;
}
interface CachedTranslationEntity {
export interface CachedTranslationEntity {
l?: string;
t?: string;
u?: number;
@ -39,14 +41,16 @@ export class CacheService implements OnApplicationShutdown {
public localUserByNativeTokenCache: MemoryKVCache<MiLocalUser | null>;
public localUserByIdCache: MemoryKVCache<MiLocalUser>;
public uriPersonCache: MemoryKVCache<MiUser | null>;
public userProfileCache: RedisKVCache<MiUserProfile>;
public userMutingsCache: RedisKVCache<Set<string>>;
public userBlockingCache: RedisKVCache<Set<string>>;
public userBlockedCache: RedisKVCache<Set<string>>; // NOTE: 「被」Blockキャッシュ
public renoteMutingsCache: RedisKVCache<Set<string>>;
public userFollowingsCache: RedisKVCache<Record<string, Pick<MiFollowing, 'withReplies'> | undefined>>;
private readonly userFollowStatsCache = new MemoryKVCache<FollowStats>(1000 * 60 * 10); // 10 minutes
private readonly translationsCache: RedisKVCache<CachedTranslationEntity>;
public userProfileCache: QuantumKVCache<MiUserProfile>;
public userMutingsCache: QuantumKVCache<Set<string>>;
public userBlockingCache: QuantumKVCache<Set<string>>;
public userBlockedCache: QuantumKVCache<Set<string>>; // NOTE: 「被」Blockキャッシュ
public renoteMutingsCache: QuantumKVCache<Set<string>>;
public userFollowingsCache: QuantumKVCache<Map<string, Omit<MiFollowing, 'isFollowerHibernated'>>>;
public userFollowersCache: QuantumKVCache<Map<string, Omit<MiFollowing, 'isFollowerHibernated'>>>;
public hibernatedUserCache: QuantumKVCache<boolean>;
protected userFollowStatsCache = new MemoryKVCache<FollowStats>(1000 * 60 * 10); // 10 minutes
protected translationsCache: RedisKVCache<CachedTranslationEntity>;
constructor(
@Inject(DI.redis)
@ -74,6 +78,7 @@ export class CacheService implements OnApplicationShutdown {
private followingsRepository: FollowingsRepository,
private userEntityService: UserEntityService,
private readonly internalEventService: InternalEventService,
) {
//this.onMessage = this.onMessage.bind(this);
@ -82,58 +87,148 @@ export class CacheService implements OnApplicationShutdown {
this.localUserByIdCache = new MemoryKVCache<MiLocalUser>(1000 * 60 * 5); // 5m
this.uriPersonCache = new MemoryKVCache<MiUser | null>(1000 * 60 * 5); // 5m
this.userProfileCache = new RedisKVCache<MiUserProfile>(this.redisClient, 'userProfile', {
this.userProfileCache = new QuantumKVCache(this.internalEventService, 'userProfile', {
lifetime: 1000 * 60 * 30, // 30m
memoryCacheLifetime: 1000 * 60, // 1m
fetcher: (key) => this.userProfilesRepository.findOneByOrFail({ userId: key }),
toRedisConverter: (value) => JSON.stringify(value),
fromRedisConverter: (value) => JSON.parse(value), // TODO: date型の考慮
bulkFetcher: userIds => this.userProfilesRepository.findBy({ userId: In(userIds) }).then(ps => ps.map(p => [p.userId, p])),
});
this.userMutingsCache = new RedisKVCache<Set<string>>(this.redisClient, 'userMutings', {
this.userMutingsCache = new QuantumKVCache<Set<string>>(this.internalEventService, 'userMutings', {
lifetime: 1000 * 60 * 30, // 30m
memoryCacheLifetime: 1000 * 60, // 1m
fetcher: (key) => this.mutingsRepository.find({ where: { muterId: key }, select: ['muteeId'] }).then(xs => new Set(xs.map(x => x.muteeId))),
toRedisConverter: (value) => JSON.stringify(Array.from(value)),
fromRedisConverter: (value) => new Set(JSON.parse(value)),
bulkFetcher: muterIds => this.mutingsRepository
.createQueryBuilder('muting')
.select('"muting"."muterId"', 'muterId')
.addSelect('array_agg("muting"."muteeId")', 'muteeIds')
.where({ muterId: In(muterIds) })
.groupBy('muting.muterId')
.getRawMany<{ muterId: string, muteeIds: string[] }>()
.then(ms => ms.map(m => [m.muterId, new Set(m.muteeIds)])),
});
this.userBlockingCache = new RedisKVCache<Set<string>>(this.redisClient, 'userBlocking', {
this.userBlockingCache = new QuantumKVCache<Set<string>>(this.internalEventService, 'userBlocking', {
lifetime: 1000 * 60 * 30, // 30m
memoryCacheLifetime: 1000 * 60, // 1m
fetcher: (key) => this.blockingsRepository.find({ where: { blockerId: key }, select: ['blockeeId'] }).then(xs => new Set(xs.map(x => x.blockeeId))),
toRedisConverter: (value) => JSON.stringify(Array.from(value)),
fromRedisConverter: (value) => new Set(JSON.parse(value)),
bulkFetcher: blockerIds => this.blockingsRepository
.createQueryBuilder('blocking')
.select('"blocking"."blockerId"', 'blockerId')
.addSelect('array_agg("blocking"."blockeeId")', 'blockeeIds')
.where({ blockerId: In(blockerIds) })
.groupBy('blocking.blockerId')
.getRawMany<{ blockerId: string, blockeeIds: string[] }>()
.then(ms => ms.map(m => [m.blockerId, new Set(m.blockeeIds)])),
});
this.userBlockedCache = new RedisKVCache<Set<string>>(this.redisClient, 'userBlocked', {
this.userBlockedCache = new QuantumKVCache<Set<string>>(this.internalEventService, 'userBlocked', {
lifetime: 1000 * 60 * 30, // 30m
memoryCacheLifetime: 1000 * 60, // 1m
fetcher: (key) => this.blockingsRepository.find({ where: { blockeeId: key }, select: ['blockerId'] }).then(xs => new Set(xs.map(x => x.blockerId))),
toRedisConverter: (value) => JSON.stringify(Array.from(value)),
fromRedisConverter: (value) => new Set(JSON.parse(value)),
bulkFetcher: blockeeIds => this.blockingsRepository
.createQueryBuilder('blocking')
.select('"blocking"."blockeeId"', 'blockeeId')
.addSelect('array_agg("blocking"."blockeeId")', 'blockeeIds')
.where({ blockeeId: In(blockeeIds) })
.groupBy('blocking.blockeeId')
.getRawMany<{ blockeeId: string, blockerIds: string[] }>()
.then(ms => ms.map(m => [m.blockeeId, new Set(m.blockerIds)])),
});
this.renoteMutingsCache = new RedisKVCache<Set<string>>(this.redisClient, 'renoteMutings', {
this.renoteMutingsCache = new QuantumKVCache<Set<string>>(this.internalEventService, 'renoteMutings', {
lifetime: 1000 * 60 * 30, // 30m
memoryCacheLifetime: 1000 * 60, // 1m
fetcher: (key) => this.renoteMutingsRepository.find({ where: { muterId: key }, select: ['muteeId'] }).then(xs => new Set(xs.map(x => x.muteeId))),
toRedisConverter: (value) => JSON.stringify(Array.from(value)),
fromRedisConverter: (value) => new Set(JSON.parse(value)),
bulkFetcher: muterIds => this.renoteMutingsRepository
.createQueryBuilder('muting')
.select('"muting"."muterId"', 'muterId')
.addSelect('array_agg("muting"."muteeId")', 'muteeIds')
.where({ muterId: In(muterIds) })
.groupBy('muting.muterId')
.getRawMany<{ muterId: string, muteeIds: string[] }>()
.then(ms => ms.map(m => [m.muterId, new Set(m.muteeIds)])),
});
this.userFollowingsCache = new RedisKVCache<Record<string, Pick<MiFollowing, 'withReplies'> | undefined>>(this.redisClient, 'userFollowings', {
this.userFollowingsCache = new QuantumKVCache<Map<string, Omit<MiFollowing, 'isFollowerHibernated'>>>(this.internalEventService, 'userFollowings', {
lifetime: 1000 * 60 * 30, // 30m
memoryCacheLifetime: 1000 * 60, // 1m
fetcher: (key) => this.followingsRepository.find({ where: { followerId: key }, select: ['followeeId', 'withReplies'] }).then(xs => {
const obj: Record<string, Pick<MiFollowing, 'withReplies'> | undefined> = {};
for (const x of xs) {
obj[x.followeeId] = { withReplies: x.withReplies };
fetcher: (key) => this.followingsRepository.findBy({ followerId: key }).then(xs => new Map(xs.map(f => [f.followeeId, f]))),
bulkFetcher: followerIds => this.followingsRepository
.findBy({ followerId: In(followerIds) })
.then(fs => fs
.reduce((groups, f) => {
let group = groups.get(f.followerId);
if (!group) {
group = new Map();
groups.set(f.followerId, group);
}
group.set(f.followeeId, f);
return groups;
}, new Map<string, Map<string, Omit<MiFollowing, 'isFollowerHibernated'>>>)),
});
this.userFollowersCache = new QuantumKVCache<Map<string, Omit<MiFollowing, 'isFollowerHibernated'>>>(this.internalEventService, 'userFollowers', {
lifetime: 1000 * 60 * 30, // 30m
fetcher: followeeId => this.followingsRepository.findBy({ followeeId: followeeId }).then(xs => new Map(xs.map(x => [x.followerId, x]))),
bulkFetcher: followeeIds => this.followingsRepository
.findBy({ followeeId: In(followeeIds) })
.then(fs => fs
.reduce((groups, f) => {
let group = groups.get(f.followeeId);
if (!group) {
group = new Map();
groups.set(f.followeeId, group);
}
group.set(f.followerId, f);
return groups;
}, new Map<string, Map<string, Omit<MiFollowing, 'isFollowerHibernated'>>>)),
});
this.hibernatedUserCache = new QuantumKVCache<boolean>(this.internalEventService, 'hibernatedUsers', {
lifetime: 1000 * 60 * 30, // 30m
fetcher: async userId => {
const { isHibernated } = await this.usersRepository.findOneOrFail({
where: { id: userId },
select: { isHibernated: true },
});
return isHibernated;
},
bulkFetcher: async userIds => {
const results = await this.usersRepository.find({
where: { id: In(userIds) },
select: { id: true, isHibernated: true },
});
return results.map(({ id, isHibernated }) => [id, isHibernated]);
},
onChanged: async userIds => {
// We only update local copies since each process will get this event, but we can have user objects in multiple different caches.
// Before doing anything else we must "find" all the objects to update.
const userObjects = new Map<string, MiUser[]>();
const toUpdate: string[] = [];
for (const uid of userIds) {
const toAdd: MiUser[] = [];
const localUserById = this.localUserByIdCache.get(uid);
if (localUserById) toAdd.push(localUserById);
const userById = this.userByIdCache.get(uid);
if (userById) toAdd.push(userById);
if (toAdd.length > 0) {
toUpdate.push(uid);
userObjects.set(uid, toAdd);
}
}
return obj;
}),
toRedisConverter: (value) => JSON.stringify(value),
fromRedisConverter: (value) => JSON.parse(value),
// In many cases, we won't have to do anything.
// Skipping the DB fetch ensures that this remains a single-step synchronous process.
if (toUpdate.length > 0) {
const hibernations = await this.usersRepository.find({ where: { id: In(toUpdate) }, select: { id: true, isHibernated: true } });
for (const { id, isHibernated } of hibernations) {
const users = userObjects.get(id);
if (users) {
for (const u of users) {
u.isHibernated = isHibernated;
}
}
}
}
},
});
this.translationsCache = new RedisKVCache<CachedTranslationEntity>(this.redisClient, 'translations', {
@ -143,20 +238,21 @@ export class CacheService implements OnApplicationShutdown {
// NOTE: チャンネルのフォロー状況キャッシュはChannelFollowingServiceで行っている
this.redisForSub.on('message', this.onMessage);
this.internalEventService.on('userChangeSuspendedState', this.onUserEvent);
this.internalEventService.on('userChangeDeletedState', this.onUserEvent);
this.internalEventService.on('remoteUserUpdated', this.onUserEvent);
this.internalEventService.on('localUserUpdated', this.onUserEvent);
this.internalEventService.on('userChangeSuspendedState', this.onUserEvent);
this.internalEventService.on('userTokenRegenerated', this.onTokenEvent);
this.internalEventService.on('follow', this.onFollowEvent);
this.internalEventService.on('unfollow', this.onFollowEvent);
}
@bindThis
private async onMessage(_: string, data: string): Promise<void> {
const obj = JSON.parse(data);
if (obj.channel === 'internal') {
const { type, body } = obj.message as GlobalEvents['internal']['payload'];
switch (type) {
case 'userChangeSuspendedState':
case 'userChangeDeletedState':
case 'remoteUserUpdated':
case 'localUserUpdated': {
private async onUserEvent<E extends 'userChangeSuspendedState' | 'userChangeDeletedState' | 'remoteUserUpdated' | 'localUserUpdated'>(body: InternalEventTypes[E], _: E, isLocal: boolean): Promise<void> {
{
{
{
const user = await this.usersRepository.findOneBy({ id: body.id });
if (user == null) {
this.userByIdCache.delete(body.id);
@ -166,6 +262,18 @@ export class CacheService implements OnApplicationShutdown {
this.uriPersonCache.delete(k);
}
}
if (isLocal) {
await Promise.all([
this.userProfileCache.delete(body.id),
this.userMutingsCache.delete(body.id),
this.userBlockingCache.delete(body.id),
this.userBlockedCache.delete(body.id),
this.renoteMutingsCache.delete(body.id),
this.userFollowingsCache.delete(body.id),
this.userFollowersCache.delete(body.id),
this.hibernatedUserCache.delete(body.id),
]);
}
} else {
this.userByIdCache.set(user.id, user);
for (const [k, v] of this.uriPersonCache.entries) {
@ -178,20 +286,37 @@ export class CacheService implements OnApplicationShutdown {
this.localUserByIdCache.set(user.id, user);
}
}
break;
}
case 'userTokenRegenerated': {
}
}
}
@bindThis
private async onTokenEvent<E extends 'userTokenRegenerated'>(body: InternalEventTypes[E]): Promise<void> {
{
{
{
const user = await this.usersRepository.findOneByOrFail({ id: body.id }) as MiLocalUser;
this.localUserByNativeTokenCache.delete(body.oldToken);
this.localUserByNativeTokenCache.set(body.newToken, user);
break;
}
}
}
}
@bindThis
private async onFollowEvent<E extends 'follow' | 'unfollow'>(body: InternalEventTypes[E], type: E): Promise<void> {
{
switch (type) {
case 'follow': {
const follower = this.userByIdCache.get(body.followerId);
if (follower) follower.followingCount++;
const followee = this.userByIdCache.get(body.followeeId);
if (followee) followee.followersCount++;
this.userFollowingsCache.delete(body.followerId);
await Promise.all([
this.userFollowingsCache.delete(body.followerId),
this.userFollowersCache.delete(body.followeeId),
]);
this.userFollowStatsCache.delete(body.followerId);
this.userFollowStatsCache.delete(body.followeeId);
break;
@ -201,13 +326,14 @@ export class CacheService implements OnApplicationShutdown {
if (follower) follower.followingCount--;
const followee = this.userByIdCache.get(body.followeeId);
if (followee) followee.followersCount--;
this.userFollowingsCache.delete(body.followerId);
await Promise.all([
this.userFollowingsCache.delete(body.followerId),
this.userFollowersCache.delete(body.followeeId),
]);
this.userFollowStatsCache.delete(body.followerId);
this.userFollowStatsCache.delete(body.followeeId);
break;
}
default:
break;
}
}
}
@ -298,9 +424,115 @@ export class CacheService implements OnApplicationShutdown {
});
}
@bindThis
public async getUsers(userIds: Iterable<string>): Promise<Map<string, MiUser>> {
const users = new Map<string, MiUser>;
const toFetch: string[] = [];
for (const userId of userIds) {
const fromCache = this.userByIdCache.get(userId);
if (fromCache) {
users.set(userId, fromCache);
} else {
toFetch.push(userId);
}
}
if (toFetch.length > 0) {
const fetched = await this.usersRepository.findBy({
id: In(toFetch),
});
for (const user of fetched) {
users.set(user.id, user);
this.userByIdCache.set(user.id, user);
}
}
return users;
}
@bindThis
public async isFollowing(follower: string | { id: string }, followee: string | { id: string }): Promise<boolean> {
const followerId = typeof(follower) === 'string' ? follower : follower.id;
const followeeId = typeof(followee) === 'string' ? followee : followee.id;
// This lets us use whichever one is in memory, falling back to DB fetch via userFollowingsCache.
return this.userFollowersCache.get(followeeId)?.has(followerId)
?? (await this.userFollowingsCache.fetch(followerId)).has(followeeId);
}
/**
* Returns all hibernated followers.
*/
@bindThis
public async getHibernatedFollowers(followeeId: string): Promise<MiFollowing[]> {
const followers = await this.getFollowersWithHibernation(followeeId);
return followers.filter(f => f.isFollowerHibernated);
}
/**
* Returns all non-hibernated followers.
*/
@bindThis
public async getNonHibernatedFollowers(followeeId: string): Promise<MiFollowing[]> {
const followers = await this.getFollowersWithHibernation(followeeId);
return followers.filter(f => !f.isFollowerHibernated);
}
/**
* Returns follower relations with populated isFollowerHibernated.
* If you don't need this field, then please use userFollowersCache directly for reduced overhead.
*/
@bindThis
public async getFollowersWithHibernation(followeeId: string): Promise<MiFollowing[]> {
const followers = await this.userFollowersCache.fetch(followeeId);
const hibernations = await this.hibernatedUserCache.fetchMany(followers.keys()).then(fs => fs.reduce((map, f) => {
map.set(f[0], f[1]);
return map;
}, new Map<string, boolean>));
return Array.from(followers.values()).map(following => ({
...following,
isFollowerHibernated: hibernations.get(following.followerId) ?? false,
}));
}
/**
* Refreshes follower and following relations for the given user.
*/
@bindThis
public async refreshFollowRelationsFor(userId: string): Promise<void> {
const followings = await this.userFollowingsCache.refresh(userId);
const followees = Array.from(followings.values()).map(f => f.followeeId);
await this.userFollowersCache.deleteMany(followees);
}
@bindThis
public clear(): void {
this.userByIdCache.clear();
this.localUserByNativeTokenCache.clear();
this.localUserByIdCache.clear();
this.uriPersonCache.clear();
this.userProfileCache.clear();
this.userMutingsCache.clear();
this.userBlockingCache.clear();
this.userBlockedCache.clear();
this.renoteMutingsCache.clear();
this.userFollowingsCache.clear();
this.userFollowStatsCache.clear();
this.translationsCache.clear();
}
@bindThis
public dispose(): void {
this.redisForSub.off('message', this.onMessage);
this.internalEventService.off('userChangeSuspendedState', this.onUserEvent);
this.internalEventService.off('userChangeDeletedState', this.onUserEvent);
this.internalEventService.off('remoteUserUpdated', this.onUserEvent);
this.internalEventService.off('localUserUpdated', this.onUserEvent);
this.internalEventService.off('userChangeSuspendedState', this.onUserEvent);
this.internalEventService.off('userTokenRegenerated', this.onTokenEvent);
this.internalEventService.off('follow', this.onFollowEvent);
this.internalEventService.off('unfollow', this.onFollowEvent);
this.userByIdCache.dispose();
this.localUserByNativeTokenCache.dispose();
this.localUserByIdCache.dispose();

29
packages/backend/src/core/ChannelFollowingService.ts
View file

@ -9,14 +9,15 @@ import { DI } from '@/di-symbols.js';
import type { ChannelFollowingsRepository } from '@/models/_.js';
import { MiChannel } from '@/models/_.js';
import { IdService } from '@/core/IdService.js';
import { GlobalEvents, GlobalEventService } from '@/core/GlobalEventService.js';
import { GlobalEvents, GlobalEventService, InternalEventTypes } from '@/core/GlobalEventService.js';
import { bindThis } from '@/decorators.js';
import type { MiLocalUser } from '@/models/User.js';
import { RedisKVCache } from '@/misc/cache.js';
import { QuantumKVCache } from '@/misc/QuantumKVCache.js';
import { InternalEventService } from './InternalEventService.js';
@Injectable()
export class ChannelFollowingService implements OnModuleInit {
public userFollowingChannelsCache: RedisKVCache<Set<string>>;
public userFollowingChannelsCache: QuantumKVCache<Set<string>>;
constructor(
@Inject(DI.redis)
@ -27,19 +28,18 @@ export class ChannelFollowingService implements OnModuleInit {
private channelFollowingsRepository: ChannelFollowingsRepository,
private idService: IdService,
private globalEventService: GlobalEventService,
private readonly internalEventService: InternalEventService,
) {
this.userFollowingChannelsCache = new RedisKVCache<Set<string>>(this.redisClient, 'userFollowingChannels', {
this.userFollowingChannelsCache = new QuantumKVCache<Set<string>>(this.internalEventService, 'userFollowingChannels', {
lifetime: 1000 * 60 * 30, // 30m
memoryCacheLifetime: 1000 * 60, // 1m
fetcher: (key) => this.channelFollowingsRepository.find({
where: { followerId: key },
select: ['followeeId'],
}).then(xs => new Set(xs.map(x => x.followeeId))),
toRedisConverter: (value) => JSON.stringify(Array.from(value)),
fromRedisConverter: (value) => new Set(JSON.parse(value)),
});
this.redisForSub.on('message', this.onMessage);
this.internalEventService.on('followChannel', this.onMessage);
this.internalEventService.on('unfollowChannel', this.onMessage);
}
onModuleInit() {
@ -79,18 +79,15 @@ export class ChannelFollowingService implements OnModuleInit {
}
@bindThis
private async onMessage(_: string, data: string): Promise<void> {
const obj = JSON.parse(data);
if (obj.channel === 'internal') {
const { type, body } = obj.message as GlobalEvents['internal']['payload'];
private async onMessage<E extends 'followChannel' | 'unfollowChannel'>(body: InternalEventTypes[E], type: E): Promise<void> {
{
switch (type) {
case 'followChannel': {
this.userFollowingChannelsCache.refresh(body.userId);
await this.userFollowingChannelsCache.delete(body.userId);
break;
}
case 'unfollowChannel': {
this.userFollowingChannelsCache.delete(body.userId);
await this.userFollowingChannelsCache.delete(body.userId);
break;
}
}
@ -99,6 +96,8 @@ export class ChannelFollowingService implements OnModuleInit {
@bindThis
public dispose(): void {
this.internalEventService.off('followChannel', this.onMessage);
this.internalEventService.off('unfollowChannel', this.onMessage);
this.userFollowingChannelsCache.dispose();
}

6
packages/backend/src/core/CoreModule.ts
View file

@ -41,6 +41,7 @@ import { HttpRequestService } from './HttpRequestService.js';
import { IdService } from './IdService.js';
import { ImageProcessingService } from './ImageProcessingService.js';
import { SystemAccountService } from './SystemAccountService.js';
import { InternalEventService } from './InternalEventService.js';
import { InternalStorageService } from './InternalStorageService.js';
import { MetaService } from './MetaService.js';
import { MfmService } from './MfmService.js';
@ -186,6 +187,7 @@ const $HashtagService: Provider = { provide: 'HashtagService', useExisting: Hash
const $HttpRequestService: Provider = { provide: 'HttpRequestService', useExisting: HttpRequestService };
const $IdService: Provider = { provide: 'IdService', useExisting: IdService };
const $ImageProcessingService: Provider = { provide: 'ImageProcessingService', useExisting: ImageProcessingService };
const $InternalEventService: Provider = { provide: 'InternalEventService', useExisting: InternalEventService };
const $InternalStorageService: Provider = { provide: 'InternalStorageService', useExisting: InternalStorageService };
const $MetaService: Provider = { provide: 'MetaService', useExisting: MetaService };
const $MfmService: Provider = { provide: 'MfmService', useExisting: MfmService };
@ -345,6 +347,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
HttpRequestService,
IdService,
ImageProcessingService,
InternalEventService,
InternalStorageService,
MetaService,
MfmService,
@ -500,6 +503,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
$HttpRequestService,
$IdService,
$ImageProcessingService,
$InternalEventService,
$InternalStorageService,
$MetaService,
$MfmService,
@ -656,6 +660,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
HttpRequestService,
IdService,
ImageProcessingService,
InternalEventService,
InternalStorageService,
MetaService,
MfmService,
@ -810,6 +815,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
$HttpRequestService,
$IdService,
$ImageProcessingService,
$InternalEventService,
$InternalStorageService,
$MetaService,
$MfmService,

4
packages/backend/src/core/FetchInstanceMetadataService.ts
View file

@ -7,7 +7,7 @@ import { URL } from 'node:url';
import { Inject, Injectable } from '@nestjs/common';
import tinycolor from 'tinycolor2';
import * as Redis from 'ioredis';
import { load as cheerio } from 'cheerio';
import { load as cheerio } from 'cheerio/slim';
import type { MiInstance } from '@/models/Instance.js';
import type Logger from '@/logger.js';
import { DI } from '@/di-symbols.js';
@ -16,7 +16,7 @@ import { HttpRequestService } from '@/core/HttpRequestService.js';
import { bindThis } from '@/decorators.js';
import { FederatedInstanceService } from '@/core/FederatedInstanceService.js';
import { renderInlineError } from '@/misc/render-inline-error.js';
import type { CheerioAPI } from 'cheerio';
import type { CheerioAPI } from 'cheerio/slim';
type NodeInfo = {
openRegistrations?: unknown;

10
packages/backend/src/core/GlobalEventService.ts
View file

@ -265,6 +265,7 @@ export interface InternalEventTypes {
unmute: { muterId: MiUser['id']; muteeId: MiUser['id']; };
userListMemberAdded: { userListId: MiUserList['id']; memberId: MiUser['id']; };
userListMemberRemoved: { userListId: MiUserList['id']; memberId: MiUser['id']; };
quantumCacheUpdated: { name: string, keys: string[] };
}
type EventTypesToEventPayload<T> = EventUnionFromDictionary<UndefinedAsNullAll<SerializedAll<T>>>;
@ -353,12 +354,12 @@ export class GlobalEventService {
}
@bindThis
private publish(channel: StreamChannels, type: string | null, value?: any): void {
private async publish(channel: StreamChannels, type: string | null, value?: any): Promise<void> {
const message = type == null ? value : value == null ?
{ type: type, body: null } :
{ type: type, body: value };
this.redisForPub.publish(this.config.host, JSON.stringify({
await this.redisForPub.publish(this.config.host, JSON.stringify({
channel: channel,
message: message,
}));
@ -369,6 +370,11 @@ export class GlobalEventService {
this.publish('internal', type, typeof value === 'undefined' ? null : value);
}
@bindThis
public async publishInternalEventAsync<K extends keyof InternalEventTypes>(type: K, value?: InternalEventTypes[K]): Promise<void> {
await this.publish('internal', type, typeof value === 'undefined' ? null : value);
}
@bindThis
public publishBroadcastStream<K extends keyof BroadcastTypes>(type: K, value?: BroadcastTypes[K]): void {
this.publish('broadcast', type, typeof value === 'undefined' ? null : value);

103
packages/backend/src/core/InternalEventService.ts Normal file
View file

@ -0,0 +1,103 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
import Redis from 'ioredis';
import { DI } from '@/di-symbols.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
import type { GlobalEvents, InternalEventTypes } from '@/core/GlobalEventService.js';
import { bindThis } from '@/decorators.js';
export type Listener<K extends keyof InternalEventTypes> = (value: InternalEventTypes[K], key: K, isLocal: boolean) => void | Promise<void>;
export interface ListenerProps {
ignoreLocal?: boolean,
ignoreRemote?: boolean,
}
@Injectable()
export class InternalEventService implements OnApplicationShutdown {
private readonly listeners = new Map<keyof InternalEventTypes, Map<Listener<keyof InternalEventTypes>, ListenerProps>>();
constructor(
@Inject(DI.redisForSub)
private readonly redisForSub: Redis.Redis,
private readonly globalEventService: GlobalEventService,
) {
this.redisForSub.on('message', this.onMessage);
}
@bindThis
public on<K extends keyof InternalEventTypes>(type: K, listener: Listener<K>, props?: ListenerProps): void {
let set = this.listeners.get(type);
if (!set) {
set = new Map();
this.listeners.set(type, set);
}
// Functionally, this is just a set with metadata on the values.
set.set(listener as Listener<keyof InternalEventTypes>, props ?? {});
}
@bindThis
public off<K extends keyof InternalEventTypes>(type: K, listener: Listener<K>): void {
this.listeners.get(type)?.delete(listener as Listener<keyof InternalEventTypes>);
}
@bindThis
public async emit<K extends keyof InternalEventTypes>(type: K, value: InternalEventTypes[K]): Promise<void> {
await this.emitInternal(type, value, true);
await this.globalEventService.publishInternalEventAsync(type, { ...value, _pid: process.pid });
}
@bindThis
private async emitInternal<K extends keyof InternalEventTypes>(type: K, value: InternalEventTypes[K], isLocal: boolean): Promise<void> {
const listeners = this.listeners.get(type);
if (!listeners) {
return;
}
const promises: Promise<void>[] = [];
for (const [listener, props] of listeners) {
if ((isLocal && !props.ignoreLocal) || (!isLocal && !props.ignoreRemote)) {
const promise = Promise.resolve(listener(value, type, isLocal));
promises.push(promise);
}
}
await Promise.all(promises);
}
@bindThis
private async onMessage(_: string, data: string): Promise<void> {
const obj = JSON.parse(data);
if (obj.channel === 'internal') {
const { type, body } = obj.message as GlobalEvents['internal']['payload'];
if (!isLocalInternalEvent(body) || body._pid !== process.pid) {
await this.emitInternal(type, body as InternalEventTypes[keyof InternalEventTypes], false);
}
}
}
@bindThis
public dispose(): void {
this.redisForSub.off('message', this.onMessage);
this.listeners.clear();
}
@bindThis
public onApplicationShutdown(): void {
this.dispose();
}
}
interface LocalInternalEvent {
_pid: number;
}
function isLocalInternalEvent(body: object): body is LocalInternalEvent {
return '_pid' in body && typeof(body._pid) === 'number';
}

485
packages/backend/src/core/MfmService.ts
View file

@ -5,25 +5,22 @@
import { URL } from 'node:url';
import { Inject, Injectable } from '@nestjs/common';
import * as parse5 from 'parse5';
import { type Document, type HTMLParagraphElement, Window } from 'happy-dom';
import { isText, isTag, Text } from 'domhandler';
import * as htmlparser2 from 'htmlparser2';
import { Node, Document, ChildNode, Element, ParentNode } from 'domhandler';
import * as domserializer from 'dom-serializer';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { intersperse } from '@/misc/prelude/array.js';
import { normalizeForSearch } from '@/misc/normalize-for-search.js';
import type { IMentionedRemoteUsers } from '@/models/Note.js';
import { bindThis } from '@/decorators.js';
import type { DefaultTreeAdapterMap } from 'parse5';
import type * as mfm from '@transfem-org/sfm-js';
const treeAdapter = parse5.defaultTreeAdapter;
type Node = DefaultTreeAdapterMap['node'];
type ChildNode = DefaultTreeAdapterMap['childNode'];
import type * as mfm from 'mfm-js';
const urlRegex = /^https?:\/\/[\w\/:%#@$&?!()\[\]~.,=+\-]+/;
const urlRegexFull = /^https?:\/\/[\w\/:%#@$&?!()\[\]~.,=+\-]+$/;
export type Appender = (document: Document, body: HTMLParagraphElement) => void;
export type Appender = (document: Document, body: Element) => void;
@Injectable()
export class MfmService {
@ -40,7 +37,7 @@ export class MfmService {
const normalizedHashtagNames = hashtagNames == null ? undefined : new Set<string>(hashtagNames.map(x => normalizeForSearch(x)));
const dom = parse5.parseFragment(html);
const dom = htmlparser2.parseDocument(html);
let text = '';
@ -51,37 +48,31 @@ export class MfmService {
return text.trim();
function getText(node: Node): string {
if (treeAdapter.isTextNode(node)) return node.value;
if (!treeAdapter.isElementNode(node)) return '';
if (node.nodeName === 'br') return '\n';
if (isText(node)) return node.data;
if (!isTag(node)) return '';
if (node.tagName === 'br') return '\n';
if (node.childNodes) {
return node.childNodes.map(n => getText(n)).join('');
}
return '';
return node.childNodes.map(n => getText(n)).join('');
}
function appendChildren(childNodes: ChildNode[]): void {
if (childNodes) {
for (const n of childNodes) {
analyze(n);
}
for (const n of childNodes) {
analyze(n);
}
}
function analyze(node: Node) {
if (treeAdapter.isTextNode(node)) {
text += node.value;
if (isText(node)) {
text += node.data;
return;
}
// Skip comment or document type node
if (!treeAdapter.isElementNode(node)) {
if (!isTag(node)) {
return;
}
switch (node.nodeName) {
switch (node.tagName) {
case 'br': {
text += '\n';
break;
@ -89,19 +80,19 @@ export class MfmService {
case 'a': {
const txt = getText(node);
const rel = node.attrs.find(x => x.name === 'rel');
const href = node.attrs.find(x => x.name === 'href');
const rel = node.attribs.rel;
const href = node.attribs.href;
// ハッシュタグ
if (normalizedHashtagNames && href && normalizedHashtagNames.has(normalizeForSearch(txt))) {
text += txt;
// メンション
} else if (txt.startsWith('@') && !(rel && rel.value.startsWith('me '))) {
} else if (txt.startsWith('@') && !(rel && rel.startsWith('me '))) {
const part = txt.split('@');
if (part.length === 2 && href) {
//#region ホスト名部分が省略されているので復元する
const acct = `${txt}@${(new URL(href.value)).hostname}`;
const acct = `${txt}@${(new URL(href)).hostname}`;
text += acct;
//#endregion
} else if (part.length === 3) {
@ -116,17 +107,17 @@ export class MfmService {
if (!href) {
return txt;
}
if (!txt || txt === href.value) { // #6383: Missing text node
if (href.value.match(urlRegexFull)) {
return href.value;
if (!txt || txt === href) { // #6383: Missing text node
if (href.match(urlRegexFull)) {
return href;
} else {
return `<${href.value}>`;
return `<${href}>`;
}
}
if (href.value.match(urlRegex) && !href.value.match(urlRegexFull)) {
return `[${txt}](<${href.value}>)`; // #6846
if (href.match(urlRegex) && !href.match(urlRegexFull)) {
return `[${txt}](<${href}>)`; // #6846
} else {
return `[${txt}](${href.value})`;
return `[${txt}](${href})`;
}
};
@ -185,14 +176,17 @@ export class MfmService {
case 'ruby--': {
let ruby: [string, string][] = [];
for (const child of node.childNodes) {
if (child.nodeName === 'rp') {
if (isText(child) && !/\s|\[|\]/.test(child.data)) {
ruby.push([child.data, '']);
continue;
}
if (treeAdapter.isTextNode(child) && !/\s|\[|\]/.test(child.value)) {
ruby.push([child.value, '']);
if (!isTag(child)) {
continue;
}
if (child.nodeName === 'rt' && ruby.length > 0) {
if (child.tagName === 'rp') {
continue;
}
if (child.tagName === 'rt' && ruby.length > 0) {
const rt = getText(child);
if (/\s|\[|\]/.test(rt)) {
// If any space is included in rt, it is treated as a normal text
@ -217,7 +211,7 @@ export class MfmService {
// block code (<pre><code>)
case 'pre': {
if (node.childNodes.length === 1 && node.childNodes[0].nodeName === 'code') {
if (node.childNodes.length === 1 && isTag(node.childNodes[0]) && node.childNodes[0].tagName === 'code') {
text += '\n```\n';
text += getText(node.childNodes[0]);
text += '\n```\n';
@ -302,17 +296,17 @@ export class MfmService {
let nonRtNodes = [];
// scan children, ignore `rp`, split on `rt`
for (const child of node.childNodes) {
if (treeAdapter.isTextNode(child)) {
if (isText(child)) {
nonRtNodes.push(child);
continue;
}
if (!treeAdapter.isElementNode(child)) {
if (!isTag(child)) {
continue;
}
if (child.nodeName === 'rp') {
if (child.tagName === 'rp') {
continue;
}
if (child.nodeName === 'rt') {
if (child.tagName === 'rt') {
// the only case in which we don't need a `$[group ]`
// is when both sides of the ruby are simple words
const needsGroup = nonRtNodes.length > 1 ||
@ -350,45 +344,44 @@ export class MfmService {
return null;
}
const { happyDOM, window } = new Window();
const doc = new Document([]);
const doc = window.document;
const body = new Element('p', {});
doc.childNodes.push(body);
const body = doc.createElement('p');
function appendChildren(children: mfm.MfmNode[], targetElement: any): void {
if (children) {
for (const child of children.map(x => (handlers as any)[x.type](x))) targetElement.appendChild(child);
function appendChildren(children: mfm.MfmNode[], targetElement: ParentNode): void {
for (const child of children.map(x => handle(x))) {
targetElement.childNodes.push(child);
}
}
function fnDefault(node: mfm.MfmFn) {
const el = doc.createElement('i');
const el = new Element('i', {});
appendChildren(node.children, el);
return el;
}
const handlers: { [K in mfm.MfmNode['type']]: (node: mfm.NodeType<K>) => any } = {
const handlers: { [K in mfm.MfmNode['type']]: (node: mfm.NodeType<K>) => ChildNode } = {
bold: (node) => {
const el = doc.createElement('b');
const el = new Element('b', {});
appendChildren(node.children, el);
return el;
},
small: (node) => {
const el = doc.createElement('small');
const el = new Element('small', {});
appendChildren(node.children, el);
return el;
},
strike: (node) => {
const el = doc.createElement('del');
const el = new Element('del', {});
appendChildren(node.children, el);
return el;
},
italic: (node) => {
const el = doc.createElement('i');
const el = new Element('i', {});
appendChildren(node.children, el);
return el;
},
@ -399,11 +392,12 @@ export class MfmService {
const text = node.children[0].type === 'text' ? node.children[0].props.text : '';
try {
const date = new Date(parseInt(text, 10) * 1000);
const el = doc.createElement('time');
el.setAttribute('datetime', date.toISOString());
el.textContent = date.toISOString();
const el = new Element('time', {
datetime: date.toISOString(),
});
el.childNodes.push(new Text(date.toISOString()));
return el;
} catch (err) {
} catch {
return fnDefault(node);
}
}
@ -412,20 +406,20 @@ export class MfmService {
if (node.children.length === 1) {
const child = node.children[0];
const text = child.type === 'text' ? child.props.text : '';
const rubyEl = doc.createElement('ruby');
const rtEl = doc.createElement('rt');
const rubyEl = new Element('ruby', {});
const rtEl = new Element('rt', {});
// ruby未対応のHTMLサニタイザーを通したときにルビが「劉備（りゅうび）」となるようにする
const rpStartEl = doc.createElement('rp');
rpStartEl.appendChild(doc.createTextNode('('));
const rpEndEl = doc.createElement('rp');
rpEndEl.appendChild(doc.createTextNode(')'));
const rpStartEl = new Element('rp', {});
rpStartEl.childNodes.push(new Text('('));
const rpEndEl = new Element('rp', {});
rpEndEl.childNodes.push(new Text(')'));
rubyEl.appendChild(doc.createTextNode(text.split(' ')[0]));
rtEl.appendChild(doc.createTextNode(text.split(' ')[1]));
rubyEl.appendChild(rpStartEl);
rubyEl.appendChild(rtEl);
rubyEl.appendChild(rpEndEl);
rubyEl.childNodes.push(new Text(text.split(' ')[0]));
rtEl.childNodes.push(new Text(text.split(' ')[1]));
rubyEl.childNodes.push(rpStartEl);
rubyEl.childNodes.push(rtEl);
rubyEl.childNodes.push(rpEndEl);
return rubyEl;
} else {
const rt = node.children.at(-1);
@ -435,20 +429,20 @@ export class MfmService {
}
const text = rt.type === 'text' ? rt.props.text : '';
const rubyEl = doc.createElement('ruby');
const rtEl = doc.createElement('rt');
const rubyEl = new Element('ruby', {});
const rtEl = new Element('rt', {});
// ruby未対応のHTMLサニタイザーを通したときにルビが「劉備（りゅうび）」となるようにする
const rpStartEl = doc.createElement('rp');
rpStartEl.appendChild(doc.createTextNode('('));
const rpEndEl = doc.createElement('rp');
rpEndEl.appendChild(doc.createTextNode(')'));
const rpStartEl = new Element('rp', {});
rpStartEl.childNodes.push(new Text('('));
const rpEndEl = new Element('rp', {});
rpEndEl.childNodes.push(new Text(')'));
appendChildren(node.children.slice(0, node.children.length - 1), rubyEl);
rtEl.appendChild(doc.createTextNode(text.trim()));
rubyEl.appendChild(rpStartEl);
rubyEl.appendChild(rtEl);
rubyEl.appendChild(rpEndEl);
rtEl.childNodes.push(new Text(text.trim()));
rubyEl.childNodes.push(rpStartEl);
rubyEl.childNodes.push(rtEl);
rubyEl.childNodes.push(rpEndEl);
return rubyEl;
}
}
@ -456,7 +450,7 @@ export class MfmService {
// hack for ruby, should never be needed because we should
// never send this out to other instances
case 'group': {
const el = doc.createElement('span');
const el = new Element('span', {});
appendChildren(node.children, el);
return el;
}
@ -468,125 +462,135 @@ export class MfmService {
},
blockCode: (node) => {
const pre = doc.createElement('pre');
const inner = doc.createElement('code');
inner.textContent = node.props.code;
pre.appendChild(inner);
const pre = new Element('pre', {});
const inner = new Element('code', {});
inner.childNodes.push(new Text(node.props.code));
pre.childNodes.push(inner);
return pre;
},
center: (node) => {
const el = doc.createElement('div');
const el = new Element('div', {});
appendChildren(node.children, el);
return el;
},
emojiCode: (node) => {
return doc.createTextNode(`\u200B:${node.props.name}:\u200B`);
return new Text(`\u200B:${node.props.name}:\u200B`);
},
unicodeEmoji: (node) => {
return doc.createTextNode(node.props.emoji);
return new Text(node.props.emoji);
},
hashtag: (node) => {
const a = doc.createElement('a');
a.setAttribute('href', `${this.config.url}/tags/${node.props.hashtag}`);
a.textContent = `#${node.props.hashtag}`;
a.setAttribute('rel', 'tag');
const a = new Element('a', {
href: `${this.config.url}/tags/${node.props.hashtag}`,
rel: 'tag',
});
a.childNodes.push(new Text(`#${node.props.hashtag}`));
return a;
},
inlineCode: (node) => {
const el = doc.createElement('code');
el.textContent = node.props.code;
const el = new Element('code', {});
el.childNodes.push(new Text(node.props.code));
return el;
},
mathInline: (node) => {
const el = doc.createElement('code');
el.textContent = node.props.formula;
const el = new Element('code', {});
el.childNodes.push(new Text(node.props.formula));
return el;
},
mathBlock: (node) => {
const el = doc.createElement('code');
el.textContent = node.props.formula;
const el = new Element('code', {});
el.childNodes.push(new Text(node.props.formula));
return el;
},
link: (node) => {
const a = doc.createElement('a');
a.setAttribute('href', node.props.url);
const a = new Element('a', {
href: node.props.url,
});
appendChildren(node.children, a);
return a;
},
mention: (node) => {
const a = doc.createElement('a');
const { username, host, acct } = node.props;
const remoteUserInfo = mentionedRemoteUsers.find(remoteUser => remoteUser.username.toLowerCase() === username.toLowerCase() && remoteUser.host?.toLowerCase() === host?.toLowerCase());
a.setAttribute('href', remoteUserInfo
? (remoteUserInfo.url ? remoteUserInfo.url : remoteUserInfo.uri)
: `${this.config.url}/${acct.endsWith(`@${this.config.url}`) ? acct.substring(0, acct.length - this.config.url.length - 1) : acct}`);
a.className = 'u-url mention';
a.textContent = acct;
const a = new Element('a', {
href: remoteUserInfo
? (remoteUserInfo.url ? remoteUserInfo.url : remoteUserInfo.uri)
: `${this.config.url}/${acct.endsWith(`@${this.config.url}`) ? acct.substring(0, acct.length - this.config.url.length - 1) : acct}`,
class: 'u-url mention',
});
a.childNodes.push(new Text(acct));
return a;
},
quote: (node) => {
const el = doc.createElement('blockquote');
const el = new Element('blockquote', {});
appendChildren(node.children, el);
return el;
},
text: (node) => {
if (!node.props.text.match(/[\r\n]/)) {
return doc.createTextNode(node.props.text);
return new Text(node.props.text);
}
const el = doc.createElement('span');
const nodes = node.props.text.split(/\r\n|\r|\n/).map(x => doc.createTextNode(x));
const el = new Element('span', {});
const nodes = node.props.text.split(/\r\n|\r|\n/).map(x => new Text(x));
for (const x of intersperse<FIXME | 'br'>('br', nodes)) {
el.appendChild(x === 'br' ? doc.createElement('br') : x);
el.childNodes.push(x === 'br' ? new Element('br', {}) : x);
}
return el;
},
url: (node) => {
const a = doc.createElement('a');
a.setAttribute('href', node.props.url);
a.textContent = node.props.url;
const a = new Element('a', {
href: node.props.url,
});
a.childNodes.push(new Text(node.props.url));
return a;
},
search: (node) => {
const a = doc.createElement('a');
a.setAttribute('href', `https://www.google.com/search?q=${node.props.query}`);
a.textContent = node.props.content;
const a = new Element('a', {
href: `https://www.google.com/search?q=${node.props.query}`,
});
a.childNodes.push(new Text(node.props.content));
return a;
},
plain: (node) => {
const el = doc.createElement('span');
const el = new Element('span', {});
appendChildren(node.children, el);
return el;
},
};
// Utility function to make TypeScript behave
function handle<T extends mfm.MfmNode>(node: T): ChildNode {
const handler = handlers[node.type] as (node: T) => ChildNode;
return handler(node);
}
appendChildren(nodes, body);
for (const additionalAppender of additionalAppenders) {
additionalAppender(doc, body);
}
const serialized = body.outerHTML;
happyDOM.close().catch(err => {});
return serialized;
return domserializer.render(body, {
encodeEntities: 'utf8'
});
}
// the toMastoApiHtml function was taken from Iceshrimp and written by zotan and modified by marie to work with the current MK version
@ -598,55 +602,55 @@ export class MfmService {
return null;
}
const { happyDOM, window } = new Window();
const doc = new Document([]);
const doc = window.document;
const body = new Element('p', {});
doc.childNodes.push(body);
const body = doc.createElement('p');
function appendChildren(children: mfm.MfmNode[], targetElement: any): void {
if (children) {
for (const child of children.map((x) => (handlers as any)[x.type](x))) targetElement.appendChild(child);
function appendChildren(children: mfm.MfmNode[], targetElement: ParentNode): void {
for (const child of children) {
const result = handle(child);
targetElement.childNodes.push(result);
}
}
const handlers: {
[K in mfm.MfmNode['type']]: (node: mfm.NodeType<K>) => any;
[K in mfm.MfmNode['type']]: (node: mfm.NodeType<K>) => ChildNode;
} = {
bold(node) {
const el = doc.createElement('span');
el.textContent = '**';
const el = new Element('span', {});
el.childNodes.push(new Text('**'));
appendChildren(node.children, el);
el.textContent += '**';
el.childNodes.push(new Text('**'));
return el;
},
small(node) {
const el = doc.createElement('small');
const el = new Element('small', {});
appendChildren(node.children, el);
return el;
},
strike(node) {
const el = doc.createElement('span');
el.textContent = '~~';
const el = new Element('span', {});
el.childNodes.push(new Text('~~'));
appendChildren(node.children, el);
el.textContent += '~~';
el.childNodes.push(new Text('~~'));
return el;
},
italic(node) {
const el = doc.createElement('span');
el.textContent = '*';
const el = new Element('span', {});
el.childNodes.push(new Text('*'));
appendChildren(node.children, el);
el.textContent += '*';
el.childNodes.push(new Text('*'));
return el;
},
fn(node) {
switch (node.props.name) {
case 'group': { // hack for ruby
const el = doc.createElement('span');
const el = new Element('span', {});
appendChildren(node.children, el);
return el;
}
@ -654,119 +658,121 @@ export class MfmService {
if (node.children.length === 1) {
const child = node.children[0];
const text = child.type === 'text' ? child.props.text : '';
const rubyEl = doc.createElement('ruby');
const rtEl = doc.createElement('rt');
const rubyEl = new Element('ruby', {});
const rtEl = new Element('rt', {});
const rpStartEl = doc.createElement('rp');
rpStartEl.appendChild(doc.createTextNode('('));
const rpEndEl = doc.createElement('rp');
rpEndEl.appendChild(doc.createTextNode(')'));
const rpStartEl = new Element('rp', {});
rpStartEl.childNodes.push(new Text('('));
const rpEndEl = new Element('rp', {});
rpEndEl.childNodes.push(new Text(')'));
rubyEl.appendChild(doc.createTextNode(text.split(' ')[0]));
rtEl.appendChild(doc.createTextNode(text.split(' ')[1]));
rubyEl.appendChild(rpStartEl);
rubyEl.appendChild(rtEl);
rubyEl.appendChild(rpEndEl);
rubyEl.childNodes.push(new Text(text.split(' ')[0]));
rtEl.childNodes.push(new Text(text.split(' ')[1]));
rubyEl.childNodes.push(rpStartEl);
rubyEl.childNodes.push(rtEl);
rubyEl.childNodes.push(rpEndEl);
return rubyEl;
} else {
const rt = node.children.at(-1);
if (!rt) {
const el = doc.createElement('span');
const el = new Element('span', {});
appendChildren(node.children, el);
return el;
}
const text = rt.type === 'text' ? rt.props.text : '';
const rubyEl = doc.createElement('ruby');
const rtEl = doc.createElement('rt');
const rubyEl = new Element('ruby', {});
const rtEl = new Element('rt', {});
const rpStartEl = doc.createElement('rp');
rpStartEl.appendChild(doc.createTextNode('('));
const rpEndEl = doc.createElement('rp');
rpEndEl.appendChild(doc.createTextNode(')'));
const rpStartEl = new Element('rp', {});
rpStartEl.childNodes.push(new Text('('));
const rpEndEl = new Element('rp', {});
rpEndEl.childNodes.push(new Text(')'));
appendChildren(node.children.slice(0, node.children.length - 1), rubyEl);
rtEl.appendChild(doc.createTextNode(text.trim()));
rubyEl.appendChild(rpStartEl);
rubyEl.appendChild(rtEl);
rubyEl.appendChild(rpEndEl);
rtEl.childNodes.push(new Text(text.trim()));
rubyEl.childNodes.push(rpStartEl);
rubyEl.childNodes.push(rtEl);
rubyEl.childNodes.push(rpEndEl);
return rubyEl;
}
}
default: {
const el = doc.createElement('span');
el.textContent = '*';
const el = new Element('span', {});
el.childNodes.push(new Text('*'));
appendChildren(node.children, el);
el.textContent += '*';
el.childNodes.push(new Text('*'));
return el;
}
}
},
blockCode(node) {
const pre = doc.createElement('pre');
const inner = doc.createElement('code');
const pre = new Element('pre', {});
const inner = new Element('code', {});
const nodes = node.props.code
.split(/\r\n|\r|\n/)
.map((x) => doc.createTextNode(x));
.map((x) => new Text(x));
for (const x of intersperse<FIXME | 'br'>('br', nodes)) {
inner.appendChild(x === 'br' ? doc.createElement('br') : x);
inner.childNodes.push(x === 'br' ? new Element('br', {}) : x);
}
pre.appendChild(inner);
pre.childNodes.push(inner);
return pre;
},
center(node) {
const el = doc.createElement('div');
const el = new Element('div', {});
appendChildren(node.children, el);
return el;
},
emojiCode(node) {
return doc.createTextNode(`\u200B:${node.props.name}:\u200B`);
return new Text(`\u200B:${node.props.name}:\u200B`);
},
unicodeEmoji(node) {
return doc.createTextNode(node.props.emoji);
return new Text(node.props.emoji);
},
hashtag: (node) => {
const a = doc.createElement('a');
a.setAttribute('href', `${this.config.url}/tags/${node.props.hashtag}`);
a.textContent = `#${node.props.hashtag}`;
a.setAttribute('rel', 'tag');
a.setAttribute('class', 'hashtag');
const a = new Element('a', {
href: `${this.config.url}/tags/${node.props.hashtag}`,
rel: 'tag',
class: 'hashtag',
});
a.childNodes.push(new Text(`#${node.props.hashtag}`));
return a;
},
inlineCode(node) {
const el = doc.createElement('code');
el.textContent = node.props.code;
const el = new Element('code', {});
el.childNodes.push(new Text(node.props.code));
return el;
},
mathInline(node) {
const el = doc.createElement('code');
el.textContent = node.props.formula;
const el = new Element('code', {});
el.childNodes.push(new Text(node.props.formula));
return el;
},
mathBlock(node) {
const el = doc.createElement('code');
el.textContent = node.props.formula;
const el = new Element('code', {});
el.childNodes.push(new Text(node.props.formula));
return el;
},
link(node) {
const a = doc.createElement('a');
a.setAttribute('rel', 'nofollow noopener noreferrer');
a.setAttribute('target', '_blank');
a.setAttribute('href', node.props.url);
const a = new Element('a', {
rel: 'nofollow noopener noreferrer',
target: '_blank',
href: node.props.url,
});
appendChildren(node.children, a);
return a;
},
@ -775,92 +781,107 @@ export class MfmService {
const { username, host, acct } = node.props;
const resolved = mentionedRemoteUsers.find(remoteUser => remoteUser.username === username && remoteUser.host === host);
const el = doc.createElement('span');
const el = new Element('span', {});
if (!resolved) {
el.textContent = acct;
el.childNodes.push(new Text(acct));
} else {
el.setAttribute('class', 'h-card');
el.setAttribute('translate', 'no');
const a = doc.createElement('a');
a.setAttribute('href', resolved.url ? resolved.url : resolved.uri);
a.className = 'u-url mention';
const span = doc.createElement('span');
span.textContent = resolved.username || username;
a.textContent = '@';
a.appendChild(span);
el.appendChild(a);
el.attribs.class = 'h-card';
el.attribs.translate = 'no';
const a = new Element('a', {
href: resolved.url ? resolved.url : resolved.uri,
class: 'u-url mention',
});
const span = new Element('span', {});
span.childNodes.push(new Text(resolved.username || username));
a.childNodes.push(new Text('@'));
a.childNodes.push(span);
el.childNodes.push(a);
}
return el;
},
quote(node) {
const el = doc.createElement('blockquote');
const el = new Element('blockquote', {});
appendChildren(node.children, el);
return el;
},
text(node) {
const el = doc.createElement('span');
if (!node.props.text.match(/[\r\n]/)) {
return new Text(node.props.text);
}
const el = new Element('span', {});
const nodes = node.props.text
.split(/\r\n|\r|\n/)
.map((x) => doc.createTextNode(x));
.map((x) => new Text(x));
for (const x of intersperse<FIXME | 'br'>('br', nodes)) {
el.appendChild(x === 'br' ? doc.createElement('br') : x);
el.childNodes.push(x === 'br' ? new Element('br', {}) : x);
}
return el;
},
url(node) {
const a = doc.createElement('a');
a.setAttribute('rel', 'nofollow noopener noreferrer');
a.setAttribute('target', '_blank');
a.setAttribute('href', node.props.url);
a.textContent = node.props.url.replace(/^https?:\/\//, '');
const a = new Element('a', {
rel: 'nofollow noopener noreferrer',
target: '_blank',
href: node.props.url,
});
a.childNodes.push(new Text(node.props.url.replace(/^https?:\/\//, '')));
return a;
},
search: (node) => {
const a = doc.createElement('a');
a.setAttribute('href', `https://www.google.com/search?q=${node.props.query}`);
a.textContent = node.props.content;
const a = new Element('a', {
href: `https://www.google.com/search?q=${node.props.query}`,
});
a.childNodes.push(new Text(node.props.content));
return a;
},
plain(node) {
const el = doc.createElement('span');
const el = new Element('span', {});
appendChildren(node.children, el);
return el;
},
};
// Utility function to make TypeScript behave
function handle<T extends mfm.MfmNode>(node: T): ChildNode {
const handler = handlers[node.type] as (node: T) => ChildNode;
return handler(node);
}
appendChildren(nodes, body);
if (quoteUri !== null) {
const a = doc.createElement('a');
a.setAttribute('href', quoteUri);
a.textContent = quoteUri.replace(/^https?:\/\//, '');
const a = new Element('a', {
href: quoteUri,
});
a.childNodes.push(new Text(quoteUri.replace(/^https?:\/\//, '')));
const quote = doc.createElement('span');
quote.setAttribute('class', 'quote-inline');
quote.appendChild(doc.createElement('br'));
quote.appendChild(doc.createElement('br'));
quote.innerHTML += 'RE: ';
quote.appendChild(a);
const quote = new Element('span', {
class: 'quote-inline',
});
quote.childNodes.push(new Element('br', {}));
quote.childNodes.push(new Element('br', {}));
quote.childNodes.push(new Text('RE: '));
quote.childNodes.push(a);
body.appendChild(quote);
body.childNodes.push(quote);
}
let result = body.outerHTML;
let result = domserializer.render(body, {
encodeEntities: 'utf8'
});
if (inline) {
result = result.replace(/^<p>/, '').replace(/<\/p>$/, '');
}
happyDOM.close().catch(() => {});
return result;
}
}

45
packages/backend/src/core/NoteCreateService.ts
View file

@ -4,7 +4,7 @@
*/
import { setImmediate } from 'node:timers/promises';
import * as mfm from '@transfem-org/sfm-js';
import * as mfm from 'mfm-js';
import { In, DataSource, IsNull, LessThan } from 'typeorm';
import * as Redis from 'ioredis';
import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
@ -606,11 +606,11 @@ export class NoteCreateService implements OnApplicationShutdown {
}
if (data.reply == null) {
// TODO: キャッシュ
this.followingsRepository.findBy({
followeeId: user.id,
notify: 'normal',
}).then(async followings => {
this.cacheService.userFollowersCache.fetch(user.id).then(async followingsMap => {
const followings = Array
.from(followingsMap.values())
.filter(f => f.notify === 'normal');
if (note.visibility !== 'specified') {
const isPureRenote = this.isRenote(data) && !this.isQuote(data) ? true : false;
for (const following of followings) {
@ -948,14 +948,7 @@ export class NoteCreateService implements OnApplicationShutdown {
// TODO: キャッシュ？
// eslint-disable-next-line prefer-const
let [followings, userListMemberships] = await Promise.all([
this.followingsRepository.find({
where: {
followeeId: user.id,
followerHost: IsNull(),
isFollowerHibernated: false,
},
select: ['followerId', 'withReplies'],
}),
this.cacheService.getNonHibernatedFollowers(user.id),
this.userListMembershipsRepository.find({
where: {
userId: user.id,
@ -1072,17 +1065,19 @@ export class NoteCreateService implements OnApplicationShutdown {
});
if (hibernatedUsers.length > 0) {
this.usersRepository.update({
id: In(hibernatedUsers.map(x => x.id)),
}, {
isHibernated: true,
});
this.followingsRepository.update({
followerId: In(hibernatedUsers.map(x => x.id)),
}, {
isFollowerHibernated: true,
});
await Promise.all([
this.usersRepository.update({
id: In(hibernatedUsers.map(x => x.id)),
}, {
isHibernated: true,
}),
this.followingsRepository.update({
followerId: In(hibernatedUsers.map(x => x.id)),
}, {
isFollowerHibernated: true,
}),
this.cacheService.hibernatedUserCache.setMany(hibernatedUsers.map(x => [x.id, true])),
]);
}
}

35
packages/backend/src/core/NoteEditService.ts
View file

@ -4,7 +4,7 @@
*/
import { setImmediate } from 'node:timers/promises';
import * as mfm from '@transfem-org/sfm-js';
import * as mfm from 'mfm-js';
import { DataSource, In, IsNull, LessThan } from 'typeorm';
import * as Redis from 'ioredis';
import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
@ -833,14 +833,7 @@ export class NoteEditService implements OnApplicationShutdown {
// TODO: キャッシュ？
// eslint-disable-next-line prefer-const
let [followings, userListMemberships] = await Promise.all([
this.followingsRepository.find({
where: {
followeeId: user.id,
followerHost: IsNull(),
isFollowerHibernated: false,
},
select: ['followerId', 'withReplies'],
}),
this.cacheService.getNonHibernatedFollowers(user.id),
this.userListMembershipsRepository.find({
where: {
userId: user.id,
@ -957,17 +950,19 @@ export class NoteEditService implements OnApplicationShutdown {
});
if (hibernatedUsers.length > 0) {
this.usersRepository.update({
id: In(hibernatedUsers.map(x => x.id)),
}, {
isHibernated: true,
});
this.followingsRepository.update({
followerId: In(hibernatedUsers.map(x => x.id)),
}, {
isFollowerHibernated: true,
});
await Promise.all([
this.usersRepository.update({
id: In(hibernatedUsers.map(x => x.id)),
}, {
isHibernated: true,
}),
this.followingsRepository.update({
followerId: In(hibernatedUsers.map(x => x.id)),
}, {
isFollowerHibernated: true,
}),
this.cacheService.hibernatedUserCache.setMany(hibernatedUsers.map(x => [x.id, true])),
]);
}
}

12
packages/backend/src/core/NotificationService.ts
View file

@ -113,27 +113,27 @@ export class NotificationService implements OnApplicationShutdown {
}
if (recieveConfig?.type === 'following') {
const isFollowing = await this.cacheService.userFollowingsCache.fetch(notifieeId).then(followings => Object.hasOwn(followings, notifierId));
const isFollowing = await this.cacheService.userFollowingsCache.fetch(notifieeId).then(followings => followings.has(notifierId));
if (!isFollowing) {
return null;
}
} else if (recieveConfig?.type === 'follower') {
const isFollower = await this.cacheService.userFollowingsCache.fetch(notifierId).then(followings => Object.hasOwn(followings, notifieeId));
const isFollower = await this.cacheService.userFollowingsCache.fetch(notifierId).then(followings => followings.has(notifieeId));
if (!isFollower) {
return null;
}
} else if (recieveConfig?.type === 'mutualFollow') {
const [isFollowing, isFollower] = await Promise.all([
this.cacheService.userFollowingsCache.fetch(notifieeId).then(followings => Object.hasOwn(followings, notifierId)),
this.cacheService.userFollowingsCache.fetch(notifierId).then(followings => Object.hasOwn(followings, notifieeId)),
this.cacheService.userFollowingsCache.fetch(notifieeId).then(followings => followings.has(notifierId)),
this.cacheService.userFollowingsCache.fetch(notifierId).then(followings => followings.has(notifieeId)),
]);
if (!(isFollowing && isFollower)) {
return null;
}
} else if (recieveConfig?.type === 'followingOrFollower') {
const [isFollowing, isFollower] = await Promise.all([
this.cacheService.userFollowingsCache.fetch(notifieeId).then(followings => Object.hasOwn(followings, notifierId)),
this.cacheService.userFollowingsCache.fetch(notifierId).then(followings => Object.hasOwn(followings, notifieeId)),
this.cacheService.userFollowingsCache.fetch(notifieeId).then(followings => followings.has(notifierId)),
this.cacheService.userFollowingsCache.fetch(notifierId).then(followings => followings.has(notifieeId)),
]);
if (!isFollowing && !isFollower) {
return null;

19
packages/backend/src/core/PushNotificationService.ts
View file

@ -12,7 +12,8 @@ import type { Packed } from '@/misc/json-schema.js';
import { getNoteSummary } from '@/misc/get-note-summary.js';
import type { MiMeta, MiSwSubscription, SwSubscriptionsRepository } from '@/models/_.js';
import { bindThis } from '@/decorators.js';
import { RedisKVCache } from '@/misc/cache.js';
import { QuantumKVCache } from '@/misc/QuantumKVCache.js';
import { InternalEventService } from '@/core/InternalEventService.js';
// Defined also packages/sw/types.ts#L13
type PushNotificationsTypes = {
@ -48,7 +49,7 @@ function truncateBody<T extends keyof PushNotificationsTypes>(type: T, body: Pus
@Injectable()
export class PushNotificationService implements OnApplicationShutdown {
private subscriptionsCache: RedisKVCache<MiSwSubscription[]>;
private subscriptionsCache: QuantumKVCache<MiSwSubscription[]>;
constructor(
@Inject(DI.config)
@ -62,13 +63,11 @@ export class PushNotificationService implements OnApplicationShutdown {
@Inject(DI.swSubscriptionsRepository)
private swSubscriptionsRepository: SwSubscriptionsRepository,
private readonly internalEventService: InternalEventService,
) {
this.subscriptionsCache = new RedisKVCache<MiSwSubscription[]>(this.redisClient, 'userSwSubscriptions', {
this.subscriptionsCache = new QuantumKVCache<MiSwSubscription[]>(this.internalEventService, 'userSwSubscriptions', {
lifetime: 1000 * 60 * 60 * 1, // 1h
memoryCacheLifetime: 1000 * 60 * 3, // 3m
fetcher: (key) => this.swSubscriptionsRepository.findBy({ userId: key }),
toRedisConverter: (value) => JSON.stringify(value),
fromRedisConverter: (value) => JSON.parse(value),
});
}
@ -114,8 +113,8 @@ export class PushNotificationService implements OnApplicationShutdown {
endpoint: subscription.endpoint,
auth: subscription.auth,
publickey: subscription.publickey,
}).then(() => {
this.refreshCache(userId);
}).then(async () => {
await this.refreshCache(userId);
});
}
});
@ -123,8 +122,8 @@ export class PushNotificationService implements OnApplicationShutdown {
}
@bindThis
public refreshCache(userId: string): void {
this.subscriptionsCache.refresh(userId);
public async refreshCache(userId: string): Promise<void> {
await this.subscriptionsCache.refresh(userId);
}
@bindThis

18
packages/backend/src/core/QueryService.ts
View file

@ -164,17 +164,17 @@ export class QueryService {
qb
// My post
.orWhere(':meId = note.userId')
// Reply to me
.orWhere(':meId = note.replyUserId')
// DM to me
// Visible to me
.orWhere(':meIdAsList <@ note.visibleUserIds')
// Mentions me (and not a direct message)
.orWhere(new Brackets(qb =>
qb.andWhere(':meIdAsList <@ note.mentions')
.andWhere('note.visibility != \'specified\'')))
// Followers-only post
.orWhere(new Brackets(qb => this
.andFollowingUser(qb, ':meId', 'note.userId')
.orWhere(new Brackets(qb => qb
.andWhere(new Brackets(qbb => this
// Following author
.orFollowingUser(qbb, ':meId', 'note.userId')
// Mentions me
.orWhere(':meIdAsList <@ note.mentions')
// Reply to me
.orWhere(':meId = note.replyUserId')))
.andWhere('note.visibility = \'followers\'')));
q.setParameters({ meId: me.id, meIdAsList: [me.id] });

2
packages/backend/src/core/ReactionService.ts
View file

@ -122,7 +122,7 @@ export class ReactionService {
}
// check visibility
if (!await this.noteEntityService.isVisibleForMe(note, user.id)) {
if (!await this.noteEntityService.isVisibleForMe(note, user.id, { me: user })) {
throw new IdentifiableError('68e9d2d1-48bf-42c2-b90a-b20e09fd3d48', 'Note not accessible for you.');
}

12
packages/backend/src/core/UserBlockingService.ts
View file

@ -77,8 +77,10 @@ export class UserBlockingService implements OnModuleInit {
await this.blockingsRepository.insert(blocking);
this.cacheService.userBlockingCache.refresh(blocker.id);
this.cacheService.userBlockedCache.refresh(blockee.id);
await Promise.all([
this.cacheService.userBlockingCache.delete(blocker.id),
this.cacheService.userBlockedCache.delete(blockee.id),
]);
this.globalEventService.publishInternalEvent('blockingCreated', {
blockerId: blocker.id,
@ -168,8 +170,10 @@ export class UserBlockingService implements OnModuleInit {
await this.blockingsRepository.delete(blocking.id);
this.cacheService.userBlockingCache.refresh(blocker.id);
this.cacheService.userBlockedCache.refresh(blockee.id);
await Promise.all([
this.cacheService.userBlockingCache.delete(blocker.id),
this.cacheService.userBlockedCache.delete(blockee.id),
]);
this.globalEventService.publishInternalEvent('blockingDeleted', {
blockerId: blocker.id,

125
packages/backend/src/core/UserFollowingService.ts
View file

@ -29,6 +29,7 @@ import { AccountMoveService } from '@/core/AccountMoveService.js';
import { UtilityService } from '@/core/UtilityService.js';
import type { ThinUser } from '@/queue/types.js';
import { LoggerService } from '@/core/LoggerService.js';
import { InternalEventService } from '@/core/InternalEventService.js';
import type Logger from '../logger.js';
type Local = MiLocalUser | {
@ -86,6 +87,7 @@ export class UserFollowingService implements OnModuleInit {
private accountMoveService: AccountMoveService,
private perUserFollowingChart: PerUserFollowingChart,
private instanceChart: InstanceChart,
private readonly internalEventService: InternalEventService,
loggerService: LoggerService,
) {
@ -145,12 +147,7 @@ export class UserFollowingService implements OnModuleInit {
if (blocked) throw new IdentifiableError('3338392a-f764-498d-8855-db939dcf8c48', 'blocked');
}
if (await this.followingsRepository.exists({
where: {
followerId: follower.id,
followeeId: followee.id,
},
})) {
if (await this.cacheService.isFollowing(follower, followee)) {
// すでにフォロー関係が存在している場合
if (this.userEntityService.isRemoteUser(follower) && this.userEntityService.isLocalUser(followee)) {
// リモート → ローカル: acceptを送り返しておしまい
@ -178,24 +175,14 @@ export class UserFollowingService implements OnModuleInit {
let autoAccept = false;
// 鍵アカウントであっても、既にフォローされていた場合はスルー
const isFollowing = await this.followingsRepository.exists({
where: {
followerId: follower.id,
followeeId: followee.id,
},
});
const isFollowing = await this.cacheService.isFollowing(follower, followee);
if (isFollowing) {
autoAccept = true;
}
// フォローしているユーザーは自動承認オプション
if (!autoAccept && (this.userEntityService.isLocalUser(followee) && followeeProfile.autoAcceptFollowed)) {
const isFollowed = await this.followingsRepository.exists({
where: {
followerId: followee.id,
followeeId: follower.id,
},
});
const isFollowed = await this.cacheService.isFollowing(followee, follower); // intentionally reversed parameters
if (isFollowed) autoAccept = true;
}
@ -204,12 +191,7 @@ export class UserFollowingService implements OnModuleInit {
if (followee.isLocked && !autoAccept) {
autoAccept = !!(await this.accountMoveService.validateAlsoKnownAs(
follower,
(oldSrc, newSrc) => this.followingsRepository.exists({
where: {
followeeId: followee.id,
followerId: newSrc.id,
},
}),
(oldSrc, newSrc) => this.cacheService.isFollowing(newSrc, followee),
true,
));
}
@ -264,7 +246,8 @@ export class UserFollowingService implements OnModuleInit {
}
});
this.cacheService.userFollowingsCache.refresh(follower.id);
// Handled by CacheService
//this.cacheService.userFollowingsCache.refresh(follower.id);
const requestExist = await this.followRequestsRepository.exists({
where: {
@ -291,7 +274,7 @@ export class UserFollowingService implements OnModuleInit {
}, followee.id);
}
this.globalEventService.publishInternalEvent('follow', { followerId: follower.id, followeeId: followee.id });
await this.internalEventService.emit('follow', { followerId: follower.id, followeeId: followee.id });
const [followeeUser, followerUser] = await Promise.all([
this.usersRepository.findOneByOrFail({ id: followee.id }),
@ -363,31 +346,29 @@ export class UserFollowingService implements OnModuleInit {
},
silent = false,
): Promise<void> {
const following = await this.followingsRepository.findOne({
relations: {
follower: true,
followee: true,
},
where: {
followerId: follower.id,
followeeId: followee.id,
},
});
const [
followerUser,
followeeUser,
following,
] = await Promise.all([
this.cacheService.findUserById(follower.id),
this.cacheService.findUserById(followee.id),
this.cacheService.userFollowingsCache.fetch(follower.id).then(fs => fs.get(followee.id)),
]);
if (following === null || !following.follower || !following.followee) {
if (following == null) {
this.logger.warn('フォロー解除がリクエストされましたがフォローしていませんでした');
return;
}
await this.followingsRepository.delete(following.id);
await this.internalEventService.emit('unfollow', { followerId: follower.id, followeeId: followee.id });
this.cacheService.userFollowingsCache.refresh(follower.id);
this.decrementFollowing(following.follower, following.followee);
this.decrementFollowing(followerUser, followeeUser);
if (!silent && this.userEntityService.isLocalUser(follower)) {
// Publish unfollow event
this.userEntityService.pack(followee.id, follower, {
this.userEntityService.pack(followeeUser, follower, {
schema: 'UserDetailedNotMe',
}).then(async packed => {
this.globalEventService.publishMainStream(follower.id, 'unfollow', packed);
@ -412,8 +393,6 @@ export class UserFollowingService implements OnModuleInit {
follower: MiUser,
followee: MiUser,
): Promise<void> {
this.globalEventService.publishInternalEvent('unfollow', { followerId: follower.id, followeeId: followee.id });
// Neither followee nor follower has moved.
if (!follower.movedToUri && !followee.movedToUri) {
//#region Decrement following / followers counts
@ -687,22 +666,22 @@ export class UserFollowingService implements OnModuleInit {
*/
@bindThis
private async removeFollow(followee: Both, follower: Both): Promise<void> {
const following = await this.followingsRepository.findOne({
relations: {
followee: true,
follower: true,
},
where: {
followeeId: followee.id,
followerId: follower.id,
},
});
const [
followerUser,
followeeUser,
following,
] = await Promise.all([
this.cacheService.findUserById(follower.id),
this.cacheService.findUserById(followee.id),
this.cacheService.userFollowingsCache.fetch(follower.id).then(fs => fs.get(followee.id)),
]);
if (!following || !following.followee || !following.follower) return;
if (!following) return;
await this.followingsRepository.delete(following.id);
await this.internalEventService.emit('unfollow', { followerId: follower.id, followeeId: followee.id });
this.decrementFollowing(following.follower, following.followee);
this.decrementFollowing(followerUser, followeeUser);
}
/**
@ -733,36 +712,26 @@ export class UserFollowingService implements OnModuleInit {
}
@bindThis
public getFollowees(userId: MiUser['id']) {
return this.followingsRepository.createQueryBuilder('following')
.select('following.followeeId')
.where('following.followerId = :followerId', { followerId: userId })
.getMany();
public async getFollowees(userId: MiUser['id']) {
const followings = await this.cacheService.userFollowingsCache.fetch(userId);
return Array.from(followings.values());
}
@bindThis
public isFollowing(followerId: MiUser['id'], followeeId: MiUser['id']) {
return this.followingsRepository.exists({
where: {
followerId,
followeeId,
},
});
public async isFollowing(followerId: MiUser['id'], followeeId: MiUser['id']) {
return this.cacheService.isFollowing(followerId, followeeId);
}
@bindThis
public async isMutual(aUserId: MiUser['id'], bUserId: MiUser['id']) {
const count = await this.followingsRepository.createQueryBuilder('following')
.where(new Brackets(qb => {
qb.where('following.followerId = :aUserId', { aUserId })
.andWhere('following.followeeId = :bUserId', { bUserId });
}))
.orWhere(new Brackets(qb => {
qb.where('following.followerId = :bUserId', { bUserId })
.andWhere('following.followeeId = :aUserId', { aUserId });
}))
.getCount();
const [
isFollowing,
isFollowed,
] = await Promise.all([
this.isFollowing(aUserId, bUserId),
this.isFollowing(bUserId, aUserId),
]);
return count === 2;
return isFollowing && isFollowed;
}
}

14
packages/backend/src/core/UserKeypairService.ts
View file

@ -7,14 +7,14 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
import * as Redis from 'ioredis';
import type { MiUser } from '@/models/User.js';
import type { UserKeypairsRepository } from '@/models/_.js';
import { RedisKVCache } from '@/misc/cache.js';
import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js';
import type { MiUserKeypair } from '@/models/UserKeypair.js';
import { DI } from '@/di-symbols.js';
import { bindThis } from '@/decorators.js';
@Injectable()
export class UserKeypairService implements OnApplicationShutdown {
private cache: RedisKVCache<MiUserKeypair>;
private cache: MemoryKVCache<MiUserKeypair>;
constructor(
@Inject(DI.redis)
@ -23,18 +23,12 @@ export class UserKeypairService implements OnApplicationShutdown {
@Inject(DI.userKeypairsRepository)
private userKeypairsRepository: UserKeypairsRepository,
) {
this.cache = new RedisKVCache<MiUserKeypair>(this.redisClient, 'userKeypair', {
lifetime: 1000 * 60 * 60 * 24, // 24h
memoryCacheLifetime: 1000 * 60 * 60, // 1h
fetcher: (key) => this.userKeypairsRepository.findOneByOrFail({ userId: key }),
toRedisConverter: (value) => JSON.stringify(value),
fromRedisConverter: (value) => JSON.parse(value),
});
this.cache = new MemoryKVCache<MiUserKeypair>(1000 * 60 * 60 * 24); // 24h
}
@bindThis
public async getUserKeypair(userId: MiUser['id']): Promise<MiUserKeypair> {
return await this.cache.fetch(userId);
return await this.cache.fetch(userId, () => this.userKeypairsRepository.findOneByOrFail({ userId }));
}
@bindThis

30
packages/backend/src/core/UserListService.ts
View file

@ -11,21 +11,22 @@ import type { MiUser } from '@/models/User.js';
import type { MiUserList } from '@/models/UserList.js';
import type { MiUserListMembership } from '@/models/UserListMembership.js';
import { IdService } from '@/core/IdService.js';
import type { GlobalEvents } from '@/core/GlobalEventService.js';
import type { GlobalEvents, InternalEventTypes } from '@/core/GlobalEventService.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
import { DI } from '@/di-symbols.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { bindThis } from '@/decorators.js';
import { QueueService } from '@/core/QueueService.js';
import { RedisKVCache } from '@/misc/cache.js';
import { QuantumKVCache } from '@/misc/QuantumKVCache.js';
import { RoleService } from '@/core/RoleService.js';
import { SystemAccountService } from '@/core/SystemAccountService.js';
import { InternalEventService } from '@/core/InternalEventService.js';
@Injectable()
export class UserListService implements OnApplicationShutdown, OnModuleInit {
public static TooManyUsersError = class extends Error {};
public membersCache: RedisKVCache<Set<string>>;
public membersCache: QuantumKVCache<Set<string>>;
private roleService: RoleService;
constructor(
@ -48,16 +49,15 @@ export class UserListService implements OnApplicationShutdown, OnModuleInit {
private globalEventService: GlobalEventService,
private queueService: QueueService,
private systemAccountService: SystemAccountService,
private readonly internalEventService: InternalEventService,
) {
this.membersCache = new RedisKVCache<Set<string>>(this.redisClient, 'userListMembers', {
this.membersCache = new QuantumKVCache<Set<string>>(this.internalEventService, 'userListMembers', {
lifetime: 1000 * 60 * 30, // 30m
memoryCacheLifetime: 1000 * 60, // 1m
fetcher: (key) => this.userListMembershipsRepository.find({ where: { userListId: key }, select: ['userId'] }).then(xs => new Set(xs.map(x => x.userId))),
toRedisConverter: (value) => JSON.stringify(Array.from(value)),
fromRedisConverter: (value) => new Set(JSON.parse(value)),
});
this.redisForSub.on('message', this.onMessage);
this.internalEventService.on('userListMemberAdded', this.onMessage);
this.internalEventService.on('userListMemberRemoved', this.onMessage);
}
async onModuleInit() {
@ -65,15 +65,12 @@ export class UserListService implements OnApplicationShutdown, OnModuleInit {
}
@bindThis
private async onMessage(_: string, data: string): Promise<void> {
const obj = JSON.parse(data);
if (obj.channel === 'internal') {
const { type, body } = obj.message as GlobalEvents['internal']['payload'];
private async onMessage<E extends 'userListMemberAdded' | 'userListMemberRemoved'>(body: InternalEventTypes[E], type: E): Promise<void> {
{
switch (type) {
case 'userListMemberAdded': {
const { userListId, memberId } = body;
const members = await this.membersCache.get(userListId);
const members = this.membersCache.get(userListId);
if (members) {
members.add(memberId);
}
@ -81,7 +78,7 @@ export class UserListService implements OnApplicationShutdown, OnModuleInit {
}
case 'userListMemberRemoved': {
const { userListId, memberId } = body;
const members = await this.membersCache.get(userListId);
const members = this.membersCache.get(userListId);
if (members) {
members.delete(memberId);
}
@ -150,7 +147,8 @@ export class UserListService implements OnApplicationShutdown, OnModuleInit {
@bindThis
public dispose(): void {
this.redisForSub.off('message', this.onMessage);
this.internalEventService.off('userListMemberAdded', this.onMessage);
this.internalEventService.off('userListMemberRemoved', this.onMessage);
this.membersCache.dispose();
}

7
packages/backend/src/core/UserMutingService.ts
View file

@ -32,7 +32,7 @@ export class UserMutingService {
muteeId: target.id,
});
this.cacheService.userMutingsCache.refresh(user.id);
await this.cacheService.userMutingsCache.delete(user.id);
}
@bindThis
@ -43,9 +43,6 @@ export class UserMutingService {
id: In(mutings.map(m => m.id)),
});
const muterIds = [...new Set(mutings.map(m => m.muterId))];
for (const muterId of muterIds) {
this.cacheService.userMutingsCache.refresh(muterId);
}
await this.cacheService.userMutingsCache.deleteMany(mutings.map(m => m.muterId));
}
}

7
packages/backend/src/core/UserRenoteMutingService.ts
View file

@ -33,7 +33,7 @@ export class UserRenoteMutingService {
muteeId: target.id,
});
await this.cacheService.renoteMutingsCache.refresh(user.id);
await this.cacheService.renoteMutingsCache.delete(user.id);
}
@bindThis
@ -44,9 +44,6 @@ export class UserRenoteMutingService {
id: In(mutings.map(m => m.id)),
});
const muterIds = [...new Set(mutings.map(m => m.muterId))];
for (const muterId of muterIds) {
await this.cacheService.renoteMutingsCache.refresh(muterId);
}
await this.cacheService.renoteMutingsCache.deleteMany(mutings.map(m => m.muterId));
}
}

21
packages/backend/src/core/UserService.ts
View file

@ -10,6 +10,7 @@ import { DI } from '@/di-symbols.js';
import { bindThis } from '@/decorators.js';
import { SystemWebhookService } from '@/core/SystemWebhookService.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { CacheService } from '@/core/CacheService.js';
@Injectable()
export class UserService {
@ -20,6 +21,7 @@ export class UserService {
private followingsRepository: FollowingsRepository,
private systemWebhookService: SystemWebhookService,
private userEntityService: UserEntityService,
private readonly cacheService: CacheService,
) {
}
@ -38,14 +40,17 @@ export class UserService {
});
const wokeUp = result.isHibernated;
if (wokeUp) {
this.usersRepository.update(user.id, {
isHibernated: false,
});
this.followingsRepository.update({
followerId: user.id,
}, {
isFollowerHibernated: false,
});
await Promise.all([
this.usersRepository.update(user.id, {
isHibernated: false,
}),
this.followingsRepository.update({
followerId: user.id,
}, {
isFollowerHibernated: false,
}),
this.cacheService.hibernatedUserCache.set(user.id, false),
]);
}
} else {
this.usersRepository.update(user.id, {

10
packages/backend/src/core/UserSuspendService.ts
View file

@ -16,6 +16,7 @@ import { bindThis } from '@/decorators.js';
import { RelationshipJobData } from '@/queue/types.js';
import { ModerationLogService } from '@/core/ModerationLogService.js';
import { isSystemAccount } from '@/misc/is-system-account.js';
import { CacheService } from '@/core/CacheService.js';
@Injectable()
export class UserSuspendService {
@ -34,6 +35,7 @@ export class UserSuspendService {
private globalEventService: GlobalEventService,
private apRendererService: ApRendererService,
private moderationLogService: ModerationLogService,
private readonly cacheService: CacheService,
) {
}
@ -143,12 +145,8 @@ export class UserSuspendService {
@bindThis
private async unFollowAll(follower: MiUser) {
const followings = await this.followingsRepository.find({
where: {
followerId: follower.id,
followeeId: Not(IsNull()),
},
});
const followings = await this.cacheService.userFollowingsCache.fetch(follower.id)
.then(fs => Array.from(fs.values()).filter(f => f.followeeHost != null));
const jobs: RelationshipJobData[] = [];
for (const following of followings) {

16
packages/backend/src/core/WebfingerService.ts
View file

@ -5,7 +5,7 @@
import { URL } from 'node:url';
import { Injectable } from '@nestjs/common';
import { XMLParser } from 'fast-xml-parser';
import { load as cheerio } from 'cheerio/slim';
import { HttpRequestService } from '@/core/HttpRequestService.js';
import { bindThis } from '@/decorators.js';
import type Logger from '@/logger.js';
@ -101,14 +101,12 @@ export class WebfingerService {
private async fetchWebFingerTemplateFromHostMeta(url: string): Promise<string | null> {
try {
const res = await this.httpRequestService.getHtml(url, 'application/xrd+xml');
const options = {
ignoreAttributes: false,
isArray: (_name: string, jpath: string) => jpath === 'XRD.Link',
};
const parser = new XMLParser(options);
const hostMeta = parser.parse(res);
const template = (hostMeta['XRD']['Link'] as Array<any>).filter(p => p['@_rel'] === 'lrdd')[0]['@_template'];
return template.indexOf('{uri}') < 0 ? null : template;
const hostMeta = cheerio(res, {
xml: true,
});
const template = hostMeta('XRD > Link[rel="lrdd"][template*="{uri}"]').attr('template');
return template ?? null;
} catch (err) {
this.logger.error(`error while request host-meta for ${url}: ${renderInlineError(err)}`);
return null;

55
packages/backend/src/core/activitypub/ApDeliverManagerService.ts
View file

@ -5,7 +5,6 @@
import { Inject, Injectable } from '@nestjs/common';
import { IsNull, Not } from 'typeorm';
import { UnrecoverableError } from 'bullmq';
import { DI } from '@/di-symbols.js';
import type { FollowingsRepository } from '@/models/_.js';
import type { MiLocalUser, MiRemoteUser, MiUser } from '@/models/User.js';
@ -14,6 +13,7 @@ import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { bindThis } from '@/decorators.js';
import type { IActivity } from '@/core/activitypub/type.js';
import { ThinUser } from '@/queue/types.js';
import { CacheService } from '@/core/CacheService.js';
interface IRecipe {
type: string;
@ -41,16 +41,14 @@ class DeliverManager {
/**
* Constructor
* @param userEntityService
* @param followingsRepository
* @param queueService
* @param cacheService
* @param actor Actor
* @param activity Activity to deliver
*/
constructor(
private userEntityService: UserEntityService,
private followingsRepository: FollowingsRepository,
private queueService: QueueService,
private readonly cacheService: CacheService,
actor: { id: MiUser['id']; host: null; },
activity: IActivity | null,
@ -114,27 +112,23 @@ class DeliverManager {
// Process follower recipes first to avoid duplication when processing direct recipes later.
if (this.recipes.some(r => isFollowers(r))) {
// followers deliver
// TODO: SELECT DISTINCT ON ("followerSharedInbox") "followerSharedInbox" みたいな問い合わせにすればよりパフォーマンス向上できそう
// ただ、sharedInboxがnullなリモートユーザーも稀におり、その対応ができなさそう？
const followers = await this.followingsRepository.find({
where: {
followeeId: this.actor.id,
followerHost: Not(IsNull()),
},
select: {
followerSharedInbox: true,
followerInbox: true,
followerId: true,
},
});
const followers = await this.cacheService.userFollowersCache
.fetch(this.actor.id)
.then(f => Array
.from(f.values())
.filter(f => f.followerHost != null)
.map(f => ({
followerInbox: f.followerInbox,
followerSharedInbox: f.followerSharedInbox,
})));
for (const following of followers) {
const inbox = following.followerSharedInbox ?? following.followerInbox;
if (inbox === null) {
if (process.env.NODE_ENV === 'test') continue;
throw new UnrecoverableError(`deliver failed for ${this.actor.id}: follower ${following.followerId} inbox is null`);
if (following.followerSharedInbox) {
inboxes.set(following.followerSharedInbox, true);
} else if (following.followerInbox) {
inboxes.set(following.followerInbox, false);
}
inboxes.set(inbox, following.followerSharedInbox != null);
}
}
@ -156,11 +150,8 @@ class DeliverManager {
@Injectable()
export class ApDeliverManagerService {
constructor(
@Inject(DI.followingsRepository)
private followingsRepository: FollowingsRepository,
private userEntityService: UserEntityService,
private queueService: QueueService,
private readonly cacheService: CacheService,
) {
}
@ -172,9 +163,8 @@ export class ApDeliverManagerService {
@bindThis
public async deliverToFollowers(actor: { id: MiLocalUser['id']; host: null; }, activity: IActivity): Promise<void> {
const manager = new DeliverManager(
this.userEntityService,
this.followingsRepository,
this.queueService,
this.cacheService,
actor,
activity,
);
@ -191,9 +181,8 @@ export class ApDeliverManagerService {
@bindThis
public async deliverToUser(actor: { id: MiLocalUser['id']; host: null; }, activity: IActivity, to: MiRemoteUser): Promise<void> {
const manager = new DeliverManager(
this.userEntityService,
this.followingsRepository,
this.queueService,
this.cacheService,
actor,
activity,
);
@ -210,9 +199,8 @@ export class ApDeliverManagerService {
@bindThis
public async deliverToUsers(actor: { id: MiLocalUser['id']; host: null; }, activity: IActivity, targets: MiRemoteUser[]): Promise<void> {
const manager = new DeliverManager(
this.userEntityService,
this.followingsRepository,
this.queueService,
this.cacheService,
actor,
activity,
);
@ -223,9 +211,8 @@ export class ApDeliverManagerService {
@bindThis
public createDeliverManager(actor: { id: MiUser['id']; host: null; }, activity: IActivity | null): DeliverManager {
return new DeliverManager(
this.userEntityService,
this.followingsRepository,
this.queueService,
this.cacheService,
actor,
activity,

18
packages/backend/src/core/activitypub/ApInboxService.ts
View file

@ -37,6 +37,7 @@ import InstanceChart from '@/core/chart/charts/instance.js';
import FederationChart from '@/core/chart/charts/federation.js';
import { FetchInstanceMetadataService } from '@/core/FetchInstanceMetadataService.js';
import { UpdateInstanceQueue } from '@/core/UpdateInstanceQueue.js';
import { CacheService } from '@/core/CacheService.js';
import { getApHrefNullable, getApId, getApIds, getApType, getNullableApId, isAccept, isActor, isAdd, isAnnounce, isApObject, isBlock, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isDislike, isMove, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost, isActivity, IObjectWithId } from './type.js';
import { ApNoteService } from './models/ApNoteService.js';
import { ApLoggerService } from './ApLoggerService.js';
@ -98,6 +99,7 @@ export class ApInboxService {
private readonly instanceChart: InstanceChart,
private readonly federationChart: FederationChart,
private readonly updateInstanceQueue: UpdateInstanceQueue,
private readonly cacheService: CacheService,
) {
this.logger = this.apLoggerService.logger;
}
@ -365,7 +367,7 @@ export class ApInboxService {
const renote = await this.apNoteService.resolveNote(target, { resolver, sentFrom: getApId(target) });
if (renote == null) return 'announce target is null';
if (!await this.noteEntityService.isVisibleForMe(renote, actor.id)) {
if (!await this.noteEntityService.isVisibleForMe(renote, actor.id, { me: actor })) {
return 'skip: invalid actor for this activity';
}
@ -766,12 +768,7 @@ export class ApInboxService {
return 'skip: follower not found';
}
const isFollowing = await this.followingsRepository.exists({
where: {
followerId: follower.id,
followeeId: actor.id,
},
});
const isFollowing = await this.cacheService.userFollowingsCache.fetch(follower.id).then(f => f.has(actor.id));
if (isFollowing) {
await this.userFollowingService.unfollow(follower, actor);
@ -830,12 +827,7 @@ export class ApInboxService {
},
});
const isFollowing = await this.followingsRepository.exists({
where: {
followerId: actor.id,
followeeId: followee.id,
},
});
const isFollowing = await this.cacheService.userFollowingsCache.fetch(actor.id).then(f => f.has(followee.id));
if (requestExist) {
await this.userFollowingService.cancelFollowRequest(followee, actor);

2
packages/backend/src/core/activitypub/ApMfmService.ts
View file

@ -4,7 +4,7 @@
*/
import { Injectable } from '@nestjs/common';
import * as mfm from '@transfem-org/sfm-js';
import * as mfm from 'mfm-js';
import { MfmService, Appender } from '@/core/MfmService.js';
import type { MiNote } from '@/models/Note.js';
import { bindThis } from '@/decorators.js';

85
packages/backend/src/core/activitypub/ApRendererService.ts
View file

@ -6,8 +6,9 @@
import { createPublicKey, randomUUID } from 'node:crypto';
import { Inject, Injectable } from '@nestjs/common';
import { In } from 'typeorm';
import * as mfm from '@transfem-org/sfm-js';
import * as mfm from 'mfm-js';
import { UnrecoverableError } from 'bullmq';
import { Element, Text } from 'domhandler';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import type { MiPartialLocalUser, MiLocalUser, MiPartialRemoteUser, MiRemoteUser, MiUser } from '@/models/User.js';
@ -34,7 +35,7 @@ import { UtilityService } from '@/core/UtilityService.js';
import { JsonLdService } from './JsonLdService.js';
import { ApMfmService } from './ApMfmService.js';
import { CONTEXT } from './misc/contexts.js';
import { getApId, IOrderedCollection, IOrderedCollectionPage } from './type.js';
import { getApId, ILink, IOrderedCollection, IOrderedCollectionPage } from './type.js';
import type { IAccept, IActivity, IAdd, IAnnounce, IApDocument, IApEmoji, IApHashtag, IApImage, IApMention, IBlock, ICreate, IDelete, IFlag, IFollow, IKey, ILike, IMove, IObject, IPost, IQuestion, IReject, IRemove, ITombstone, IUndo, IUpdate } from './type.js';
@Injectable()
@ -419,7 +420,7 @@ export class ApRendererService {
inReplyTo = null;
}
let quote;
let quote: string | undefined = undefined;
if (note.renoteId) {
const renote = await this.notesRepository.findOneBy({ id: note.renoteId });
@ -475,16 +476,18 @@ export class ApRendererService {
// the claas name `quote-inline` is used in non-misskey clients for styling quote notes.
// For compatibility, the span part should be kept as possible.
apAppend.push((doc, body) => {
body.appendChild(doc.createElement('br'));
body.appendChild(doc.createElement('br'));
const span = doc.createElement('span');
span.className = 'quote-inline';
span.appendChild(doc.createTextNode('RE: '));
const link = doc.createElement('a');
link.setAttribute('href', quote);
link.textContent = quote;
span.appendChild(link);
body.appendChild(span);
body.childNodes.push(new Element('br', {}));
body.childNodes.push(new Element('br', {}));
const span = new Element('span', {
class: 'quote-inline',
});
span.childNodes.push(new Text('RE: '));
const link = new Element('a', {
href: quote,
});
link.childNodes.push(new Text(quote));
span.childNodes.push(link);
body.childNodes.push(span);
});
}
@ -500,12 +503,22 @@ export class ApRendererService {
const emojis = await this.getEmojis(note.emojis);
const apemojis = emojis.filter(emoji => !emoji.localOnly).map(emoji => this.renderEmoji(emoji));
const tag = [
const tag: IObject[] = [
...hashtagTags,
...mentionTags,
...apemojis,
];
// https://codeberg.org/fediverse/fep/src/branch/main/fep/e232/fep-e232.md
if (quote) {
tag.push({
type: 'Link',
mediaType: 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
rel: 'https://misskey-hub.net/ns#_misskey_quote',
href: quote,
} satisfies ILink);
}
const asPoll = poll ? {
type: 'Question',
[poll.expiresAt && poll.expiresAt < new Date() ? 'closed' : 'endTime']: poll.expiresAt,
@ -537,6 +550,8 @@ export class ApRendererService {
_misskey_quote: quote,
quoteUrl: quote,
quoteUri: quote,
// https://codeberg.org/fediverse/fep/src/branch/main/fep/044f/fep-044f.md
quote: quote,
published: this.idService.parse(note.id).date.toISOString(),
to,
cc,
@ -774,7 +789,7 @@ export class ApRendererService {
inReplyTo = null;
}
let quote;
let quote: string | undefined = undefined;
if (note.renoteId) {
const renote = await this.notesRepository.findOneBy({ id: note.renoteId });
@ -827,16 +842,18 @@ export class ApRendererService {
// the claas name `quote-inline` is used in non-misskey clients for styling quote notes.
// For compatibility, the span part should be kept as possible.
apAppend.push((doc, body) => {
body.appendChild(doc.createElement('br'));
body.appendChild(doc.createElement('br'));
const span = doc.createElement('span');
span.className = 'quote-inline';
span.appendChild(doc.createTextNode('RE: '));
const link = doc.createElement('a');
link.setAttribute('href', quote);
link.textContent = quote;
span.appendChild(link);
body.appendChild(span);
body.childNodes.push(new Element('br', {}));
body.childNodes.push(new Element('br', {}));
const span = new Element('span', {
class: 'quote-inline',
});
span.childNodes.push(new Text('RE: '));
const link = new Element('a', {
href: quote,
});
link.childNodes.push(new Text(quote));
span.childNodes.push(link);
body.childNodes.push(span);
});
}
@ -852,12 +869,22 @@ export class ApRendererService {
const emojis = await this.getEmojis(note.emojis);
const apemojis = emojis.filter(emoji => !emoji.localOnly).map(emoji => this.renderEmoji(emoji));
const tag = [
const tag: IObject[] = [
...hashtagTags,
...mentionTags,
...apemojis,
];
// https://codeberg.org/fediverse/fep/src/branch/main/fep/e232/fep-e232.md
if (quote) {
tag.push({
type: 'Link',
mediaType: 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
rel: 'https://misskey-hub.net/ns#_misskey_quote',
href: quote,
} satisfies ILink);
}
const asPoll = poll ? {
type: 'Question',
[poll.expiresAt && poll.expiresAt < new Date() ? 'closed' : 'endTime']: poll.expiresAt,
@ -886,6 +913,8 @@ export class ApRendererService {
_misskey_quote: quote,
quoteUrl: quote,
quoteUri: quote,
// https://codeberg.org/fediverse/fep/src/branch/main/fep/044f/fep-044f.md
quote: quote,
published: this.idService.parse(note.id).date.toISOString(),
to,
cc,
@ -936,9 +965,7 @@ export class ApRendererService {
const keypair = await this.userKeypairService.getUserKeypair(user.id);
const jsonLd = this.jsonLdService.use();
jsonLd.debug = false;
activity = await jsonLd.signRsaSignature2017(activity, keypair.privateKey, `${this.config.url}/users/${user.id}#main-key`);
activity = await this.jsonLdService.signRsaSignature2017(activity, keypair.privateKey, `${this.config.url}/users/${user.id}#main-key`);
return activity;
}

73
packages/backend/src/core/activitypub/ApRequestService.ts
View file

@ -6,7 +6,7 @@
import * as crypto from 'node:crypto';
import { URL } from 'node:url';
import { Inject, Injectable } from '@nestjs/common';
import { Window } from 'happy-dom';
import { load as cheerio } from 'cheerio/slim';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import type { MiUser } from '@/models/User.js';
@ -18,6 +18,8 @@ import { bindThis } from '@/decorators.js';
import type Logger from '@/logger.js';
import { validateContentTypeSetAsActivityPub } from '@/core/activitypub/misc/validator.js';
import type { IObject, IObjectWithId } from './type.js';
import type { Cheerio, CheerioAPI } from 'cheerio/slim';
import type { AnyNode } from 'domhandler';
type Request = {
url: string;
@ -219,53 +221,33 @@ export class ApRequestService {
(contentType ?? '').split(';')[0].trimEnd().toLowerCase() === 'text/html' &&
_followAlternate === true
) {
const html = await res.text();
const { window, happyDOM } = new Window({
settings: {
disableJavaScriptEvaluation: true,
disableJavaScriptFileLoading: true,
disableCSSFileLoading: true,
disableComputedStyleRendering: true,
handleDisabledFileLoadingAsSuccess: true,
navigation: {
disableMainFrameNavigation: true,
disableChildFrameNavigation: true,
disableChildPageNavigation: true,
disableFallbackToSetURL: true,
},
timer: {
maxTimeout: 0,
maxIntervalTime: 0,
maxIntervalIterations: 0,
},
},
});
const document = window.document;
let alternate: Cheerio<AnyNode> | null;
try {
document.documentElement.innerHTML = html;
const html = await res.text();
const document = cheerio(html);
// Search for any matching value in priority order:
// 1. Type=AP > Type=none > Type=anything
// 2. Alternate > Canonical
// 3. Page order (fallback)
const alternate =
document.querySelector('head > link[href][rel="alternate"][type="application/activity+json"]') ??
document.querySelector('head > link[href][rel="canonical"][type="application/activity+json"]') ??
document.querySelector('head > link[href][rel="alternate"]:not([type])') ??
document.querySelector('head > link[href][rel="canonical"]:not([type])') ??
document.querySelector('head > link[href][rel="alternate"]') ??
document.querySelector('head > link[href][rel="canonical"]');
if (alternate) {
const href = alternate.getAttribute('href');
if (href && this.apUtilityService.haveSameAuthority(url, href)) {
return await this.signedGet(href, user, allowAnonymous, false);
}
}
alternate = selectFirst(document, [
'head > link[href][rel="alternate"][type="application/activity+json"]',
'head > link[href][rel="canonical"][type="application/activity+json"]',
'head > link[href][rel="alternate"]:not([type])',
'head > link[href][rel="canonical"]:not([type])',
'head > link[href][rel="alternate"]',
'head > link[href][rel="canonical"]',
]);
} catch {
// something went wrong parsing the HTML, ignore the whole thing
} finally {
happyDOM.close().catch(err => {});
alternate = null;
}
if (alternate) {
const href = alternate.attr('href');
if (href && this.apUtilityService.haveSameAuthority(url, href)) {
return await this.signedGet(href, user, allowAnonymous, false);
}
}
}
//#endregion
@ -285,3 +267,14 @@ export class ApRequestService {
return activity as IObjectWithId;
}
}
function selectFirst($: CheerioAPI, selectors: string[]): Cheerio<AnyNode> | null {
for (const selector of selectors) {
const selection = $(selector);
if (selection.length > 0) {
return selection;
}
}
return null;
}

80
packages/backend/src/core/activitypub/JsonLdService.ts
View file

@ -8,26 +8,61 @@ import { Injectable } from '@nestjs/common';
import { UnrecoverableError } from 'bullmq';
import { HttpRequestService } from '@/core/HttpRequestService.js';
import { bindThis } from '@/decorators.js';
import Logger from '@/logger.js';
import { LoggerService } from '@/core/LoggerService.js';
import { StatusError } from '@/misc/status-error.js';
import { CONTEXT, PRELOADED_CONTEXTS } from './misc/contexts.js';
import { validateContentTypeSetAsJsonLD } from './misc/validator.js';
import type { JsonLdDocument } from 'jsonld';
import type { ContextDefinition, JsonLdDocument } from 'jsonld';
import type { JsonLd as JsonLdObject, RemoteDocument } from 'jsonld/jsonld-spec.js';
// https://stackoverflow.com/a/66252656
type RemoveIndex<T> = {
[ K in keyof T as string extends K
? never
: number extends K
? never
: symbol extends K
? never
: K
] : T[K];
};
export type Document = RemoveIndex<JsonLdDocument>;
export type Signature = {
id?: string;
type: string;
creator: string;
domain?: string;
nonce: string;
created: string;
signatureValue: string;
};
export type Signed<T extends Document> = T & {
signature: Signature;
};
export function isSigned<T extends Document>(doc: T): doc is Signed<T> {
return 'signature' in doc && typeof(doc.signature) === 'object';
}
// RsaSignature2017 implementation is based on https://github.com/transmute-industries/RsaSignature2017
class JsonLd {
public debug = false;
public preLoad = true;
public loderTimeout = 5000;
@Injectable()
export class JsonLdService {
private readonly logger: Logger;
constructor(
private httpRequestService: HttpRequestService,
loggerService: LoggerService,
) {
this.logger = loggerService.getLogger('json-ld');
}
@bindThis
public async signRsaSignature2017(data: any, privateKey: string, creator: string, domain?: string, created?: Date): Promise<any> {
public async signRsaSignature2017<T extends Document>(data: T, privateKey: string, creator: string, domain?: string, created?: Date): Promise<Signed<T>> {
const options: {
type: string;
creator: string;
@ -63,7 +98,7 @@ class JsonLd {
}
@bindThis
public async verifyRsaSignature2017(data: any, publicKey: string): Promise<boolean> {
public async verifyRsaSignature2017(data: Signed<Document>, publicKey: string): Promise<boolean> {
const toBeSigned = await this.createVerifyData(data, data.signature);
const verifier = crypto.createVerify('sha256');
verifier.update(toBeSigned);
@ -71,7 +106,7 @@ class JsonLd {
}
@bindThis
public async createVerifyData(data: any, options: any): Promise<string> {
public async createVerifyData<T extends Document>(data: T, options: Partial<Signature>): Promise<string> {
const transformedOptions = {
...options,
'@context': 'https://w3id.org/identity/v1',
@ -81,17 +116,18 @@ class JsonLd {
delete transformedOptions['signatureValue'];
const canonizedOptions = await this.normalize(transformedOptions);
const optionsHash = this.sha256(canonizedOptions.toString());
const transformedData = { ...data };
const transformedData = { ...data } as T & { signature?: unknown };
delete transformedData['signature'];
const cannonidedData = await this.normalize(transformedData);
if (this.debug) console.debug(`cannonidedData: ${cannonidedData}`);
this.logger.debug('cannonidedData', cannonidedData);
const documentHash = this.sha256(cannonidedData.toString());
const verifyData = `${optionsHash}${documentHash}`;
return verifyData;
}
@bindThis
public async compact(data: any, context: any = CONTEXT): Promise<JsonLdDocument> {
// TODO our default CONTEXT isn't valid for the library, is this a bug?
public async compact(data: Document, context: ContextDefinition = CONTEXT as unknown as ContextDefinition): Promise<Document> {
const customLoader = this.getLoader();
// XXX: Importing jsonld dynamically since Jest frequently fails to import it statically
// https://github.com/misskey-dev/misskey/pull/9894#discussion_r1103753595
@ -101,7 +137,7 @@ class JsonLd {
}
@bindThis
public async normalize(data: JsonLdDocument): Promise<string> {
public async normalize(data: Document): Promise<string> {
const customLoader = this.getLoader();
return (await import('jsonld')).default.normalize(data, {
documentLoader: customLoader,
@ -113,9 +149,9 @@ class JsonLd {
return async (url: string): Promise<RemoteDocument> => {
if (!/^https?:\/\//.test(url)) throw new UnrecoverableError(`Invalid URL: ${url}`);
if (this.preLoad) {
{
if (url in PRELOADED_CONTEXTS) {
if (this.debug) console.debug(`HIT: ${url}`);
this.logger.debug(`Preload HIT: ${url}`);
return {
contextUrl: undefined,
document: PRELOADED_CONTEXTS[url],
@ -124,7 +160,7 @@ class JsonLd {
}
}
if (this.debug) console.debug(`MISS: ${url}`);
this.logger.debug(`Preload MISS: ${url}`);
const document = await this.fetchDocument(url);
return {
contextUrl: undefined,
@ -142,7 +178,6 @@ class JsonLd {
headers: {
Accept: 'application/ld+json, application/json',
},
timeout: this.loderTimeout,
},
{
throwErrorWhenResponseNotOk: false,
@ -166,16 +201,3 @@ class JsonLd {
return hash.digest('hex');
}
}
@Injectable()
export class JsonLdService {
constructor(
private httpRequestService: HttpRequestService,
) {
}
@bindThis
public use(): JsonLd {
return new JsonLd(this.httpRequestService);
}
}

4
packages/backend/src/core/activitypub/misc/contexts.ts
View file

@ -540,6 +540,10 @@ const extension_context_definition = {
quoteUrl: 'as:quoteUrl',
fedibird: 'http://fedibird.com/ns#',
quoteUri: 'fedibird:quoteUri',
quote: {
'@id': 'https://w3id.org/fep/044f#quote',
'@type': '@id',
},
// Mastodon
toot: 'http://joinmastodon.org/ns#',
Emoji: 'toot:Emoji',

28
packages/backend/src/core/activitypub/models/ApNoteService.ts
View file

@ -27,7 +27,7 @@ import { checkHttps } from '@/misc/check-https.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import { isRetryableError } from '@/misc/is-retryable-error.js';
import { renderInlineError } from '@/misc/render-inline-error.js';
import { getOneApId, getApId, validPost, isEmoji, getApType, isApObject, isDocument, IApDocument } from '../type.js';
import { getOneApId, getApId, validPost, isEmoji, getApType, isApObject, isDocument, IApDocument, isLink } from '../type.js';
import { ApLoggerService } from '../ApLoggerService.js';
import { ApMfmService } from '../ApMfmService.js';
import { ApDbResolverService } from '../ApDbResolverService.js';
@ -657,9 +657,29 @@ export class ApNoteService {
*/
private async getQuote(note: IPost, entryUri: string, resolver: Resolver): Promise<MiNote | null | undefined> {
const quoteUris = new Set<string>();
if (note._misskey_quote) quoteUris.add(note._misskey_quote);
if (note.quoteUrl) quoteUris.add(note.quoteUrl);
if (note.quoteUri) quoteUris.add(note.quoteUri);
if (note._misskey_quote && typeof(note._misskey_quote as unknown) === 'string') quoteUris.add(note._misskey_quote);
if (note.quoteUrl && typeof(note.quoteUrl as unknown) === 'string') quoteUris.add(note.quoteUrl);
if (note.quoteUri && typeof(note.quoteUri as unknown) === 'string') quoteUris.add(note.quoteUri);
// https://codeberg.org/fediverse/fep/src/branch/main/fep/044f/fep-044f.md
if (note.quote && typeof(note.quote as unknown) === 'string') quoteUris.add(note.quote);
// https://codeberg.org/fediverse/fep/src/branch/main/fep/e232/fep-e232.md
const tags = toArray(note.tag).filter(tag => typeof(tag) === 'object' && isLink(tag));
for (const tag of tags) {
if (!tag.href || typeof (tag.href as unknown) !== 'string') continue;
const mediaTypes = toArray(tag.mediaType);
if (
!mediaTypes.includes('application/ld+json; profile="https://www.w3.org/ns/activitystreams"') &&
!mediaTypes.includes('application/activity+json')
) continue;
const rels = toArray(tag.rel);
if (!rels.includes('https://misskey-hub.net/ns#_misskey_quote')) continue;
quoteUris.add(tag.href);
}
// No quote, return undefined
if (quoteUris.size < 1) return undefined;

15
packages/backend/src/core/activitypub/models/ApPersonService.ts
View file

@ -741,10 +741,17 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
this.hashtagService.updateUsertags(exist, tags);
// 該当ユーザーが既にフォロワーになっていた場合はFollowingもアップデートする
await this.followingsRepository.update(
{ followerId: exist.id },
{ followerSharedInbox: person.sharedInbox ?? person.endpoints?.sharedInbox ?? null },
);
if (exist.inbox !== person.inbox || exist.sharedInbox !== (person.sharedInbox ?? person.endpoints?.sharedInbox)) {
await this.followingsRepository.update(
{ followerId: exist.id },
{
followerInbox: person.inbox,
followerSharedInbox: person.sharedInbox ?? person.endpoints?.sharedInbox ?? null,
},
);
await this.cacheService.refreshFollowRelationsFor(exist.id);
}
await this.updateFeatured(exist.id, resolver).catch(err => {
// Permanent error implies hidden or inaccessible, which is a normal thing.

15
packages/backend/src/core/activitypub/type.ts
View file

@ -35,6 +35,7 @@ export interface IObject {
mediaType?: string;
url?: ApObject | string;
href?: string;
rel?: string | string[];
tag?: IObject | IObject[];
sensitive?: boolean;
}
@ -55,6 +56,16 @@ export function isAnonymousObject(object: IObject): object is IAnonymousObject {
return object.id === undefined;
}
export interface ILink extends IObject {
'@context'?: string | string[] | Obj | Obj[];
type: 'Link' | 'Mention';
href: string;
}
export const isLink = (object: IObject): object is ILink =>
(getApType(object) === 'Link' || getApType(object) === 'Link') &&
typeof object.href === 'string';
/**
* Get array of ActivityStreams Objects id
*/
@ -204,6 +215,7 @@ export interface IPost extends IObject {
_misskey_content?: string;
quoteUrl?: string;
quoteUri?: string;
quote?: string;
updated?: string;
}
@ -306,9 +318,8 @@ export const isPropertyValue = (object: IObject): object is IApPropertyValue =>
'value' in object &&
typeof object.value === 'string';
export interface IApMention extends IObject {
export interface IApMention extends ILink {
type: 'Mention';
href: string;
name: string;
}

1
packages/backend/src/core/chart/charts/federation.ts
View file

@ -44,6 +44,7 @@ export default class FederationChart extends Chart<typeof schema> { // eslint-di
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
// TODO optimization: replace these with exists()
const pubsubSubQuery = this.followingsRepository.createQueryBuilder('f')
.select('f.followerHost')
.where('f.followerHost IS NOT NULL');

19
packages/backend/src/core/chart/charts/per-user-following.ts
View file

@ -15,6 +15,7 @@ import Chart from '../core.js';
import { ChartLoggerService } from '../ChartLoggerService.js';
import { name, schema } from './entities/per-user-following.js';
import type { KVs } from '../core.js';
import { CacheService } from '@/core/CacheService.js';
/**
* ユーザーごとのフォローに関するチャート
@ -31,23 +32,25 @@ export default class PerUserFollowingChart extends Chart<typeof schema> { // esl
private appLockService: AppLockService,
private userEntityService: UserEntityService,
private chartLoggerService: ChartLoggerService,
private readonly cacheService: CacheService,
) {
super(db, (k) => appLockService.getChartInsertLock(k), chartLoggerService.logger, name, schema, true);
}
protected async tickMajor(group: string): Promise<Partial<KVs<typeof schema>>> {
const [
localFollowingsCount,
localFollowersCount,
remoteFollowingsCount,
remoteFollowersCount,
followees,
followers,
] = await Promise.all([
this.followingsRepository.countBy({ followerId: group, followeeHost: IsNull() }),
this.followingsRepository.countBy({ followeeId: group, followerHost: IsNull() }),
this.followingsRepository.countBy({ followerId: group, followeeHost: Not(IsNull()) }),
this.followingsRepository.countBy({ followeeId: group, followerHost: Not(IsNull()) }),
this.cacheService.userFollowingsCache.fetch(group).then(fs => Array.from(fs.values())),
this.cacheService.userFollowersCache.fetch(group).then(fs => Array.from(fs.values())),
]);
const localFollowingsCount = followees.reduce((sum, f) => sum + (f.followeeHost == null ? 1 : 0), 0);
const localFollowersCount = followers.reduce((sum, f) => sum + (f.followerHost == null ? 1 : 0), 0);
const remoteFollowingsCount = followees.reduce((sum, f) => sum + (f.followeeHost == null ? 0 : 1), 0);
const remoteFollowersCount = followers.reduce((sum, f) => sum + (f.followerHost == null ? 0 : 1), 0);
return {
'local.followings.total': localFollowingsCount,
'local.followers.total': localFollowersCount,

340
packages/backend/src/core/entities/NoteEntityService.ts
View file

@ -11,7 +11,7 @@ import type { Packed } from '@/misc/json-schema.js';
import { awaitAll } from '@/misc/prelude/await-all.js';
import type { MiUser } from '@/models/User.js';
import type { MiNote } from '@/models/Note.js';
import type { UsersRepository, NotesRepository, FollowingsRepository, PollsRepository, PollVotesRepository, NoteReactionsRepository, ChannelsRepository, MiMeta } from '@/models/_.js';
import type { UsersRepository, NotesRepository, FollowingsRepository, PollsRepository, PollVotesRepository, NoteReactionsRepository, ChannelsRepository, MiMeta, MiPollVote, MiPoll, MiChannel, MiFollowing } from '@/models/_.js';
import { bindThis } from '@/decorators.js';
import { DebounceLoader } from '@/misc/loader.js';
import { IdService } from '@/core/IdService.js';
@ -26,13 +26,13 @@ import type { UserEntityService } from './UserEntityService.js';
import type { DriveFileEntityService } from './DriveFileEntityService.js';
// is-renote.tsとよしなにリンク
function isPureRenote(note: MiNote): note is MiNote & { renoteId: MiNote['id']; renote: MiNote } {
function isPureRenote(note: MiNote): note is MiNote & { renoteId: MiNote['id'] } {
return (
note.renote != null &&
note.reply == null &&
note.renoteId != null &&
note.replyId == null &&
note.text == null &&
note.cw == null &&
(note.fileIds == null || note.fileIds.length === 0) &&
note.fileIds.length === 0 &&
!note.hasPoll
);
}
@ -132,7 +132,10 @@ export class NoteEntityService implements OnModuleInit {
}
@bindThis
public async hideNote(packedNote: Packed<'Note'>, meId: MiUser['id'] | null): Promise<void> {
public async hideNote(packedNote: Packed<'Note'>, meId: MiUser['id'] | null, hint?: {
myFollowing?: ReadonlyMap<string, unknown>,
myBlockers?: ReadonlySet<string>,
}): Promise<void> {
if (meId === packedNote.userId) return;
// TODO: isVisibleForMe を使うようにしても良さそう(型違うけど)
@ -188,14 +191,9 @@ export class NoteEntityService implements OnModuleInit {
} else if (packedNote.renote && (meId === packedNote.renote.userId)) {
hide = false;
} else {
// フォロワーかどうか
// TODO: 当関数呼び出しごとにクエリが走るのは重そうだからなんとかする
const isFollowing = await this.followingsRepository.exists({
where: {
followeeId: packedNote.userId,
followerId: meId,
},
});
const isFollowing = hint?.myFollowing
? hint.myFollowing.has(packedNote.userId)
: (await this.cacheService.userFollowingsCache.fetch(meId)).has(packedNote.userId);
hide = !isFollowing;
}
@ -211,7 +209,8 @@ export class NoteEntityService implements OnModuleInit {
}
if (!hide && meId && packedNote.userId !== meId) {
const isBlocked = (await this.cacheService.userBlockedCache.fetch(meId)).has(packedNote.userId);
const blockers = hint?.myBlockers ?? await this.cacheService.userBlockedCache.fetch(meId);
const isBlocked = blockers.has(packedNote.userId);
if (isBlocked) hide = true;
}
@ -235,8 +234,11 @@ export class NoteEntityService implements OnModuleInit {
}
@bindThis
private async populatePoll(note: MiNote, meId: MiUser['id'] | null) {
const poll = await this.pollsRepository.findOneByOrFail({ noteId: note.id });
private async populatePoll(note: MiNote, meId: MiUser['id'] | null, hint?: {
poll?: MiPoll,
myVotes?: MiPollVote[],
}) {
const poll = hint?.poll ?? await this.pollsRepository.findOneByOrFail({ noteId: note.id });
const choices = poll.choices.map(c => ({
text: c,
votes: poll.votes[poll.choices.indexOf(c)],
@ -245,7 +247,7 @@ export class NoteEntityService implements OnModuleInit {
if (meId) {
if (poll.multiple) {
const votes = await this.pollVotesRepository.findBy({
const votes = hint?.myVotes ?? await this.pollVotesRepository.findBy({
userId: meId,
noteId: note.id,
});
@ -255,7 +257,7 @@ export class NoteEntityService implements OnModuleInit {
choices[myChoice].isVoted = true;
}
} else {
const vote = await this.pollVotesRepository.findOneBy({
const vote = hint?.myVotes ? hint.myVotes[0] : await this.pollVotesRepository.findOneBy({
userId: meId,
noteId: note.id,
});
@ -317,7 +319,12 @@ export class NoteEntityService implements OnModuleInit {
}
@bindThis
public async isVisibleForMe(note: MiNote, meId: MiUser['id'] | null): Promise<boolean> {
public async isVisibleForMe(note: MiNote, meId: MiUser['id'] | null, hint?: {
myFollowing?: ReadonlySet<string>,
myBlocking?: ReadonlySet<string>,
myBlockers?: ReadonlySet<string>,
me?: Pick<MiUser, 'host'> | null,
}): Promise<boolean> {
// This code must always be synchronized with the checks in generateVisibilityQuery.
// visibility が specified かつ自分が指定されていなかったら非表示
if (note.visibility === 'specified') {
@ -345,16 +352,16 @@ export class NoteEntityService implements OnModuleInit {
return true;
} else {
// フォロワーかどうか
const [blocked, following, user] = await Promise.all([
this.cacheService.userBlockingCache.fetch(meId).then((ids) => ids.has(note.userId)),
this.followingsRepository.count({
where: {
followeeId: note.userId,
followerId: meId,
},
take: 1,
}),
this.usersRepository.findOneByOrFail({ id: meId }),
const [blocked, following, userHost] = await Promise.all([
hint?.myBlocking
? hint.myBlocking.has(note.userId)
: this.cacheService.userBlockingCache.fetch(meId).then((ids) => ids.has(note.userId)),
hint?.myFollowing
? hint.myFollowing.has(note.userId)
: this.cacheService.userFollowingsCache.fetch(meId).then(ids => ids.has(note.userId)),
hint?.me !== undefined
? (hint.me?.host ?? null)
: this.cacheService.findUserById(meId).then(me => me.host),
]);
if (blocked) return false;
@ -366,12 +373,13 @@ export class NoteEntityService implements OnModuleInit {
in which case we can never know the following. Instead we have
to assume that the users are following each other.
*/
return following > 0 || (note.userHost != null && user.host != null);
return following || (note.userHost != null && userHost != null);
}
}
if (meId != null) {
const isBlocked = (await this.cacheService.userBlockedCache.fetch(meId)).has(note.userId);
const blockers = hint?.myBlockers ?? await this.cacheService.userBlockedCache.fetch(meId);
const isBlocked = blockers.has(note.userId);
if (isBlocked) return false;
}
@ -408,6 +416,12 @@ export class NoteEntityService implements OnModuleInit {
packedFiles: Map<MiNote['fileIds'][number], Packed<'DriveFile'> | null>;
packedUsers: Map<MiUser['id'], Packed<'UserLite'>>;
mentionHandles: Record<string, string | undefined>;
userFollowings: Map<string, Map<string, Omit<MiFollowing, 'isFollowerHibernated'>>>;
userBlockers: Map<string, Set<string>>;
polls: Map<string, MiPoll>;
pollVotes: Map<string, Map<string, MiPollVote[]>>;
channels: Map<string, MiChannel>;
notes: Map<string, MiNote>;
};
},
): Promise<Packed<'Note'>> {
@ -437,9 +451,7 @@ export class NoteEntityService implements OnModuleInit {
}
const channel = note.channelId
? note.channel
? note.channel
: await this.channelsRepository.findOneBy({ id: note.channelId })
? (opts._hint_?.channels.get(note.channelId) ?? note.channel ?? await this.channelsRepository.findOneBy({ id: note.channelId }))
: null;
const reactionEmojiNames = Object.keys(reactions)
@ -485,7 +497,10 @@ export class NoteEntityService implements OnModuleInit {
mentionHandles: note.mentions.length > 0 ? this.getUserHandles(note.mentions, options?._hint_?.mentionHandles) : undefined,
uri: note.uri ?? undefined,
url: note.url ?? undefined,
poll: note.hasPoll ? this.populatePoll(note, meId) : undefined,
poll: note.hasPoll ? this.populatePoll(note, meId, {
poll: opts._hint_?.polls.get(note.id),
myVotes: opts._hint_?.pollVotes.get(note.id)?.get(note.userId),
}) : undefined,
...(meId && Object.keys(reactions).length > 0 ? {
myReaction: this.populateMyReaction({
@ -499,14 +514,14 @@ export class NoteEntityService implements OnModuleInit {
clippedCount: note.clippedCount,
processErrors: note.processErrors,
reply: note.replyId ? this.pack(note.reply ?? note.replyId, me, {
reply: note.replyId ? this.pack(note.reply ?? opts._hint_?.notes.get(note.replyId) ?? note.replyId, me, {
detail: false,
skipHide: opts.skipHide,
withReactionAndUserPairCache: opts.withReactionAndUserPairCache,
_hint_: options?._hint_,
}) : undefined,
renote: note.renoteId ? this.pack(note.renote ?? note.renoteId, me, {
renote: note.renoteId ? this.pack(note.renote ?? opts._hint_?.notes.get(note.renoteId) ?? note.renoteId, me, {
detail: true,
skipHide: opts.skipHide,
withReactionAndUserPairCache: opts.withReactionAndUserPairCache,
@ -518,7 +533,10 @@ export class NoteEntityService implements OnModuleInit {
this.treatVisibility(packed);
if (!opts.skipHide) {
await this.hideNote(packed, meId);
await this.hideNote(packed, meId, meId == null ? undefined : {
myFollowing: opts._hint_?.userFollowings.get(meId),
myBlockers: opts._hint_?.userBlockers.get(meId),
});
}
return packed;
@ -535,79 +553,139 @@ export class NoteEntityService implements OnModuleInit {
) {
if (notes.length === 0) return [];
const targetNotes: MiNote[] = [];
const targetNotesMap = new Map<string, MiNote>();
const targetNotesToFetch : string[] = [];
for (const note of notes) {
if (isPureRenote(note)) {
// we may need to fetch 'my reaction' for renote target.
targetNotes.push(note.renote);
if (note.renote.reply) {
// idem if the renote is also a reply.
targetNotes.push(note.renote.reply);
if (note.renote) {
targetNotesMap.set(note.renote.id, note.renote);
if (note.renote.reply) {
// idem if the renote is also a reply.
targetNotesMap.set(note.renote.reply.id, note.renote.reply);
}
} else if (options?.detail) {
targetNotesToFetch.push(note.renoteId);
}
} else {
if (note.reply) {
// idem for OP of a regular reply.
targetNotes.push(note.reply);
targetNotesMap.set(note.reply.id, note.reply);
} else if (note.replyId && options?.detail) {
targetNotesToFetch.push(note.replyId);
}
targetNotes.push(note);
targetNotesMap.set(note.id, note);
}
}
const bufferedReactions = this.meta.enableReactionsBuffering ? await this.reactionsBufferingService.getMany([...getAppearNoteIds(notes)]) : null;
// Don't fetch notes that were added by ID and then found inline in another note.
for (let i = targetNotesToFetch.length - 1; i >= 0; i--) {
if (targetNotesMap.has(targetNotesToFetch[i])) {
targetNotesToFetch.splice(i, 1);
}
}
const meId = me ? me.id : null;
const myReactionsMap = new Map<MiNote['id'], string | null>();
if (meId) {
const idsNeedFetchMyReaction = new Set<MiNote['id']>();
// Populate any relations that weren't included in the source
if (targetNotesToFetch.length > 0) {
const newNotes = await this.notesRepository.find({
where: {
id: In(targetNotesToFetch),
},
relations: {
user: {
userProfile: true,
},
reply: {
user: {
userProfile: true,
},
},
renote: {
user: {
userProfile: true,
},
reply: {
user: {
userProfile: true,
},
},
},
channel: true,
},
});
for (const note of targetNotes) {
const reactionsCount = Object.values(this.reactionsBufferingService.mergeReactions(note.reactions, bufferedReactions?.get(note.id)?.deltas ?? {})).reduce((a, b) => a + b, 0);
if (reactionsCount === 0) {
myReactionsMap.set(note.id, null);
} else if (reactionsCount <= note.reactionAndUserPairCache.length + (bufferedReactions?.get(note.id)?.pairs.length ?? 0)) {
const pairInBuffer = bufferedReactions?.get(note.id)?.pairs.find(p => p[0] === meId);
if (pairInBuffer) {
myReactionsMap.set(note.id, pairInBuffer[1]);
} else {
const pair = note.reactionAndUserPairCache.find(p => p.startsWith(meId));
myReactionsMap.set(note.id, pair ? pair.split('/')[1] : null);
for (const note of newNotes) {
targetNotesMap.set(note.id, note);
}
}
const targetNotes = Array.from(targetNotesMap.values());
const noteIds = Array.from(targetNotesMap.keys());
const usersMap = new Map<string, MiUser | string>();
const allUsers = notes.flatMap(note => [
note.user ?? note.userId,
note.reply?.user ?? note.replyUserId,
note.renote?.user ?? note.renoteUserId,
]);
for (const user of allUsers) {
if (!user) continue;
if (typeof(user) === 'object') {
// ID -> Entity
usersMap.set(user.id, user);
} else if (!usersMap.has(user)) {
// ID -> ID
usersMap.set(user, user);
}
}
const users = Array.from(usersMap.values());
const userIds = Array.from(usersMap.keys());
const fileIds = new Set(targetNotes.flatMap(n => n.fileIds));
const mentionedUsers = new Set(targetNotes.flatMap(note => note.mentions));
const [{ bufferedReactions, myReactionsMap }, packedFiles, packedUsers, mentionHandles, userFollowings, userBlockers, polls, pollVotes, channels] = await Promise.all([
// bufferedReactions & myReactionsMap
this.getReactions(targetNotes, me),
// packedFiles
this.driveFileEntityService.packManyByIdsMap(Array.from(fileIds)),
// packedUsers
this.userEntityService.packMany(users, me)
.then(users => new Map(users.map(u => [u.id, u]))),
// mentionHandles
this.getUserHandles(Array.from(mentionedUsers)),
// userFollowings
this.cacheService.userFollowingsCache.fetchMany(userIds).then(fs => new Map(fs)),
// userBlockers
this.cacheService.userBlockedCache.fetchMany(userIds).then(bs => new Map(bs)),
// polls
this.pollsRepository.findBy({ noteId: In(noteIds) })
.then(polls => new Map(polls.map(p => [p.noteId, p]))),
// pollVotes
this.pollVotesRepository.findBy({ noteId: In(noteIds), userId: In(userIds) })
.then(votes => votes.reduce((noteMap, vote) => {
let userMap = noteMap.get(vote.noteId);
if (!userMap) {
userMap = new Map<string, MiPollVote[]>();
noteMap.set(vote.noteId, userMap);
}
} else {
idsNeedFetchMyReaction.add(note.id);
}
}
const myReactions = idsNeedFetchMyReaction.size > 0 ? await this.noteReactionsRepository.findBy({
userId: meId,
noteId: In(Array.from(idsNeedFetchMyReaction)),
}) : [];
for (const id of idsNeedFetchMyReaction) {
myReactionsMap.set(id, myReactions.find(reaction => reaction.noteId === id)?.reaction ?? null);
}
}
await this.customEmojiService.prefetchEmojis(this.aggregateNoteEmojis(notes));
// TODO: 本当は renote とか reply がないのに renoteId とか replyId があったらここで解決しておく
const fileIds = notes.map(n => [n.fileIds, n.renote?.fileIds, n.reply?.fileIds]).flat(2).filter(x => x != null);
const packedFiles = fileIds.length > 0 ? await this.driveFileEntityService.packManyByIdsMap(fileIds) : new Map();
const users = [
...notes.map(({ user, userId }) => user ?? userId),
...notes.map(({ replyUserId }) => replyUserId).filter(x => x != null),
...notes.map(({ renoteUserId }) => renoteUserId).filter(x => x != null),
];
const packedUsers = await this.userEntityService.packMany(users, me)
.then(users => new Map(users.map(u => [u.id, u])));
// Recursively add all mentioned users from all notes + replies + renotes
const allMentionedUsers = targetNotes.reduce((users, note) => {
for (const user of note.mentions) {
users.add(user);
}
return users;
}, new Set<string>());
const mentionHandles = await this.getUserHandles(Array.from(allMentionedUsers));
let voteList = userMap.get(vote.userId);
if (!voteList) {
voteList = [];
userMap.set(vote.userId, voteList);
}
voteList.push(vote);
return noteMap;
}, new Map<string, Map<string, MiPollVote[]>>)),
// channels
this.getChannels(targetNotes),
// (not returned)
this.customEmojiService.prefetchEmojis(this.aggregateNoteEmojis(notes)),
]);
return await Promise.all(notes.map(n => this.pack(n, me, {
...options,
@ -617,6 +695,12 @@ export class NoteEntityService implements OnModuleInit {
packedFiles,
packedUsers,
mentionHandles,
userFollowings,
userBlockers,
polls,
pollVotes,
channels,
notes: new Map(targetNotes.map(n => [n.id, n])),
},
})));
}
@ -685,6 +769,68 @@ export class NoteEntityService implements OnModuleInit {
}, {} as Record<string, string | undefined>);
}
private async getChannels(notes: MiNote[]): Promise<Map<string, MiChannel>> {
const channels = new Map<string, MiChannel>();
const channelsToFetch = new Set<string>();
for (const note of notes) {
if (note.channel) {
channels.set(note.channel.id, note.channel);
} else if (note.channelId) {
channelsToFetch.add(note.channelId);
}
}
if (channelsToFetch.size > 0) {
const newChannels = await this.channelsRepository.findBy({
id: In(Array.from(channelsToFetch)),
});
for (const channel of newChannels) {
channels.set(channel.id, channel);
}
}
return channels;
}
private async getReactions(notes: MiNote[], me: { id: string } | null | undefined) {
const bufferedReactions = this.meta.enableReactionsBuffering ? await this.reactionsBufferingService.getMany([...getAppearNoteIds(notes)]) : null;
const meId = me ? me.id : null;
const myReactionsMap = new Map<MiNote['id'], string | null>();
if (meId) {
const idsNeedFetchMyReaction = new Set<MiNote['id']>();
for (const note of notes) {
const reactionsCount = Object.values(this.reactionsBufferingService.mergeReactions(note.reactions, bufferedReactions?.get(note.id)?.deltas ?? {})).reduce((a, b) => a + b, 0);
if (reactionsCount === 0) {
myReactionsMap.set(note.id, null);
} else if (reactionsCount <= note.reactionAndUserPairCache.length + (bufferedReactions?.get(note.id)?.pairs.length ?? 0)) {
const pairInBuffer = bufferedReactions?.get(note.id)?.pairs.find(p => p[0] === meId);
if (pairInBuffer) {
myReactionsMap.set(note.id, pairInBuffer[1]);
} else {
const pair = note.reactionAndUserPairCache.find(p => p.startsWith(meId));
myReactionsMap.set(note.id, pair ? pair.split('/')[1] : null);
}
} else {
idsNeedFetchMyReaction.add(note.id);
}
}
const myReactions = idsNeedFetchMyReaction.size > 0 ? await this.noteReactionsRepository.findBy({
userId: meId,
noteId: In(Array.from(idsNeedFetchMyReaction)),
}) : [];
for (const id of idsNeedFetchMyReaction) {
myReactionsMap.set(id, myReactions.find(reaction => reaction.noteId === id)?.reaction ?? null);
}
}
return { bufferedReactions, myReactionsMap };
}
@bindThis
public genLocalNoteUri(noteId: string): string {
return `${this.config.url}/notes/${noteId}`;

325
packages/backend/src/core/entities/UserEntityService.ts
View file

@ -30,6 +30,7 @@ import type {
FollowingsRepository,
FollowRequestsRepository,
MiFollowing,
MiInstance,
MiMeta,
MiUserNotePining,
MiUserProfile,
@ -42,7 +43,7 @@ import type {
UsersRepository,
} from '@/models/_.js';
import { bindThis } from '@/decorators.js';
import { RoleService } from '@/core/RoleService.js';
import { RolePolicies, RoleService } from '@/core/RoleService.js';
import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js';
import { FederatedInstanceService } from '@/core/FederatedInstanceService.js';
import { IdService } from '@/core/IdService.js';
@ -52,6 +53,7 @@ import { AvatarDecorationService } from '@/core/AvatarDecorationService.js';
import { ChatService } from '@/core/ChatService.js';
import { isSystemAccount } from '@/misc/is-system-account.js';
import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js';
import type { CacheService } from '@/core/CacheService.js';
import type { OnModuleInit } from '@nestjs/common';
import type { NoteEntityService } from './NoteEntityService.js';
import type { PageEntityService } from './PageEntityService.js';
@ -77,7 +79,7 @@ function isRemoteUser(user: MiUser | { host: MiUser['host'] }): boolean {
export type UserRelation = {
id: MiUser['id']
following: MiFollowing | null,
following: Omit<MiFollowing, 'isFollowerHibernated'> | null,
isFollowing: boolean
isFollowed: boolean
hasPendingFollowRequestFromYou: boolean
@ -103,6 +105,7 @@ export class UserEntityService implements OnModuleInit {
private idService: IdService;
private avatarDecorationService: AvatarDecorationService;
private chatService: ChatService;
private cacheService: CacheService;
constructor(
private moduleRef: ModuleRef,
@ -163,6 +166,7 @@ export class UserEntityService implements OnModuleInit {
this.idService = this.moduleRef.get('IdService');
this.avatarDecorationService = this.moduleRef.get('AvatarDecorationService');
this.chatService = this.moduleRef.get('ChatService');
this.cacheService = this.moduleRef.get('CacheService');
}
//#region Validators
@ -193,16 +197,8 @@ export class UserEntityService implements OnModuleInit {
memo,
mutedInstances,
] = await Promise.all([
this.followingsRepository.findOneBy({
followerId: me,
followeeId: target,
}),
this.followingsRepository.exists({
where: {
followerId: target,
followeeId: me,
},
}),
this.cacheService.userFollowingsCache.fetch(me).then(f => f.get(target) ?? null),
this.cacheService.userFollowingsCache.fetch(target).then(f => f.has(me)),
this.followRequestsRepository.exists({
where: {
followerId: me,
@ -215,45 +211,22 @@ export class UserEntityService implements OnModuleInit {
followeeId: me,
},
}),
this.blockingsRepository.exists({
where: {
blockerId: me,
blockeeId: target,
},
}),
this.blockingsRepository.exists({
where: {
blockerId: target,
blockeeId: me,
},
}),
this.mutingsRepository.exists({
where: {
muterId: me,
muteeId: target,
},
}),
this.renoteMutingsRepository.exists({
where: {
muterId: me,
muteeId: target,
},
}),
this.usersRepository.createQueryBuilder('u')
.select('u.host')
.where({ id: target })
.getRawOne<{ u_host: string }>()
.then(it => it?.u_host ?? null),
this.cacheService.userBlockingCache.fetch(me)
.then(blockees => blockees.has(target)),
this.cacheService.userBlockedCache.fetch(me)
.then(blockers => blockers.has(target)),
this.cacheService.userMutingsCache.fetch(me)
.then(mutings => mutings.has(target)),
this.cacheService.renoteMutingsCache.fetch(me)
.then(mutings => mutings.has(target)),
this.cacheService.findUserById(target).then(u => u.host),
this.userMemosRepository.createQueryBuilder('m')
.select('m.memo')
.where({ userId: me, targetUserId: target })
.getRawOne<{ m_memo: string | null }>()
.then(it => it?.m_memo ?? null),
this.userProfilesRepository.createQueryBuilder('p')
.select('p.mutedInstances')
.where({ userId: me })
.getRawOne<{ p_mutedInstances: string[] }>()
.then(it => it?.p_mutedInstances ?? []),
this.cacheService.userProfileCache.fetch(me)
.then(profile => profile.mutedInstances),
]);
const isInstanceMuted = !!host && mutedInstances.includes(host);
@ -277,8 +250,8 @@ export class UserEntityService implements OnModuleInit {
@bindThis
public async getRelations(me: MiUser['id'], targets: MiUser['id'][]): Promise<Map<MiUser['id'], UserRelation>> {
const [
followers,
followees,
myFollowing,
myFollowers,
followersRequests,
followeesRequests,
blockers,
@ -289,13 +262,8 @@ export class UserEntityService implements OnModuleInit {
memos,
mutedInstances,
] = await Promise.all([
this.followingsRepository.findBy({ followerId: me })
.then(f => new Map(f.map(it => [it.followeeId, it]))),
this.followingsRepository.createQueryBuilder('f')
.select('f.followerId')
.where('f.followeeId = :me', { me })
.getRawMany<{ f_followerId: string }>()
.then(it => it.map(it => it.f_followerId)),
this.cacheService.userFollowingsCache.fetch(me),
this.cacheService.userFollowersCache.fetch(me),
this.followRequestsRepository.createQueryBuilder('f')
.select('f.followeeId')
.where('f.followerId = :me', { me })
@ -306,34 +274,18 @@ export class UserEntityService implements OnModuleInit {
.where('f.followeeId = :me', { me })
.getRawMany<{ f_followerId: string }>()
.then(it => it.map(it => it.f_followerId)),
this.blockingsRepository.createQueryBuilder('b')
.select('b.blockeeId')
.where('b.blockerId = :me', { me })
.getRawMany<{ b_blockeeId: string }>()
.then(it => it.map(it => it.b_blockeeId)),
this.blockingsRepository.createQueryBuilder('b')
.select('b.blockerId')
.where('b.blockeeId = :me', { me })
.getRawMany<{ b_blockerId: string }>()
.then(it => it.map(it => it.b_blockerId)),
this.mutingsRepository.createQueryBuilder('m')
.select('m.muteeId')
.where('m.muterId = :me', { me })
.getRawMany<{ m_muteeId: string }>()
.then(it => it.map(it => it.m_muteeId)),
this.renoteMutingsRepository.createQueryBuilder('m')
.select('m.muteeId')
.where('m.muterId = :me', { me })
.getRawMany<{ m_muteeId: string }>()
.then(it => it.map(it => it.m_muteeId)),
this.usersRepository.createQueryBuilder('u')
.select(['u.id', 'u.host'])
.where({ id: In(targets) } )
.getRawMany<{ m_id: string, m_host: string }>()
.then(it => it.reduce((map, it) => {
map[it.m_id] = it.m_host;
return map;
}, {} as Record<string, string>)),
this.cacheService.userBlockedCache.fetch(me),
this.cacheService.userBlockingCache.fetch(me),
this.cacheService.userMutingsCache.fetch(me),
this.cacheService.renoteMutingsCache.fetch(me),
this.cacheService.getUsers(targets)
.then(users => {
const record: Record<string, string | null> = {};
for (const [id, user] of users) {
record[id] = user.host;
}
return record;
}),
this.userMemosRepository.createQueryBuilder('m')
.select(['m.targetUserId', 'm.memo'])
.where({ userId: me, targetUserId: In(targets) })
@ -342,16 +294,13 @@ export class UserEntityService implements OnModuleInit {
map[it.m_targetUserId] = it.m_memo;
return map;
}, {} as Record<string, string | null>)),
this.userProfilesRepository.createQueryBuilder('p')
.select('p.mutedInstances')
.where({ userId: me })
.getRawOne<{ p_mutedInstances: string[] }>()
.then(it => it?.p_mutedInstances ?? []),
this.cacheService.userProfileCache.fetch(me)
.then(p => p.mutedInstances),
]);
return new Map(
targets.map(target => {
const following = followers.get(target) ?? null;
const following = myFollowing.get(target) ?? null;
return [
target,
@ -359,14 +308,14 @@ export class UserEntityService implements OnModuleInit {
id: target,
following: following,
isFollowing: following != null,
isFollowed: followees.includes(target),
isFollowed: myFollowers.has(target),
hasPendingFollowRequestFromYou: followersRequests.includes(target),
hasPendingFollowRequestToYou: followeesRequests.includes(target),
isBlocking: blockers.includes(target),
isBlocked: blockees.includes(target),
isMuted: muters.includes(target),
isRenoteMuted: renoteMuters.includes(target),
isInstanceMuted: mutedInstances.includes(hosts[target]),
isBlocking: blockees.has(target),
isBlocked: blockers.has(target),
isMuted: muters.has(target),
isRenoteMuted: renoteMuters.has(target),
isInstanceMuted: hosts[target] != null && mutedInstances.includes(hosts[target]),
memo: memos[target] ?? null,
},
];
@ -391,6 +340,7 @@ export class UserEntityService implements OnModuleInit {
return false; // TODO
}
// TODO optimization: make redis calls in MULTI
@bindThis
public async getNotificationsInfo(userId: MiUser['id']): Promise<{
hasUnread: boolean;
@ -424,16 +374,14 @@ export class UserEntityService implements OnModuleInit {
@bindThis
public async getHasPendingReceivedFollowRequest(userId: MiUser['id']): Promise<boolean> {
const count = await this.followRequestsRepository.countBy({
return await this.followRequestsRepository.existsBy({
followeeId: userId,
});
return count > 0;
}
@bindThis
public async getHasPendingSentFollowRequest(userId: MiUser['id']): Promise<boolean> {
return this.followRequestsRepository.existsBy({
return await this.followRequestsRepository.existsBy({
followerId: userId,
});
}
@ -480,6 +428,12 @@ export class UserEntityService implements OnModuleInit {
userRelations?: Map<MiUser['id'], UserRelation>,
userMemos?: Map<MiUser['id'], string | null>,
pinNotes?: Map<MiUser['id'], MiUserNotePining[]>,
iAmModerator?: boolean,
userIdsByUri?: Map<string, string>,
instances?: Map<string, MiInstance | null>,
securityKeyCounts?: Map<string, number>,
pendingReceivedFollows?: Set<string>,
pendingSentFollows?: Set<string>,
},
): Promise<Packed<S>> {
const opts = Object.assign({
@ -521,7 +475,7 @@ export class UserEntityService implements OnModuleInit {
const isDetailed = opts.schema !== 'UserLite';
const meId = me ? me.id : null;
const isMe = meId === user.id;
const iAmModerator = me ? await this.roleService.isModerator(me as MiUser) : false;
const iAmModerator = opts.iAmModerator ?? (me ? await this.roleService.isModerator(me as MiUser) : false);
const profile = isDetailed
? (opts.userProfile ?? user.userProfile ?? await this.userProfilesRepository.findOneByOrFail({ userId: user.id }))
@ -582,6 +536,9 @@ export class UserEntityService implements OnModuleInit {
const checkHost = user.host == null ? this.config.host : user.host;
const notificationsInfo = isMe && isDetailed ? await this.getNotificationsInfo(user.id) : null;
let fetchPoliciesPromise: Promise<RolePolicies> | null = null;
const fetchPolicies = () => fetchPoliciesPromise ??= this.roleService.getUserPolicies(user);
const packed = {
id: user.id,
name: user.name,
@ -607,13 +564,13 @@ export class UserEntityService implements OnModuleInit {
mandatoryCW: user.mandatoryCW,
rejectQuotes: user.rejectQuotes,
attributionDomains: user.attributionDomains,
isSilenced: user.isSilenced || this.roleService.getUserPolicies(user.id).then(r => !r.canPublicNote),
isSilenced: user.isSilenced || fetchPolicies().then(r => !r.canPublicNote),
speakAsCat: user.speakAsCat ?? false,
approved: user.approved,
requireSigninToViewContents: user.requireSigninToViewContents === false ? undefined : true,
makeNotesFollowersOnlyBefore: user.makeNotesFollowersOnlyBefore ?? undefined,
makeNotesHiddenBefore: user.makeNotesHiddenBefore ?? undefined,
instance: user.host ? this.federatedInstanceService.fetch(user.host).then(instance => instance ? {
instance: user.host ? Promise.resolve(opts.instances?.has(user.host) ? opts.instances.get(user.host) : this.federatedInstanceService.fetch(user.host)).then(instance => instance ? {
name: instance.name,
softwareName: instance.softwareName,
softwareVersion: instance.softwareVersion,
@ -628,7 +585,7 @@ export class UserEntityService implements OnModuleInit {
emojis: this.customEmojiService.populateEmojis(user.emojis, checkHost),
onlineStatus: this.getOnlineStatus(user),
// パフォーマンス上の理由でローカルユーザーのみ
badgeRoles: user.host == null ? this.roleService.getUserBadgeRoles(user.id).then((rs) => rs
badgeRoles: user.host == null ? this.roleService.getUserBadgeRoles(user).then((rs) => rs
.filter((r) => r.isPublic || iAmModerator)
.sort((a, b) => b.displayOrder - a.displayOrder)
.map((r) => ({
@ -641,9 +598,9 @@ export class UserEntityService implements OnModuleInit {
...(isDetailed ? {
url: profile!.url,
uri: user.uri,
movedTo: user.movedToUri ? this.apPersonService.resolvePerson(user.movedToUri).then(user => user.id).catch(() => null) : null,
movedTo: user.movedToUri ? Promise.resolve(opts.userIdsByUri?.get(user.movedToUri) ?? this.apPersonService.resolvePerson(user.movedToUri).then(user => user.id).catch(() => null)) : null,
alsoKnownAs: user.alsoKnownAs
? Promise.all(user.alsoKnownAs.map(uri => this.apPersonService.fetchPerson(uri).then(user => user?.id).catch(() => null)))
? Promise.all(user.alsoKnownAs.map(uri => Promise.resolve(opts.userIdsByUri?.get(uri) ?? this.apPersonService.fetchPerson(uri).then(user => user?.id).catch(() => null))))
.then(xs => xs.length === 0 ? null : xs.filter(x => x != null))
: null,
updatedAt: user.updatedAt ? user.updatedAt.toISOString() : null,
@ -670,8 +627,8 @@ export class UserEntityService implements OnModuleInit {
followersVisibility: profile!.followersVisibility,
followingVisibility: profile!.followingVisibility,
chatScope: user.chatScope,
canChat: this.roleService.getUserPolicies(user.id).then(r => r.chatAvailability === 'available'),
roles: this.roleService.getUserRoles(user.id).then(roles => roles.filter(role => role.isPublic).sort((a, b) => b.displayOrder - a.displayOrder).map(role => ({
canChat: fetchPolicies().then(r => r.chatAvailability === 'available'),
roles: this.roleService.getUserRoles(user).then(roles => roles.filter(role => role.isPublic).sort((a, b) => b.displayOrder - a.displayOrder).map(role => ({
id: role.id,
name: role.name,
color: role.color,
@ -689,7 +646,7 @@ export class UserEntityService implements OnModuleInit {
twoFactorEnabled: profile!.twoFactorEnabled,
usePasswordLessLogin: profile!.usePasswordLessLogin,
securityKeys: profile!.twoFactorEnabled
? this.userSecurityKeysRepository.countBy({ userId: user.id }).then(result => result >= 1)
? Promise.resolve(opts.securityKeyCounts?.get(user.id) ?? this.userSecurityKeysRepository.countBy({ userId: user.id })).then(result => result >= 1)
: false,
} : {}),
@ -722,8 +679,8 @@ export class UserEntityService implements OnModuleInit {
hasUnreadAntenna: this.getHasUnreadAntenna(user.id),
hasUnreadChannel: false, // 後方互換性のため
hasUnreadNotification: notificationsInfo?.hasUnread, // 後方互換性のため
hasPendingReceivedFollowRequest: this.getHasPendingReceivedFollowRequest(user.id),
hasPendingSentFollowRequest: this.getHasPendingSentFollowRequest(user.id),
hasPendingReceivedFollowRequest: opts.pendingReceivedFollows?.has(user.id) ?? this.getHasPendingReceivedFollowRequest(user.id),
hasPendingSentFollowRequest: opts.pendingSentFollows?.has(user.id) ?? this.getHasPendingSentFollowRequest(user.id),
unreadNotificationsCount: notificationsInfo?.unreadCount,
mutedWords: profile!.mutedWords,
hardMutedWords: profile!.hardMutedWords,
@ -733,7 +690,7 @@ export class UserEntityService implements OnModuleInit {
emailNotificationTypes: profile!.emailNotificationTypes,
achievements: profile!.achievements,
loggedInDays: profile!.loggedInDates.length,
policies: this.roleService.getUserPolicies(user.id),
policies: fetchPolicies(),
defaultCW: profile!.defaultCW,
defaultCWPriority: profile!.defaultCWPriority,
allowUnsignedFetch: user.allowUnsignedFetch,
@ -783,6 +740,8 @@ export class UserEntityService implements OnModuleInit {
includeSecrets?: boolean,
},
): Promise<Packed<S>[]> {
if (users.length === 0) return [];
// -- IDのみの要素を補完して完全なエンティティ一覧を作る
const _users = users.filter((user): user is MiUser => typeof user !== 'string');
@ -800,57 +759,105 @@ export class UserEntityService implements OnModuleInit {
}
const _userIds = _users.map(u => u.id);
// -- 実行者の有無や指定スキーマの種別によって要否が異なる値群を取得
const iAmModerator = await this.roleService.isModerator(me as MiUser);
const meId = me ? me.id : null;
const isMe = meId && _userIds.includes(meId);
const isDetailed = options && options.schema !== 'UserLite';
const isDetailedAndMe = isDetailed && isMe;
const isDetailedAndMeOrMod = isDetailed && (isMe || iAmModerator);
const isDetailedAndNotMe = isDetailed && !isMe;
let profilesMap: Map<MiUser['id'], MiUserProfile> = new Map();
let userRelations: Map<MiUser['id'], UserRelation> = new Map();
let userMemos: Map<MiUser['id'], string | null> = new Map();
let pinNotes: Map<MiUser['id'], MiUserNotePining[]> = new Map();
const userUris = new Set(_users
.flatMap(user => [user.uri, user.movedToUri])
.filter((uri): uri is string => uri != null));
if (options?.schema !== 'UserLite') {
const _profiles: MiUserProfile[] = [];
const _profilesToFetch: string[] = [];
for (const user of _users) {
if (user.userProfile) {
_profiles.push(user.userProfile);
} else {
_profilesToFetch.push(user.id);
}
}
if (_profilesToFetch.length > 0) {
const fetched = await this.userProfilesRepository.findBy({ userId: In(_profilesToFetch) });
_profiles.push(...fetched);
}
profilesMap = new Map(_profiles.map(p => [p.userId, p]));
const userHosts = new Set(_users
.map(user => user.host)
.filter((host): host is string => host != null));
const meId = me ? me.id : null;
if (meId) {
userMemos = await this.userMemosRepository.findBy({ userId: meId })
.then(memos => new Map(memos.map(memo => [memo.targetUserId, memo.memo])));
if (_userIds.length > 0) {
userRelations = await this.getRelations(meId, _userIds);
pinNotes = await this.userNotePiningsRepository.createQueryBuilder('pin')
.where('pin.userId IN (:...userIds)', { userIds: _userIds })
.innerJoinAndSelect('pin.note', 'note')
.getMany()
.then(pinsNotes => {
const map = new Map<MiUser['id'], MiUserNotePining[]>();
for (const note of pinsNotes) {
const notes = map.get(note.userId) ?? [];
notes.push(note);
map.set(note.userId, notes);
}
for (const [, notes] of map.entries()) {
// pack側ではDESCで取得しているので、それに合わせて降順に並び替えておく
notes.sort((a, b) => b.id.localeCompare(a.id));
}
return map;
});
}
const _profilesFromUsers: [string, MiUserProfile][] = [];
const _profilesToFetch: string[] = [];
for (const user of _users) {
if (user.userProfile) {
_profilesFromUsers.push([user.id, user.userProfile]);
} else {
_profilesToFetch.push(user.id);
}
}
// -- 実行者の有無や指定スキーマの種別によって要否が異なる値群を取得
const [profilesMap, userMemos, userRelations, pinNotes, userIdsByUri, instances, securityKeyCounts, pendingReceivedFollows, pendingSentFollows] = await Promise.all([
// profilesMap
this.cacheService.userProfileCache.fetchMany(_profilesToFetch).then(profiles => new Map(profiles.concat(_profilesFromUsers))),
// userMemos
isDetailed && meId ? this.userMemosRepository.findBy({ userId: meId })
.then(memos => new Map(memos.map(memo => [memo.targetUserId, memo.memo]))) : new Map(),
// userRelations
isDetailedAndNotMe && meId ? this.getRelations(meId, _userIds) : new Map(),
// pinNotes
isDetailed ? this.userNotePiningsRepository.createQueryBuilder('pin')
.where('pin.userId IN (:...userIds)', { userIds: _userIds })
.innerJoinAndSelect('pin.note', 'note')
.getMany()
.then(pinsNotes => {
const map = new Map<MiUser['id'], MiUserNotePining[]>();
for (const note of pinsNotes) {
const notes = map.get(note.userId) ?? [];
notes.push(note);
map.set(note.userId, notes);
}
for (const [, notes] of map.entries()) {
// pack側ではDESCで取得しているので、それに合わせて降順に並び替えておく
notes.sort((a, b) => b.id.localeCompare(a.id));
}
return map;
}) : new Map(),
// userIdsByUrl
isDetailed ? this.usersRepository.createQueryBuilder('user')
.select([
'user.id',
'user.uri',
])
.where({
uri: In(Array.from(userUris)),
})
.getRawMany<{ user_uri: string, user_id: string }>()
.then(users => new Map(users.map(u => [u.user_uri, u.user_id]))) : new Map(),
// instances
Promise.all(Array.from(userHosts).map(async host => [host, await this.federatedInstanceService.fetch(host)] as const))
.then(hosts => new Map(hosts)),
// securityKeyCounts
isDetailedAndMeOrMod ? this.userSecurityKeysRepository.createQueryBuilder('key')
.select('key.userId', 'userId')
.addSelect('count(key.id)', 'userCount')
.where({
userId: In(_userIds),
})
.groupBy('key.userId')
.getRawMany<{ userId: string, userCount: number }>()
.then(counts => new Map(counts.map(c => [c.userId, c.userCount]))) : new Map(),
// TODO optimization: cache follow requests
// pendingReceivedFollows
isDetailedAndMe ? this.followRequestsRepository.createQueryBuilder('req')
.select('req.followeeId', 'followeeId')
.where({
followeeId: In(_userIds),
})
.groupBy('req.followeeId')
.getRawMany<{ followeeId: string }>()
.then(reqs => new Set(reqs.map(r => r.followeeId))) : new Set<string>(),
// pendingSentFollows
isDetailedAndMe ? this.followRequestsRepository.createQueryBuilder('req')
.select('req.followerId', 'followerId')
.where({
followerId: In(_userIds),
})
.groupBy('req.followerId')
.getRawMany<{ followerId: string }>()
.then(reqs => new Set(reqs.map(r => r.followerId))) : new Set<string>(),
]);
return Promise.all(
_users.map(u => this.pack(
u,
@ -861,6 +868,12 @@ export class UserEntityService implements OnModuleInit {
userRelations: userRelations,
userMemos: userMemos,
pinNotes: pinNotes,
iAmModerator,
userIdsByUri,
instances,
securityKeyCounts,
pendingReceivedFollows,
pendingSentFollows,
},
)),
);

10
packages/backend/src/logger.ts
View file

@ -23,6 +23,14 @@ export type DataElement = DataObject | Error | string | null;
// https://stackoverflow.com/questions/61148466/typescript-type-that-matches-any-object-but-not-arrays
export type DataObject = Record<string, unknown> | (object & { length?: never; });
const levelFuncs = {
error: 'error',
warning: 'warn',
success: 'info',
info: 'log',
debug: 'debug',
} as const satisfies Record<Level, keyof typeof console>;
// eslint-disable-next-line import/no-default-export
export default class Logger {
private context: Context;
@ -86,7 +94,7 @@ export default class Logger {
} else if (data != null) {
args.push(data);
}
console.log(...args);
console[levelFuncs[level]](...args);
}
@bindThis

385
packages/backend/src/misc/QuantumKVCache.ts Normal file
View file

@ -0,0 +1,385 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { InternalEventService } from '@/core/InternalEventService.js';
import { bindThis } from '@/decorators.js';
import { InternalEventTypes } from '@/core/GlobalEventService.js';
import { MemoryKVCache } from '@/misc/cache.js';
export interface QuantumKVOpts<T> {
/**
* Memory cache lifetime in milliseconds.
*/
lifetime: number;
/**
* Callback to fetch the value for a key that wasn't found in the cache.
* May be synchronous or async.
*/
fetcher: (key: string, cache: QuantumKVCache<T>) => T | Promise<T>;
/**
* Optional callback to fetch the value for multiple keys that weren't found in the cache.
* May be synchronous or async.
* If not provided, then the implementation will fall back on repeated calls to fetcher().
*/
bulkFetcher?: (keys: string[], cache: QuantumKVCache<T>) => Iterable<[key: string, value: T]> | Promise<Iterable<[key: string, value: T]>>;
/**
* Optional callback when one or more values are changed (created, updated, or deleted) in the cache, either locally or elsewhere in the cluster.
* This is called *after* the cache state is updated.
* Implementations may be synchronous or async.
*/
onChanged?: (keys: string[], cache: QuantumKVCache<T>) => void | Promise<void>;
}
/**
* QuantumKVCache is a lifetime-bounded memory cache (like MemoryKVCache) with automatic cross-cluster synchronization via Redis.
* All nodes in the cluster are guaranteed to have a *subset* view of the current accurate state, though individual processes may have different items in their local cache.
* This ensures that a call to get() will never return stale data.
*/
export class QuantumKVCache<T> implements Iterable<[key: string, value: T]> {
private readonly memoryCache: MemoryKVCache<T>;
public readonly fetcher: QuantumKVOpts<T>['fetcher'];
public readonly bulkFetcher: QuantumKVOpts<T>['bulkFetcher'];
public readonly onChanged: QuantumKVOpts<T>['onChanged'];
/**
* @param internalEventService Service bus to synchronize events.
* @param name Unique name of the cache - must be the same in all processes.
* @param opts Cache options
*/
constructor(
private readonly internalEventService: InternalEventService,
private readonly name: string,
opts: QuantumKVOpts<T>,
) {
this.memoryCache = new MemoryKVCache(opts.lifetime);
this.fetcher = opts.fetcher;
this.bulkFetcher = opts.bulkFetcher;
this.onChanged = opts.onChanged;
this.internalEventService.on('quantumCacheUpdated', this.onQuantumCacheUpdated, {
// Ignore our own events, otherwise we'll immediately erase any set value.
ignoreLocal: true,
});
}
/**
* The number of items currently in memory.
* This applies to the local subset view, not the cross-cluster cache state.
*/
public get size() {
return this.memoryCache.size;
}
/**
* Iterates all [key, value] pairs in memory.
* This applies to the local subset view, not the cross-cluster cache state.
*/
@bindThis
public *entries(): Generator<[key: string, value: T]> {
for (const entry of this.memoryCache.entries) {
yield [entry[0], entry[1].value];
}
}
/**
* Iterates all keys in memory.
* This applies to the local subset view, not the cross-cluster cache state.
*/
@bindThis
public *keys() {
for (const entry of this.memoryCache.entries) {
yield entry[0];
}
}
/**
* Iterates all values pairs in memory.
* This applies to the local subset view, not the cross-cluster cache state.
*/
@bindThis
public *values() {
for (const entry of this.memoryCache.entries) {
yield entry[1].value;
}
}
/**
* Creates or updates a value in the cache, and erases any stale caches across the cluster.
* Fires an onSet event after the cache has been updated in all processes.
* Skips if the value is unchanged.
*/
@bindThis
public async set(key: string, value: T): Promise<void> {
if (this.memoryCache.get(key) === value) {
return;
}
this.memoryCache.set(key, value);
await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, keys: [key] });
if (this.onChanged) {
await this.onChanged([key], this);
}
}
/**
* Creates or updates multiple value in the cache, and erases any stale caches across the cluster.
* Fires an onSet for each changed item event after the cache has been updated in all processes.
* Skips if all values are unchanged.
*/
@bindThis
public async setMany(items: Iterable<[key: string, value: T]>): Promise<void> {
const changedKeys: string[] = [];
for (const item of items) {
if (this.memoryCache.get(item[0]) !== item[1]) {
changedKeys.push(item[0]);
this.memoryCache.set(item[0], item[1]);
}
}
if (changedKeys.length > 0) {
await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, keys: changedKeys });
if (this.onChanged) {
await this.onChanged(changedKeys, this);
}
}
}
/**
* Adds a value to the local memory cache without notifying other process.
* Neither a Redis event nor onSet callback will be fired, as the value has not actually changed.
* This should only be used when the value is known to be current, like after fetching from the database.
*/
@bindThis
public add(key: string, value: T): void {
this.memoryCache.set(key, value);
}
/**
* Adds multiple values to the local memory cache without notifying other process.
* Neither a Redis event nor onSet callback will be fired, as the value has not actually changed.
* This should only be used when the value is known to be current, like after fetching from the database.
*/
@bindThis
public addMany(items: Iterable<[key: string, value: T]>): void {
for (const [key, value] of items) {
this.memoryCache.set(key, value);
}
}
/**
* Gets a value from the local memory cache, or returns undefined if not found.
* Returns cached data only - does not make any fetches.
*/
@bindThis
public get(key: string): T | undefined {
return this.memoryCache.get(key);
}
/**
* Gets multiple values from the local memory cache; returning undefined for any missing keys.
* Returns cached data only - does not make any fetches.
*/
@bindThis
public getMany(keys: Iterable<string>): [key: string, value: T | undefined][] {
const results: [key: string, value: T | undefined][] = [];
for (const key of keys) {
results.push([key, this.get(key)]);
}
return results;
}
/**
* Gets or fetches a value from the cache.
* Fires an onSet event, but does not emit an update event to other processes.
*/
@bindThis
public async fetch(key: string): Promise<T> {
let value = this.memoryCache.get(key);
if (value === undefined) {
value = await this.fetcher(key, this);
this.memoryCache.set(key, value);
if (this.onChanged) {
await this.onChanged([key], this);
}
}
return value;
}
/**
* Gets or fetches multiple values from the cache.
* Fires onSet events, but does not emit any update events to other processes.
*/
@bindThis
public async fetchMany(keys: Iterable<string>): Promise<[key: string, value: T][]> {
const results: [key: string, value: T][] = [];
const toFetch: string[] = [];
// Spliterate into cached results / uncached keys.
for (const key of keys) {
const fromCache = this.get(key);
if (fromCache) {
results.push([key, fromCache]);
} else {
toFetch.push(key);
}
}
// Fetch any uncached keys
if (toFetch.length > 0) {
const fetched = await this.bulkFetch(toFetch);
// Add to cache and return set
this.addMany(fetched);
results.push(...fetched);
// Emit event
if (this.onChanged) {
await this.onChanged(toFetch, this);
}
}
return results;
}
/**
* Returns true is a key exists in memory.
* This applies to the local subset view, not the cross-cluster cache state.
*/
@bindThis
public has(key: string): boolean {
return this.memoryCache.get(key) !== undefined;
}
/**
* Deletes a value from the cache, and erases any stale caches across the cluster.
* Fires an onDelete event after the cache has been updated in all processes.
*/
@bindThis
public async delete(key: string): Promise<void> {
this.memoryCache.delete(key);
await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, keys: [key] });
if (this.onChanged) {
await this.onChanged([key], this);
}
}
/**
* Deletes multiple values from the cache, and erases any stale caches across the cluster.
* Fires an onDelete event for each key after the cache has been updated in all processes.
* Skips if the input is empty.
*/
@bindThis
public async deleteMany(keys: Iterable<string>): Promise<void> {
const deleted: string[] = [];
for (const key of keys) {
this.memoryCache.delete(key);
deleted.push(key);
}
if (deleted.length === 0) {
return;
}
await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, keys: deleted });
if (this.onChanged) {
await this.onChanged(deleted, this);
}
}
/**
* Refreshes the value of a key from the fetcher, and erases any stale caches across the cluster.
* Fires an onSet event after the cache has been updated in all processes.
*/
@bindThis
public async refresh(key: string): Promise<T> {
const value = await this.fetcher(key, this);
await this.set(key, value);
return value;
}
@bindThis
public async refreshMany(keys: Iterable<string>): Promise<[key: string, value: T][]> {
const values = await this.bulkFetch(keys);
await this.setMany(values);
return values;
}
/**
* Erases all entries from the local memory cache.
* Does not send any events or update other processes.
*/
@bindThis
public clear() {
this.memoryCache.clear();
}
/**
* Removes expired cache entries from the local view.
* Does not send any events or update other processes.
*/
@bindThis
public gc() {
this.memoryCache.gc();
}
/**
* Erases all data and disconnects from the cluster.
* This *must* be called when shutting down to prevent memory leaks!
*/
@bindThis
public dispose() {
this.internalEventService.off('quantumCacheUpdated', this.onQuantumCacheUpdated);
this.memoryCache.dispose();
}
@bindThis
private async bulkFetch(keys: Iterable<string>): Promise<[key: string, value: T][]> {
if (this.bulkFetcher) {
const results = await this.bulkFetcher(Array.from(keys), this);
return Array.from(results);
}
const results: [key: string, value: T][] = [];
for (const key of keys) {
const value = await this.fetcher(key, this);
results.push([key, value]);
}
return results;
}
@bindThis
private async onQuantumCacheUpdated(data: InternalEventTypes['quantumCacheUpdated']): Promise<void> {
if (data.name === this.name) {
for (const key of data.keys) {
this.memoryCache.delete(key);
}
if (this.onChanged) {
await this.onChanged(data.keys, this);
}
}
}
/**
* Iterates all [key, value] pairs in memory.
* This applies to the local subset view, not the cross-cluster cache state.
*/
[Symbol.iterator](): Iterator<[key: string, value: T]> {
return this.entries();
}
}

44
packages/backend/src/misc/cache.ts
View file

@ -9,9 +9,9 @@ import { bindThis } from '@/decorators.js';
export class RedisKVCache<T> {
private readonly lifetime: number;
private readonly memoryCache: MemoryKVCache<T>;
private readonly fetcher: (key: string) => Promise<T>;
private readonly toRedisConverter: (value: T) => string;
private readonly fromRedisConverter: (value: string) => T | undefined;
public readonly fetcher: (key: string) => Promise<T>;
public readonly toRedisConverter: (value: T) => string;
public readonly fromRedisConverter: (value: string) => T | undefined;
constructor(
private redisClient: Redis.Redis,
@ -99,6 +99,11 @@ export class RedisKVCache<T> {
// TODO: イベント発行して他プロセスのメモリキャッシュも更新できるようにする
}
@bindThis
public clear() {
this.memoryCache.clear();
}
@bindThis
public gc() {
this.memoryCache.gc();
@ -113,9 +118,9 @@ export class RedisKVCache<T> {
export class RedisSingleCache<T> {
private readonly lifetime: number;
private readonly memoryCache: MemorySingleCache<T>;
private readonly fetcher: () => Promise<T>;
private readonly toRedisConverter: (value: T) => string;
private readonly fromRedisConverter: (value: string) => T | undefined;
public readonly fetcher: () => Promise<T>;
public readonly toRedisConverter: (value: T) => string;
public readonly fromRedisConverter: (value: string) => T | undefined;
constructor(
private redisClient: Redis.Redis,
@ -123,16 +128,17 @@ export class RedisSingleCache<T> {
opts: {
lifetime: number;
memoryCacheLifetime: number;
fetcher: RedisSingleCache<T>['fetcher'];
toRedisConverter: RedisSingleCache<T>['toRedisConverter'];
fromRedisConverter: RedisSingleCache<T>['fromRedisConverter'];
fetcher?: RedisSingleCache<T>['fetcher'];
toRedisConverter?: RedisSingleCache<T>['toRedisConverter'];
fromRedisConverter?: RedisSingleCache<T>['fromRedisConverter'];
},
) {
this.lifetime = opts.lifetime;
this.memoryCache = new MemorySingleCache(opts.memoryCacheLifetime);
this.fetcher = opts.fetcher;
this.toRedisConverter = opts.toRedisConverter;
this.fromRedisConverter = opts.fromRedisConverter;
this.fetcher = opts.fetcher ?? (() => { throw new Error('fetch not supported - use get/set directly'); });
this.toRedisConverter = opts.toRedisConverter ?? ((value) => JSON.stringify(value));
this.fromRedisConverter = opts.fromRedisConverter ?? ((value) => JSON.parse(value));
}
@bindThis
@ -237,6 +243,16 @@ export class MemoryKVCache<T> {
return cached.value;
}
public has(key: string): boolean {
const cached = this.cache.get(key);
if (cached == null) return false;
if ((Date.now() - cached.date) > this.lifetime) {
this.cache.delete(key);
return false;
}
return true;
}
@bindThis
public delete(key: string): void {
this.cache.delete(key);
@ -322,6 +338,10 @@ export class MemoryKVCache<T> {
clearInterval(this.gcIntervalHandle);
}
public get size() {
return this.cache.size;
}
public get entries() {
return this.cache.entries();
}

2
packages/backend/src/misc/extract-custom-emojis-from-mfm.ts
View file

@ -3,7 +3,7 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import * as mfm from '@transfem-org/sfm-js';
import * as mfm from 'mfm-js';
import { unique } from '@/misc/prelude/array.js';
export function extractCustomEmojisFromMfm(nodes: mfm.MfmNode[]): string[] {

2
packages/backend/src/misc/extract-hashtags.ts
View file

@ -3,7 +3,7 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import * as mfm from '@transfem-org/sfm-js';
import * as mfm from 'mfm-js';
import { unique } from '@/misc/prelude/array.js';
export function extractHashtags(nodes: mfm.MfmNode[]): string[] {

2
packages/backend/src/misc/extract-mentions.ts
View file

@ -5,7 +5,7 @@
// test is located in test/extract-mentions
import * as mfm from '@transfem-org/sfm-js';
import * as mfm from 'mfm-js';
export function extractMentions(nodes: mfm.MfmNode[]): mfm.MfmMention['props'][] {
// TODO: 重複を削除

4
packages/backend/src/misc/truncate.ts
View file

@ -3,14 +3,12 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { substring } from 'stringz';
export function truncate(input: string, size: number): string;
export function truncate(input: string | undefined, size: number): string | undefined;
export function truncate(input: string | undefined, size: number): string | undefined {
if (!input) {
return input;
} else {
return substring(input, 0, size);
return input.slice(0, size);
}
}

2
packages/backend/src/misc/verify-field-link.ts
View file

@ -3,7 +3,7 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { load as cheerio } from 'cheerio';
import { load as cheerio } from 'cheerio/slim';
import type { HttpRequestService } from '@/core/HttpRequestService.js';
type Field = { name: string, value: string };

18
packages/backend/src/queue/processors/DeleteAccountProcessorService.ts
View file

@ -18,6 +18,7 @@ import { SearchService } from '@/core/SearchService.js';
import { ApLogService } from '@/core/ApLogService.js';
import { ReactionService } from '@/core/ReactionService.js';
import { QueueService } from '@/core/QueueService.js';
import { CacheService } from '@/core/CacheService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
import type { DbUserDeleteJobData } from '../types.js';
@ -94,6 +95,7 @@ export class DeleteAccountProcessorService {
private searchService: SearchService,
private reactionService: ReactionService,
private readonly apLogService: ApLogService,
private readonly cacheService: CacheService,
) {
this.logger = this.queueLoggerService.logger.createSubLogger('delete-account');
}
@ -140,6 +142,22 @@ export class DeleteAccountProcessorService {
}
{ // Delete user relations
await this.cacheService.refreshFollowRelationsFor(user.id);
await this.cacheService.userFollowingsCache.delete(user.id);
await this.cacheService.userFollowingsCache.delete(user.id);
await this.cacheService.userBlockingCache.delete(user.id);
await this.cacheService.userBlockedCache.delete(user.id);
await this.cacheService.userMutingsCache.delete(user.id);
await this.cacheService.userMutingsCache.delete(user.id);
await this.cacheService.hibernatedUserCache.delete(user.id);
await this.cacheService.renoteMutingsCache.delete(user.id);
await this.cacheService.userProfileCache.delete(user.id);
this.cacheService.userByIdCache.delete(user.id);
this.cacheService.localUserByIdCache.delete(user.id);
if (user.token) {
this.cacheService.localUserByNativeTokenCache.delete(user.token);
}
await this.followingsRepository.delete({
followerId: user.id,
});

15
packages/backend/src/queue/processors/InboxProcessorService.ts
View file

@ -21,7 +21,7 @@ import { ApDbResolverService } from '@/core/activitypub/ApDbResolverService.js';
import { StatusError } from '@/misc/status-error.js';
import { UtilityService } from '@/core/UtilityService.js';
import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js';
import { JsonLdService } from '@/core/activitypub/JsonLdService.js';
import { isSigned, JsonLdService } from '@/core/activitypub/JsonLdService.js';
import { ApInboxService } from '@/core/activitypub/ApInboxService.js';
import { bindThis } from '@/decorators.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
@ -179,8 +179,8 @@ export class InboxProcessorService implements OnApplicationShutdown {
// また、signatureのsignerは、activity.actorと一致する必要がある
if (!httpSignatureValidated || authUser.user.uri !== actorId) {
// 一致しなくても、でもLD-Signatureがありそうならそっちも見る
const ldSignature = activity.signature;
if (ldSignature) {
if (isSigned(activity)) {
const ldSignature = activity.signature;
if (ldSignature.type !== 'RsaSignature2017') {
throw new Bull.UnrecoverableError(`skip: unsupported LD-signature type ${ldSignature.type}`);
}
@ -202,24 +202,21 @@ export class InboxProcessorService implements OnApplicationShutdown {
throw new Bull.UnrecoverableError('skip: LD-SignatureのユーザーはpublicKeyを持っていませんでした');
}
const jsonLd = this.jsonLdService.use();
// LD-Signature検証
const verified = await jsonLd.verifyRsaSignature2017(activity, authUser.key.keyPem).catch(() => false);
const verified = await this.jsonLdService.verifyRsaSignature2017(activity, authUser.key.keyPem).catch(() => false);
if (!verified) {
throw new Bull.UnrecoverableError('skip: LD-Signatureの検証に失敗しました');
}
// アクティビティを正規化
delete activity.signature;
const copy = { ...activity, signature: undefined };
try {
activity = await jsonLd.compact(activity) as IActivity;
activity = await this.jsonLdService.compact(copy) as IActivity;
} catch (e) {
throw new Bull.UnrecoverableError(`skip: failed to compact activity: ${e}`);
}
// TODO: 元のアクティビティと非互換な形に正規化される場合は転送をスキップする
// https://github.com/mastodon/mastodon/blob/664b0ca/app/services/activitypub/process_collection_service.rb#L24-L29
activity.signature = ldSignature;
// もう一度actorチェック
if (authUser.user.uri !== actorId) {

2
packages/backend/src/server/api/endpoints/admin/nsfw-user.ts
View file

@ -47,7 +47,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
alwaysMarkNsfw: true,
});
await this.cacheService.userProfileCache.refresh(ps.userId);
await this.cacheService.userProfileCache.delete(ps.userId);
});
}
}

215
packages/backend/src/server/api/endpoints/fetch-rss.ts
View file

@ -3,12 +3,12 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import Parser from 'rss-parser';
import { Injectable } from '@nestjs/common';
import { parseFeed } from 'htmlparser2';
import { Endpoint } from '@/server/api/endpoint-base.js';
import { HttpRequestService } from '@/core/HttpRequestService.js';
const rssParser = new Parser();
import { ApiError } from '../error.js';
import type { FeedItem } from 'domutils';
export const meta = {
tags: ['meta'],
@ -17,52 +17,32 @@ export const meta = {
allowGet: true,
cacheSec: 60 * 3,
errors: {
fetchFailed: {
id: '88f4356f-719d-4715-b4fc-703a10a812d2',
code: 'FETCH_FAILED',
message: 'Failed to fetch RSS feed',
},
},
res: {
type: 'object',
properties: {
image: {
type: 'object',
optional: true,
properties: {
link: {
type: 'string',
optional: true,
},
url: {
type: 'string',
optional: false,
},
title: {
type: 'string',
optional: true,
},
},
type: {
type: 'string',
optional: false,
},
paginationLinks: {
type: 'object',
id: {
type: 'string',
optional: true,
},
updated: {
type: 'string',
optional: true,
},
author: {
type: 'string',
optional: true,
properties: {
self: {
type: 'string',
optional: true,
},
first: {
type: 'string',
optional: true,
},
next: {
type: 'string',
optional: true,
},
last: {
type: 'string',
optional: true,
},
prev: {
type: 'string',
optional: true,
},
},
},
link: {
type: 'string',
@ -94,113 +74,42 @@ export const meta = {
type: 'string',
optional: true,
},
creator: {
description: {
type: 'string',
optional: true,
},
summary: {
type: 'string',
optional: true,
},
content: {
type: 'string',
optional: true,
},
isoDate: {
type: 'string',
optional: true,
},
categories: {
media: {
type: 'array',
optional: true,
optional: false,
items: {
type: 'string',
},
},
contentSnippet: {
type: 'string',
optional: true,
},
enclosure: {
type: 'object',
optional: true,
properties: {
url: {
type: 'string',
optional: false,
},
length: {
type: 'number',
optional: true,
},
type: {
type: 'string',
optional: true,
type: 'object',
properties: {
medium: {
type: 'string',
optional: true,
},
url: {
type: 'string',
optional: true,
},
type: {
type: 'string',
optional: true,
},
lang: {
type: 'string',
optional: true,
},
},
},
},
},
},
},
feedUrl: {
type: 'string',
optional: true,
},
description: {
type: 'string',
optional: true,
},
itunes: {
type: 'object',
optional: true,
additionalProperties: true,
properties: {
image: {
type: 'string',
optional: true,
},
owner: {
type: 'object',
optional: true,
properties: {
name: {
type: 'string',
optional: true,
},
email: {
type: 'string',
optional: true,
},
},
},
author: {
type: 'string',
optional: true,
},
summary: {
type: 'string',
optional: true,
},
explicit: {
type: 'string',
optional: true,
},
categories: {
type: 'array',
optional: true,
items: {
type: 'string',
},
},
keywords: {
type: 'array',
optional: true,
items: {
type: 'string',
},
},
},
},
},
},
@ -224,7 +133,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
constructor(
private httpRequestService: HttpRequestService,
) {
super(meta, paramDef, async (ps, me) => {
super(meta, paramDef, async (ps) => {
const res = await this.httpRequestService.send(ps.url, {
method: 'GET',
headers: {
@ -234,8 +143,38 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
});
const text = await res.text();
const feed = parseFeed(text, {
xmlMode: true,
});
return rssParser.parseString(text);
if (!feed) {
throw new ApiError(meta.errors.fetchFailed);
}
return {
type: feed.type,
id: feed.id,
title: feed.title,
link: feed.link,
description: feed.description,
updated: feed.updated?.toISOString(),
author: feed.author,
items: feed.items
.filter((item): item is FeedItem & { link: string, title: string } => !!item.link && !!item.title)
.map(item => ({
guid: item.id,
title: item.title,
link: item.link,
description: item.description,
pubDate: item.pubDate?.toISOString(),
media: item.media.map(media => ({
medium: media.medium,
url: media.url,
type: media.type,
lang: media.lang,
})),
})),
};
});
}
}

9
packages/backend/src/server/api/endpoints/following/delete.ts
View file

@ -12,6 +12,7 @@ import { UserFollowingService } from '@/core/UserFollowingService.js';
import { DI } from '@/di-symbols.js';
import { GetterService } from '@/server/api/GetterService.js';
import { ApiError } from '../../error.js';
import { CacheService } from '@/core/CacheService.js';
export const meta = {
tags: ['following', 'users'],
@ -69,6 +70,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private userEntityService: UserEntityService,
private getterService: GetterService,
private userFollowingService: UserFollowingService,
private readonly cacheService: CacheService,
) {
super(meta, paramDef, async (ps, me) => {
const follower = me;
@ -85,12 +87,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
});
// Check not following
const exist = await this.followingsRepository.exists({
where: {
followerId: follower.id,
followeeId: followee.id,
},
});
const exist = await this.cacheService.userFollowingsCache.fetch(follower.id).then(f => f.has(followee.id));
if (!exist) {
throw new ApiError(meta.errors.notFollowing);

9
packages/backend/src/server/api/endpoints/following/invalidate.ts
View file

@ -11,6 +11,7 @@ import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { UserFollowingService } from '@/core/UserFollowingService.js';
import { DI } from '@/di-symbols.js';
import { GetterService } from '@/server/api/GetterService.js';
import { CacheService } from '@/core/CacheService.js';
import { ApiError } from '../../error.js';
export const meta = {
@ -69,6 +70,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private userEntityService: UserEntityService,
private getterService: GetterService,
private userFollowingService: UserFollowingService,
private readonly cacheService: CacheService,
) {
super(meta, paramDef, async (ps, me) => {
const followee = me;
@ -85,12 +87,9 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
});
// Check not following
const exist = await this.followingsRepository.findOneBy({
followerId: follower.id,
followeeId: followee.id,
});
const isFollowing = await this.cacheService.userFollowingsCache.fetch(follower.id).then(f => f.has(followee.id));
if (exist == null) {
if (!isFollowing) {
throw new ApiError(meta.errors.notFollowing);
}

4
packages/backend/src/server/api/endpoints/following/update-all.ts
View file

@ -12,6 +12,7 @@ import { UserFollowingService } from '@/core/UserFollowingService.js';
import { DI } from '@/di-symbols.js';
import { GetterService } from '@/server/api/GetterService.js';
import { ApiError } from '../../error.js';
import { CacheService } from '@/core/CacheService.js';
export const meta = {
tags: ['following', 'users'],
@ -39,6 +40,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
constructor(
@Inject(DI.followingsRepository)
private followingsRepository: FollowingsRepository,
private readonly cacheService: CacheService,
) {
super(meta, paramDef, async (ps, me) => {
await this.followingsRepository.update({
@ -48,6 +50,8 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
withReplies: ps.withReplies != null ? ps.withReplies : undefined,
});
await this.cacheService.refreshFollowRelationsFor(me.id);
return;
});
}

9
packages/backend/src/server/api/endpoints/following/update.ts
View file

@ -11,6 +11,7 @@ import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { UserFollowingService } from '@/core/UserFollowingService.js';
import { DI } from '@/di-symbols.js';
import { GetterService } from '@/server/api/GetterService.js';
import { CacheService } from '@/core/CacheService.js';
import { ApiError } from '../../error.js';
export const meta = {
@ -71,6 +72,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private userEntityService: UserEntityService,
private getterService: GetterService,
private userFollowingService: UserFollowingService,
private readonly cacheService: CacheService,
) {
super(meta, paramDef, async (ps, me) => {
const follower = me;
@ -87,10 +89,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
});
// Check not following
const exist = await this.followingsRepository.findOneBy({
followerId: follower.id,
followeeId: followee.id,
});
const exist = await this.cacheService.userFollowingsCache.fetch(follower.id).then(f => f.get(followee.id));
if (exist == null) {
throw new ApiError(meta.errors.notFollowing);
@ -103,6 +102,8 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
withReplies: ps.withReplies != null ? ps.withReplies : undefined,
});
await this.cacheService.refreshFollowRelationsFor(follower.id);
return await this.userEntityService.pack(follower.id, me);
});
}

4
packages/backend/src/server/api/endpoints/i/update.ts
View file

@ -3,7 +3,7 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import * as mfm from '@transfem-org/sfm-js';
import * as mfm from 'mfm-js';
import { Inject, Injectable } from '@nestjs/common';
import ms from 'ms';
import { extractCustomEmojisFromMfm } from '@/misc/extract-custom-emojis-from-mfm.js';
@ -617,7 +617,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
const updatedProfile = await this.userProfilesRepository.findOneByOrFail({ userId: user.id });
this.cacheService.userProfileCache.set(user.id, updatedProfile);
await this.cacheService.userProfileCache.set(user.id, updatedProfile);
// Publish meUpdated event
this.globalEventService.publishMainStream(user.id, 'meUpdated', iObj);

2
packages/backend/src/server/api/endpoints/notes/create.ts
View file

@ -350,7 +350,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
throw new ApiError(meta.errors.noSuchReplyTarget);
} else if (isRenote(reply) && !isQuote(reply)) {
throw new ApiError(meta.errors.cannotReplyToPureRenote);
} else if (!await this.noteEntityService.isVisibleForMe(reply, me.id)) {
} else if (!await this.noteEntityService.isVisibleForMe(reply, me.id, { me })) {
throw new ApiError(meta.errors.cannotReplyToInvisibleNote);
} else if (reply.visibility === 'specified' && ps.visibility !== 'specified') {
throw new ApiError(meta.errors.cannotReplyToSpecifiedVisibilityNoteWithExtendedVisibility);

2
packages/backend/src/server/api/endpoints/notes/edit.ts
View file

@ -402,7 +402,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
throw new ApiError(meta.errors.noSuchReplyTarget);
} else if (isRenote(reply) && !isQuote(reply)) {
throw new ApiError(meta.errors.cannotReplyToPureRenote);
} else if (!await this.noteEntityService.isVisibleForMe(reply, me.id)) {
} else if (!await this.noteEntityService.isVisibleForMe(reply, me.id, { me })) {
throw new ApiError(meta.errors.cannotReplyToInvisibleNote);
} else if (reply.visibility === 'specified' && ps.visibility !== 'specified') {
throw new ApiError(meta.errors.cannotReplyToSpecifiedVisibilityNoteWithExtendedVisibility);

2
packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts
View file

@ -164,7 +164,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
excludeBots: !ps.withBots,
noteFilter: note => {
if (note.reply && note.reply.visibility === 'followers') {
if (!Object.hasOwn(followings, note.reply.userId) && note.reply.userId !== me.id) return false;
if (!followings.has(note.reply.userId) && note.reply.userId !== me.id) return false;
}
return true;

2
packages/backend/src/server/api/endpoints/notes/timeline.ts
View file

@ -115,7 +115,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
excludePureRenotes: !ps.withRenotes,
noteFilter: note => {
if (note.reply && note.reply.visibility === 'followers') {
if (!Object.hasOwn(followings, note.reply.userId) && note.reply.userId !== me.id) return false;
if (!followings.has(note.reply.userId) && note.reply.userId !== me.id) return false;
}
if (!ps.withBots && note.user?.isBot) return false;

2
packages/backend/src/server/api/endpoints/notes/translate.ts
View file

@ -91,7 +91,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
throw err;
});
if (!(await this.noteEntityService.isVisibleForMe(note, me?.id ?? null))) {
if (!(await this.noteEntityService.isVisibleForMe(note, me?.id ?? null, { me }))) {
throw new ApiError(meta.errors.cannotTranslateInvisibleNote);
}

2
packages/backend/src/server/api/endpoints/sw/register.ts
View file

@ -104,7 +104,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
sendReadMessage: ps.sendReadMessage,
});
this.pushNotificationService.refreshCache(me.id);
await this.pushNotificationService.refreshCache(me.id);
return {
state: 'subscribed' as const,

2
packages/backend/src/server/api/endpoints/sw/unregister.ts
View file

@ -46,7 +46,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
});
if (me) {
this.pushNotificationService.refreshCache(me.id);
await this.pushNotificationService.refreshCache(me.id);
}
});
}

2
packages/backend/src/server/api/endpoints/sw/update-registration.ts
View file

@ -86,7 +86,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
sendReadMessage: swSubscription.sendReadMessage,
});
this.pushNotificationService.refreshCache(me.id);
await this.pushNotificationService.refreshCache(me.id);
return {
userId: swSubscription.userId,

9
packages/backend/src/server/api/endpoints/users/followers.ts
View file

@ -12,6 +12,7 @@ import { FollowingEntityService } from '@/core/entities/FollowingEntityService.j
import { UtilityService } from '@/core/UtilityService.js';
import { DI } from '@/di-symbols.js';
import { RoleService } from '@/core/RoleService.js';
import { CacheService } from '@/core/CacheService.js';
import { ApiError } from '../../error.js';
export const meta = {
@ -89,6 +90,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private followingEntityService: FollowingEntityService,
private queryService: QueryService,
private roleService: RoleService,
private readonly cacheService: CacheService,
) {
super(meta, paramDef, async (ps, me) => {
const user = await this.usersRepository.findOneBy(ps.userId != null
@ -110,12 +112,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
if (me == null) {
throw new ApiError(meta.errors.forbidden);
} else if (me.id !== user.id) {
const isFollowing = await this.followingsRepository.exists({
where: {
followeeId: user.id,
followerId: me.id,
},
});
const isFollowing = await this.cacheService.userFollowingsCache.fetch(me.id).then(f => f.has(user.id));
if (!isFollowing) {
throw new ApiError(meta.errors.forbidden);
}

9
packages/backend/src/server/api/endpoints/users/following.ts
View file

@ -13,6 +13,7 @@ import { FollowingEntityService } from '@/core/entities/FollowingEntityService.j
import { UtilityService } from '@/core/UtilityService.js';
import { DI } from '@/di-symbols.js';
import { RoleService } from '@/core/RoleService.js';
import { CacheService } from '@/core/CacheService.js';
import { ApiError } from '../../error.js';
export const meta = {
@ -98,6 +99,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private followingEntityService: FollowingEntityService,
private queryService: QueryService,
private roleService: RoleService,
private readonly cacheService: CacheService,
) {
super(meta, paramDef, async (ps, me) => {
const user = await this.usersRepository.findOneBy(ps.userId != null
@ -119,12 +121,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
if (me == null) {
throw new ApiError(meta.errors.forbidden);
} else if (me.id !== user.id) {
const isFollowing = await this.followingsRepository.exists({
where: {
followeeId: user.id,
followerId: me.id,
},
});
const isFollowing = await this.cacheService.userFollowingsCache.fetch(me.id).then(f => f.has(user.id));
if (!isFollowing) {
throw new ApiError(meta.errors.forbidden);
}

2
packages/backend/src/server/api/endpoints/users/notes.ts
View file

@ -134,7 +134,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
if (ps.withReplies) redisTimelines.push(`userTimelineWithReplies:${ps.userId}`);
if (ps.withChannelNotes) redisTimelines.push(`userTimelineWithChannel:${ps.userId}`);
const isFollowing = me && Object.hasOwn(await this.cacheService.userFollowingsCache.fetch(me.id), ps.userId);
const isFollowing = me && (await this.cacheService.userFollowingsCache.fetch(me.id)).has(ps.userId);
const timeline = await this.fanoutTimelineEndpointService.timeline({
untilId,

1
packages/backend/src/server/api/endpoints/users/recommendation.ts
View file

@ -71,6 +71,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
this.queryService.generateBlockQueryForUsers(query, me);
this.queryService.generateBlockedUserQueryForNotes(query, me);
// TODO optimization: replace with exists()
const followingQuery = this.followingsRepository.createQueryBuilder('following')
.select('following.followeeId')
.where('following.followerId = :followerId', { followerId: me.id });

2
packages/backend/src/server/api/mastodon/MastodonConverters.ts
View file

@ -5,7 +5,7 @@
import { Inject, Injectable } from '@nestjs/common';
import { Entity, MastodonEntity, MisskeyEntity } from 'megalodon';
import mfm from '@transfem-org/sfm-js';
import mfm from 'mfm-js';
import { MastodonNotificationType } from 'megalodon/lib/src/mastodon/notification.js';
import { NotificationType } from 'megalodon/lib/src/notification.js';
import { DI } from '@/di-symbols.js';

16
packages/backend/src/server/api/mastodon/MastodonDataService.ts
View file

@ -57,19 +57,19 @@ export class MastodonDataService {
if (relations.reply) {
query.leftJoinAndSelect('note.reply', 'reply');
if (typeof(relations.reply) === 'object') {
if (relations.reply.reply) query.leftJoinAndSelect('note.reply.reply', 'replyReply');
if (relations.reply.renote) query.leftJoinAndSelect('note.reply.renote', 'replyRenote');
if (relations.reply.user) query.innerJoinAndSelect('note.reply.user', 'replyUser');
if (relations.reply.channel) query.leftJoinAndSelect('note.reply.channel', 'replyChannel');
if (relations.reply.reply) query.leftJoinAndSelect('reply.reply', 'replyReply');
if (relations.reply.renote) query.leftJoinAndSelect('reply.renote', 'replyRenote');
if (relations.reply.user) query.innerJoinAndSelect('reply.user', 'replyUser');
if (relations.reply.channel) query.leftJoinAndSelect('reply.channel', 'replyChannel');
}
}
if (relations.renote) {
query.leftJoinAndSelect('note.renote', 'renote');
if (typeof(relations.renote) === 'object') {
if (relations.renote.reply) query.leftJoinAndSelect('note.renote.reply', 'renoteReply');
if (relations.renote.renote) query.leftJoinAndSelect('note.renote.renote', 'renoteRenote');
if (relations.renote.user) query.innerJoinAndSelect('note.renote.user', 'renoteUser');
if (relations.renote.channel) query.leftJoinAndSelect('note.renote.channel', 'renoteChannel');
if (relations.renote.reply) query.leftJoinAndSelect('renote.reply', 'renoteReply');
if (relations.renote.renote) query.leftJoinAndSelect('renote.renote', 'renoteRenote');
if (relations.renote.user) query.innerJoinAndSelect('renote.user', 'renoteUser');
if (relations.renote.channel) query.leftJoinAndSelect('renote.channel', 'renoteChannel');
}
}
if (relations.user) {

2
packages/backend/src/server/api/stream/Connection.ts
View file

@ -36,7 +36,7 @@ export default class Connection {
private channels = new Map<string, Channel>();
private subscribingNotes = new Map<string, number>();
public userProfile: MiUserProfile | null = null;
public following: Record<string, Pick<MiFollowing, 'withReplies'> | undefined> = {};
public following: Map<string, Omit<MiFollowing, 'isFollowerHibernated'>> = new Map();
public followingChannels: Set<string> = new Set();
public userIdsWhoMeMuting: Set<string> = new Set();
public userIdsWhoBlockingMe: Set<string> = new Set();

6
packages/backend/src/server/api/stream/channel.ts
View file

@ -70,7 +70,7 @@ export default abstract class Channel {
if (!this.user) return false;
if (this.user.id === note.userId) return true;
if (note.visibility === 'followers') {
return this.following[note.userId] != null;
return this.following.has(note.userId);
}
if (!note.visibleUserIds) return false;
return note.visibleUserIds.includes(this.user.id);
@ -84,7 +84,7 @@ export default abstract class Channel {
if (note.user.requireSigninToViewContents && !this.user) return true;
// 流れてきたNoteがインスタンスミュートしたインスタンスが関わる
if (isInstanceMuted(note, this.userMutedInstances) && !this.following[note.userId]) return true;
if (isInstanceMuted(note, this.userMutedInstances) && !this.following.has(note.userId)) return true;
// 流れてきたNoteがミュートしているユーザーが関わる
if (isUserRelated(note, this.userIdsWhoMeMuting)) return true;
@ -101,7 +101,7 @@ export default abstract class Channel {
if (note.user.isSilenced || note.user.instance?.isSilenced) {
if (this.user == null) return true;
if (this.user.id === note.userId) return false;
if (this.following[note.userId] == null) return true;
if (!this.following.has(note.userId)) return true;
}
// TODO muted threads

2
packages/backend/src/server/api/stream/channels/bubble-timeline.ts
View file

@ -62,7 +62,7 @@ class BubbleTimelineChannel extends Channel {
const reply = note.reply;
// 自分のフォローしていないユーザーの visibility: followers な投稿への返信は弾く
if (!this.isNoteVisibleToMe(reply)) return;
if (!this.following[note.userId]?.withReplies) {
if (!this.following.get(note.userId)?.withReplies) {
// 「チャンネル接続主への返信」でもなければ、「チャンネル接続主が行った返信」でもなければ、「投稿者の投稿者自身への返信」でもない場合
if (reply.userId !== this.user?.id && !isMe && reply.userId !== note.userId) return;
}

2
packages/backend/src/server/api/stream/channels/global-timeline.ts
View file

@ -63,7 +63,7 @@ class GlobalTimelineChannel extends Channel {
const reply = note.reply;
// 自分のフォローしていないユーザーの visibility: followers な投稿への返信は弾く
if (!this.isNoteVisibleToMe(reply)) return;
if (!this.following[note.userId]?.withReplies) {
if (!this.following.get(note.userId)?.withReplies) {
// 「チャンネル接続主への返信」でもなければ、「チャンネル接続主が行った返信」でもなければ、「投稿者の投稿者自身への返信」でもない場合
if (reply.userId !== this.user?.id && !isMe && reply.userId !== note.userId) return;
}

4
packages/backend/src/server/api/stream/channels/home-timeline.ts
View file

@ -47,7 +47,7 @@ class HomeTimelineChannel extends Channel {
if (!this.followingChannels.has(note.channelId)) return;
} else {
// その投稿のユーザーをフォローしていなかったら弾く
if (!isMe && !Object.hasOwn(this.following, note.userId)) return;
if (!isMe && !this.following.has(note.userId)) return;
}
if (this.isNoteMutedOrBlocked(note)) return;
@ -57,7 +57,7 @@ class HomeTimelineChannel extends Channel {
const reply = note.reply;
// 自分のフォローしていないユーザーの visibility: followers な投稿への返信は弾く
if (!this.isNoteVisibleToMe(reply)) return;
if (!this.following[note.userId]?.withReplies) {
if (!this.following.get(note.userId)?.withReplies) {
// 「チャンネル接続主への返信」でもなければ、「チャンネル接続主が行った返信」でもなければ、「投稿者の投稿者自身への返信」でもない場合
if (reply.userId !== this.user!.id && !isMe && reply.userId !== note.userId) return;
}

4
packages/backend/src/server/api/stream/channels/hybrid-timeline.ts
View file

@ -62,7 +62,7 @@ class HybridTimelineChannel extends Channel {
// フォローしているチャンネルの投稿 の場合だけ
if (!(
(note.channelId == null && isMe) ||
(note.channelId == null && Object.hasOwn(this.following, note.userId)) ||
(note.channelId == null && this.following.has(note.userId)) ||
(note.channelId == null && (note.user.host == null && note.visibility === 'public')) ||
(note.channelId != null && this.followingChannels.has(note.channelId))
)) return;
@ -74,7 +74,7 @@ class HybridTimelineChannel extends Channel {
const reply = note.reply;
// 自分のフォローしていないユーザーの visibility: followers な投稿への返信は弾く
if (!this.isNoteVisibleToMe(reply)) return;
if (!this.following[note.userId]?.withReplies && !this.withReplies) {
if (!this.following.get(note.userId)?.withReplies && !this.withReplies) {
// 「チャンネル接続主への返信」でもなければ、「チャンネル接続主が行った返信」でもなければ、「投稿者の投稿者自身への返信」でもない場合
if (reply.userId !== this.user!.id && !isMe && reply.userId !== note.userId) return;
}

2
packages/backend/src/server/api/stream/channels/local-timeline.ts
View file

@ -67,7 +67,7 @@ class LocalTimelineChannel extends Channel {
const reply = note.reply;
// 自分のフォローしていないユーザーの visibility: followers な投稿への返信は弾く
if (!this.isNoteVisibleToMe(reply)) return;
if (!this.following[note.userId]?.withReplies) {
if (!this.following.get(note.userId)?.withReplies) {
// 「チャンネル接続主への返信」でもなければ、「チャンネル接続主が行った返信」でもなければ、「投稿者の投稿者自身への返信」でもない場合
if (reply.userId !== this.user?.id && !isMe && reply.userId !== note.userId) return;
}

2
packages/backend/src/server/api/stream/channels/role-timeline.ts
View file

@ -55,7 +55,7 @@ class RoleTimelineChannel extends Channel {
const reply = note.reply;
// 自分のフォローしていないユーザーの visibility: followers な投稿への返信は弾く
if (!this.isNoteVisibleToMe(reply)) return;
if (!this.following[note.userId]?.withReplies) {
if (!this.following.get(note.userId)?.withReplies) {
// 「チャンネル接続主への返信」でもなければ、「チャンネル接続主が行った返信」でもなければ、「投稿者の投稿者自身への返信」でもない場合
if (reply.userId !== this.user?.id && !isMe && reply.userId !== note.userId) return;
}

2
packages/backend/src/server/api/stream/channels/user-list.ts
View file

@ -98,7 +98,7 @@ class UserListChannel extends Channel {
const reply = note.reply;
// 自分のフォローしていないユーザーの visibility: followers な投稿への返信は弾く
if (!this.isNoteVisibleToMe(reply)) return;
if (!this.following[note.userId]?.withReplies) {
if (!this.following.get(note.userId)?.withReplies) {
// 「チャンネル接続主への返信」でもなければ、「チャンネル接続主が行った返信」でもなければ、「投稿者の投稿者自身への返信」でもない場合
if (reply.userId !== this.user!.id && !isMe && reply.userId !== note.userId) return;
}

2
packages/backend/src/server/web/FeedService.ts
View file

@ -15,7 +15,7 @@ import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.j
import { bindThis } from '@/decorators.js';
import { IdService } from '@/core/IdService.js';
import { MfmService } from "@/core/MfmService.js";
import { parse as mfmParse } from '@transfem-org/sfm-js';
import { parse as mfmParse } from 'mfm-js';
@Injectable()
export class FeedService {

2
packages/backend/test/e2e/oauth.ts
View file

@ -19,7 +19,7 @@ import {
ResourceOwnerPassword,
} from 'simple-oauth2';
import pkceChallenge from 'pkce-challenge';
import { load as cheerio } from 'cheerio';
import { load as cheerio } from 'cheerio/slim';
import Fastify, { type FastifyInstance, type FastifyReply } from 'fastify';
import { api, port, sendEnvUpdateRequest, signup } from '../utils.js';
import type * as misskey from 'misskey-js';

92
packages/backend/test/misc/FakeInternalEventService.ts Normal file
View file

@ -0,0 +1,92 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
import type { Listener, ListenerProps } from '@/core/InternalEventService.js';
import type Redis from 'ioredis';
import type { GlobalEventService, InternalEventTypes } from '@/core/GlobalEventService.js';
import { InternalEventService } from '@/core/InternalEventService.js';
import { bindThis } from '@/decorators.js';
type FakeCall<K extends keyof InternalEventService> = [K, Parameters<InternalEventService[K]>];
type FakeListener<K extends keyof InternalEventTypes> = [K, Listener<K>, ListenerProps];
/**
* Minimal implementation of InternalEventService meant for use in unit tests.
* There is no redis connection, and metadata is tracked in the public _calls and _listeners arrays.
* The on/off/emit methods are fully functional and can be called in tests to invoke any registered listeners.
*/
export class FakeInternalEventService extends InternalEventService {
/**
* List of calls to public methods, in chronological order.
*/
public _calls: FakeCall<keyof InternalEventService>[] = [];
/**
* List of currently registered listeners.
*/
public _listeners: FakeListener<keyof InternalEventTypes>[] = [];
/**
* Resets the mock.
* Clears all listeners and tracked calls.
*/
public _reset() {
this._calls = [];
this._listeners = [];
}
/**
* Simulates a remote event sent from another process in the cluster via redis.
*/
@bindThis
public async _emitRedis<K extends keyof InternalEventTypes>(type: K, value: InternalEventTypes[K]): Promise<void> {
await this.emit(type, value, false);
}
constructor() {
super(
{ on: () => {} } as unknown as Redis.Redis,
{} as unknown as GlobalEventService,
);
}
@bindThis
public on<K extends keyof InternalEventTypes>(type: K, listener: Listener<K>, props?: ListenerProps): void {
if (!this._listeners.some(l => l[0] === type && l[1] === listener)) {
this._listeners.push([type, listener as Listener<keyof InternalEventTypes>, props ?? {}]);
}
this._calls.push(['on', [type, listener as Listener<keyof InternalEventTypes>, props]]);
}
@bindThis
public off<K extends keyof InternalEventTypes>(type: K, listener: Listener<K>): void {
this._listeners = this._listeners.filter(l => l[0] !== type || l[1] !== listener);
this._calls.push(['off', [type, listener as Listener<keyof InternalEventTypes>]]);
}
@bindThis
public async emit<K extends keyof InternalEventTypes>(type: K, value: InternalEventTypes[K], isLocal = true): Promise<void> {
for (const listener of this._listeners) {
if (listener[0] === type) {
if ((isLocal && !listener[2].ignoreLocal) || (!isLocal && !listener[2].ignoreRemote)) {
await listener[1](value, type, isLocal);
}
}
}
this._calls.push(['emit', [type, value]]);
}
@bindThis
public dispose(): void {
this._listeners = [];
this._calls.push(['dispose', []]);
}
@bindThis
public onApplicationShutdown(): void {
this._calls.push(['onApplicationShutdown', []]);
}
}

187
packages/backend/test/misc/noOpCaches.ts Normal file
View file

@ -0,0 +1,187 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
import * as Redis from 'ioredis';
import { Inject } from '@nestjs/common';
import { FakeInternalEventService } from './FakeInternalEventService.js';
import type { BlockingsRepository, FollowingsRepository, MiUser, MutingsRepository, RenoteMutingsRepository, UserProfilesRepository, UsersRepository } from '@/models/_.js';
import type { MiLocalUser } from '@/models/User.js';
import { MemoryKVCache, MemorySingleCache, RedisKVCache, RedisSingleCache } from '@/misc/cache.js';
import { QuantumKVCache, QuantumKVOpts } from '@/misc/QuantumKVCache.js';
import { CacheService, FollowStats } from '@/core/CacheService.js';
import { DI } from '@/di-symbols.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { InternalEventService } from '@/core/InternalEventService.js';
export function noOpRedis() {
return {
set: () => Promise.resolve(),
get: () => Promise.resolve(null),
del: () => Promise.resolve(),
on: () => {},
off: () => {},
} as unknown as Redis.Redis;
}
export class NoOpCacheService extends CacheService {
public readonly fakeRedis: {
[K in keyof Redis.Redis]: Redis.Redis[K];
};
public readonly fakeInternalEventService: FakeInternalEventService;
constructor(
@Inject(DI.usersRepository)
usersRepository: UsersRepository,
@Inject(DI.userProfilesRepository)
userProfilesRepository: UserProfilesRepository,
@Inject(DI.mutingsRepository)
mutingsRepository: MutingsRepository,
@Inject(DI.blockingsRepository)
blockingsRepository: BlockingsRepository,
@Inject(DI.renoteMutingsRepository)
renoteMutingsRepository: RenoteMutingsRepository,
@Inject(DI.followingsRepository)
followingsRepository: FollowingsRepository,
@Inject(UserEntityService)
userEntityService: UserEntityService,
) {
const fakeRedis = noOpRedis();
const fakeInternalEventService = new FakeInternalEventService();
super(
fakeRedis,
fakeRedis,
usersRepository,
userProfilesRepository,
mutingsRepository,
blockingsRepository,
renoteMutingsRepository,
followingsRepository,
userEntityService,
fakeInternalEventService,
);
this.fakeRedis = fakeRedis;
this.fakeInternalEventService = fakeInternalEventService;
// Override caches
this.userByIdCache = new NoOpMemoryKVCache<MiUser>();
this.localUserByNativeTokenCache = new NoOpMemoryKVCache<MiLocalUser | null>();
this.localUserByIdCache = new NoOpMemoryKVCache<MiLocalUser>();
this.uriPersonCache = new NoOpMemoryKVCache<MiUser | null>();
this.userProfileCache = NoOpQuantumKVCache.copy(this.userProfileCache, fakeInternalEventService);
this.userMutingsCache = NoOpQuantumKVCache.copy(this.userMutingsCache, fakeInternalEventService);
this.userBlockingCache = NoOpQuantumKVCache.copy(this.userBlockingCache, fakeInternalEventService);
this.userBlockedCache = NoOpQuantumKVCache.copy(this.userBlockedCache, fakeInternalEventService);
this.renoteMutingsCache = NoOpQuantumKVCache.copy(this.renoteMutingsCache, fakeInternalEventService);
this.userFollowingsCache = NoOpQuantumKVCache.copy(this.userFollowingsCache, fakeInternalEventService);
this.userFollowersCache = NoOpQuantumKVCache.copy(this.userFollowersCache, fakeInternalEventService);
this.hibernatedUserCache = NoOpQuantumKVCache.copy(this.hibernatedUserCache, fakeInternalEventService);
this.userFollowStatsCache = new NoOpMemoryKVCache<FollowStats>();
this.translationsCache = NoOpRedisKVCache.copy(this.translationsCache, fakeRedis);
}
}
export class NoOpMemoryKVCache<T> extends MemoryKVCache<T> {
constructor() {
super(-1);
}
}
export class NoOpMemorySingleCache<T> extends MemorySingleCache<T> {
constructor() {
super(-1);
}
}
export class NoOpRedisKVCache<T> extends RedisKVCache<T> {
constructor(opts?: {
redis?: Redis.Redis;
fetcher?: RedisKVCache<T>['fetcher'];
toRedisConverter?: RedisKVCache<T>['toRedisConverter'];
fromRedisConverter?: RedisKVCache<T>['fromRedisConverter'];
}) {
super(
opts?.redis ?? noOpRedis(),
'no-op',
{
lifetime: -1,
memoryCacheLifetime: -1,
fetcher: opts?.fetcher,
toRedisConverter: opts?.toRedisConverter,
fromRedisConverter: opts?.fromRedisConverter,
},
);
}
public static copy<T>(cache: RedisKVCache<T>, redis?: Redis.Redis): NoOpRedisKVCache<T> {
return new NoOpRedisKVCache<T>({
redis,
fetcher: cache.fetcher,
toRedisConverter: cache.toRedisConverter,
fromRedisConverter: cache.fromRedisConverter,
});
}
}
export class NoOpRedisSingleCache<T> extends RedisSingleCache<T> {
constructor(opts?: {
redis?: Redis.Redis;
fetcher?: RedisSingleCache<T>['fetcher'];
toRedisConverter?: RedisSingleCache<T>['toRedisConverter'];
fromRedisConverter?: RedisSingleCache<T>['fromRedisConverter'];
}) {
super(
opts?.redis ?? noOpRedis(),
'no-op',
{
lifetime: -1,
memoryCacheLifetime: -1,
fetcher: opts?.fetcher,
toRedisConverter: opts?.toRedisConverter,
fromRedisConverter: opts?.fromRedisConverter,
},
);
}
public static copy<T>(cache: RedisSingleCache<T>, redis?: Redis.Redis): NoOpRedisSingleCache<T> {
return new NoOpRedisSingleCache<T>({
redis,
fetcher: cache.fetcher,
toRedisConverter: cache.toRedisConverter,
fromRedisConverter: cache.fromRedisConverter,
});
}
}
export class NoOpQuantumKVCache<T> extends QuantumKVCache<T> {
constructor(opts: Omit<QuantumKVOpts<T>, 'lifetime'> & {
internalEventService?: InternalEventService,
}) {
super(
opts.internalEventService ?? new FakeInternalEventService(),
'no-op',
{
...opts,
lifetime: -1,
},
);
}
public static copy<T>(cache: QuantumKVCache<T>, internalEventService?: InternalEventService): NoOpQuantumKVCache<T> {
return new NoOpQuantumKVCache<T>({
internalEventService,
fetcher: cache.fetcher,
bulkFetcher: cache.bulkFetcher,
onChanged: cache.onChanged,
});
}
}

26
packages/backend/test/unit/AnnouncementService.ts
View file

@ -8,9 +8,12 @@ process.env.NODE_ENV = 'test';
import { jest } from '@jest/globals';
import { ModuleMocker } from 'jest-mock';
import { Test } from '@nestjs/testing';
import { NoOpCacheService } from '../misc/noOpCaches.js';
import { FakeInternalEventService } from '../misc/FakeInternalEventService.js';
import { GlobalModule } from '@/GlobalModule.js';
import { AnnouncementService } from '@/core/AnnouncementService.js';
import { AnnouncementEntityService } from '@/core/entities/AnnouncementEntityService.js';
import { InternalEventService } from '@/core/InternalEventService.js';
import type {
AnnouncementReadsRepository,
AnnouncementsRepository,
@ -71,24 +74,27 @@ describe('AnnouncementService', () => {
AnnouncementEntityService,
CacheService,
IdService,
InternalEventService,
GlobalEventService,
ModerationLogService,
],
})
.useMocker((token) => {
if (token === GlobalEventService) {
return {
publishMainStream: jest.fn(),
publishBroadcastStream: jest.fn(),
};
} else if (token === ModerationLogService) {
return {
log: jest.fn(),
};
} else if (typeof token === 'function') {
if (typeof token === 'function') {
const mockMetadata = moduleMocker.getMetadata(token) as MockFunctionMetadata<any, any>;
const Mock = moduleMocker.generateFromMetadata(mockMetadata);
return new Mock();
}
})
.overrideProvider(GlobalEventService).useValue({
publishMainStream: jest.fn(),
publishBroadcastStream: jest.fn(),
} as unknown as GlobalEventService)
.overrideProvider(ModerationLogService).useValue({
log: jest.fn(),
})
.overrideProvider(InternalEventService).useClass(FakeInternalEventService)
.overrideProvider(CacheService).useClass(NoOpCacheService)
.compile();
app.enableShutdownHooks();

4
packages/backend/test/unit/MfmService.ts
View file

@ -4,7 +4,7 @@
*/
import * as assert from 'assert';
import * as mfm from '@transfem-org/sfm-js';
import * as mfm from 'mfm-js';
import { Test } from '@nestjs/testing';
import { CoreModule } from '@/core/CoreModule.js';
@ -86,7 +86,7 @@ describe('MfmService', () => {
test('ruby', async () => {
const input = '$[ruby $[group *some* text] ignore me]';
const output = '<p><ruby><span><span>*some*</span><span> text</span></span><rp>(</rp><rt>ignore me</rt><rp>)</rp></ruby></p>';
const output = '<p><ruby><span><span>*some*</span> text</span><rp>(</rp><rt>ignore me</rt><rp>)</rp></ruby></p>';
assert.equal(await mfmService.toMastoApiHtml(mfm.parse(input)), output);
});
});

16
packages/backend/test/unit/RoleService.ts
View file

@ -10,12 +10,15 @@ import { jest } from '@jest/globals';
import { ModuleMocker } from 'jest-mock';
import { Test } from '@nestjs/testing';
import * as lolex from '@sinonjs/fake-timers';
import { NoOpCacheService } from '../misc/noOpCaches.js';
import { FakeInternalEventService } from '../misc/FakeInternalEventService.js';
import type { TestingModule } from '@nestjs/testing';
import type { MockFunctionMetadata } from 'jest-mock';
import { GlobalModule } from '@/GlobalModule.js';
import { RoleService } from '@/core/RoleService.js';
import {
InstancesRepository,
MetasRepository,
MiMeta,
MiRole,
MiRoleAssignment,
@ -34,6 +37,7 @@ import { secureRndstr } from '@/misc/secure-rndstr.js';
import { NotificationService } from '@/core/NotificationService.js';
import { RoleCondFormulaValue } from '@/models/Role.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { InternalEventService } from '@/core/InternalEventService.js';
const moduleMocker = new ModuleMocker(global);
@ -45,6 +49,7 @@ describe('RoleService', () => {
let rolesRepository: RolesRepository;
let roleAssignmentsRepository: RoleAssignmentsRepository;
let meta: jest.Mocked<MiMeta>;
let metasRepository: MetasRepository;
let notificationService: jest.Mocked<NotificationService>;
let clock: lolex.InstalledClock;
@ -143,18 +148,20 @@ describe('RoleService', () => {
provide: NotificationService.name,
useExisting: NotificationService,
},
MetaService,
InternalEventService,
],
})
.useMocker((token) => {
if (token === MetaService) {
return { fetch: jest.fn() };
}
if (typeof token === 'function') {
const mockMetadata = moduleMocker.getMetadata(token) as MockFunctionMetadata<any, any>;
const Mock = moduleMocker.generateFromMetadata(mockMetadata);
return new Mock();
}
})
.overrideProvider(MetaService).useValue({ fetch: jest.fn() })
.overrideProvider(InternalEventService).useClass(FakeInternalEventService)
.overrideProvider(CacheService).useClass(NoOpCacheService)
.compile();
app.enableShutdownHooks();
@ -164,6 +171,7 @@ describe('RoleService', () => {
usersRepository = app.get<UsersRepository>(DI.usersRepository);
rolesRepository = app.get<RolesRepository>(DI.rolesRepository);
roleAssignmentsRepository = app.get<RoleAssignmentsRepository>(DI.roleAssignmentsRepository);
metasRepository = app.get<MetasRepository>(DI.metasRepository);
meta = app.get<MiMeta>(DI.meta) as jest.Mocked<MiMeta>;
notificationService = app.get<NotificationService>(NotificationService) as jest.Mocked<NotificationService>;
@ -175,7 +183,7 @@ describe('RoleService', () => {
clock.uninstall();
await Promise.all([
app.get(DI.metasRepository).delete({}),
metasRepository.delete({}),
usersRepository.delete({}),
rolesRepository.delete({}),
roleAssignmentsRepository.delete({}),

14
packages/backend/test/unit/activitypub.ts
View file

@ -9,8 +9,12 @@ import { generateKeyPair } from 'crypto';
import { Test } from '@nestjs/testing';
import { jest } from '@jest/globals';
import { NoOpCacheService } from '../misc/noOpCaches.js';
import { FakeInternalEventService } from '../misc/FakeInternalEventService.js';
import type { Config } from '@/config.js';
import type { MiLocalUser, MiRemoteUser } from '@/models/User.js';
import { InternalEventService } from '@/core/InternalEventService.js';
import { CacheService } from '@/core/CacheService.js';
import { ApImageService } from '@/core/activitypub/models/ApImageService.js';
import { ApNoteService } from '@/core/activitypub/models/ApNoteService.js';
import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js';
@ -30,7 +34,7 @@ import { genAidx } from '@/misc/id/aidx.js';
import { IdService } from '@/core/IdService.js';
import { MockResolver } from '../misc/mock-resolver.js';
import { UserKeypairService } from '@/core/UserKeypairService.js';
import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js';
import { MemoryKVCache } from '@/misc/cache.js';
const host = 'https://host1.test';
@ -154,6 +158,8 @@ describe('ActivityPub', () => {
},
})
.overrideProvider(DI.meta).useFactory({ factory: () => meta })
.overrideProvider(CacheService).useClass(NoOpCacheService)
.overrideProvider(InternalEventService).useClass(FakeInternalEventService)
.compile();
await app.init();
@ -473,8 +479,6 @@ describe('ActivityPub', () => {
describe('JSON-LD', () => {
test('Compaction', async () => {
const jsonLd = jsonLdService.use();
const object = {
'@context': [
'https://www.w3.org/ns/activitystreams',
@ -493,7 +497,7 @@ describe('ActivityPub', () => {
unknown: 'test test bar',
undefined: 'test test baz',
};
const compacted = await jsonLd.compact(object);
const compacted = await jsonLdService.compact(object);
assert.deepStrictEqual(compacted, {
'@context': CONTEXT,
@ -556,7 +560,7 @@ describe('ActivityPub', () => {
publicKey,
privateKey,
});
((userKeypairService as unknown as { cache: RedisKVCache<MiUserKeypair> }).cache as unknown as { memoryCache: MemoryKVCache<MiUserKeypair> }).memoryCache.set(author.id, keypair);
(userKeypairService as unknown as { cache: MemoryKVCache<MiUserKeypair> }).cache.set(author.id, keypair);
note = new MiNote({
id: idService.gen(),

9
packages/backend/test/unit/entities/UserEntityService.ts
View file

@ -4,6 +4,8 @@
*/
import { Test, TestingModule } from '@nestjs/testing';
import { FakeInternalEventService } from '../../misc/FakeInternalEventService.js';
import { NoOpCacheService } from '../../misc/noOpCaches.js';
import type { MiUser } from '@/models/User.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { GlobalModule } from '@/GlobalModule.js';
@ -51,6 +53,7 @@ import { ReactionService } from '@/core/ReactionService.js';
import { NotificationService } from '@/core/NotificationService.js';
import { ReactionsBufferingService } from '@/core/ReactionsBufferingService.js';
import { ChatService } from '@/core/ChatService.js';
import { InternalEventService } from '@/core/InternalEventService.js';
process.env.NODE_ENV = 'test';
@ -174,6 +177,7 @@ describe('UserEntityService', () => {
ReactionsBufferingService,
NotificationService,
ChatService,
InternalEventService,
];
app = await Test.createTestingModule({
@ -182,7 +186,10 @@ describe('UserEntityService', () => {
...services,
...services.map(x => ({ provide: x.name, useExisting: x })),
],
}).compile();
})
.overrideProvider(InternalEventService).useClass(FakeInternalEventService)
.overrideProvider(CacheService).useClass(NoOpCacheService)
.compile();
await app.init();
app.enableShutdownHooks();

2
packages/backend/test/unit/extract-mentions.ts
View file

@ -5,7 +5,7 @@
import * as assert from 'assert';
import { parse } from '@transfem-org/sfm-js';
import { parse } from 'mfm-js';
import { extractMentions } from '@/misc/extract-mentions.js';
describe('Extract mentions', () => {

799
packages/backend/test/unit/misc/QuantumKVCache.ts Normal file
View file

@ -0,0 +1,799 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { jest } from '@jest/globals';
import { FakeInternalEventService } from '../../misc/FakeInternalEventService.js';
import { QuantumKVCache, QuantumKVOpts } from '@/misc/QuantumKVCache.js';
describe(QuantumKVCache, () => {
let fakeInternalEventService: FakeInternalEventService;
let madeCaches: { dispose: () => void }[];
function makeCache<T>(opts?: Partial<QuantumKVOpts<T>> & { name?: string }): QuantumKVCache<T> {
const _opts = {
name: 'test',
lifetime: Infinity,
fetcher: () => { throw new Error('not implemented'); },
} satisfies QuantumKVOpts<T> & { name: string };
if (opts) {
Object.assign(_opts, opts);
}
const cache = new QuantumKVCache<T>(fakeInternalEventService, _opts.name, _opts);
madeCaches.push(cache);
return cache;
}
beforeEach(() => {
madeCaches = [];
fakeInternalEventService = new FakeInternalEventService();
});
afterEach(() => {
madeCaches.forEach(cache => {
cache.dispose();
});
});
it('should connect on construct', () => {
makeCache();
expect(fakeInternalEventService._calls).toContainEqual(['on', ['quantumCacheUpdated', expect.anything(), { ignoreLocal: true }]]);
});
it('should disconnect on dispose', () => {
const cache = makeCache();
cache.dispose();
const callback = fakeInternalEventService._calls
.find(c => c[0] === 'on' && c[1][0] === 'quantumCacheUpdated')
?.[1][1];
expect(fakeInternalEventService._calls).toContainEqual(['off', ['quantumCacheUpdated', callback]]);
});
it('should store in memory cache', async () => {
const cache = makeCache<string>();
await cache.set('foo', 'bar');
await cache.set('alpha', 'omega');
const result1 = await cache.get('foo');
const result2 = await cache.get('alpha');
expect(result1).toBe('bar');
expect(result2).toBe('omega');
});
it('should emit event when storing', async () => {
const cache = makeCache<string>({ name: 'fake' });
await cache.set('foo', 'bar');
expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo'] }]]);
});
it('should call onChanged when storing', async () => {
const fakeOnChanged = jest.fn(() => Promise.resolve());
const cache = makeCache<string>({
name: 'fake',
onChanged: fakeOnChanged,
});
await cache.set('foo', 'bar');
expect(fakeOnChanged).toHaveBeenCalledWith(['foo'], cache);
});
it('should not emit event when storing unchanged value', async () => {
const cache = makeCache<string>({ name: 'fake' });
await cache.set('foo', 'bar');
await cache.set('foo', 'bar');
expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(1);
});
it('should not call onChanged when storing unchanged value', async () => {
const fakeOnChanged = jest.fn(() => Promise.resolve());
const cache = makeCache<string>({
name: 'fake',
onChanged: fakeOnChanged,
});
await cache.set('foo', 'bar');
await cache.set('foo', 'bar');
expect(fakeOnChanged).toHaveBeenCalledTimes(1);
});
it('should fetch an unknown value', async () => {
const cache = makeCache<string>({
name: 'fake',
fetcher: key => `value#${key}`,
});
const result = await cache.fetch('foo');
expect(result).toBe('value#foo');
});
it('should store fetched value in memory cache', async () => {
const cache = makeCache<string>({
name: 'fake',
fetcher: key => `value#${key}`,
});
await cache.fetch('foo');
const result = cache.has('foo');
expect(result).toBe(true);
});
it('should call onChanged when fetching', async () => {
const fakeOnChanged = jest.fn(() => Promise.resolve());
const cache = makeCache<string>({
name: 'fake',
fetcher: key => `value#${key}`,
onChanged: fakeOnChanged,
});
await cache.fetch('foo');
expect(fakeOnChanged).toHaveBeenCalledWith(['foo'], cache);
});
it('should not emit event when fetching', async () => {
const cache = makeCache<string>({
name: 'fake',
fetcher: key => `value#${key}`,
});
await cache.fetch('foo');
expect(fakeInternalEventService._calls).not.toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo'] }]]);
});
it('should delete from memory cache', async () => {
const cache = makeCache<string>();
await cache.set('foo', 'bar');
await cache.delete('foo');
const result = cache.has('foo');
expect(result).toBe(false);
});
it('should call onChanged when deleting', async () => {
const fakeOnChanged = jest.fn(() => Promise.resolve());
const cache = makeCache<string>({
name: 'fake',
onChanged: fakeOnChanged,
});
await cache.set('foo', 'bar');
await cache.delete('foo');
expect(fakeOnChanged).toHaveBeenCalledWith(['foo'], cache);
});
it('should emit event when deleting', async () => {
const cache = makeCache<string>({ name: 'fake' });
await cache.set('foo', 'bar');
await cache.delete('foo');
expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo'] }]]);
});
it('should delete when receiving set event', async () => {
const cache = makeCache<string>({ name: 'fake' });
await cache.set('foo', 'bar');
await fakeInternalEventService._emitRedis('quantumCacheUpdated', { name: 'fake', keys: ['foo'] });
const result = cache.has('foo');
expect(result).toBe(false);
});
it('should call onChanged when receiving set event', async () => {
const fakeOnChanged = jest.fn(() => Promise.resolve());
const cache = makeCache<string>({
name: 'fake',
onChanged: fakeOnChanged,
});
await fakeInternalEventService._emitRedis('quantumCacheUpdated', { name: 'fake', keys: ['foo'] });
expect(fakeOnChanged).toHaveBeenCalledWith(['foo'], cache);
});
it('should delete when receiving delete event', async () => {
const cache = makeCache<string>({ name: 'fake' });
await cache.set('foo', 'bar');
await fakeInternalEventService._emitRedis('quantumCacheUpdated', { name: 'fake', keys: ['foo'] });
const result = cache.has('foo');
expect(result).toBe(false);
});
it('should call onChanged when receiving delete event', async () => {
const fakeOnChanged = jest.fn(() => Promise.resolve());
const cache = makeCache<string>({
name: 'fake',
onChanged: fakeOnChanged,
});
await cache.set('foo', 'bar');
await fakeInternalEventService._emitRedis('quantumCacheUpdated', { name: 'fake', keys: ['foo'] });
expect(fakeOnChanged).toHaveBeenCalledWith(['foo'], cache);
});
describe('get', () => {
it('should return value if present', async () => {
const cache = makeCache<string>();
await cache.set('foo', 'bar');
const result = cache.get('foo');
expect(result).toBe('bar');
});
it('should return undefined if missing', () => {
const cache = makeCache<string>();
const result = cache.get('foo');
expect(result).toBe(undefined);
});
});
describe('setMany', () => {
it('should populate all values', async () => {
const cache = makeCache<string>();
await cache.setMany([['foo', 'bar'], ['alpha', 'omega']]);
expect(cache.has('foo')).toBe(true);
expect(cache.has('alpha')).toBe(true);
});
it('should emit one event', async () => {
const cache = makeCache<string>({
name: 'fake',
});
await cache.setMany([['foo', 'bar'], ['alpha', 'omega']]);
expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo', 'alpha'] }]]);
expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(1);
});
it('should call onChanged once with all items', async () => {
const fakeOnChanged = jest.fn(() => Promise.resolve());
const cache = makeCache<string>({
name: 'fake',
onChanged: fakeOnChanged,
});
await cache.setMany([['foo', 'bar'], ['alpha', 'omega']]);
expect(fakeOnChanged).toHaveBeenCalledWith(['foo', 'alpha'], cache);
expect(fakeOnChanged).toHaveBeenCalledTimes(1);
});
it('should emit events only for changed items', async () => {
const fakeOnChanged = jest.fn(() => Promise.resolve());
const cache = makeCache<string>({
name: 'fake',
onChanged: fakeOnChanged,
});
await cache.set('foo', 'bar');
fakeOnChanged.mockClear();
fakeInternalEventService._reset();
await cache.setMany([['foo', 'bar'], ['alpha', 'omega']]);
expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['alpha'] }]]);
expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(1);
expect(fakeOnChanged).toHaveBeenCalledWith(['alpha'], cache);
expect(fakeOnChanged).toHaveBeenCalledTimes(1);
});
});
describe('getMany', () => {
it('should return empty for empty input', () => {
const cache = makeCache();
const result = cache.getMany([]);
expect(result).toEqual([]);
});
it('should return the value for all keys', () => {
const cache = makeCache();
cache.add('foo', 'bar');
cache.add('alpha', 'omega');
const result = cache.getMany(['foo', 'alpha']);
expect(result).toEqual([['foo', 'bar'], ['alpha', 'omega']]);
});
it('should return undefined for missing keys', () => {
const cache = makeCache();
cache.add('foo', 'bar');
const result = cache.getMany(['foo', 'alpha']);
expect(result).toEqual([['foo', 'bar'], ['alpha', undefined]]);
});
});
describe('fetchMany', () => {
it('should do nothing for empty input', async () => {
const fakeOnChanged = jest.fn(() => Promise.resolve());
const cache = makeCache({
onChanged: fakeOnChanged,
});
await cache.fetchMany([]);
expect(fakeOnChanged).not.toHaveBeenCalled();
expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0);
});
it('should return existing items', async () => {
const cache = makeCache();
cache.add('foo', 'bar');
cache.add('alpha', 'omega');
const result = await cache.fetchMany(['foo', 'alpha']);
expect(result).toEqual([['foo', 'bar'], ['alpha', 'omega']]);
});
it('should return existing items without events', async () => {
const fakeOnChanged = jest.fn(() => Promise.resolve());
const cache = makeCache({
onChanged: fakeOnChanged,
});
cache.add('foo', 'bar');
cache.add('alpha', 'omega');
await cache.fetchMany(['foo', 'alpha']);
expect(fakeOnChanged).not.toHaveBeenCalled();
expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0);
});
it('should call bulkFetcher for missing items', async () => {
const cache = makeCache({
bulkFetcher: keys => keys.map(k => [k, `${k}#many`]),
fetcher: key => `${key}#single`,
});
const results = await cache.fetchMany(['foo', 'alpha']);
expect(results).toEqual([['foo', 'foo#many'], ['alpha', 'alpha#many']]);
});
it('should call bulkFetcher only once', async () => {
const mockBulkFetcher = jest.fn((keys: string[]) => keys.map(k => [k, `${k}#value`] as [string, string]));
const cache = makeCache({
bulkFetcher: mockBulkFetcher,
});
await cache.fetchMany(['foo', 'bar']);
expect(mockBulkFetcher).toHaveBeenCalledTimes(1);
});
it('should call fetcher when fetchMany is undefined', async () => {
const cache = makeCache({
fetcher: key => `${key}#single`,
});
const results = await cache.fetchMany(['foo', 'alpha']);
expect(results).toEqual([['foo', 'foo#single'], ['alpha', 'alpha#single']]);
});
it('should call onChanged', async () => {
const fakeOnChanged = jest.fn(() => Promise.resolve());
const cache = makeCache({
onChanged: fakeOnChanged,
fetcher: k => k,
});
await cache.fetchMany(['foo', 'alpha']);
expect(fakeOnChanged).toHaveBeenCalledWith(['foo', 'alpha'], cache);
expect(fakeOnChanged).toHaveBeenCalledTimes(1);
});
it('should call onChanged only for changed', async () => {
const fakeOnChanged = jest.fn(() => Promise.resolve());
const cache = makeCache({
onChanged: fakeOnChanged,
fetcher: k => k,
});
cache.add('foo', 'bar');
await cache.fetchMany(['foo', 'alpha']);
expect(fakeOnChanged).toHaveBeenCalledWith(['alpha'], cache);
expect(fakeOnChanged).toHaveBeenCalledTimes(1);
});
it('should not emit event', async () => {
const cache = makeCache({
fetcher: k => k,
});
await cache.fetchMany(['foo', 'alpha']);
expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0);
});
});
describe('refreshMany', () => {
it('should do nothing for empty input', async () => {
const fakeOnChanged = jest.fn(() => Promise.resolve());
const cache = makeCache({
onChanged: fakeOnChanged,
});
const result = await cache.refreshMany([]);
expect(result).toEqual([]);
expect(fakeOnChanged).not.toHaveBeenCalled();
expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0);
});
it('should call bulkFetcher for all keys', async () => {
const mockBulkFetcher = jest.fn((keys: string[]) => keys.map(k => [k, `${k}#value`] as [string, string]));
const cache = makeCache({
bulkFetcher: mockBulkFetcher,
});
const result = await cache.refreshMany(['foo', 'alpha']);
expect(result).toEqual([['foo', 'foo#value'], ['alpha', 'alpha#value']]);
expect(mockBulkFetcher).toHaveBeenCalledWith(['foo', 'alpha'], cache);
expect(mockBulkFetcher).toHaveBeenCalledTimes(1);
});
it('should replace any existing keys', async () => {
const mockBulkFetcher = jest.fn((keys: string[]) => keys.map(k => [k, `${k}#value`] as [string, string]));
const cache = makeCache({
bulkFetcher: mockBulkFetcher,
});
cache.add('foo', 'bar');
const result = await cache.refreshMany(['foo', 'alpha']);
expect(result).toEqual([['foo', 'foo#value'], ['alpha', 'alpha#value']]);
expect(mockBulkFetcher).toHaveBeenCalledWith(['foo', 'alpha'], cache);
expect(mockBulkFetcher).toHaveBeenCalledTimes(1);
});
it('should call onChanged for all keys', async () => {
const fakeOnChanged = jest.fn(() => Promise.resolve());
const cache = makeCache({
bulkFetcher: keys => keys.map(k => [k, `${k}#value`]),
onChanged: fakeOnChanged,
});
cache.add('foo', 'bar');
await cache.refreshMany(['foo', 'alpha']);
expect(fakeOnChanged).toHaveBeenCalledWith(['foo', 'alpha'], cache);
expect(fakeOnChanged).toHaveBeenCalledTimes(1);
});
it('should emit event for all keys', async () => {
const cache = makeCache({
name: 'fake',
bulkFetcher: keys => keys.map(k => [k, `${k}#value`]),
});
cache.add('foo', 'bar');
await cache.refreshMany(['foo', 'alpha']);
expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo', 'alpha'] }]]);
expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(1);
});
});
describe('deleteMany', () => {
it('should remove keys from memory cache', async () => {
const cache = makeCache<string>();
await cache.set('foo', 'bar');
await cache.set('alpha', 'omega');
await cache.deleteMany(['foo', 'alpha']);
expect(cache.has('foo')).toBe(false);
expect(cache.has('alpha')).toBe(false);
});
it('should emit only one event', async () => {
const cache = makeCache<string>({
name: 'fake',
});
await cache.deleteMany(['foo', 'alpha']);
expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo', 'alpha'] }]]);
expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(1);
});
it('should call onChanged once with all items', async () => {
const fakeOnChanged = jest.fn(() => Promise.resolve());
const cache = makeCache<string>({
name: 'fake',
onChanged: fakeOnChanged,
});
await cache.deleteMany(['foo', 'alpha']);
expect(fakeOnChanged).toHaveBeenCalledWith(['foo', 'alpha'], cache);
expect(fakeOnChanged).toHaveBeenCalledTimes(1);
});
it('should do nothing if no keys are provided', async () => {
const fakeOnChanged = jest.fn(() => Promise.resolve());
const cache = makeCache<string>({
name: 'fake',
onChanged: fakeOnChanged,
});
await cache.deleteMany([]);
expect(fakeOnChanged).not.toHaveBeenCalled();
expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0);
});
});
describe('refresh', () => {
it('should populate the value', async () => {
const cache = makeCache<string>({
name: 'fake',
fetcher: key => `value#${key}`,
});
await cache.refresh('foo');
const result = cache.has('foo');
expect(result).toBe(true);
});
it('should return the value', async () => {
const cache = makeCache<string>({
name: 'fake',
fetcher: key => `value#${key}`,
});
const result = await cache.refresh('foo');
expect(result).toBe('value#foo');
});
it('should replace the value if it exists', async () => {
const cache = makeCache<string>({
name: 'fake',
fetcher: key => `value#${key}`,
});
await cache.set('foo', 'bar');
const result = await cache.refresh('foo');
expect(result).toBe('value#foo');
});
it('should call onChanged', async () => {
const fakeOnChanged = jest.fn(() => Promise.resolve());
const cache = makeCache<string>({
name: 'fake',
fetcher: key => `value#${key}`,
onChanged: fakeOnChanged,
});
await cache.refresh('foo');
expect(fakeOnChanged).toHaveBeenCalledWith(['foo'], cache);
});
it('should emit event', async () => {
const cache = makeCache<string>({
name: 'fake',
fetcher: key => `value#${key}`,
});
await cache.refresh('foo');
expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo'] }]]);
});
});
describe('add', () => {
it('should add the item', () => {
const cache = makeCache();
cache.add('foo', 'bar');
expect(cache.has('foo')).toBe(true);
});
it('should not emit event', () => {
const cache = makeCache({
name: 'fake',
});
cache.add('foo', 'bar');
expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0);
});
it('should not call onChanged', () => {
const fakeOnChanged = jest.fn(() => Promise.resolve());
const cache = makeCache({
onChanged: fakeOnChanged,
});
cache.add('foo', 'bar');
expect(fakeOnChanged).not.toHaveBeenCalled();
});
});
describe('addMany', () => {
it('should add all items', () => {
const cache = makeCache();
cache.addMany([['foo', 'bar'], ['alpha', 'omega']]);
expect(cache.has('foo')).toBe(true);
expect(cache.has('alpha')).toBe(true);
});
it('should not emit event', () => {
const cache = makeCache({
name: 'fake',
});
cache.addMany([['foo', 'bar'], ['alpha', 'omega']]);
expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0);
});
it('should not call onChanged', () => {
const fakeOnChanged = jest.fn(() => Promise.resolve());
const cache = makeCache({
onChanged: fakeOnChanged,
});
cache.addMany([['foo', 'bar'], ['alpha', 'omega']]);
expect(fakeOnChanged).not.toHaveBeenCalled();
});
});
describe('has', () => {
it('should return false when empty', () => {
const cache = makeCache();
const result = cache.has('foo');
expect(result).toBe(false);
});
it('should return false when value is not in memory', async () => {
const cache = makeCache<string>();
await cache.set('foo', 'bar');
const result = cache.has('alpha');
expect(result).toBe(false);
});
it('should return true when value is in memory', async () => {
const cache = makeCache<string>();
await cache.set('foo', 'bar');
const result = cache.has('foo');
expect(result).toBe(true);
});
});
describe('size', () => {
it('should return 0 when empty', () => {
const cache = makeCache();
expect(cache.size).toBe(0);
});
it('should return correct size when populated', async () => {
const cache = makeCache<string>();
await cache.set('foo', 'bar');
expect(cache.size).toBe(1);
});
});
describe('entries', () => {
it('should return empty when empty', () => {
const cache = makeCache();
const result = Array.from(cache.entries());
expect(result).toHaveLength(0);
});
it('should return all entries when populated', async () => {
const cache = makeCache<string>();
await cache.set('foo', 'bar');
const result = Array.from(cache.entries());
expect(result).toEqual([['foo', 'bar']]);
});
});
describe('keys', () => {
it('should return empty when empty', () => {
const cache = makeCache();
const result = Array.from(cache.keys());
expect(result).toHaveLength(0);
});
it('should return all keys when populated', async () => {
const cache = makeCache<string>();
await cache.set('foo', 'bar');
const result = Array.from(cache.keys());
expect(result).toEqual(['foo']);
});
});
describe('values', () => {
it('should return empty when empty', () => {
const cache = makeCache();
const result = Array.from(cache.values());
expect(result).toHaveLength(0);
});
it('should return all values when populated', async () => {
const cache = makeCache<string>();
await cache.set('foo', 'bar');
const result = Array.from(cache.values());
expect(result).toEqual(['bar']);
});
});
describe('[Symbol.iterator]', () => {
it('should return empty when empty', () => {
const cache = makeCache();
const result = Array.from(cache);
expect(result).toHaveLength(0);
});
it('should return all entries when populated', async () => {
const cache = makeCache<string>();
await cache.set('foo', 'bar');
const result = Array.from(cache);
expect(result).toEqual([['foo', 'bar']]);
});
});
});

4
packages/backend/test/utils.ts
View file

@ -11,12 +11,12 @@ import { inspect } from 'node:util';
import WebSocket, { ClientOptions } from 'ws';
import fetch, { File, RequestInit, type Headers } from 'node-fetch';
import { DataSource } from 'typeorm';
import { load as cheerio } from 'cheerio';
import { load as cheerio } from 'cheerio/slim';
import { type Response } from 'node-fetch';
import Fastify from 'fastify';
import { entities } from '../src/postgres.js';
import { loadConfig } from '../src/config.js';
import type { CheerioAPI } from 'cheerio';
import type { CheerioAPI } from 'cheerio/slim';
import type * as misskey from 'misskey-js';
import { DEFAULT_POLICIES } from '@/core/RoleService.js';
import { validateContentTypeSetAsActivityPub } from '@/core/activitypub/misc/validator.js';

34
packages/frontend-embed/package.json
View file

@ -11,35 +11,25 @@
},
"dependencies": {
"@discordapp/twemoji": "15.1.0",
"@phosphor-icons/web": "^2.0.3",
"@rollup/plugin-json": "6.1.0",
"@rollup/plugin-replace": "6.0.2",
"@rollup/pluginutils": "5.1.4",
"@transfem-org/sfm-js": "0.24.5",
"@twemoji/parser": "15.1.1",
"@vitejs/plugin-vue": "5.2.3",
"@vue/compiler-sfc": "3.5.14",
"astring": "1.9.0",
"@phosphor-icons/web": "2.1.2",
"mfm-js": "npm:@transfem-org/sfm-js@0.24.6",
"buraha": "0.0.1",
"estree-walker": "3.0.3",
"frontend-shared": "workspace:*",
"json5": "2.2.3",
"misskey-js": "workspace:*",
"punycode.js": "2.3.1",
"rollup": "4.40.0",
"sass": "1.87.0",
"shiki": "3.3.0",
"tinycolor2": "1.6.0",
"tsc-alias": "1.8.15",
"tsconfig-paths": "4.2.0",
"typescript": "5.8.3",
"uuid": "11.1.0",
"vite": "6.3.4",
"vue": "3.5.14"
},
"devDependencies": {
"@misskey-dev/summaly": "5.2.1",
"@misskey-dev/summaly": "npm:@transfem-org/summaly@5.2.2",
"@rollup/plugin-json": "6.1.0",
"@rollup/plugin-replace": "6.0.2",
"@rollup/pluginutils": "5.1.4",
"@testing-library/vue": "8.1.0",
"@twemoji/parser": "15.1.1",
"@types/estree": "1.0.7",
"@types/micromatch": "4.0.9",
"@types/node": "22.15.2",
@ -48,12 +38,16 @@
"@types/ws": "8.18.1",
"@typescript-eslint/eslint-plugin": "8.31.0",
"@typescript-eslint/parser": "8.31.0",
"@vitejs/plugin-vue": "5.2.3",
"@vitest/coverage-v8": "3.1.2",
"@vue/compiler-sfc": "3.5.14",
"@vue/runtime-core": "3.5.14",
"acorn": "8.14.1",
"astring": "1.9.0",
"cross-env": "7.0.3",
"eslint-plugin-import": "2.31.0",
"eslint-plugin-vue": "10.0.0",
"estree-walker": "3.0.3",
"fast-glob": "3.3.3",
"happy-dom": "17.4.4",
"intersection-observer": "0.12.2",
@ -61,7 +55,13 @@
"msw": "2.7.5",
"nodemon": "3.1.10",
"prettier": "3.5.3",
"rollup": "4.40.0",
"sass": "1.87.0",
"start-server-and-test": "2.0.11",
"tsc-alias": "1.8.15",
"tsconfig-paths": "4.2.0",
"typescript": "5.8.3",
"vite": "6.3.4",
"vite-plugin-turbosnap": "1.0.3",
"vue-component-type-helpers": "2.2.10",
"vue-eslint-parser": "10.1.3",

2
packages/frontend-embed/src/components/EmMfm.ts
View file

@ -5,7 +5,7 @@
import { h, provide } from 'vue';
import type { VNode, SetupContext } from 'vue';
import * as mfm from '@transfem-org/sfm-js';
import * as mfm from 'mfm-js';
import * as Misskey from 'misskey-js';
import { host } from '@@/js/config.js';
import EmUrl from '@/components/EmUrl.vue';

2
packages/frontend-embed/src/components/EmNote.vue
View file

@ -105,7 +105,7 @@ SPDX-License-Identifier: AGPL-3.0-only
<script lang="ts" setup>
import { computed, inject, ref, shallowRef } from 'vue';
import * as mfm from '@transfem-org/sfm-js';
import * as mfm from 'mfm-js';
import * as Misskey from 'misskey-js';
import { shouldCollapsed } from '@@/js/collapsed.js';
import { url } from '@@/js/config.js';

2
packages/frontend-embed/src/components/EmNoteDetailed.vue
View file

@ -128,7 +128,7 @@ SPDX-License-Identifier: AGPL-3.0-only
<script lang="ts" setup>
import { computed, inject, ref } from 'vue';
import * as mfm from '@transfem-org/sfm-js';
import * as mfm from 'mfm-js';
import * as Misskey from 'misskey-js';
import { computeMergedCw } from '@@/js/compute-merged-cw.js';
import I18n from '@/components/I18n.vue';

2
packages/frontend-shared/build.js
View file

@ -101,7 +101,7 @@ async function watchSrc() {
process.on('SIGHUP', resolve);
process.on('SIGINT', resolve);
process.on('SIGTERM', resolve);
process.on('uncaughtException', reject);
process.on('uncaughtExceptionMonitor', reject);
process.on('exit', resolve);
}).finally(async () => {
await context.dispose();

1
packages/frontend-shared/package.json
View file

@ -35,7 +35,6 @@
],
"dependencies": {
"misskey-js": "workspace:*",
"nodemon": "3.1.7",
"vue": "3.5.13"
}
}

Some files were not shown because too many files have changed in this diff Show more