Merge branch 'develop' into upstream/2025.5.0

This commit is contained in:
dakkar 2025-05-30 11:13:37 +01:00
commit 46bb75d274
116 changed files with 2636 additions and 973 deletions

View file

@ -8,9 +8,11 @@ import { fileURLToPath } from 'node:url';
import { dirname, resolve } from 'node:path';
import * as yaml from 'js-yaml';
import { globSync } from 'glob';
import ipaddr from 'ipaddr.js';
import type * as Sentry from '@sentry/node';
import type * as SentryVue from '@sentry/vue';
import type { RedisOptions } from 'ioredis';
import type { IPv4, IPv6 } from 'ipaddr.js';
type RedisOptionsSource = Partial<RedisOptions> & {
host?: string;
@ -82,7 +84,7 @@ type Source = {
proxySmtp?: string;
proxyBypassHosts?: string[];
allowedPrivateNetworks?: string[];
allowedPrivateNetworks?: PrivateNetworkSource[];
disallowExternalApRedirect?: boolean;
maxFileSize?: number;
@ -109,6 +111,7 @@ type Source = {
deliverJobMaxAttempts?: number;
inboxJobMaxAttempts?: number;
mediaDirectory?: string;
mediaProxy?: string;
proxyRemoteFiles?: boolean;
videoThumbnailGenerator?: string;
@ -152,6 +155,60 @@ type Source = {
}
};
export type PrivateNetworkSource = string | { network?: string, ports?: number[] };
export type PrivateNetwork = {
/**
* CIDR IP/netmask definition of the IP range to match.
*/
cidr: CIDR;
/**
* List of ports to match.
* If undefined, then all ports match.
* If empty, then NO ports match.
*/
ports?: number[];
};
export type CIDR = [ip: IPv4 | IPv6, prefixLength: number];
export function parsePrivateNetworks(patterns: PrivateNetworkSource[]): PrivateNetwork[];
export function parsePrivateNetworks(patterns: undefined): undefined;
export function parsePrivateNetworks(patterns: PrivateNetworkSource[] | undefined): PrivateNetwork[] | undefined;
export function parsePrivateNetworks(patterns: PrivateNetworkSource[] | undefined): PrivateNetwork[] | undefined {
if (!patterns) return undefined;
return patterns
.map(e => {
if (typeof(e) === 'string') {
const cidr = parseIpOrMask(e);
if (cidr) {
return { cidr } satisfies PrivateNetwork;
}
} else if (e.network) {
const cidr = parseIpOrMask(e.network);
if (cidr) {
return { cidr, ports: e.ports } satisfies PrivateNetwork;
}
}
console.warn('[config] Skipping invalid entry in allowedPrivateNetworks: ', e);
return null;
})
.filter(p => p != null);
}
function parseIpOrMask(ipOrMask: string): CIDR | null {
if (ipaddr.isValidCIDR(ipOrMask)) {
return ipaddr.parseCIDR(ipOrMask);
}
if (ipaddr.isValid(ipOrMask)) {
const ip = ipaddr.parse(ipOrMask);
return [ip, 32];
}
return null;
}
export type Config = {
url: string;
port: number;
@ -190,7 +247,7 @@ export type Config = {
proxy: string | undefined;
proxySmtp: string | undefined;
proxyBypassHosts: string[] | undefined;
allowedPrivateNetworks: string[] | undefined;
allowedPrivateNetworks: PrivateNetwork[] | undefined;
disallowExternalApRedirect: boolean;
maxFileSize: number;
maxNoteLength: number;
@ -241,6 +298,7 @@ export type Config = {
frontendManifestExists: boolean;
frontendEmbedEntry: string;
frontendEmbedManifestExists: boolean;
mediaDirectory: string;
mediaProxy: string;
externalMediaProxyEnabled: boolean;
videoThumbnailGenerator: string | null;
@ -290,7 +348,7 @@ const _dirname = dirname(_filename);
/**
* Path of configuration directory
*/
const dir = `${_dirname}/../../../.config`;
const dir = process.env.MISSKEY_CONFIG_DIR ?? `${_dirname}/../../../.config`;
/**
* Path of configuration file
@ -382,7 +440,7 @@ export function loadConfig(): Config {
proxy: config.proxy,
proxySmtp: config.proxySmtp,
proxyBypassHosts: config.proxyBypassHosts,
allowedPrivateNetworks: config.allowedPrivateNetworks,
allowedPrivateNetworks: parsePrivateNetworks(config.allowedPrivateNetworks),
disallowExternalApRedirect: config.disallowExternalApRedirect ?? false,
maxFileSize: config.maxFileSize ?? 262144000,
maxNoteLength: config.maxNoteLength ?? 3000,
@ -407,6 +465,7 @@ export function loadConfig(): Config {
signToActivityPubGet: config.signToActivityPubGet ?? true,
attachLdSignatureForRelays: config.attachLdSignatureForRelays ?? true,
checkActivityPubGetSignature: config.checkActivityPubGetSignature,
mediaDirectory: config.mediaDirectory ?? resolve(_dirname, '../../../files'),
mediaProxy: externalMediaProxy ?? internalMediaProxy,
externalMediaProxyEnabled: externalMediaProxy !== null && externalMediaProxy !== internalMediaProxy,
videoThumbnailGenerator: config.videoThumbnailGenerator ?
@ -575,14 +634,14 @@ function applyEnvOverrides(config: Source) {
['host', 'port', 'username', 'pass', 'db', 'prefix'],
]);
_apply_top(['fulltextSearch', 'provider']);
_apply_top(['meilisearch', ['host', 'port', 'apikey', 'ssl', 'index', 'scope']]);
_apply_top(['meilisearch', ['host', 'port', 'apiKey', 'ssl', 'index', 'scope']]);
_apply_top([['sentryForFrontend', 'sentryForBackend'], 'options', ['dsn', 'profileSampleRate', 'serverName', 'includeLocalVariables', 'proxy', 'keepAlive', 'caCerts']]);
_apply_top(['sentryForBackend', 'enableNodeProfiling']);
_apply_top(['sentryForFrontend', 'vueIntegration', ['attachProps', 'attachErrorHandler']]);
_apply_top(['sentryForFrontend', 'vueIntegration', 'tracingOptions', 'timeout']);
_apply_top(['sentryForFrontend', 'browserTracingIntegration', 'routeLabel']);
_apply_top([['clusterLimit', 'deliverJobConcurrency', 'inboxJobConcurrency', 'relashionshipJobConcurrency', 'deliverJobPerSec', 'inboxJobPerSec', 'relashionshipJobPerSec', 'deliverJobMaxAttempts', 'inboxJobMaxAttempts']]);
_apply_top([['outgoingAddress', 'outgoingAddressFamily', 'proxy', 'proxySmtp', 'mediaProxy', 'proxyRemoteFiles', 'videoThumbnailGenerator']]);
_apply_top([['outgoingAddress', 'outgoingAddressFamily', 'proxy', 'proxySmtp', 'mediaDirectory', 'mediaProxy', 'proxyRemoteFiles', 'videoThumbnailGenerator']]);
_apply_top([['maxFileSize', 'maxNoteLength', 'maxRemoteNoteLength', 'maxAltTextLength', 'maxRemoteAltTextLength', 'pidFile', 'filePermissionBits']]);
_apply_top(['import', ['downloadTimeout', 'maxFileSize']]);
_apply_top([['signToActivityPubGet', 'checkActivityPubGetSignature', 'setupPassword', 'disallowExternalApRedirect']]);

View file

@ -6,7 +6,7 @@
import { Inject, Injectable } from '@nestjs/common';
import * as Redis from 'ioredis';
import { IsNull } from 'typeorm';
import type { BlockingsRepository, FollowingsRepository, MutingsRepository, RenoteMutingsRepository, MiUserProfile, UserProfilesRepository, UsersRepository, MiFollowing } from '@/models/_.js';
import type { BlockingsRepository, FollowingsRepository, MutingsRepository, RenoteMutingsRepository, MiUserProfile, UserProfilesRepository, UsersRepository, MiFollowing, MiNote } from '@/models/_.js';
import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js';
import type { MiLocalUser, MiUser } from '@/models/User.js';
import { DI } from '@/di-symbols.js';
@ -22,6 +22,17 @@ export interface FollowStats {
remoteFollowers: number;
}
export interface CachedTranslation {
sourceLang: string | undefined;
text: string | undefined;
}
interface CachedTranslationEntity {
l?: string;
t?: string;
u?: number;
}
@Injectable()
export class CacheService implements OnApplicationShutdown {
public userByIdCache: MemoryKVCache<MiUser>;
@ -35,6 +46,7 @@ export class CacheService implements OnApplicationShutdown {
public renoteMutingsCache: RedisKVCache<Set<string>>;
public userFollowingsCache: RedisKVCache<Record<string, Pick<MiFollowing, 'withReplies'> | undefined>>;
private readonly userFollowStatsCache = new MemoryKVCache<FollowStats>(1000 * 60 * 10); // 10 minutes
private readonly translationsCache: RedisKVCache<CachedTranslationEntity>;
constructor(
@Inject(DI.redis)
@ -124,6 +136,11 @@ export class CacheService implements OnApplicationShutdown {
fromRedisConverter: (value) => JSON.parse(value),
});
this.translationsCache = new RedisKVCache<CachedTranslationEntity>(this.redisClient, 'translations', {
lifetime: 1000 * 60 * 60 * 24 * 7, // 1 week,
memoryCacheLifetime: 1000 * 60, // 1 minute
});
// NOTE: チャンネルのフォロー状況キャッシュはChannelFollowingServiceで行っている
this.redisForSub.on('message', this.onMessage);
@ -253,6 +270,34 @@ export class CacheService implements OnApplicationShutdown {
});
}
@bindThis
public async getCachedTranslation(note: MiNote, targetLang: string): Promise<CachedTranslation | null> {
const cacheKey = `${note.id}@${targetLang}`;
// Use cached translation, if present and up-to-date
const cached = await this.translationsCache.get(cacheKey);
if (cached && cached.u === note.updatedAt?.valueOf()) {
return {
sourceLang: cached.l,
text: cached.t,
};
}
// No cache entry :(
return null;
}
@bindThis
public async setCachedTranslation(note: MiNote, targetLang: string, translation: CachedTranslation): Promise<void> {
const cacheKey = `${note.id}@${targetLang}`;
await this.translationsCache.set(cacheKey, {
l: translation.sourceLang,
t: translation.text,
u: note.updatedAt?.valueOf(),
});
}
@bindThis
public dispose(): void {
this.redisForSub.off('message', this.onMessage);

View file

@ -63,8 +63,6 @@ export class DeleteAccountService {
// 知り得る全SharedInboxにDelete配信
const content = this.apRendererService.addContext(this.apRendererService.renderDelete(this.userEntityService.genLocalUserUri(user.id), user));
const queue: string[] = [];
const followings = await this.followingsRepository.find({
where: [
{ followerSharedInbox: Not(IsNull()) },
@ -73,22 +71,17 @@ export class DeleteAccountService {
select: ['followerSharedInbox', 'followeeSharedInbox'],
});
const inboxes = followings.map(x => x.followerSharedInbox ?? x.followeeSharedInbox);
const inboxes = followings.map(x => [x.followerSharedInbox ?? x.followeeSharedInbox as string, true] as const);
const queue = new Map<string, true>(inboxes);
for (const inbox of inboxes) {
if (inbox != null && !queue.includes(inbox)) queue.push(inbox);
}
await this.queueService.deliverMany(user, content, queue);
for (const inbox of queue) {
this.queueService.deliver(user, content, inbox, true);
}
this.queueService.createDeleteAccountJob(user, {
await this.queueService.createDeleteAccountJob(user, {
soft: false,
});
} else {
// リモートユーザーの削除は、完全にDBから物理削除してしまうと再度連合してきてアカウントが復活する可能性があるため、soft指定する
this.queueService.createDeleteAccountJob(user, {
await this.queueService.createDeleteAccountJob(user, {
soft: true,
});
}

View file

@ -8,6 +8,7 @@ import * as Redis from 'ioredis';
import type { MiGalleryPost, MiNote, MiUser } from '@/models/_.js';
import { DI } from '@/di-symbols.js';
import { bindThis } from '@/decorators.js';
import { RoleService } from '@/core/RoleService.js';
const GLOBAL_NOTES_RANKING_WINDOW = 1000 * 60 * 60 * 24 * 3; // 3日ごと
export const GALLERY_POSTS_RANKING_WINDOW = 1000 * 60 * 60 * 24 * 3; // 3日ごと
@ -21,6 +22,8 @@ export class FeaturedService {
constructor(
@Inject(DI.redis)
private redisClient: Redis.Redis, // TODO: 専用のRedisサーバーを設定できるようにする
private readonly roleService: RoleService,
) {
}
@ -31,7 +34,14 @@ export class FeaturedService {
}
@bindThis
private async updateRankingOf(name: string, windowRange: number, element: string, score = 1): Promise<void> {
private async updateRankingOf(name: string, windowRange: number, element: string, score: number, userId: string | null): Promise<void> {
if (userId) {
const policies = await this.roleService.getUserPolicies(userId);
if (!policies.canTrend) {
return;
}
}
const currentWindow = this.getCurrentWindow(windowRange);
const redisTransaction = this.redisClient.multi();
redisTransaction.zincrby(
@ -89,28 +99,28 @@ export class FeaturedService {
}
@bindThis
public updateGlobalNotesRanking(noteId: MiNote['id'], score = 1): Promise<void> {
return this.updateRankingOf('featuredGlobalNotesRanking', GLOBAL_NOTES_RANKING_WINDOW, noteId, score);
public updateGlobalNotesRanking(note: Pick<MiNote, 'id' | 'userId'>, score = 1): Promise<void> {
return this.updateRankingOf('featuredGlobalNotesRanking', GLOBAL_NOTES_RANKING_WINDOW, note.id, score, note.userId);
}
@bindThis
public updateGalleryPostsRanking(galleryPostId: MiGalleryPost['id'], score = 1): Promise<void> {
return this.updateRankingOf('featuredGalleryPostsRanking', GALLERY_POSTS_RANKING_WINDOW, galleryPostId, score);
public updateGalleryPostsRanking(galleryPost: Pick<MiGalleryPost, 'id' | 'userId'>, score = 1): Promise<void> {
return this.updateRankingOf('featuredGalleryPostsRanking', GALLERY_POSTS_RANKING_WINDOW, galleryPost.id, score, galleryPost.userId);
}
@bindThis
public updateInChannelNotesRanking(channelId: MiNote['channelId'], noteId: MiNote['id'], score = 1): Promise<void> {
return this.updateRankingOf(`featuredInChannelNotesRanking:${channelId}`, GLOBAL_NOTES_RANKING_WINDOW, noteId, score);
public updateInChannelNotesRanking(channelId: MiNote['channelId'], note: Pick<MiNote, 'id' | 'userId'>, score = 1): Promise<void> {
return this.updateRankingOf(`featuredInChannelNotesRanking:${channelId}`, GLOBAL_NOTES_RANKING_WINDOW, note.id, score, note.userId);
}
@bindThis
public updatePerUserNotesRanking(userId: MiUser['id'], noteId: MiNote['id'], score = 1): Promise<void> {
return this.updateRankingOf(`featuredPerUserNotesRanking:${userId}`, PER_USER_NOTES_RANKING_WINDOW, noteId, score);
public updatePerUserNotesRanking(userId: MiUser['id'], note: Pick<MiNote, 'id'>, score = 1): Promise<void> {
return this.updateRankingOf(`featuredPerUserNotesRanking:${userId}`, PER_USER_NOTES_RANKING_WINDOW, note.id, score, userId);
}
@bindThis
public updateHashtagsRanking(hashtag: string, score = 1): Promise<void> {
return this.updateRankingOf('featuredHashtagsRanking', HASHTAG_RANKING_WINDOW, hashtag, score);
return this.updateRankingOf('featuredHashtagsRanking', HASHTAG_RANKING_WINDOW, hashtag, score, null);
}
@bindThis

View file

@ -12,7 +12,7 @@ import fetch from 'node-fetch';
import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent';
import { Inject, Injectable } from '@nestjs/common';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import type { Config, PrivateNetwork } from '@/config.js';
import { StatusError } from '@/misc/status-error.js';
import { bindThis } from '@/decorators.js';
import { validateContentTypeSetAsActivityPub } from '@/core/activitypub/misc/validator.js';
@ -20,12 +20,36 @@ import type { IObject, IObjectWithId } from '@/core/activitypub/type.js';
import { ApUtilityService } from './activitypub/ApUtilityService.js';
import type { Response } from 'node-fetch';
import type { URL } from 'node:url';
import type { Socket } from 'node:net';
export type HttpRequestSendOptions = {
throwErrorWhenResponseNotOk: boolean;
validators?: ((res: Response) => void)[];
};
export function isPrivateIp(allowedPrivateNetworks: PrivateNetwork[] | undefined, ip: string, port?: number): boolean {
const parsedIp = ipaddr.parse(ip);
for (const { cidr, ports } of allowedPrivateNetworks ?? []) {
if (cidr[0].kind() === parsedIp.kind() && parsedIp.match(cidr)) {
if (ports == null || (port != null && ports.includes(port))) {
return false;
}
}
}
return parsedIp.range() !== 'unicast';
}
export function validateSocketConnect(allowedPrivateNetworks: PrivateNetwork[] | undefined, socket: Socket): void {
const address = socket.remoteAddress;
if (address && ipaddr.isValid(address)) {
if (isPrivateIp(allowedPrivateNetworks, address, socket.remotePort)) {
socket.destroy(new Error(`Blocked address: ${address}`));
}
}
}
declare module 'node:http' {
interface Agent {
createConnection(options: net.NetConnectOpts, callback?: (err: unknown, stream: net.Socket) => void): net.Socket;
@ -44,31 +68,12 @@ class HttpRequestServiceAgent extends http.Agent {
public createConnection(options: net.NetConnectOpts, callback?: (err: unknown, stream: net.Socket) => void): net.Socket {
const socket = super.createConnection(options, callback)
.on('connect', () => {
const address = socket.remoteAddress;
if (process.env.NODE_ENV === 'production') {
if (address && ipaddr.isValid(address)) {
if (this.isPrivateIp(address)) {
socket.destroy(new Error(`Blocked address: ${address}`));
}
}
validateSocketConnect(this.config.allowedPrivateNetworks, socket);
}
});
return socket;
}
@bindThis
private isPrivateIp(ip: string): boolean {
const parsedIp = ipaddr.parse(ip);
for (const net of this.config.allowedPrivateNetworks ?? []) {
const cidr = ipaddr.parseCIDR(net);
if (cidr[0].kind() === parsedIp.kind() && parsedIp.match(ipaddr.parseCIDR(net))) {
return false;
}
}
return parsedIp.range() !== 'unicast';
}
}
class HttpsRequestServiceAgent extends https.Agent {
@ -83,31 +88,12 @@ class HttpsRequestServiceAgent extends https.Agent {
public createConnection(options: net.NetConnectOpts, callback?: (err: unknown, stream: net.Socket) => void): net.Socket {
const socket = super.createConnection(options, callback)
.on('connect', () => {
const address = socket.remoteAddress;
if (process.env.NODE_ENV === 'production') {
if (address && ipaddr.isValid(address)) {
if (this.isPrivateIp(address)) {
socket.destroy(new Error(`Blocked address: ${address}`));
}
}
validateSocketConnect(this.config.allowedPrivateNetworks, socket);
}
});
return socket;
}
@bindThis
private isPrivateIp(ip: string): boolean {
const parsedIp = ipaddr.parse(ip);
for (const net of this.config.allowedPrivateNetworks ?? []) {
const cidr = ipaddr.parseCIDR(net);
if (cidr[0].kind() === parsedIp.kind() && parsedIp.match(ipaddr.parseCIDR(net))) {
return false;
}
}
return parsedIp.range() !== 'unicast';
}
}
@Injectable()
@ -250,6 +236,8 @@ export class HttpRequestService {
@bindThis
public async getActivityJson(url: string, isLocalAddressAllowed = false): Promise<IObjectWithId> {
this.apUtilityService.assertApUrl(url);
const res = await this.send(url, {
method: 'GET',
headers: {

View file

@ -6,18 +6,11 @@
import * as fs from 'node:fs';
import { copyFile, unlink, writeFile, chmod } from 'node:fs/promises';
import * as Path from 'node:path';
import { fileURLToPath } from 'node:url';
import { dirname } from 'node:path';
import { Inject, Injectable } from '@nestjs/common';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { bindThis } from '@/decorators.js';
const _filename = fileURLToPath(import.meta.url);
const _dirname = dirname(_filename);
const path = Path.resolve(_dirname, '../../../../files');
@Injectable()
export class InternalStorageService {
constructor(
@ -25,12 +18,12 @@ export class InternalStorageService {
private config: Config,
) {
// No one should erase the working directory *while the server is running*.
fs.mkdirSync(path, { recursive: true });
fs.mkdirSync(this.config.mediaDirectory, { recursive: true });
}
@bindThis
public resolvePath(key: string) {
return Path.resolve(path, key);
return Path.resolve(this.config.mediaDirectory, key);
}
@bindThis

View file

@ -592,6 +592,8 @@ export class NoteCreateService implements OnApplicationShutdown {
if (!this.isRenote(note) || this.isQuote(note)) {
// Increment notes count (user)
this.incNotesCountOfUser(user);
} else {
this.usersRepository.update({ id: user.id }, { updatedAt: new Date() });
}
this.pushToTl(note, user);
@ -631,7 +633,7 @@ export class NoteCreateService implements OnApplicationShutdown {
}
if (this.isRenote(data) && !this.isQuote(data) && data.renote.userId !== user.id && !user.isBot) {
this.incRenoteCount(data.renote);
this.incRenoteCount(data.renote, user);
}
if (data.poll && data.poll.expiresAt) {
@ -814,8 +816,8 @@ export class NoteCreateService implements OnApplicationShutdown {
}
@bindThis
private incRenoteCount(renote: MiNote) {
this.notesRepository.createQueryBuilder().update()
private async incRenoteCount(renote: MiNote, user: MiUser) {
await this.notesRepository.createQueryBuilder().update()
.set({
renoteCount: () => '"renoteCount" + 1',
})
@ -823,15 +825,18 @@ export class NoteCreateService implements OnApplicationShutdown {
.execute();
// 30%の確率、3日以内に投稿されたートの場合ハイライト用ランキング更新
if (Math.random() < 0.3 && (Date.now() - this.idService.parse(renote.id).date.getTime()) < 1000 * 60 * 60 * 24 * 3) {
if (renote.channelId != null) {
if (renote.replyId == null) {
this.featuredService.updateInChannelNotesRanking(renote.channelId, renote.id, 5);
}
} else {
if (renote.visibility === 'public' && renote.userHost == null && renote.replyId == null) {
this.featuredService.updateGlobalNotesRanking(renote.id, 5);
this.featuredService.updatePerUserNotesRanking(renote.userId, renote.id, 5);
if (user.isExplorable && Math.random() < 0.3 && (Date.now() - this.idService.parse(renote.id).date.getTime()) < 1000 * 60 * 60 * 24 * 3) {
const policies = await this.roleService.getUserPolicies(user);
if (policies.canTrend) {
if (renote.channelId != null) {
if (renote.replyId == null) {
this.featuredService.updateInChannelNotesRanking(renote.channelId, renote, 5);
}
} else {
if (renote.visibility === 'public' && renote.userHost == null && renote.replyId == null) {
this.featuredService.updateGlobalNotesRanking(renote, 5);
this.featuredService.updatePerUserNotesRanking(renote.userId, renote, 5);
}
}
}
}

View file

@ -124,9 +124,11 @@ export class NoteDeleteService {
this.perUserNotesChart.update(user, note, false);
}
if (note.renoteId && note.text || !note.renoteId) {
if (!isRenote(note) || isQuote(note)) {
// Decrement notes count (user)
this.decNotesCountOfUser(user);
} else {
this.usersRepository.update({ id: user.id }, { updatedAt: new Date() });
}
if (this.meta.enableStatsForFederatedInstances) {
@ -165,8 +167,11 @@ export class NoteDeleteService {
});
}
if (note.uri) {
this.apLogService.deleteObjectLogs(note.uri)
const deletedUris = [note, ...cascadingNotes]
.map(n => n.uri)
.filter((u): u is string => u != null);
if (deletedUris.length > 0) {
this.apLogService.deleteObjectLogs(deletedUris)
.catch(err => this.logger.error(err, `Failed to delete AP logs for note '${note.uri}'`));
}
}

View file

@ -610,6 +610,8 @@ export class NoteEditService implements OnApplicationShutdown {
}
}
this.usersRepository.update({ id: user.id }, { updatedAt: new Date() });
// ハッシュタグ更新
this.pushToTl(note, user);

View file

@ -30,6 +30,7 @@ import { trackPromise } from '@/misc/promise-tracker.js';
import { isQuote, isRenote } from '@/misc/is-renote.js';
import { ReactionsBufferingService } from '@/core/ReactionsBufferingService.js';
import { PER_NOTE_REACTION_USER_PAIR_CACHE_MAX } from '@/const.js';
import { CacheService } from '@/core/CacheService.js';
const FALLBACK = '\u2764';
@ -102,6 +103,7 @@ export class ReactionService {
private apDeliverManagerService: ApDeliverManagerService,
private notificationService: NotificationService,
private perUserReactionsChart: PerUserReactionsChart,
private readonly cacheService: CacheService,
) {
}
@ -212,20 +214,28 @@ export class ReactionService {
.execute();
}
this.usersRepository.update({ id: user.id }, { updatedAt: new Date() });
// 30%の確率、セルフではない、3日以内に投稿されたートの場合ハイライト用ランキング更新
if (
Math.random() < 0.3 &&
note.userId !== user.id &&
(Date.now() - this.idService.parse(note.id).date.getTime()) < 1000 * 60 * 60 * 24 * 3
) {
if (note.channelId != null) {
if (note.replyId == null) {
this.featuredService.updateInChannelNotesRanking(note.channelId, note.id, 1);
}
} else {
if (note.visibility === 'public' && note.userHost == null && note.replyId == null) {
this.featuredService.updateGlobalNotesRanking(note.id, 1);
this.featuredService.updatePerUserNotesRanking(note.userId, note.id, 1);
const author = await this.cacheService.findUserById(note.userId);
if (author.isExplorable) {
const policies = await this.roleService.getUserPolicies(author);
if (policies.canTrend) {
if (note.channelId != null) {
if (note.replyId == null) {
this.featuredService.updateInChannelNotesRanking(note.channelId, note, 1);
}
} else {
if (note.visibility === 'public' && note.userHost == null && note.replyId == null) {
this.featuredService.updateGlobalNotesRanking(note, 1);
this.featuredService.updatePerUserNotesRanking(note.userId, note, 1);
}
}
}
}
}
@ -298,9 +308,9 @@ export class ReactionService {
}
@bindThis
public async delete(user: { id: MiUser['id']; host: MiUser['host']; isBot: MiUser['isBot']; }, note: MiNote) {
public async delete(user: { id: MiUser['id']; host: MiUser['host']; isBot: MiUser['isBot']; }, note: MiNote, exist?: MiNoteReaction | null) {
// if already unreacted
const exist = await this.noteReactionsRepository.findOneBy({
exist ??= await this.noteReactionsRepository.findOneBy({
noteId: note.id,
userId: user.id,
});
@ -330,6 +340,8 @@ export class ReactionService {
.execute();
}
this.usersRepository.update({ id: user.id }, { updatedAt: new Date() });
this.globalEventService.publishNoteStream(note.id, 'unreacted', {
reaction: this.decodeReaction(exist.reaction).reaction,
userId: user.id,

View file

@ -69,6 +69,7 @@ export type RolePolicies = {
canImportMuting: boolean;
canImportUserLists: boolean;
chatAvailability: 'available' | 'readonly' | 'unavailable';
canTrend: boolean;
};
export const DEFAULT_POLICIES: RolePolicies = {
@ -108,6 +109,7 @@ export const DEFAULT_POLICIES: RolePolicies = {
canImportMuting: true,
canImportUserLists: true,
chatAvailability: 'available',
canTrend: true,
};
@Injectable()
@ -149,6 +151,7 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit {
) {
this.rolesCache = new MemorySingleCache<MiRole[]>(1000 * 60 * 60); // 1h
this.roleAssignmentByUserIdCache = new MemoryKVCache<MiRoleAssignment[]>(1000 * 60 * 5); // 5m
// TODO additional cache for final calculation?
this.redisForSub.on('message', this.onMessage);
}
@ -358,8 +361,9 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit {
}
@bindThis
public async getUserAssigns(userId: MiUser['id']) {
public async getUserAssigns(userOrId: MiUser | MiUser['id']) {
const now = Date.now();
const userId = typeof(userOrId) === 'object' ? userOrId.id : userOrId;
let assigns = await this.roleAssignmentByUserIdCache.fetch(userId, () => this.roleAssignmentsRepository.findBy({ userId }));
// 期限切れのロールを除外
assigns = assigns.filter(a => a.expiresAt == null || (a.expiresAt.getTime() > now));
@ -367,12 +371,13 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit {
}
@bindThis
public async getUserRoles(userId: MiUser['id']) {
public async getUserRoles(userOrId: MiUser | MiUser['id']) {
const roles = await this.rolesCache.fetch(() => this.rolesRepository.findBy({}));
const userId = typeof(userOrId) === 'object' ? userOrId.id : userOrId;
const followStats = await this.cacheService.getFollowStats(userId);
const assigns = await this.getUserAssigns(userId);
const assigns = await this.getUserAssigns(userOrId);
const assignedRoles = roles.filter(r => assigns.map(x => x.roleId).includes(r.id));
const user = roles.some(r => r.target === 'conditional') ? await this.cacheService.findUserById(userId) : null;
const user = typeof(userOrId) === 'object' ? userOrId : roles.some(r => r.target === 'conditional') ? await this.cacheService.findUserById(userOrId) : null;
const matchedCondRoles = roles.filter(r => r.target === 'conditional' && this.evalCond(user!, assignedRoles, r.condFormula, followStats));
return [...assignedRoles, ...matchedCondRoles];
}
@ -381,8 +386,9 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit {
*
*/
@bindThis
public async getUserBadgeRoles(userId: MiUser['id']) {
public async getUserBadgeRoles(userOrId: MiUser | MiUser['id']) {
const now = Date.now();
const userId = typeof(userOrId) === 'object' ? userOrId.id : userOrId;
let assigns = await this.roleAssignmentByUserIdCache.fetch(userId, () => this.roleAssignmentsRepository.findBy({ userId }));
// 期限切れのロールを除外
assigns = assigns.filter(a => a.expiresAt == null || (a.expiresAt.getTime() > now));
@ -392,7 +398,7 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit {
const assignedBadgeRoles = assignedRoles.filter(r => r.asBadge);
const badgeCondRoles = roles.filter(r => r.asBadge && (r.target === 'conditional'));
if (badgeCondRoles.length > 0) {
const user = roles.some(r => r.target === 'conditional') ? await this.cacheService.findUserById(userId) : null;
const user = typeof(userOrId) === 'object' ? userOrId : roles.some(r => r.target === 'conditional') ? await this.cacheService.findUserById(userOrId) : null;
const matchedBadgeCondRoles = badgeCondRoles.filter(r => this.evalCond(user!, assignedRoles, r.condFormula, followStats));
return [...assignedBadgeRoles, ...matchedBadgeCondRoles];
} else {
@ -401,12 +407,12 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit {
}
@bindThis
public async getUserPolicies(userId: MiUser['id'] | null): Promise<RolePolicies> {
public async getUserPolicies(userOrId: MiUser | MiUser['id'] | null): Promise<RolePolicies> {
const basePolicies = { ...DEFAULT_POLICIES, ...this.meta.policies };
if (userId == null) return basePolicies;
if (userOrId == null) return basePolicies;
const roles = await this.getUserRoles(userId);
const roles = await this.getUserRoles(userOrId);
function calc<T extends keyof RolePolicies>(name: T, aggregate: (values: RolePolicies[T][]) => RolePolicies[T]) {
if (roles.length === 0) return basePolicies[name];
@ -465,6 +471,7 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit {
canImportMuting: calc('canImportMuting', vs => vs.some(v => v === true)),
canImportUserLists: calc('canImportUserLists', vs => vs.some(v => v === true)),
chatAvailability: calc('chatAvailability', aggregateChatAvailability),
canTrend: calc('canTrend', vs => vs.some(v => v === true)),
};
}

View file

@ -155,6 +155,8 @@ export class ApRequestService {
@bindThis
public async signedPost(user: { id: MiUser['id'] }, url: string, object: unknown, digest?: string): Promise<void> {
this.apUtilityService.assertApUrl(url);
const body = typeof object === 'string' ? object : JSON.stringify(object);
const keypair = await this.userKeypairService.getUserKeypair(user.id);
@ -186,6 +188,8 @@ export class ApRequestService {
*/
@bindThis
public async signedGet(url: string, user: { id: MiUser['id'] }, followAlternate?: boolean): Promise<IObjectWithId> {
this.apUtilityService.assertApUrl(url);
const _followAlternate = followAlternate ?? true;
const keypair = await this.userKeypairService.getUserKeypair(user.id);

View file

@ -77,16 +77,48 @@ export class ApUtilityService {
return acceptableUrls[0]?.url ?? null;
}
/**
* Verifies that a provided URL is in a format acceptable for federation.
* @throws {IdentifiableError} If URL cannot be parsed
* @throws {IdentifiableError} If URL contains a fragment
* @throws {IdentifiableError} If URL is not HTTPS
*/
public assertApUrl(url: string | URL): void {
// If string, parse and validate
if (typeof(url) === 'string') {
try {
url = new URL(url);
} catch {
throw new IdentifiableError('0bedd29b-e3bf-4604-af51-d3352e2518af', `invalid AP url ${url}: not a valid URL`);
}
}
// Hash component breaks federation
if (url.hash) {
throw new IdentifiableError('0bedd29b-e3bf-4604-af51-d3352e2518af', `invalid AP url ${url}: contains a fragment (#)`);
}
// Must be HTTPS
if (!this.checkHttps(url)) {
throw new IdentifiableError('0bedd29b-e3bf-4604-af51-d3352e2518af', `invalid AP url ${url}: unsupported protocol ${url.protocol}`);
}
}
/**
* Checks if the URL contains HTTPS.
* Additionally, allows HTTP in non-production environments.
* Based on check-https.ts.
*/
private checkHttps(url: string): boolean {
private checkHttps(url: string | URL): boolean {
const isNonProd = this.envService.env.NODE_ENV !== 'production';
// noinspection HttpUrlsUsage
return url.startsWith('https://') || (url.startsWith('http://') && isNonProd);
try {
const proto = new URL(url).protocol;
return proto === 'https:' || (proto === 'http:' && isNonProd);
} catch {
// Invalid URLs don't "count" as HTTPS
return false;
}
}
}

View file

@ -95,6 +95,7 @@ export class ApNoteService {
actor?: MiRemoteUser,
user?: MiRemoteUser,
): Error | null {
this.apUtilityService.assertApUrl(uri);
const expectHost = this.utilityService.extractDbHost(uri);
const apType = getApType(object);

View file

@ -153,6 +153,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
*/
@bindThis
private validateActor(x: IObject, uri: string): IActor {
this.apUtilityService.assertApUrl(uri);
const expectHost = this.utilityService.punyHostPSLDomain(uri);
if (!isActor(x)) {
@ -167,6 +168,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
throw new UnrecoverableError(`invalid Actor ${uri} - wrong inbox type`);
}
this.apUtilityService.assertApUrl(x.inbox);
const inboxHost = this.utilityService.punyHostPSLDomain(x.inbox);
if (inboxHost !== expectHost) {
throw new UnrecoverableError(`invalid Actor ${uri} - wrong inbox ${inboxHost}`);
@ -175,6 +177,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
const sharedInboxObject = x.sharedInbox ?? (x.endpoints ? x.endpoints.sharedInbox : undefined);
if (sharedInboxObject != null) {
const sharedInbox = getApId(sharedInboxObject);
this.apUtilityService.assertApUrl(sharedInbox);
if (!(typeof sharedInbox === 'string' && sharedInbox.length > 0 && this.utilityService.punyHostPSLDomain(sharedInbox) === expectHost)) {
throw new UnrecoverableError(`invalid Actor ${uri} - wrong shared inbox ${sharedInbox}`);
}
@ -185,6 +188,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
if (xCollection != null) {
const collectionUri = getApId(xCollection);
if (typeof collectionUri === 'string' && collectionUri.length > 0) {
this.apUtilityService.assertApUrl(collectionUri);
if (this.utilityService.punyHostPSLDomain(collectionUri) !== expectHost) {
throw new UnrecoverableError(`invalid Actor ${uri} - wrong ${collection} ${collectionUri}`);
}

View file

@ -23,6 +23,13 @@ import type { NoteEntityService } from './NoteEntityService.js';
const NOTE_REQUIRED_NOTIFICATION_TYPES = new Set(['note', 'mention', 'reply', 'renote', 'renote:grouped', 'quote', 'reaction', 'reaction:grouped', 'pollEnded', 'edited', 'scheduledNotePosted'] as (typeof groupedNotificationTypes[number])[]);
function undefOnMissing<T>(packPromise: Promise<T>): Promise<T | undefined> {
return packPromise.catch(err => {
if (err instanceof EntityNotFoundError) return undefined;
throw err;
});
}
@Injectable()
export class NotificationEntityService implements OnModuleInit {
private userEntityService: UserEntityService;
@ -75,9 +82,9 @@ export class NotificationEntityService implements OnModuleInit {
const noteIfNeed = needsNote ? (
hint?.packedNotes != null
? hint.packedNotes.get(notification.noteId)
: this.noteEntityService.pack(notification.noteId, { id: meId }, {
: undefOnMissing(this.noteEntityService.pack(notification.noteId, { id: meId }, {
detail: true,
})
}))
) : undefined;
// if the note has been deleted, don't show this notification
if (needsNote && !noteIfNeed) return null;
@ -86,7 +93,7 @@ export class NotificationEntityService implements OnModuleInit {
const userIfNeed = needsUser ? (
hint?.packedUsers != null
? hint.packedUsers.get(notification.notifierId)
: this.userEntityService.pack(notification.notifierId, { id: meId })
: undefOnMissing(this.userEntityService.pack(notification.notifierId, { id: meId }))
) : undefined;
// if the user has been deleted, don't show this notification
if (needsUser && !userIfNeed) return null;
@ -96,7 +103,7 @@ export class NotificationEntityService implements OnModuleInit {
const reactions = (await Promise.all(notification.reactions.map(async reaction => {
const user = hint?.packedUsers != null
? hint.packedUsers.get(reaction.userId)!
: await this.userEntityService.pack(reaction.userId, { id: meId });
: await undefOnMissing(this.userEntityService.pack(reaction.userId, { id: meId }));
return {
user,
reaction: reaction.reaction,
@ -121,7 +128,7 @@ export class NotificationEntityService implements OnModuleInit {
return packedUser;
}
return this.userEntityService.pack(userId, { id: meId });
return undefOnMissing(this.userEntityService.pack(userId, { id: meId }));
}))).filter(x => x != null);
// if all users have been deleted, don't show this notification
if (users.length === 0) {
@ -140,10 +147,7 @@ export class NotificationEntityService implements OnModuleInit {
const needsRole = notification.type === 'roleAssigned';
const role = needsRole
? await this.roleEntityService.pack(notification.roleId).catch(err => {
if (err instanceof EntityNotFoundError) return undefined;
throw err;
})
? await undefOnMissing(this.roleEntityService.pack(notification.roleId))
: undefined;
// if the role has been deleted, don't show this notification
if (needsRole && !role) {

View file

@ -19,16 +19,16 @@ export class RedisKVCache<T> {
opts: {
lifetime: RedisKVCache<T>['lifetime'];
memoryCacheLifetime: number;
fetcher: RedisKVCache<T>['fetcher'];
toRedisConverter: RedisKVCache<T>['toRedisConverter'];
fromRedisConverter: RedisKVCache<T>['fromRedisConverter'];
fetcher?: RedisKVCache<T>['fetcher'];
toRedisConverter?: RedisKVCache<T>['toRedisConverter'];
fromRedisConverter?: RedisKVCache<T>['fromRedisConverter'];
},
) {
this.lifetime = opts.lifetime;
this.memoryCache = new MemoryKVCache(opts.memoryCacheLifetime);
this.fetcher = opts.fetcher;
this.toRedisConverter = opts.toRedisConverter;
this.fromRedisConverter = opts.fromRedisConverter;
this.fetcher = opts.fetcher ?? (() => { throw new Error('fetch not supported - use get/set directly'); });
this.toRedisConverter = opts.toRedisConverter ?? ((value) => JSON.stringify(value));
this.fromRedisConverter = opts.fromRedisConverter ?? ((value) => JSON.parse(value));
}
@bindThis

View file

@ -22,7 +22,7 @@ export class MiAbuseReportNotificationRecipient {
/**
* .
*/
@Index()
@Index('IDX_abuse_report_notification_recipient_isActive')
@Column('boolean', {
default: true,
})
@ -47,7 +47,7 @@ export class MiAbuseReportNotificationRecipient {
/**
* .
*/
@Index()
@Index('IDX_abuse_report_notification_recipient_method')
@Column('varchar', {
length: 64,
})
@ -56,7 +56,7 @@ export class MiAbuseReportNotificationRecipient {
/**
* ID.
*/
@Index()
@Index('IDX_abuse_report_notification_recipient_userId')
@Column({
...id(),
nullable: true,
@ -75,14 +75,16 @@ export class MiAbuseReportNotificationRecipient {
/**
* .
*/
@ManyToOne(type => MiUserProfile, {})
@ManyToOne(type => MiUserProfile, {
onDelete: 'CASCADE',
})
@JoinColumn({ name: 'userId', referencedColumnName: 'userId', foreignKeyConstraintName: 'FK_abuse_report_notification_recipient_userId2' })
public userProfile: MiUserProfile | null;
/**
* WebhookId.
*/
@Index()
@Index('IDX_abuse_report_notification_recipient_systemWebhookId')
@Column({
...id(),
nullable: true,
@ -95,6 +97,8 @@ export class MiAbuseReportNotificationRecipient {
@ManyToOne(type => MiSystemWebhook, {
onDelete: 'CASCADE',
})
@JoinColumn()
@JoinColumn({
foreignKeyConstraintName: 'FK_abuse_report_notification_recipient_systemWebhookId',
})
public systemWebhook: MiSystemWebhook | null;
}

View file

@ -29,6 +29,7 @@ export class MiEmoji {
})
public host: string | null;
@Index('IDX_EMOJI_CATEGORY')
@Column('varchar', {
length: 128, nullable: true,
})
@ -77,6 +78,8 @@ export class MiEmoji {
public isSensitive: boolean;
// TODO: 定期ジョブで存在しなくなったロールIDを除去するようにする
// Synchronize: false is needed because TypeORM doesn't understand GIN indexes
@Index('IDX_EMOJI_ROLE_IDS', { synchronize: false })
@Column('varchar', {
array: true, length: 128, default: '{}',
})

View file

@ -45,6 +45,7 @@ export class SkLatestNote {
})
@JoinColumn({
name: 'user_id',
foreignKeyConstraintName: 'FK_20e346fffe4a2174585005d6d80',
})
public user: MiUser | null;
@ -60,6 +61,7 @@ export class SkLatestNote {
})
@JoinColumn({
name: 'note_id',
foreignKeyConstraintName: 'FK_47a38b1c13de6ce4e5090fb1acd',
})
public note: MiNote | null;

View file

@ -60,7 +60,7 @@ export class MiMeta {
public maintainerEmail: string | null;
@Column('boolean', {
default: false,
default: true,
})
public disableRegistration: boolean;
@ -382,6 +382,12 @@ export class MiMeta {
})
public swPrivateKey: string | null;
@Column('integer', {
default: 5000,
comment: 'Timeout in milliseconds for translation API requests',
})
public translationTimeout: number;
@Column('varchar', {
length: 1024,
nullable: true,
@ -425,7 +431,7 @@ export class MiMeta {
@Column('varchar', {
length: 1024,
default: 'https://activitypub.software/TransFem-org/Sharkey/',
nullable: false,
nullable: true,
})
public repositoryUrl: string | null;
@ -612,8 +618,8 @@ export class MiMeta {
})
public enableAchievements: boolean;
@Column('varchar', {
length: 2048, nullable: true,
@Column('text', {
nullable: true,
})
public robotsTxt: string | null;
@ -643,7 +649,7 @@ export class MiMeta {
public bannedEmailDomains: string[];
@Column('varchar', {
length: 1024, array: true, default: '{ "admin", "administrator", "root", "system", "maintainer", "host", "mod", "moderator", "owner", "superuser", "staff", "auth", "i", "me", "everyone", "all", "mention", "mentions", "example", "user", "users", "account", "accounts", "official", "help", "helps", "support", "supports", "info", "information", "informations", "announce", "announces", "announcement", "announcements", "notice", "notification", "notifications", "dev", "developer", "developers", "tech", "misskey" }',
length: 1024, array: true, default: '{admin,administrator,root,system,maintainer,host,mod,moderator,owner,superuser,staff,auth,i,me,everyone,all,mention,mentions,example,user,users,account,accounts,official,help,helps,support,supports,info,information,informations,announce,announces,announcement,announcements,notice,notification,notifications,dev,developer,developers,tech,misskey}',
})
public preservedUsernames: string[];
@ -658,22 +664,22 @@ export class MiMeta {
public enableFanoutTimelineDbFallback: boolean;
@Column('integer', {
default: 300,
default: 800,
})
public perLocalUserUserTimelineCacheMax: number;
@Column('integer', {
default: 100,
default: 800,
})
public perRemoteUserUserTimelineCacheMax: number;
@Column('integer', {
default: 300,
default: 800,
})
public perUserHomeTimelineCacheMax: number;
@Column('integer', {
default: 300,
default: 800,
})
public perUserListTimelineCacheMax: number;
@ -689,9 +695,9 @@ export class MiMeta {
@Column('varchar', {
length: 500,
nullable: true,
default: '❤️',
})
public defaultLike: string | null;
public defaultLike: string;
@Column('varchar', {
length: 256, array: true, default: '{}',
@ -714,7 +720,7 @@ export class MiMeta {
public urlPreviewMaximumContentLength: number;
@Column('boolean', {
default: true,
default: false,
})
public urlPreviewRequireContentLength: boolean;

View file

@ -20,7 +20,7 @@ import type { MiDriveFile } from './DriveFile.js';
// You should not use `@Index({ concurrent: true })` decorator because database initialization for test will fail
// because it will always run CREATE INDEX in transaction based on decorators.
// Not appending `{ concurrent: true }` to `@Index` will not cause any problem in production,
@Index(['userId', 'id'])
@Index('IDX_724b311e6f883751f261ebe378', ['userId', 'id'])
@Entity('note')
export class MiNote {
@PrimaryColumn(id())
@ -273,3 +273,7 @@ export type IMentionedRemoteUsers = {
username: string;
host: string;
}[];
export function hasText(note: MiNote): note is MiNote & { text: string } {
return note.text != null;
}

View file

@ -129,7 +129,9 @@ export class MiUser {
@OneToOne(() => MiDriveFile, {
onDelete: 'SET NULL',
})
@JoinColumn()
@JoinColumn({
foreignKeyConstraintName: 'FK_q5lm0tbgejtfskzg0rc4wd7t1n',
})
public background: MiDriveFile | null;
// avatarId が null になったとしてもこれが null でない可能性があるため、このフィールドを使うときは avatarId の non-null チェックをすること
@ -345,7 +347,7 @@ export class MiUser {
*/
@Column('boolean', {
name: 'enable_rss',
default: true,
default: false,
})
public enableRss: boolean;

View file

@ -24,7 +24,9 @@ export class MiUserListMembership {
@ManyToOne(type => MiUser, {
onDelete: 'CASCADE',
})
@JoinColumn()
@JoinColumn({
foreignKeyConstraintName: 'FK_d844bfc6f3f523a05189076efaa',
})
public user: MiUser | null;
@Index()
@ -37,7 +39,9 @@ export class MiUserListMembership {
@ManyToOne(type => MiUserList, {
onDelete: 'CASCADE',
})
@JoinColumn()
@JoinColumn({
foreignKeyConstraintName: 'FK_605472305f26818cc93d1baaa74',
})
public userList: MiUserList | null;
// タイムラインにその人のリプライまで含めるかどうか

View file

@ -34,6 +34,7 @@ export class MiUserPending {
@Column('varchar', {
length: 1000,
nullable: true,
})
public reason: string;
}

View file

@ -110,12 +110,14 @@ export class MiUserProfile {
@Column('enum', {
enum: followingVisibilities,
enumName: 'user_profile_followingVisibility_enum',
default: 'public',
})
public followingVisibility: typeof followingVisibilities[number];
@Column('enum', {
enum: followersVisibilities,
enumName: 'user_profile_followersVisibility_enum',
default: 'public',
})
public followersVisibility: typeof followersVisibilities[number];

View file

@ -309,6 +309,10 @@ export const packedRolePoliciesSchema = {
optional: false, nullable: false,
enum: ['available', 'readonly', 'unavailable'],
},
canTrend: {
type: 'boolean',
optional: false, nullable: false,
},
},
} as const;

View file

@ -4,9 +4,9 @@
*/
import { Inject, Injectable } from '@nestjs/common';
import { MoreThan } from 'typeorm';
import { In, MoreThan } from 'typeorm';
import { DI } from '@/di-symbols.js';
import type { DriveFilesRepository, NoteReactionsRepository, NotesRepository, UserProfilesRepository, UsersRepository, NoteScheduleRepository, MiNoteSchedule } from '@/models/_.js';
import type { DriveFilesRepository, NoteReactionsRepository, NotesRepository, UserProfilesRepository, UsersRepository, NoteScheduleRepository, MiNoteSchedule, FollowingsRepository, FollowRequestsRepository, BlockingsRepository, MutingsRepository, ClipsRepository, ClipNotesRepository, LatestNotesRepository, NoteEditRepository, NoteFavoritesRepository, PollVotesRepository, PollsRepository, SigninsRepository, UserIpsRepository, RegistryItemsRepository } from '@/models/_.js';
import type Logger from '@/logger.js';
import { DriveService } from '@/core/DriveService.js';
import type { MiDriveFile } from '@/models/DriveFile.js';
@ -17,10 +17,10 @@ import { bindThis } from '@/decorators.js';
import { SearchService } from '@/core/SearchService.js';
import { ApLogService } from '@/core/ApLogService.js';
import { ReactionService } from '@/core/ReactionService.js';
import { QueueService } from '@/core/QueueService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
import type { DbUserDeleteJobData } from '../types.js';
import { QueueService } from '@/core/QueueService.js';
@Injectable()
export class DeleteAccountProcessorService {
@ -45,6 +45,48 @@ export class DeleteAccountProcessorService {
@Inject(DI.noteScheduleRepository)
private noteScheduleRepository: NoteScheduleRepository,
@Inject(DI.followingsRepository)
private readonly followingsRepository: FollowingsRepository,
@Inject(DI.followRequestsRepository)
private readonly followRequestsRepository: FollowRequestsRepository,
@Inject(DI.blockingsRepository)
private readonly blockingsRepository: BlockingsRepository,
@Inject(DI.mutingsRepository)
private readonly mutingsRepository: MutingsRepository,
@Inject(DI.clipsRepository)
private readonly clipsRepository: ClipsRepository,
@Inject(DI.clipNotesRepository)
private readonly clipNotesRepository: ClipNotesRepository,
@Inject(DI.latestNotesRepository)
private readonly latestNotesRepository: LatestNotesRepository,
@Inject(DI.noteEditRepository)
private readonly noteEditRepository: NoteEditRepository,
@Inject(DI.noteFavoritesRepository)
private readonly noteFavoritesRepository: NoteFavoritesRepository,
@Inject(DI.pollVotesRepository)
private readonly pollVotesRepository: PollVotesRepository,
@Inject(DI.pollsRepository)
private readonly pollsRepository: PollsRepository,
@Inject(DI.signinsRepository)
private readonly signinsRepository: SigninsRepository,
@Inject(DI.userIpsRepository)
private readonly userIpsRepository: UserIpsRepository,
@Inject(DI.registryItemsRepository)
private readonly registryItemsRepository: RegistryItemsRepository,
private queueService: QueueService,
private driveService: DriveService,
private emailService: EmailService,
@ -65,6 +107,140 @@ export class DeleteAccountProcessorService {
return;
}
{ // Delete user clips
const userClips = await this.clipsRepository.find({
select: {
id: true,
},
where: {
userId: user.id,
},
}) as { id: string }[];
// Delete one-at-a-time because there can be a lot
for (const clip of userClips) {
await this.clipNotesRepository.delete({
id: clip.id,
});
}
await this.clipsRepository.delete({
userId: user.id,
});
this.logger.succ('All clips have been deleted.');
}
{ // Delete favorites
await this.noteFavoritesRepository.delete({
userId: user.id,
});
this.logger.succ('All favorites have been deleted.');
}
{ // Delete user relations
await this.followingsRepository.delete({
followerId: user.id,
});
await this.followingsRepository.delete({
followeeId: user.id,
});
await this.followRequestsRepository.delete({
followerId: user.id,
});
await this.followRequestsRepository.delete({
followeeId: user.id,
});
await this.blockingsRepository.delete({
blockerId: user.id,
});
await this.blockingsRepository.delete({
blockeeId: user.id,
});
await this.mutingsRepository.delete({
muterId: user.id,
});
await this.mutingsRepository.delete({
muteeId: user.id,
});
this.logger.succ('All user relations have been deleted.');
}
{ // Delete reactions
let cursor: MiNoteReaction['id'] | null = null;
while (true) {
const reactions = await this.noteReactionsRepository.find({
where: {
userId: user.id,
...(cursor ? { id: MoreThan(cursor) } : {}),
},
take: 100,
order: {
id: 1,
},
relations: {
note: true,
},
}) as MiNoteReaction[];
if (reactions.length === 0) {
break;
}
cursor = reactions.at(-1)?.id ?? null;
for (const reaction of reactions) {
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
const note = reaction.note!;
await this.reactionService.delete(user, note, reaction);
}
}
this.logger.succ('All reactions have been deleted');
}
{ // Poll votes
let cursor: MiNoteReaction['id'] | null = null;
while (true) {
const votes = await this.pollVotesRepository.find({
where: {
userId: user.id,
...(cursor ? { id: MoreThan(cursor) } : {}),
},
select: {
id: true,
},
take: 100,
order: {
id: 1,
},
}) as { id: string }[];
if (votes.length === 0) {
break;
}
cursor = votes.at(-1)?.id ?? null;
await this.pollVotesRepository.delete({
id: In(votes.map(v => v.id)),
});
}
this.logger.succ('All poll votes have been deleted');
}
{ // Delete scheduled notes
const scheduledNotes = await this.noteScheduleRepository.findBy({
userId: user.id,
@ -82,6 +258,10 @@ export class DeleteAccountProcessorService {
}
{ // Delete notes
await this.latestNotesRepository.delete({
userId: user.id,
});
let cursor: MiNote['id'] | null = null;
while (true) {
@ -102,7 +282,23 @@ export class DeleteAccountProcessorService {
cursor = notes.at(-1)?.id ?? null;
await this.notesRepository.delete(notes.map(note => note.id));
// Delete associated polls one-at-a-time, since it can cascade to a LOT of vote entries
for (const note of notes) {
if (note.hasPoll) {
await this.pollsRepository.delete({
noteId: note.id,
});
}
}
const ids = notes.map(note => note.id);
await this.noteEditRepository.delete({
noteId: In(ids),
});
await this.notesRepository.delete({
id: In(ids),
});
for (const note of notes) {
await this.searchService.unindexNote(note);
@ -119,37 +315,6 @@ export class DeleteAccountProcessorService {
this.logger.succ('All of notes deleted');
}
{ // Delete reactions
let cursor: MiNoteReaction['id'] | null = null;
while (true) {
const reactions = await this.noteReactionsRepository.find({
where: {
userId: user.id,
...(cursor ? { id: MoreThan(cursor) } : {}),
},
take: 100,
order: {
id: 1,
},
}) as MiNoteReaction[];
if (reactions.length === 0) {
break;
}
cursor = reactions.at(-1)?.id ?? null;
for (const reaction of reactions) {
const note = await this.notesRepository.findOneBy({ id: reaction.noteId }) as MiNote;
await this.reactionService.delete(user, note);
}
}
this.logger.succ('All reactions have been deleted');
}
{ // Delete files
let cursor: MiDriveFile['id'] | null = null;
@ -191,20 +356,42 @@ export class DeleteAccountProcessorService {
this.logger.succ('All AP logs deleted');
}
{ // Send email notification
const profile = await this.userProfilesRepository.findOneByOrFail({ userId: user.id });
if (profile.email && profile.emailVerified) {
this.emailService.sendEmail(profile.email, 'Account deleted',
'Your account has been deleted.',
'Your account has been deleted.');
// Do this BEFORE deleting the account!
const profile = await this.userProfilesRepository.findOneBy({ userId: user.id });
{ // Delete the actual account
await this.userIpsRepository.delete({
userId: user.id,
});
await this.signinsRepository.delete({
userId: user.id,
});
await this.registryItemsRepository.delete({
userId: user.id,
});
// soft指定されている場合は物理削除しない
if (job.data.soft) {
// nop
} else {
await this.usersRepository.delete(user.id);
}
this.logger.succ('Account data deleted');
}
// soft指定されている場合は物理削除しない
if (job.data.soft) {
// nop
} else {
await this.usersRepository.delete(job.data.user.id);
{ // Send email notification
if (profile && profile.email && profile.emailVerified) {
try {
await this.emailService.sendEmail(profile.email, 'Account deleted',
'Your account has been deleted.',
'Your account has been deleted.');
} catch (e) {
this.logger.warn('Failed to send account deletion message:', { e });
}
}
}
return 'Account deleted';

View file

@ -675,9 +675,11 @@ export class FileServerService {
if (info.blocked) {
reply.code(429);
reply.send({
message: 'Rate limit exceeded. Please try again later.',
code: 'RATE_LIMIT_EXCEEDED',
id: 'd5826d14-3982-4d2e-8011-b9e9f02499ef',
error: {
message: 'Rate limit exceeded. Please try again later.',
code: 'RATE_LIMIT_EXCEEDED',
id: 'd5826d14-3982-4d2e-8011-b9e9f02499ef',
},
});
return false;

View file

@ -445,6 +445,10 @@ export const meta = {
type: 'string',
optional: false, nullable: true,
},
translationTimeout: {
type: 'number',
optional: false, nullable: false,
},
deeplAuthKey: {
type: 'string',
optional: false, nullable: true,
@ -477,6 +481,10 @@ export const meta = {
type: 'string',
optional: false, nullable: true,
},
defaultLike: {
type: 'string',
optional: false, nullable: false,
},
description: {
type: 'string',
optional: false, nullable: true,
@ -741,6 +749,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
objectStorageUseProxy: instance.objectStorageUseProxy,
objectStorageSetPublicRead: instance.objectStorageSetPublicRead,
objectStorageS3ForcePathStyle: instance.objectStorageS3ForcePathStyle,
translationTimeout: instance.translationTimeout,
deeplAuthKey: instance.deeplAuthKey,
deeplIsPro: instance.deeplIsPro,
deeplFreeMode: instance.deeplFreeMode,

View file

@ -12,6 +12,7 @@ import { RoleEntityService } from '@/core/entities/RoleEntityService.js';
import { IdService } from '@/core/IdService.js';
import { notificationRecieveConfig } from '@/models/json-schema/user.js';
import { isSystemAccount } from '@/misc/is-system-account.js';
import { CacheService } from '@/core/CacheService.js';
export const meta = {
tags: ['admin'],
@ -186,6 +187,36 @@ export const meta = {
},
},
},
followStats: {
type: 'object',
optional: false, nullable: false,
properties: {
totalFollowing: {
type: 'number',
optional: false, nullable: false,
},
totalFollowers: {
type: 'number',
optional: false, nullable: false,
},
localFollowing: {
type: 'number',
optional: false, nullable: false,
},
localFollowers: {
type: 'number',
optional: false, nullable: false,
},
remoteFollowing: {
type: 'number',
optional: false, nullable: false,
},
remoteFollowers: {
type: 'number',
optional: false, nullable: false,
},
},
},
},
},
} as const;
@ -213,6 +244,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private roleService: RoleService,
private roleEntityService: RoleEntityService,
private idService: IdService,
private readonly cacheService: CacheService,
) {
super(meta, paramDef, async (ps, me) => {
const [user, profile] = await Promise.all([
@ -237,6 +269,8 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
const roleAssigns = await this.roleService.getUserAssigns(user.id);
const roles = await this.roleService.getUserRoles(user.id);
const followStats = await this.cacheService.getFollowStats(user.id);
return {
email: profile.email,
emailVerified: profile.emailVerified,
@ -269,6 +303,11 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
expiresAt: a.expiresAt ? a.expiresAt.toISOString() : null,
roleId: a.roleId,
})),
followStats: {
...followStats,
totalFollowers: Math.max(user.followersCount, followStats.localFollowers + followStats.remoteFollowers),
totalFollowing: Math.max(user.followingCount, followStats.localFollowing + followStats.remoteFollowing),
},
};
});
}

View file

@ -69,7 +69,7 @@ export const paramDef = {
description: { type: 'string', nullable: true },
defaultLightTheme: { type: 'string', nullable: true },
defaultDarkTheme: { type: 'string', nullable: true },
defaultLike: { type: 'string', nullable: true },
defaultLike: { type: 'string' },
cacheRemoteFiles: { type: 'boolean' },
cacheRemoteSensitiveFiles: { type: 'boolean' },
emailRequiredForSignup: { type: 'boolean' },
@ -103,6 +103,7 @@ export const paramDef = {
type: 'string',
},
},
translationTimeout: { type: 'number' },
deeplAuthKey: { type: 'string', nullable: true },
deeplIsPro: { type: 'boolean' },
deeplFreeMode: { type: 'boolean' },
@ -571,6 +572,10 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
set.objectStorageS3ForcePathStyle = ps.objectStorageS3ForcePathStyle;
}
if (ps.translationTimeout !== undefined) {
set.translationTimeout = ps.translationTimeout;
}
if (ps.deeplAuthKey !== undefined) {
if (ps.deeplAuthKey === '') {
set.deeplAuthKey = null;

View file

@ -98,7 +98,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
// ランキング更新
if (Date.now() - this.idService.parse(post.id).date.getTime() < GALLERY_POSTS_RANKING_WINDOW) {
await this.featuredService.updateGalleryPostsRanking(post.id, 1);
await this.featuredService.updateGalleryPostsRanking(post, 1);
}
this.galleryPostsRepository.increment({ id: post.id }, 'likedCount', 1);

View file

@ -81,7 +81,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
// ランキング更新
if (Date.now() - this.idService.parse(post.id).date.getTime() < GALLERY_POSTS_RANKING_WINDOW) {
await this.featuredService.updateGalleryPostsRanking(post.id, -1);
await this.featuredService.updateGalleryPostsRanking(post, -1);
}
this.galleryPostsRepository.decrement({ id: post.id }, 'likedCount', 1);

View file

@ -10,6 +10,7 @@ import { safeForSql } from "@/misc/safe-for-sql.js";
import { normalizeForSearch } from '@/misc/normalize-for-search.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { DI } from '@/di-symbols.js';
import { RoleService } from '@/core/RoleService.js';
export const meta = {
requireCredential: false,
@ -41,6 +42,7 @@ export const paramDef = {
sort: { type: 'string', enum: ['+follower', '-follower', '+createdAt', '-createdAt', '+updatedAt', '-updatedAt'] },
state: { type: 'string', enum: ['all', 'alive'], default: 'all' },
origin: { type: 'string', enum: ['combined', 'local', 'remote'], default: 'local' },
trending: { type: 'boolean', default: false },
},
required: ['tag', 'sort'],
} as const;
@ -52,6 +54,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private usersRepository: UsersRepository,
private userEntityService: UserEntityService,
private readonly roleService: RoleService,
) {
super(meta, paramDef, async (ps, me) => {
if (!safeForSql(normalizeForSearch(ps.tag))) throw new Error('Injection');
@ -80,7 +83,18 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
case '-updatedAt': query.orderBy('user.updatedAt', 'ASC'); break;
}
const users = await query.limit(ps.limit).getMany();
let users = await query.limit(ps.limit).getMany();
// This is not ideal, for a couple of reasons:
// 1. It may return less than "limit" results.
// 2. A span of more than "limit" consecutive non-trendable users may cause the pagination to stop early.
// Unfortunately, there's no better solution unless we refactor role policies to be persisted to the DB.
if (ps.trending) {
const usersWithRoles = await Promise.all(users.map(async u => [u, await this.roleService.getUserPolicies(u)] as const));
users = usersWithRoles
.filter(([,p]) => p.canTrend)
.map(([u]) => u);
}
return await this.userEntityService.packMany(users, me, { schema: 'UserDetailed' });
});

View file

@ -117,7 +117,8 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
.leftJoinAndSelect('note.renote', 'renote')
.leftJoinAndSelect('reply.user', 'replyUser')
.leftJoinAndSelect('renote.user', 'renoteUser')
.leftJoinAndSelect('note.channel', 'channel');
.leftJoinAndSelect('note.channel', 'channel')
.andWhere('user.isExplorable = TRUE');
this.queryService.generateBlockedHostQueryForNote(query);
this.queryService.generateSuspendedUserQueryForNote(query);

View file

@ -10,22 +10,28 @@ import { NoteEntityService } from '@/core/entities/NoteEntityService.js';
import { HttpRequestService } from '@/core/HttpRequestService.js';
import { GetterService } from '@/server/api/GetterService.js';
import { RoleService } from '@/core/RoleService.js';
import { ApiError } from '../../error.js';
import { MiMeta } from '@/models/_.js';
import type { MiMeta, MiNote } from '@/models/_.js';
import { DI } from '@/di-symbols.js';
import { CacheService } from '@/core/CacheService.js';
import { hasText } from '@/models/Note.js';
import { ApiLoggerService } from '@/server/api/ApiLoggerService.js';
import { ApiError } from '../../error.js';
export const meta = {
tags: ['notes'],
// TODO allow unauthenticated if default template allows?
// Maybe a value 'optional' that allows unauthenticated OR a token w/ appropriate role.
// This will allow unauthenticated requests without leaking post data to restricted clients.
requireCredential: true,
kind: 'read:account',
res: {
type: 'object',
optional: true, nullable: false,
optional: false, nullable: false,
properties: {
sourceLang: { type: 'string' },
text: { type: 'string' },
sourceLang: { type: 'string', optional: true, nullable: false },
text: { type: 'string', optional: true, nullable: false },
},
},
@ -45,6 +51,11 @@ export const meta = {
code: 'CANNOT_TRANSLATE_INVISIBLE_NOTE',
id: 'ea29f2ca-c368-43b3-aaf1-5ac3e74bbe5d',
},
translationFailed: {
message: 'Failed to translate note. Please try again later or contact an administrator for assistance.',
code: 'TRANSLATION_FAILED',
id: '4e7a1a4f-521c-4ba2-b10a-69e5e2987b2f',
},
},
// 10 calls per 5 seconds
@ -73,6 +84,8 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private getterService: GetterService,
private httpRequestService: HttpRequestService,
private roleService: RoleService,
private readonly cacheService: CacheService,
private readonly loggerService: ApiLoggerService,
) {
super(meta, paramDef, async (ps, me) => {
const policies = await this.roleService.getUserPolicies(me.id);
@ -89,8 +102,8 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
throw new ApiError(meta.errors.cannotTranslateInvisibleNote);
}
if (note.text == null) {
return;
if (!hasText(note)) {
return {};
}
const canDeeplFree = this.serverSettings.deeplFreeMode && !!this.serverSettings.deeplFreeInstance;
@ -101,13 +114,33 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
let targetLang = ps.targetLang;
if (targetLang.includes('-')) targetLang = targetLang.split('-')[0];
let response = await this.cacheService.getCachedTranslation(note, targetLang);
if (!response) {
this.loggerService.logger.debug(`Fetching new translation for note=${note.id} lang=${targetLang}`);
response = await this.fetchTranslation(note, targetLang);
if (!response) {
throw new ApiError(meta.errors.translationFailed);
}
await this.cacheService.setCachedTranslation(note, targetLang, response);
}
return response;
});
}
private async fetchTranslation(note: MiNote & { text: string }, targetLang: string) {
// Load-bearing try/catch - removing this will shift indentation and cause ~80 lines of upstream merge conflicts
try {
// Ignore deeplFreeInstance unless deeplFreeMode is set
const deeplFreeInstance = this.serverSettings.deeplFreeMode ? this.serverSettings.deeplFreeInstance : null;
// DeepL/DeepLX handling
if (canDeepl) {
if (this.serverSettings.deeplAuthKey || deeplFreeInstance) {
const params = new URLSearchParams();
if (this.serverSettings.deeplAuthKey) params.append('auth_key', this.serverSettings.deeplAuthKey);
params.append('text', note.text);
params.append('target_lang', targetLang);
const endpoint = canDeeplFree ? this.serverSettings.deeplFreeInstance as string : this.serverSettings.deeplIsPro ? 'https://api.deepl.com/v2/translate' : 'https://api-free.deepl.com/v2/translate';
const endpoint = deeplFreeInstance ?? this.serverSettings.deeplIsPro ? 'https://api.deepl.com/v2/translate' : 'https://api-free.deepl.com/v2/translate';
const res = await this.httpRequestService.send(endpoint, {
method: 'POST',
@ -116,6 +149,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
Accept: 'application/json, */*',
},
body: params.toString(),
timeout: this.serverSettings.translationTimeout,
});
if (this.serverSettings.deeplAuthKey) {
const json = (await res.json()) as {
@ -151,8 +185,8 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
}
// LibreTranslate handling
if (canLibre) {
const res = await this.httpRequestService.send(this.serverSettings.libreTranslateURL as string, {
if (this.serverSettings.libreTranslateURL) {
const res = await this.httpRequestService.send(this.serverSettings.libreTranslateURL, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
@ -165,6 +199,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
format: 'text',
api_key: this.serverSettings.libreTranslateKey ?? '',
}),
timeout: this.serverSettings.translationTimeout,
});
const json = (await res.json()) as {
@ -182,8 +217,10 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
text: json.translatedText,
};
}
} catch (e) {
this.loggerService.logger.error('Unhandled error from translation API: ', { e });
}
return;
});
return null;
}
}

View file

@ -4,11 +4,14 @@
*/
import { Inject, Injectable } from '@nestjs/common';
import type { UsersRepository } from '@/models/_.js';
import { MiFollowing } from '@/models/_.js';
import type { MiUser, UsersRepository } from '@/models/_.js';
import { Endpoint } from '@/server/api/endpoint-base.js';
import { QueryService } from '@/core/QueryService.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { DI } from '@/di-symbols.js';
import { RoleService } from '@/core/RoleService.js';
import type { SelectQueryBuilder } from 'typeorm';
export const meta = {
tags: ['users'],
@ -38,7 +41,7 @@ export const paramDef = {
properties: {
limit: { type: 'integer', minimum: 1, maximum: 100, default: 10 },
offset: { type: 'integer', default: 0 },
sort: { type: 'string', enum: ['+follower', '-follower', '+createdAt', '-createdAt', '+updatedAt', '-updatedAt'] },
sort: { type: 'string', enum: ['+follower', '-follower', '+localFollower', '-localFollower', '+createdAt', '-createdAt', '+updatedAt', '-updatedAt'] },
state: { type: 'string', enum: ['all', 'alive'], default: 'all' },
origin: { type: 'string', enum: ['combined', 'local', 'remote'], default: 'local' },
hostname: {
@ -59,6 +62,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private userEntityService: UserEntityService,
private queryService: QueryService,
private readonly roleService: RoleService,
) {
super(meta, paramDef, async (ps, me) => {
const query = this.usersRepository.createQueryBuilder('user')
@ -81,6 +85,8 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
switch (ps.sort) {
case '+follower': query.orderBy('user.followersCount', 'DESC'); break;
case '-follower': query.orderBy('user.followersCount', 'ASC'); break;
case '+localFollower': this.addLocalFollowers(query); query.orderBy('f."localFollowers"', 'DESC'); break;
case '-localFollower': this.addLocalFollowers(query); query.orderBy('f."localFollowers"', 'ASC'); break;
case '+createdAt': query.orderBy('user.id', 'DESC'); break;
case '-createdAt': query.orderBy('user.id', 'ASC'); break;
case '+updatedAt': query.andWhere('user.updatedAt IS NOT NULL').orderBy('user.updatedAt', 'DESC'); break;
@ -94,9 +100,29 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
query.limit(ps.limit);
query.offset(ps.offset);
const users = await query.getMany();
const allUsers = await query.getMany();
// This is not ideal, for a couple of reasons:
// 1. It may return less than "limit" results.
// 2. A span of more than "limit" consecutive non-trendable users may cause the pagination to stop early.
// Unfortunately, there's no better solution unless we refactor role policies to be persisted to the DB.
const usersWithRoles = await Promise.all(allUsers.map(async u => [u, await this.roleService.getUserPolicies(u)] as const));
const users = usersWithRoles
.filter(([,p]) => p.canTrend)
.map(([u]) => u);
return await this.userEntityService.packMany(users, me, { schema: 'UserDetailed' });
});
}
private addLocalFollowers(query: SelectQueryBuilder<MiUser>) {
query.innerJoin(qb => {
return qb
.from(MiFollowing, 'f')
.addSelect('f."followeeId"')
.addSelect('COUNT(*) FILTER (where f."followerHost" IS NULL)', 'localFollowers')
.addSelect('COUNT(*) FILTER (where f."followeeHost" IS NOT NULL)', 'remoteFollowers')
.groupBy('"followeeId"');
}, 'f', 'user.id = f."followeeId"');
}
}

View file

@ -890,6 +890,7 @@ export class ClientServerService {
return await reply.view('info-card', {
version: this.config.version,
host: this.config.host,
url: this.config.url,
meta: this.meta,
originalUsersCount: await this.usersRepository.countBy({ host: IsNull() }),
originalNotesCount: await this.notesRepository.countBy({ userHost: IsNull() }),

View file

@ -15,15 +15,21 @@ import type Logger from '@/logger.js';
import { query } from '@/misc/prelude/url.js';
import { LoggerService } from '@/core/LoggerService.js';
import { bindThis } from '@/decorators.js';
import { ApiError } from '@/server/api/error.js';
import { MiMeta } from '@/models/Meta.js';
import { RedisKVCache } from '@/misc/cache.js';
import { UtilityService } from '@/core/UtilityService.js';
import { ApDbResolverService } from '@/core/activitypub/ApDbResolverService.js';
import type { NotesRepository } from '@/models/_.js';
import type { MiAccessToken, NotesRepository } from '@/models/_.js';
import { ApUtilityService } from '@/core/activitypub/ApUtilityService.js';
import { ApRequestService } from '@/core/activitypub/ApRequestService.js';
import { SystemAccountService } from '@/core/SystemAccountService.js';
import { ApNoteService } from '@/core/activitypub/models/ApNoteService.js';
import { AuthenticateService, AuthenticationError } from '@/server/api/AuthenticateService.js';
import { SkRateLimiterService } from '@/server/SkRateLimiterService.js';
import { BucketRateLimit, Keyed, sendRateLimitHeaders } from '@/misc/rate-limit-utils.js';
import type { MiLocalUser } from '@/models/User.js';
import { getIpHash } from '@/misc/get-ip-hash.js';
import { isRetryableError } from '@/misc/is-retryable-error.js';
import type { FastifyRequest, FastifyReply } from 'fastify';
export type LocalSummalyResult = SummalyResult & {
@ -31,7 +37,27 @@ export type LocalSummalyResult = SummalyResult & {
};
// Increment this to invalidate cached previews after a major change.
const cacheFormatVersion = 2;
const cacheFormatVersion = 3;
type PreviewRoute = {
Querystring: {
url?: string
lang?: string,
fetch?: string,
i?: string,
},
};
type AuthArray = [user: MiLocalUser | null | undefined, app: MiAccessToken | null | undefined, actor: MiLocalUser | string];
// Up to 50 requests, then 10 / second (at 2 / 200ms rate)
const previewLimit: Keyed<BucketRateLimit> = {
key: '/url',
type: 'bucket',
size: 50,
dripSize: 2,
dripRate: 200,
};
@Injectable()
export class UrlPreviewService {
@ -58,6 +84,9 @@ export class UrlPreviewService {
private readonly apDbResolverService: ApDbResolverService,
private readonly apRequestService: ApRequestService,
private readonly systemAccountService: SystemAccountService,
private readonly apNoteService: ApNoteService,
private readonly authenticateService: AuthenticateService,
private readonly rateLimiterService: SkRateLimiterService,
) {
this.logger = this.loggerService.getLogger('url-preview');
this.previewCache = new RedisKVCache<LocalSummalyResult>(this.redisClient, 'summaly', {
@ -85,9 +114,9 @@ export class UrlPreviewService {
@bindThis
public async handle(
request: FastifyRequest<{ Querystring: { url?: string; lang?: string; } }>,
request: FastifyRequest<PreviewRoute>,
reply: FastifyReply,
): Promise<object | undefined> {
): Promise<void> {
const url = request.query.url;
if (typeof url !== 'string' || !URL.canParse(url)) {
reply.code(400);
@ -101,38 +130,39 @@ export class UrlPreviewService {
}
if (!this.meta.urlPreviewEnabled) {
reply.code(403);
return {
error: new ApiError({
return reply.code(403).send({
error: {
message: 'URL preview is disabled',
code: 'URL_PREVIEW_DISABLED',
id: '58b36e13-d2f5-0323-b0c6-76aa9dabefb8',
}),
};
},
});
}
// Check rate limit
const auth = await this.authenticate(request);
if (!await this.checkRateLimit(auth, reply)) {
return;
}
if (this.utilityService.isBlockedHost(this.meta.blockedHosts, new URL(url).host)) {
reply.code(403);
return {
error: new ApiError({
return reply.code(403).send({
error: {
message: 'URL is blocked',
code: 'URL_PREVIEW_BLOCKED',
id: '50294652-857b-4b13-9700-8e5c7a8deae8',
}),
};
},
});
}
const fetch = !!request.query.fetch;
if (fetch && !await this.checkFetchPermissions(auth, reply)) {
return;
}
const cacheKey = `${url}@${lang}@${cacheFormatVersion}`;
const cached = await this.previewCache.get(cacheKey);
if (cached !== undefined) {
// Cache 1 day (matching redis)
reply.header('Cache-Control', 'public, max-age=86400');
if (cached.activityPub) {
cached.haveNoteLocally = !! await this.apDbResolverService.getNoteFromApId(cached.activityPub);
}
return cached;
if (await this.sendCachedPreview(cacheKey, reply, fetch)) {
return;
}
try {
@ -144,14 +174,13 @@ export class UrlPreviewService {
// Repeat check, since redirects are allowed.
if (this.utilityService.isBlockedHost(this.meta.blockedHosts, new URL(summary.url).host)) {
reply.code(403);
return {
error: new ApiError({
return reply.code(403).send({
error: {
message: 'URL is blocked',
code: 'URL_PREVIEW_BLOCKED',
id: '50294652-857b-4b13-9700-8e5c7a8deae8',
}),
};
},
});
}
this.logger.info(`Got preview of ${url} in ${lang}: ${summary.title}`);
@ -164,33 +193,76 @@ export class UrlPreviewService {
await this.inferActivityPubLink(summary);
}
if (summary.activityPub) {
if (summary.activityPub && !summary.haveNoteLocally) {
// Avoid duplicate checks in case inferActivityPubLink already set this.
summary.haveNoteLocally ||= !!await this.apDbResolverService.getNoteFromApId(summary.activityPub);
const exists = await this.noteExists(summary.activityPub, fetch);
// Remove the AP flag if we encounter a permanent error fetching the note.
if (exists === false) {
summary.activityPub = null;
summary.haveNoteLocally = undefined;
} else {
summary.haveNoteLocally = exists ?? false;
}
}
// Await this to avoid hammering redis when a bunch of URLs are fetched at once
await this.previewCache.set(cacheKey, summary);
// Cache 1 day (matching redis)
reply.header('Cache-Control', 'public, max-age=86400');
// Cache 1 day (matching redis), but only once we finalize the result
if (!summary.activityPub || summary.haveNoteLocally) {
reply.header('Cache-Control', 'public, max-age=86400');
}
return summary;
return reply.code(200).send(summary);
} catch (err) {
this.logger.warn(`Failed to get preview of ${url} for ${lang}: ${err}`);
reply.code(422);
reply.header('Cache-Control', 'max-age=3600');
return {
error: new ApiError({
return reply.code(422).send({
error: {
message: 'Failed to get preview',
code: 'URL_PREVIEW_FAILED',
id: '09d01cb5-53b9-4856-82e5-38a50c290a3b',
}),
};
},
});
}
}
private async sendCachedPreview(cacheKey: string, reply: FastifyReply, fetch: boolean): Promise<boolean> {
const summary = await this.previewCache.get(cacheKey);
if (summary === undefined) {
return false;
}
// Check if note has loaded since we last cached the preview
if (summary.activityPub && !summary.haveNoteLocally) {
// Avoid duplicate checks in case inferActivityPubLink already set this.
const exists = await this.noteExists(summary.activityPub, fetch);
// Remove the AP flag if we encounter a permanent error fetching the note.
if (exists === false) {
summary.activityPub = null;
summary.haveNoteLocally = undefined;
} else {
summary.haveNoteLocally = exists ?? false;
}
// Persist the result once we finalize the result
if (!summary.activityPub || summary.haveNoteLocally) {
await this.previewCache.set(cacheKey, summary);
}
}
// Cache 1 day (matching redis), but only once we finalize the result
if (!summary.activityPub || summary.haveNoteLocally) {
reply.header('Cache-Control', 'public, max-age=86400');
}
reply.code(200).send(summary);
return true;
}
private fetchSummary(url: string, meta: MiMeta, lang?: string): Promise<SummalyResult> {
const agent = this.config.proxy
? {
@ -211,6 +283,7 @@ export class UrlPreviewService {
}
private fetchSummaryFromProxy(url: string, meta: MiMeta, lang?: string): Promise<SummalyResult> {
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
const proxy = meta.urlPreviewSummaryProxyUrl!;
const queryStr = query({
followRedirects: true,
@ -302,4 +375,129 @@ export class UrlPreviewService {
return;
}
}
// true = exists, false = does not exist (permanently), null = does not exist (temporarily)
private async noteExists(uri: string, fetch = false): Promise<boolean | null> {
try {
// Local note or cached remote note
if (await this.apDbResolverService.getNoteFromApId(uri)) {
return true;
}
// Un-cached remote note
if (!fetch) {
return null;
}
// Newly cached remote note
if (await this.apNoteService.resolveNote(uri)) {
return true;
}
// Non-existent or deleted note
return false;
} catch (err) {
// Errors, including invalid notes and network errors
return isRetryableError(err) ? null : false;
}
}
// Adapted from ApiCallService
private async authenticate(request: FastifyRequest<{ Querystring?: { i?: string | string[] }, Body?: { i?: string | string[] } }>): Promise<AuthArray> {
const body = request.method === 'GET' ? request.query : request.body;
// https://datatracker.ietf.org/doc/html/rfc6750.html#section-2.1 (case sensitive)
const token = request.headers.authorization?.startsWith('Bearer ')
? request.headers.authorization.slice(7)
: body?.['i'];
if (token != null && typeof token !== 'string') {
return [undefined, undefined, getIpHash(request.ip)];
}
try {
const auth = await this.authenticateService.authenticate(token);
return [auth[0], auth[1], auth[0] ?? getIpHash(request.ip)];
} catch (err) {
if (err instanceof AuthenticationError) {
return [undefined, undefined, getIpHash(request.ip)];
} else {
throw err;
}
}
}
// Adapted from ApiCallService
private async checkFetchPermissions(auth: AuthArray, reply: FastifyReply): Promise<boolean> {
const [user, app] = auth;
// Authentication
if (user === undefined) {
reply.code(401).send({
error: {
message: 'Authentication failed. Please ensure your token is correct.',
code: 'AUTHENTICATION_FAILED',
id: 'b0a7f5f8-dc2f-4171-b91f-de88ad238e14',
},
});
return false;
}
if (user === null) {
reply.code(401).send({
error: {
message: 'Credential required.',
code: 'CREDENTIAL_REQUIRED',
id: '1384574d-a912-4b81-8601-c7b1c4085df1',
},
});
return false;
}
// Authorization
if (user.isSuspended || user.isDeleted) {
reply.code(403).send({
error: {
message: 'Your account has been suspended.',
code: 'YOUR_ACCOUNT_SUSPENDED',
kind: 'permission',
id: 'a8c724b3-6e9c-4b46-b1a8-bc3ed6258370',
},
});
return false;
}
if (app && !app.permission.includes('read:account')) {
reply.code(403).send({
error: {
message: 'Your app does not have the necessary permissions to use this endpoint.',
code: 'PERMISSION_DENIED',
kind: 'permission',
id: '1370e5b7-d4eb-4566-bb1d-7748ee6a1838',
},
});
return false;
}
return true;
}
private async checkRateLimit(auth: AuthArray, reply: FastifyReply): Promise<boolean> {
const info = await this.rateLimiterService.limit(previewLimit, auth[2]);
// Always send headers, even if not blocked
sendRateLimitHeaders(reply, info);
if (info.blocked) {
reply.code(429).send({
error: {
message: 'Rate limit exceeded. Please try again later.',
code: 'RATE_LIMIT_EXCEEDED',
id: 'd5826d14-3982-4d2e-8011-b9e9f02499ef',
},
});
return false;
}
return true;
}
}

View file

@ -43,7 +43,7 @@ html
}
body
a#a(href=`https://${host}` target="_blank")
a#a(href=url target="_blank")
header#banner(style=`background-image: url(${meta.bannerUrl})`)
div#title= meta.name || host
div#content