Merge branch 'develop' into 'feature/dialog-announcement-cap'

# Conflicts:
#   packages/backend/src/core/AnnouncementService.ts
#   packages/backend/src/server/api/endpoints/admin/announcements/create.ts
#   packages/backend/src/server/api/endpoints/admin/announcements/update.ts

packages/backend/src/boot/entry.ts

@@ -55,44 +55,42 @@ async function main() {
 	// Display detail of unhandled promise rejection
 	if (!envOption.quiet) {
 		process.on('unhandledRejection', e => {
-			try {
-				logger.error('Unhandled rejection:', inspect(e));
-			} catch {
-				console.error('Unhandled rejection:', inspect(e));
-			}
+			logger.error('Unhandled rejection:', inspect(e));
 		});
 	}
 
-	// Display detail of uncaught exception
-	process.on('uncaughtExceptionMonitor', ((err, origin) => {
-		try {
-			logger.error(`Uncaught exception (${origin}):`, err);
-		} catch {
-			console.error(`Uncaught exception (${origin}):`, err);
+	process.on('uncaughtException', (err) => {
+		// Workaround for https://github.com/node-fetch/node-fetch/issues/954
+		if (String(err).match(/^TypeError: .+ is an? url with embedded credentials.$/)) {
+			logger.debug('Suppressed node-fetch issue#954, but the current job may fail.');
+			return;
 		}
-	}));
+
+		// Workaround for https://github.com/node-fetch/node-fetch/issues/1845
+		if (String(err) === 'TypeError: Cannot read properties of undefined (reading \'body\')') {
+			logger.debug('Suppressed node-fetch issue#1845, but the current job may fail.');
+			return;
+		}
+
+		// Throw all other errors to avoid inconsistent state.
+		// (per NodeJS docs, it's unsafe to suppress arbitrary errors in an uncaughtException handler.)
+		throw err;
+	});
+
+	// Display detail of uncaught exception
+	process.on('uncaughtExceptionMonitor', (err, origin) => {
+		logger.error(`Uncaught exception (${origin}):`, err);
+	});
 
 	// Dying away...
 	process.on('disconnect', () => {
-		try {
-			logger.warn('IPC channel disconnected! The process may soon die.');
-		} catch {
-			console.warn('IPC channel disconnected! The process may soon die.');
-		}
+		logger.warn('IPC channel disconnected! The process may soon die.');
 	});
 	process.on('beforeExit', code => {
-		try {
-			logger.warn(`Event loop died! Process will exit with code ${code}.`);
-		} catch {
-			console.warn(`Event loop died! Process will exit with code ${code}.`);
-		}
+		logger.warn(`Event loop died! Process will exit with code ${code}.`);
 	});
 	process.on('exit', code => {
-		try {
-			logger.info(`The process is going to exit with code ${code}`);
-		} catch {
-			console.info(`The process is going to exit with code ${code}`);
-		}
+		logger.info(`The process is going to exit with code ${code}`);
 	});
 	//#endregion
 

packages/backend/src/boot/master.ts

@@ -51,7 +51,7 @@ function greet() {
 	}
 
 	bootLogger.info('Welcome to Sharkey!');
-	bootLogger.info(`Sharkey v${meta.version}`, null, true);
+	bootLogger.info(`Sharkey v${meta.gitVersion ?? meta.version}`, null, true);
 }
 
 /**
@@ -91,7 +91,7 @@ export async function masterMain() {
 			maxBreadcrumbs: 0,
 
 			// Set release version
-			release: 'Sharkey@' + meta.version,
+			release: 'Sharkey@' + (meta.gitVersion ?? meta.version),
 
 			...config.sentryForBackend.options,
 		});

packages/backend/src/boot/worker.ts

@@ -37,7 +37,7 @@ export async function workerMain() {
 			maxBreadcrumbs: 0,
 
 			// Set release version
-			release: "Sharkey@" + meta.version,
+			release: "Sharkey@" + (meta.gitVersion ?? meta.version),
 
 			...config.sentryForBackend.options,
 		});

packages/backend/src/config.ts

@@ -403,7 +403,7 @@ export function loadConfig(): Config {
 	applyEnvOverrides(config);
 
 	const url = tryCreateUrl(config.url ?? process.env.MISSKEY_URL ?? '');
-	const version = meta.version;
+	const version = meta.gitVersion ?? meta.version;
 	const host = url.host;
 	const hostname = url.hostname;
 	const scheme = url.protocol.replace(/:$/, '');

packages/backend/src/core/AbuseReportService.ts

@@ -14,6 +14,7 @@ import { ApRendererService } from '@/core/activitypub/ApRendererService.js';
 import { ModerationLogService } from '@/core/ModerationLogService.js';
 import { SystemAccountService } from '@/core/SystemAccountService.js';
 import { IdentifiableError } from '@/misc/identifiable-error.js';
+import { trackPromise } from '@/misc/promise-tracker.js';
 import { IdService } from './IdService.js';
 
 @Injectable()
@@ -68,11 +69,11 @@ export class AbuseReportService {
 			reports.push(report);
 		}
 
-		return Promise.all([
+		trackPromise(Promise.all([
 			this.abuseReportNotificationService.notifyAdminStream(reports),
 			this.abuseReportNotificationService.notifySystemWebhook(reports, 'abuseReport'),
 			this.abuseReportNotificationService.notifyMail(reports),
-		]);
+		]));
 	}
 
 	/**

packages/backend/src/core/AnnouncementService.ts

@@ -16,6 +16,7 @@ import { GlobalEventService } from '@/core/GlobalEventService.js';
 import { ModerationLogService } from '@/core/ModerationLogService.js';
 import { IdentifiableError } from '@/misc/identifiable-error.js';
 import type { Config } from '@/config.js';
+import { RoleService } from '@/core/RoleService.js';
 
 @Injectable()
 export class AnnouncementService {
@@ -36,6 +37,7 @@ export class AnnouncementService {
 		private globalEventService: GlobalEventService,
 		private moderationLogService: ModerationLogService,
 		private announcementEntityService: AnnouncementEntityService,
+		private roleService: RoleService,
 	) {
 	}
 
@@ -51,6 +53,7 @@ export class AnnouncementService {
 		const readsQuery = this.announcementReadsRepository.createQueryBuilder('read')
 			.select('read.announcementId')
 			.where('read.userId = :userId', { userId: user.id });
+		const roles = await this.roleService.getUserRoles(user);
 
 		const q = this.announcementsRepository.createQueryBuilder('announcement')
 			.where('announcement.isActive = true')
@@ -63,6 +66,10 @@ export class AnnouncementService {
 				qb.orWhere('announcement.forExistingUsers = false');
 				qb.orWhere('announcement.id > :userId', { userId: user.id });
 			}))
+			.andWhere(new Brackets(qb => {
+				qb.orWhere('announcement.forRoles && :roles', { roles: roles.map((r) => r.id) });
+				qb.orWhere('announcement.forRoles = \'{}\'');
+			}))
 			.andWhere(`announcement.id NOT IN (${ readsQuery.getQuery() })`);
 
 		q.setParameters(readsQuery.getParameters());
@@ -85,6 +92,7 @@ export class AnnouncementService {
 			icon: values.icon,
 			display: values.display,
 			forExistingUsers: values.forExistingUsers,
+			forRoles: values.forRoles,
 			silence: values.silence,
 			needConfirmationToRead: values.needConfirmationToRead,
 			confetti: values.confetti,
@@ -143,6 +151,7 @@ export class AnnouncementService {
 			display: values.display,
 			icon: values.icon,
 			forExistingUsers: values.forExistingUsers,
+			forRoles: values.forRoles,
 			silence: values.silence,
 			needConfirmationToRead: values.needConfirmationToRead,
 			confetti: values.confetti,

packages/backend/src/core/CacheService.ts

@@ -9,7 +9,7 @@ import { In, IsNull } from 'typeorm';
 import type { BlockingsRepository, FollowingsRepository, MutingsRepository, RenoteMutingsRepository, MiUserProfile, UserProfilesRepository, UsersRepository, MiNote, MiFollowing, NoteThreadMutingsRepository } from '@/models/_.js';
 import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js';
 import { QuantumKVCache } from '@/misc/QuantumKVCache.js';
-import type { MiLocalUser, MiUser } from '@/models/User.js';
+import type { MiLocalUser, MiRemoteUser, MiUser } from '@/models/User.js';
 import { DI } from '@/di-symbols.js';
 import { UserEntityService } from '@/core/entities/UserEntityService.js';
 import { bindThis } from '@/decorators.js';
@@ -387,6 +387,22 @@ export class CacheService implements OnApplicationShutdown {
 		}) ?? null;
 	}
 
+	@bindThis
+	public async findRemoteUserById(userId: MiUser['id']): Promise<MiRemoteUser | null> {
+		const user = await this.findUserById(userId);
+
+		if (user.host == null) {
+			return null;
+		}
+
+		return user as MiRemoteUser;
+	}
+
+	@bindThis
+	public findOptionalUserById(userId: MiUser['id']) {
+		return this.userByIdCache.fetchMaybe(userId, async () => await this.usersRepository.findOneBy({ id: userId }) ?? undefined);
+	}
+
 	@bindThis
 	public async getFollowStats(userId: MiUser['id']): Promise<FollowStats> {
 		return await this.userFollowStatsCache.fetch(userId, async () => {

packages/backend/src/core/CaptchaService.ts

@@ -9,7 +9,10 @@ import { bindThis } from '@/decorators.js';
 import { MetaService } from '@/core/MetaService.js';
 import { MiMeta } from '@/models/Meta.js';
 import Logger from '@/logger.js';
-import { LoggerService } from './LoggerService.js';
+import { LoggerService } from '@/core/LoggerService.js';
+import { CaptchaError } from '@/misc/captcha-error.js';
+
+export { CaptchaError } from '@/misc/captcha-error.js';
 
 export const supportedCaptchaProviders = ['none', 'hcaptcha', 'mcaptcha', 'recaptcha', 'turnstile', 'fc', 'testcaptcha'] as const;
 export type CaptchaProvider = typeof supportedCaptchaProviders[number];
@@ -49,18 +52,6 @@ export type CaptchaSetting = {
 	}
 };
 
-export class CaptchaError extends Error {
-	public readonly code: CaptchaErrorCode;
-	public readonly cause?: unknown;
-
-	constructor(code: CaptchaErrorCode, message: string, cause?: unknown) {
-		super(message, cause ? { cause } : undefined);
-		this.code = code;
-		this.cause = cause;
-		this.name = 'CaptchaError';
-	}
-}
-
 export type CaptchaSaveSuccess = {
 	success: true;
 };

packages/backend/src/core/ChatService.ts

@@ -580,11 +580,20 @@ export class ChatService {
 	public async deleteRoom(room: MiChatRoom, deleter?: MiUser) {
 		await this.chatRoomsRepository.delete(room.id);
 
+		// Erase any message notifications for this room
+		const redisPipeline = this.redisClient.pipeline();
+		const memberships = await this.chatRoomMembershipsRepository.findBy({ roomId: room.id });
+		for (const membership of memberships) {
+			redisPipeline.del(`newRoomChatMessageExists:${membership.userId}:${room.id}`);
+			redisPipeline.srem(`newChatMessagesExists:${membership.userId}`, `room:${room.id}`);
+		}
+		await redisPipeline.exec();
+
 		if (deleter) {
 			const deleterIsModerator = await this.roleService.isModerator(deleter);
 
 			if (deleterIsModerator) {
-				this.moderationLogService.log(deleter, 'deleteChatRoom', {
+				await this.moderationLogService.log(deleter, 'deleteChatRoom', {
 					roomId: room.id,
 					room: room,
 				});

packages/backend/src/core/DownloadService.ts

@@ -19,6 +19,7 @@ import type Logger from '@/logger.js';
 
 import { bindThis } from '@/decorators.js';
 import { renderInlineError } from '@/misc/render-inline-error.js';
+import { UtilityService } from '@/core/UtilityService.js';
 
 @Injectable()
 export class DownloadService {
@@ -30,6 +31,7 @@ export class DownloadService {
 
 		private httpRequestService: HttpRequestService,
 		private loggerService: LoggerService,
+		private readonly utilityService: UtilityService,
 	) {
 		this.logger = this.loggerService.getLogger('download');
 	}
@@ -38,6 +40,8 @@ export class DownloadService {
 	public async downloadUrl(url: string, path: string, options: { timeout?: number, operationTimeout?: number, maxSize?: number } = {} ): Promise<{
 		filename: string;
 	}> {
+		this.utilityService.assertUrl(url);
+
 		this.logger.debug(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`);
 
 		const timeout = options.timeout ?? 30 * 1000;

packages/backend/src/core/DriveService.ts

@@ -154,8 +154,8 @@ export class DriveService {
 	@bindThis
 	private async save(file: MiDriveFile, path: string, name: string, info: FileInfo): Promise<MiDriveFile> {
 		const type = info.type.mime;
-		const hash = info.md5;
-		const size = info.size;
+		let hash = info.md5;
+		let size = info.size;
 
 		// thunbnail, webpublic を必要なら生成
 		const alts = await this.generateAlts(path, type, !file.uri);
@@ -163,6 +163,9 @@ export class DriveService {
 		if (type && type.startsWith('video/')) {
 			try {
 				await this.videoProcessingService.webOptimizeVideo(path, type);
+				const newInfo = await this.fileInfoService.getFileInfo(path);
+				hash = newInfo.md5;
+				size = newInfo.size;
 			} catch (err) {
 				this.registerLogger.warn(`Video optimization failed: ${renderInlineError(err)}`);
 			}
@@ -738,14 +741,14 @@ export class DriveService {
 	@bindThis
 	public async deleteFile(file: MiDriveFile, isExpired = false, deleter?: MiUser) {
 		if (file.storedInternal) {
-			this.internalStorageService.del(file.accessKey!);
+			this.deleteLocalFile(file.accessKey!);
 
 			if (file.thumbnailUrl) {
-				this.internalStorageService.del(file.thumbnailAccessKey!);
+				this.deleteLocalFile(file.thumbnailAccessKey!);
 			}
 
 			if (file.webpublicUrl) {
-				this.internalStorageService.del(file.webpublicAccessKey!);
+				this.deleteLocalFile(file.webpublicAccessKey!);
 			}
 		} else if (!file.isLink) {
 			this.queueService.createDeleteObjectStorageFileJob(file.accessKey!);
@@ -767,14 +770,14 @@ export class DriveService {
 		const promises = [];
 
 		if (file.storedInternal) {
-			promises.push(this.internalStorageService.del(file.accessKey!));
+			promises.push(this.deleteLocalFile(file.accessKey!));
 
 			if (file.thumbnailUrl) {
-				promises.push(this.internalStorageService.del(file.thumbnailAccessKey!));
+				promises.push(this.deleteLocalFile(file.thumbnailAccessKey!));
 			}
 
 			if (file.webpublicUrl) {
-				promises.push(this.internalStorageService.del(file.webpublicAccessKey!));
+				promises.push(this.deleteLocalFile(file.webpublicAccessKey!));
 			}
 		} else if (!file.isLink) {
 			promises.push(this.deleteObjectStorageFile(file.accessKey!));
@@ -861,6 +864,22 @@ export class DriveService {
 		}
 	}
 
+	@bindThis
+	public async deleteLocalFile(key: string) {
+		try {
+			await this.internalStorageService.del(key);
+		} catch (err: any) {
+			if (err.code === 'ENOENT') {
+				this.deleteLogger.warn(`The file to delete did not exist: ${key}. Skipping this.`);
+				return;
+			} else {
+				throw new Error(`Failed to delete the file: ${key}`, {
+					cause: err,
+				});
+			}
+		}
+	}
+
 	@bindThis
 	public async uploadFromUrl({
 		url,

packages/backend/src/core/HttpRequestService.ts

@@ -17,9 +17,9 @@ import { StatusError } from '@/misc/status-error.js';
 import { bindThis } from '@/decorators.js';
 import { validateContentTypeSetAsActivityPub } from '@/core/activitypub/misc/validator.js';
 import type { IObject, IObjectWithId } from '@/core/activitypub/type.js';
-import { ApUtilityService } from './activitypub/ApUtilityService.js';
+import { UtilityService } from '@/core/UtilityService.js';
+import { ApUtilityService } from '@/core/activitypub/ApUtilityService.js';
 import type { Response } from 'node-fetch';
-import type { URL } from 'node:url';
 import type { Socket } from 'node:net';
 
 export type HttpRequestSendOptions = {
@@ -27,7 +27,27 @@ export type HttpRequestSendOptions = {
 	validators?: ((res: Response) => void)[];
 };
 
-export function isPrivateIp(allowedPrivateNetworks: PrivateNetwork[] | undefined, ip: string, port?: number): boolean {
+export async function isPrivateUrl(url: URL, lookup: net.LookupFunction): Promise<boolean> {
+	const ip = await resolveIp(url, lookup);
+	return ip.range() !== 'unicast';
+}
+
+export async function resolveIp(url: URL, lookup: net.LookupFunction) {
+	if (ipaddr.isValid(url.hostname)) {
+		return ipaddr.parse(url.hostname);
+	}
+
+	const resolvedIp = await new Promise<string>((resolve, reject) => {
+		lookup(url.hostname, {}, (err, address) => {
+			if (err) reject(err);
+			else resolve(address as string);
+		});
+	});
+
+	return ipaddr.parse(resolvedIp);
+}
+
+export function isAllowedPrivateIp(allowedPrivateNetworks: PrivateNetwork[] | undefined, ip: string, port?: number): boolean {
 	const parsedIp = ipaddr.parse(ip);
 
 	for (const { cidr, ports } of allowedPrivateNetworks ?? []) {
@@ -44,7 +64,7 @@ export function isPrivateIp(allowedPrivateNetworks: PrivateNetwork[] | undefined
 export function validateSocketConnect(allowedPrivateNetworks: PrivateNetwork[] | undefined, socket: Socket): void {
 	const address = socket.remoteAddress;
 	if (address && ipaddr.isValid(address)) {
-		if (isPrivateIp(allowedPrivateNetworks, address, socket.remotePort)) {
+		if (isAllowedPrivateIp(allowedPrivateNetworks, address, socket.remotePort)) {
 			socket.destroy(new Error(`Blocked address: ${address}`));
 		}
 	}
@@ -128,10 +148,16 @@ export class HttpRequestService {
 	 */
 	public readonly httpsAgent: https.Agent;
 
+	/**
+	 * Get shared DNS resolver
+	 */
+	public readonly lookup: net.LookupFunction;
+
 	constructor(
 		@Inject(DI.config)
 		private config: Config,
 		private readonly apUtilityService: ApUtilityService,
+		private readonly utilityService: UtilityService,
 	) {
 		const cache = new CacheableLookup({
 			maxTtl: 3600,	// 1hours
@@ -139,6 +165,8 @@ export class HttpRequestService {
 			lookup: false,	// nativeのdns.lookupにfallbackしない
 		});
 
+		this.lookup = cache.lookup as unknown as net.LookupFunction;
+
 		const agentOption = {
 			keepAlive: true,
 			keepAliveMsecs: 30 * 1000,
@@ -236,8 +264,6 @@ export class HttpRequestService {
 
 	@bindThis
 	public async getActivityJson(url: string, isLocalAddressAllowed = false, allowAnonymous = false): Promise<IObjectWithId> {
-		this.apUtilityService.assertApUrl(url);
-
 		const res = await this.send(url, {
 			method: 'GET',
 			headers: {
@@ -303,6 +329,7 @@ export class HttpRequestService {
 			timeout?: number,
 			size?: number,
 			isLocalAddressAllowed?: boolean,
+			allowHttp?: boolean,
 		} = {},
 		extra: HttpRequestSendOptions = {
 			throwErrorWhenResponseNotOk: true,
@@ -311,6 +338,10 @@ export class HttpRequestService {
 	): Promise<Response> {
 		const timeout = args.timeout ?? 5000;
 
+		const parsedUrl = new URL(url);
+		const allowHttp = args.allowHttp || await isPrivateUrl(parsedUrl, this.lookup);
+		this.utilityService.assertUrl(parsedUrl, allowHttp);
+
 		const controller = new AbortController();
 		setTimeout(() => {
 			controller.abort();
@@ -318,7 +349,7 @@ export class HttpRequestService {
 
 		const isLocalAddressAllowed = args.isLocalAddressAllowed ?? false;
 
-		const res = await fetch(url, {
+		const res = await fetch(parsedUrl, {
 			method: args.method ?? 'GET',
 			headers: {
 				'User-Agent': this.config.userAgent,

packages/backend/src/core/MfmService.ts

@@ -377,7 +377,7 @@ export class MfmService {
 	}
 
 	@bindThis
-	public toHtml(nodes: mfm.MfmNode[] | null, mentionedRemoteUsers: IMentionedRemoteUsers = [], additionalAppenders: Appender[] = []) {
+	public toHtml(nodes: mfm.MfmNode[] | null, mentionedRemoteUsers: IMentionedRemoteUsers = [], additionalAppenders: Appender[] = [], inline = false) {
 		if (nodes == null) {
 			return null;
 		}
@@ -626,9 +626,15 @@ export class MfmService {
 			additionalAppender(doc, body);
 		}
 
-		return domserializer.render(body, {
+		let result = domserializer.render(body, {
 			encodeEntities: 'utf8'
 		});
+
+		if (inline) {
+			result = result.replace(/^<p>/, '').replace(/<\/p>$/, '');
+		}
+
+		return result;
 	}
 
 	// the toMastoApiHtml function was taken from Iceshrimp and written by zotan and modified by marie to work with the current MK version

packages/backend/src/core/QueueService.ts

@@ -17,6 +17,7 @@ import { bindThis } from '@/decorators.js';
 import type { Antenna } from '@/server/api/endpoints/i/import-antennas.js';
 import { ApRequestCreator } from '@/core/activitypub/ApRequestService.js';
 import { type SystemWebhookPayload } from '@/core/SystemWebhookService.js';
+import { MiNote } from '@/models/Note.js';
 import { type UserWebhookPayload } from './UserWebhookService.js';
 import type {
 	DbJobData,
@@ -40,7 +41,6 @@ import type {
 } from './QueueModule.js';
 import type httpSignature from '@peertube/http-signature';
 import type * as Bull from 'bullmq';
-import { MiNote } from '@/models/Note.js';
 
 export const QUEUE_TYPES = [
 	'system',
@@ -231,6 +231,9 @@ export class QueueService {
 				age: 3600 * 24 * 7, // keep up to 7 days
 				count: 100,
 			},
+			deduplication: activity.id ? {
+				id: activity.id,
+			} : undefined,
 		});
 	}
 
@@ -247,6 +250,9 @@ export class QueueService {
 				age: 3600 * 24 * 7, // keep up to 7 days
 				count: 100,
 			},
+			deduplication: {
+				id: user.id,
+			},
 		});
 	}
 
@@ -263,6 +269,9 @@ export class QueueService {
 				age: 3600 * 24 * 7, // keep up to 7 days
 				count: 100,
 			},
+			deduplication: {
+				id: user.id,
+			},
 		});
 	}
 
@@ -273,6 +282,9 @@ export class QueueService {
 		}, {
 			removeOnComplete: true,
 			removeOnFail: true,
+			deduplication: {
+				id: user.id,
+			},
 		});
 	}
 
@@ -289,6 +301,9 @@ export class QueueService {
 				age: 3600 * 24 * 7, // keep up to 7 days
 				count: 100,
 			},
+			deduplication: {
+				id: user.id,
+			},
 		});
 	}
 
@@ -305,6 +320,9 @@ export class QueueService {
 				age: 3600 * 24 * 7, // keep up to 7 days
 				count: 100,
 			},
+			deduplication: {
+				id: user.id,
+			},
 		});
 	}
 
@@ -321,6 +339,9 @@ export class QueueService {
 				age: 3600 * 24 * 7, // keep up to 7 days
 				count: 100,
 			},
+			deduplication: {
+				id: user.id,
+			},
 		});
 	}
 
@@ -339,6 +360,9 @@ export class QueueService {
 				age: 3600 * 24 * 7, // keep up to 7 days
 				count: 100,
 			},
+			deduplication: {
+				id: user.id,
+			},
 		});
 	}
 
@@ -355,6 +379,9 @@ export class QueueService {
 				age: 3600 * 24 * 7, // keep up to 7 days
 				count: 100,
 			},
+			deduplication: {
+				id: user.id,
+			},
 		});
 	}
 
@@ -371,6 +398,9 @@ export class QueueService {
 				age: 3600 * 24 * 7, // keep up to 7 days
 				count: 100,
 			},
+			deduplication: {
+				id: user.id,
+			},
 		});
 	}
 
@@ -387,6 +417,9 @@ export class QueueService {
 				age: 3600 * 24 * 7, // keep up to 7 days
 				count: 100,
 			},
+			deduplication: {
+				id: user.id,
+			},
 		});
 	}
 
@@ -403,6 +436,9 @@ export class QueueService {
 				age: 3600 * 24 * 7, // keep up to 7 days
 				count: 100,
 			},
+			deduplication: {
+				id: user.id,
+			},
 		});
 	}
 
@@ -421,6 +457,9 @@ export class QueueService {
 				age: 3600 * 24 * 7, // keep up to 7 days
 				count: 100,
 			},
+			deduplication: {
+				id: `${user.id}_${fileId}_${withReplies ?? false}`,
+			},
 		});
 	}
 
@@ -433,6 +472,9 @@ export class QueueService {
 		}, {
 			removeOnComplete: true,
 			removeOnFail: true,
+			deduplication: {
+				id: `${user.id}_${fileId}_${type ?? null}`,
+			},
 		});
 	}
 
@@ -492,6 +534,9 @@ export class QueueService {
 				age: 3600 * 24 * 7, // keep up to 7 days
 				count: 100,
 			},
+			deduplication: {
+				id: `${user.id}_${fileId}`,
+			},
 		});
 	}
 
@@ -509,6 +554,9 @@ export class QueueService {
 				age: 3600 * 24 * 7, // keep up to 7 days
 				count: 100,
 			},
+			deduplication: {
+				id: `${user.id}_${fileId}`,
+			},
 		});
 	}
 
@@ -554,6 +602,9 @@ export class QueueService {
 				age: 3600 * 24 * 7, // keep up to 7 days
 				count: 100,
 			},
+			deduplication: {
+				id: `${user.id}_${fileId}`,
+			},
 		});
 	}
 
@@ -571,14 +622,18 @@ export class QueueService {
 				age: 3600 * 24 * 7, // keep up to 7 days
 				count: 100,
 			},
+			deduplication: {
+				id: `${user.id}_${fileId}`,
+			},
 		});
 	}
 
 	@bindThis
-	public createImportAntennasJob(user: ThinUser, antenna: Antenna) {
+	public createImportAntennasJob(user: ThinUser, antenna: Antenna, fileId: MiDriveFile['id']) {
 		return this.dbQueue.add('importAntennas', {
 			user: { id: user.id },
 			antenna,
+			fileId,
 		}, {
 			removeOnComplete: {
 				age: 3600 * 24 * 7, // keep up to 7 days
@@ -588,6 +643,9 @@ export class QueueService {
 				age: 3600 * 24 * 7, // keep up to 7 days
 				count: 100,
 			},
+			deduplication: {
+				id: `${user.id}_${fileId}`,
+			},
 		});
 	}
 
@@ -605,6 +663,9 @@ export class QueueService {
 				age: 3600 * 24 * 7, // keep up to 7 days
 				count: 100,
 			},
+			deduplication: {
+				id: user.id,
+			},
 		});
 	}
 
@@ -663,6 +724,9 @@ export class QueueService {
 					count: 100,
 				},
 				...opts,
+				deduplication: {
+					id: `${data.from.id}_${data.to.id}_${data.requestId ?? ''}_${data.silent ?? false}_${data.withReplies ?? false}`,
+				},
 			},
 		};
 	}
@@ -680,6 +744,9 @@ export class QueueService {
 				age: 3600 * 24 * 7, // keep up to 7 days
 				count: 100,
 			},
+			deduplication: {
+				id: key,
+			},
 		});
 	}
 
@@ -697,6 +764,9 @@ export class QueueService {
 				age: 3600 * 24 * 7, // keep up to 7 days
 				count: 100,
 			},
+			deduplication: {
+				id: `${olderThanSeconds}_${keepFilesInUse}`,
+			},
 		});
 	}
 
@@ -879,7 +949,7 @@ export class QueueService {
 	public async queueGetJobs(queueType: typeof QUEUE_TYPES[number], jobTypes: JobType[], search?: string) {
 		const RETURN_LIMIT = 100;
 		const queue = this.getQueue(queueType);
-		let jobs: Bull.Job[];
+		let jobs: (Bull.Job | null)[];
 
 		if (search) {
 			jobs = await queue.getJobs(jobTypes, 0, 1000);
@@ -896,7 +966,9 @@ export class QueueService {
 			jobs = await queue.getJobs(jobTypes, 0, RETURN_LIMIT);
 		}
 
-		return jobs.map(job => this.packJobData(job));
+		return jobs
+			.filter(job => job != null) // not sure how this happens, but it does
+			.map(job => this.packJobData(job));
 	}
 
 	@bindThis

packages/backend/src/core/UserSuspendService.ts

@@ -4,7 +4,7 @@
  */
 
 import { Inject, Injectable } from '@nestjs/common';
-import { Not, IsNull } from 'typeorm';
+import { Not, IsNull, DataSource } from 'typeorm';
 import type { FollowingsRepository, FollowRequestsRepository, UsersRepository } from '@/models/_.js';
 import { MiUser } from '@/models/User.js';
 import { QueueService } from '@/core/QueueService.js';
@@ -21,6 +21,7 @@ import { LoggerService } from '@/core/LoggerService.js';
 import type Logger from '@/logger.js';
 import { renderInlineError } from '@/misc/render-inline-error.js';
 import { trackPromise } from '@/misc/promise-tracker.js';
+import { InternalEventService } from '@/core/InternalEventService.js';
 
 @Injectable()
 export class UserSuspendService {
@@ -36,12 +37,16 @@ export class UserSuspendService {
 		@Inject(DI.followRequestsRepository)
 		private followRequestsRepository: FollowRequestsRepository,
 
+		@Inject(DI.db)
+		private db: DataSource,
+
 		private userEntityService: UserEntityService,
 		private queueService: QueueService,
 		private globalEventService: GlobalEventService,
 		private apRendererService: ApRendererService,
 		private moderationLogService: ModerationLogService,
 		private readonly cacheService: CacheService,
+		private readonly internalEventService: InternalEventService,
 
 		loggerService: LoggerService,
 	) {
@@ -56,6 +61,8 @@ export class UserSuspendService {
 			isSuspended: true,
 		});
 
+		await this.internalEventService.emit(user.host == null ? 'localUserUpdated' : 'remoteUserUpdated', { id: user.id });
+
 		await this.moderationLogService.log(moderator, 'suspend', {
 			userId: user.id,
 			userUsername: user.username,
@@ -74,6 +81,8 @@ export class UserSuspendService {
 			isSuspended: false,
 		});
 
+		await this.internalEventService.emit(user.host == null ? 'localUserUpdated' : 'remoteUserUpdated', { id: user.id });
+
 		await this.moderationLogService.log(moderator, 'unsuspend', {
 			userId: user.id,
 			userUsername: user.username,
@@ -178,30 +187,29 @@ export class UserSuspendService {
 		// Freeze follow relations with all remote users
 		await this.followingsRepository
 			.createQueryBuilder('following')
-			.orWhere({
-				followeeId: user.id,
-				followerHost: Not(IsNull()),
-			})
 			.update({
 				isFollowerHibernated: true,
 			})
+			.where({
+				followeeId: user.id,
+				followerHost: Not(IsNull()),
+			})
 			.execute();
 	}
 
 	@bindThis
 	private async unFreezeAll(user: MiUser): Promise<void> {
 		// Restore follow relations with all remote users
-		await this.followingsRepository
-			.createQueryBuilder('following')
-			.innerJoin(MiUser, 'follower', 'user.id = following.followerId')
-			.andWhere('follower.isHibernated = false') // Don't unfreeze if the follower is *actually* frozen
-			.andWhere({
-				followeeId: user.id,
-				followerHost: Not(IsNull()),
-			})
-			.update({
-				isFollowerHibernated: false,
-			})
-			.execute();
+
+		// TypeORM does not support UPDATE with JOIN: https://github.com/typeorm/typeorm/issues/564#issuecomment-310331468
+		await this.db.query(`
+			UPDATE "following"
+				SET "isFollowerHibernated" = false
+			FROM "user"
+			WHERE "user"."id" = "following"."followerId"
+				AND "user"."isHibernated" = false -- Don't unfreeze if the follower is *actually* frozen
+				AND "followeeId" = $1
+				AND "followeeHost" IS NOT NULL
+		`, [user.id]);
 	}
 }

packages/backend/src/core/UtilityService.ts

@@ -11,8 +11,10 @@ import semver from 'semver';
 import { DI } from '@/di-symbols.js';
 import type { Config } from '@/config.js';
 import { bindThis } from '@/decorators.js';
-import { MiMeta, SoftwareSuspension } from '@/models/Meta.js';
-import { MiInstance } from '@/models/Instance.js';
+import type { MiMeta, SoftwareSuspension } from '@/models/Meta.js';
+import type { MiInstance } from '@/models/Instance.js';
+import { IdentifiableError } from '@/misc/identifiable-error.js';
+import { EnvService } from '@/core/EnvService.js';
 
 @Injectable()
 export class UtilityService {
@@ -22,6 +24,8 @@ export class UtilityService {
 
 		@Inject(DI.meta)
 		private meta: MiMeta,
+
+		private readonly envService: EnvService,
 	) {
 	}
 
@@ -183,8 +187,8 @@ export class UtilityService {
 	}
 
 	@bindThis
-	public punyHostPSLDomain(url: string): string {
-		const urlObj = new URL(url);
+	public punyHostPSLDomain(url: string | URL): string {
+		const urlObj = typeof(url) === 'object' ? url : new URL(url);
 		const hostname = urlObj.hostname;
 		const domain = this.specialSuffix(hostname) ?? psl.get(hostname) ?? hostname;
 		const host = `${this.toPuny(domain)}${urlObj.port.length > 0 ? ':' + urlObj.port : ''}`;
@@ -218,17 +222,84 @@ export class UtilityService {
 
 	@bindThis
 	public isDeliverSuspendedSoftware(software: Pick<MiInstance, 'softwareName' | 'softwareVersion'>): SoftwareSuspension | undefined {
-		if (software.softwareName == null) return undefined;
-		if (software.softwareVersion == null) {
-			// software version is null; suspend iff versionRange is *
-			return this.meta.deliverSuspendedSoftware.find(x =>
-				x.software === software.softwareName
-				&& x.versionRange.trim() === '*');
-		} else {
-			const softwareVersion = software.softwareVersion;
-			return this.meta.deliverSuspendedSoftware.find(x =>
-				x.software === software.softwareName
-				&& semver.satisfies(softwareVersion, x.versionRange, { includePrerelease: true }));
+		// a missing name or version is treated as the empty string
+		const softwareName = software.softwareName ?? '';
+		const softwareVersion = software.softwareVersion ?? '';
+
+		function maybeRegexpMatch(test: string, target: string): boolean {
+			const regexpStrPair = test.trim().match(/^\/(.+)\/(.*)$/);
+			if (!regexpStrPair) return false; // not a regexp, can't match
+
+			try {
+				return new RE2(regexpStrPair[1], regexpStrPair[2]).test(target);
+			} catch (err) {
+				return false; // not a well-formed regexp, can't match
+			}
+		}
+
+		// each element of `meta.deliverSuspendedSoftware` can have a
+		// normal string, a `*`, or a `/regexp/` for software or
+		// versionRange
+		return this.meta.deliverSuspendedSoftware.find(
+			x => (
+				(
+					x.software.trim() === '*' ||
+						x.software === softwareName ||
+						maybeRegexpMatch(x.software, softwareName)
+				) && (
+					x.versionRange.trim() === '*' ||
+						semver.satisfies(softwareVersion, x.versionRange, { includePrerelease: true }) ||
+						maybeRegexpMatch(x.versionRange, softwareVersion)
+				)
+			)
+		);
+	}
+
+	/**
+	 * Verifies that a provided URL is in a format acceptable for federation.
+	 * @throws {IdentifiableError} If URL cannot be parsed
+	 * @throws {IdentifiableError} If URL is not HTTPS
+	 * @throws {IdentifiableError} If URL contains credentials
+	 */
+	@bindThis
+	public assertUrl(url: string | URL, allowHttp?: boolean): URL | never {
+		// If string, parse and validate
+		if (typeof(url) === 'string') {
+			try {
+				url = new URL(url);
+			} catch {
+				throw new IdentifiableError('0bedd29b-e3bf-4604-af51-d3352e2518af', `invalid url ${url}: not a valid URL`);
+			}
+		}
+
+		// Must be HTTPS
+		if (!this.checkHttps(url, allowHttp)) {
+			throw new IdentifiableError('0bedd29b-e3bf-4604-af51-d3352e2518af', `invalid url ${url}: unsupported protocol ${url.protocol}`);
+		}
+
+		// Must not have credentials
+		if (url.username || url.password) {
+			throw new IdentifiableError('0bedd29b-e3bf-4604-af51-d3352e2518af', `invalid url ${url}: contains embedded credentials`);
+		}
+
+		return url;
+	}
+
+	/**
+	 * Checks if the URL contains HTTPS.
+	 * Additionally, allows HTTP in non-production environments.
+	 * Based on check-https.ts.
+	 */
+	@bindThis
+	public checkHttps(url: string | URL, allowHttp = false): boolean {
+		const isNonProd = this.envService.env.NODE_ENV !== 'production';
+
+		try {
+			const proto = new URL(url).protocol;
+			return proto === 'https:' || (proto === 'http:' && (isNonProd || allowHttp));
+		} catch {
+			// Invalid URLs don't "count" as HTTPS
+			return false;
 		}
 	}
 }

packages/backend/src/core/VideoProcessingService.ts

@@ -102,6 +102,7 @@ export class VideoProcessingService {
 					.format(outputFormat) // Specify output format
 					.addOutputOptions('-c copy') // Copy streams without re-encoding
 					.addOutputOptions('-movflags +faststart')
+					.addOutputOptions('-map 0')
 					.on('error', reject)
 					.on('end', async () => {
 						try {

packages/backend/src/core/WebhookTestService.ts

@@ -443,6 +443,8 @@ export class WebhookTestService {
 		return {
 			...user,
 			createdAt: this.idService.parse(user.id).date.toISOString(),
+			updatedAt: null,
+			lastFetchedAt: null,
 			id: user.id,
 			name: user.name,
 			username: user.username,

packages/backend/src/core/activitypub/ApRendererService.ts

@@ -585,7 +585,7 @@ export class ApRendererService {
 		const attachment = profile.fields.map(field => ({
 			type: 'PropertyValue',
 			name: field.name,
-			value: this.mfmService.toHtml(mfm.parse(field.value)),
+			value: this.mfmService.toHtml(mfm.parse(field.value), [], [], true),
 		}));
 
 		const emojis = await this.getEmojis(user.emojis);
@@ -750,9 +750,11 @@ export class ApRendererService {
 	}
 
 	@bindThis
-	public renderUpdate(object: string | IObject, user: { id: MiUser['id'] }): IUpdate {
+	public renderUpdate(object: IObject, user: { id: MiUser['id'] }): IUpdate {
+		// Deterministic activity IDs to allow de-duplication by remote instances
+		const updatedAt = object.updated ? new Date(object.updated).getTime() : Date.now();
 		return {
-			id: `${this.config.url}/users/${user.id}#updates/${new Date().getTime()}`,
+			id: `${this.config.url}/users/${user.id}#updates/${updatedAt}`,
 			actor: this.userEntityService.genLocalUserUri(user.id),
 			type: 'Update',
 			to: ['https://www.w3.org/ns/activitystreams#Public'],

packages/backend/src/core/activitypub/ApRequestService.ts

@@ -157,8 +157,6 @@ export class ApRequestService {
 
 	@bindThis
 	public async signedPost(user: { id: MiUser['id'] }, url: string, object: unknown, digest?: string): Promise<void> {
-		this.apUtilityService.assertApUrl(url);
-
 		const body = typeof object === 'string' ? object : JSON.stringify(object);
 
 		const keypair = await this.userKeypairService.getUserKeypair(user.id);
@@ -191,8 +189,6 @@ export class ApRequestService {
 	 */
 	@bindThis
 	public async signedGet(url: string, user: { id: MiUser['id'] }, allowAnonymous = false, followAlternate?: boolean): Promise<IObjectWithId> {
-		this.apUtilityService.assertApUrl(url);
-
 		const _followAlternate = followAlternate ?? true;
 		const keypair = await this.userKeypairService.getUserKeypair(user.id);
 

packages/backend/src/core/activitypub/ApUtilityService.ts

@@ -7,20 +7,29 @@ import { Injectable } from '@nestjs/common';
 import { UtilityService } from '@/core/UtilityService.js';
 import { IdentifiableError } from '@/misc/identifiable-error.js';
 import { toArray } from '@/misc/prelude/array.js';
-import { EnvService } from '@/core/EnvService.js';
-import { getApId, getOneApHrefNullable, IObject } from './type.js';
+import { getApId, getNullableApId, getOneApHrefNullable } from '@/core/activitypub/type.js';
+import type { IObject, IObjectWithId } from '@/core/activitypub/type.js';
+import { bindThis } from '@/decorators.js';
+import { renderInlineError } from '@/misc/render-inline-error.js';
+import { LoggerService } from '@/core/LoggerService.js';
+import type Logger from '@/logger.js';
 
 @Injectable()
 export class ApUtilityService {
+	private readonly logger: Logger;
+
 	constructor(
 		private readonly utilityService: UtilityService,
-		private readonly envService: EnvService,
-	) {}
+		loggerService: LoggerService,
+	) {
+		this.logger = loggerService.getLogger('ap-utility');
+	}
 
 	/**
 	 * Verifies that the object's ID has the same authority as the provided URL.
 	 * Returns on success, throws on any validation error.
 	 */
+	@bindThis
 	public assertIdMatchesUrlAuthority(object: IObject, url: string): void {
 		// This throws if the ID is missing or invalid, but that's ok.
 		// Anonymous objects are impossible to verify, so we don't allow fetching them.
@@ -36,11 +45,15 @@ export class ApUtilityService {
 	/**
 	 * Checks if two URLs have the same host authority
 	 */
+	@bindThis
 	public haveSameAuthority(url1: string, url2: string): boolean {
 		if (url1 === url2) return true;
 
-		const authority1 = this.utilityService.punyHostPSLDomain(url1);
-		const authority2 = this.utilityService.punyHostPSLDomain(url2);
+		const parsed1 = this.utilityService.assertUrl(url1);
+		const parsed2 = this.utilityService.assertUrl(url2);
+
+		const authority1 = this.utilityService.punyHostPSLDomain(parsed1);
+		const authority2 = this.utilityService.punyHostPSLDomain(parsed2);
 		return authority1 === authority2;
 	}
 
@@ -50,6 +63,7 @@ export class ApUtilityService {
 	 * @throws {IdentifiableError} if object does not have an ID
 	 * @returns the best URL, or null if none were found
 	 */
+	@bindThis
 	public findBestObjectUrl(object: IObject): string | null {
 		const targetUrl = getApId(object);
 		const targetAuthority = this.utilityService.punyHostPSLDomain(targetUrl);
@@ -63,12 +77,16 @@ export class ApUtilityService {
 					: undefined,
 			}))
 			.filter(({ url, type }) => {
-				if (!url) return false;
-				if (!this.checkHttps(url)) return false;
-				if (!isAcceptableUrlType(type)) return false;
+				try {
+					if (!url) return false;
+					if (!isAcceptableUrlType(type)) return false;
+					const parsed = this.utilityService.assertUrl(url);
 
-				const urlAuthority = this.utilityService.punyHostPSLDomain(url);
-				return urlAuthority === targetAuthority;
+					const urlAuthority = this.utilityService.punyHostPSLDomain(parsed);
+					return urlAuthority === targetAuthority;
+				} catch {
+					return false;
+				}
 			})
 			.sort((a, b) => {
 				return rankUrlType(a.type) - rankUrlType(b.type);
@@ -78,41 +96,72 @@ export class ApUtilityService {
 	}
 
 	/**
-	 * Verifies that a provided URL is in a format acceptable for federation.
-	 * @throws {IdentifiableError} If URL cannot be parsed
-	 * @throws {IdentifiableError} If URL is not HTTPS
+	 * Sanitizes an inline / nested Object property within an AP object.
+	 *
+	 * Returns true if the property contains a valid string URL, object w/ valid ID, or an array containing one of those.
+	 * Returns false and erases the property if it doesn't contain a valid value.
+	 *
+	 * Arrays are automatically flattened.
+	 * Falsy values (including null) are collapsed to undefined.
+	 * @param obj Object containing the property to validate
+	 * @param key Key of the property in obj
+	 * @param parentUri URI of the object that contains this inline object.
+	 * @param parentHost PSL host of parentUri
+	 * @param keyPath If obj is *itself* a nested object, set this to the property path from root to obj (including the trailing '.'). This does not affect the logic, but improves the clarity of logs.
 	 */
-	public assertApUrl(url: string | URL): void {
-		// If string, parse and validate
-		if (typeof(url) === 'string') {
-			try {
-				url = new URL(url);
-			} catch {
-				throw new IdentifiableError('0bedd29b-e3bf-4604-af51-d3352e2518af', `invalid AP url ${url}: not a valid URL`);
-			}
+	@bindThis
+	public sanitizeInlineObject<Key extends string>(obj: Partial<Record<Key, string | { id?: string } | (string | { id?: string })[]>>, key: Key, parentUri: string | URL, parentHost: string, keyPath = ''): obj is Partial<Record<Key, string | { id: string }>> {
+		let value: unknown = obj[key];
+
+		// Unpack arrays
+		if (Array.isArray(value)) {
+			value = value[0];
 		}
 
-		// Must be HTTPS
-		if (!this.checkHttps(url)) {
-			throw new IdentifiableError('0bedd29b-e3bf-4604-af51-d3352e2518af', `invalid AP url ${url}: unsupported protocol ${url.protocol}`);
-		}
-	}
+		// Clear the value - we'll add it back once we have a confirmed ID
+		obj[key] = undefined;
 
-	/**
-	 * Checks if the URL contains HTTPS.
-	 * Additionally, allows HTTP in non-production environments.
-	 * Based on check-https.ts.
-	 */
-	private checkHttps(url: string | URL): boolean {
-		const isNonProd = this.envService.env.NODE_ENV !== 'production';
-
-		try {
-			const proto = new URL(url).protocol;
-			return proto === 'https:' || (proto === 'http:' && isNonProd);
-		} catch {
-			// Invalid URLs don't "count" as HTTPS
+		// Collapse falsy values to undefined
+		if (!value) {
 			return false;
 		}
+
+		// Exclude nested arrays
+		if (Array.isArray(value)) {
+			this.logger.warn(`Excluding ${keyPath}${key} from object ${parentUri}: nested arrays are prohibited`);
+			return false;
+		}
+
+		// Exclude incorrect types
+		if (typeof(value) !== 'string' && typeof(value) !== 'object') {
+			this.logger.warn(`Excluding ${keyPath}${key} from object ${parentUri}: incorrect type ${typeof(value)}`);
+			return false;
+		}
+
+		const valueId = getNullableApId(value);
+		if (!valueId) {
+			// Exclude missing ID
+			this.logger.warn(`Excluding ${keyPath}${key} from object ${parentUri}: missing or invalid ID`);
+			return false;
+		}
+
+		try {
+			const parsed = this.utilityService.assertUrl(valueId);
+			const parsedHost = this.utilityService.punyHostPSLDomain(parsed);
+			if (parsedHost !== parentHost) {
+				// Exclude wrong host
+				this.logger.warn(`Excluding ${keyPath}${key} from object ${parentUri}: wrong host in ${valueId} (got ${parsedHost}, expected ${parentHost})`);
+				return false;
+			}
+		} catch (err) {
+			// Exclude invalid URLs
+			this.logger.warn(`Excluding ${keyPath}${key} from object ${parentUri}: invalid URL ${valueId}: ${renderInlineError(err)}`);
+			return false;
+		}
+
+		// Success - store the sanitized value and return
+		obj[key] = value as string | IObjectWithId;
+		return true;
 	}
 }
 

packages/backend/src/core/activitypub/models/ApNoteService.ts

@@ -100,7 +100,7 @@ export class ApNoteService {
 		actor?: MiRemoteUser,
 		user?: MiRemoteUser,
 	): Error | null {
-		this.apUtilityService.assertApUrl(uri);
+		this.utilityService.assertUrl(uri);
 		const expectHost = this.utilityService.extractDbHost(uri);
 		const apType = getApType(object);
 

packages/backend/src/core/activitypub/models/ApPersonService.ts

@@ -153,89 +153,88 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
 	 */
 	@bindThis
 	private validateActor(x: IObject, uri: string): IActor {
-		this.apUtilityService.assertApUrl(uri);
-		const expectHost = this.utilityService.punyHostPSLDomain(uri);
+		const parsedUri = this.utilityService.assertUrl(uri);
+		const expectHost = this.utilityService.punyHostPSLDomain(parsedUri);
 
+		// Validate type
 		if (!isActor(x)) {
 			throw new UnrecoverableError(`invalid Actor ${uri}: unknown type '${x.type}'`);
 		}
 
-		if (!(typeof x.id === 'string' && x.id.length > 0)) {
-			throw new UnrecoverableError(`invalid Actor ${uri}: wrong id type`);
+		// Validate id
+		if (!x.id) {
+			throw new UnrecoverableError(`invalid Actor ${uri}: missing id`);
+		}
+		if (typeof(x.id) !== 'string') {
+			throw new UnrecoverableError(`invalid Actor ${uri}: wrong id type ${typeof(x.id)}`);
+		}
+		const parsedId = this.utilityService.assertUrl(x.id);
+		const idHost = this.utilityService.punyHostPSLDomain(parsedId);
+		if (idHost !== expectHost) {
+			throw new UnrecoverableError(`invalid Actor ${uri}: wrong host in id ${x.id} (got ${parsedId}, expected ${expectHost})`);
 		}
 
-		if (!(typeof x.inbox === 'string' && x.inbox.length > 0)) {
-			throw new UnrecoverableError(`invalid Actor ${uri}: wrong inbox type`);
+		// Validate inbox
+		this.apUtilityService.sanitizeInlineObject(x, 'inbox', parsedUri, expectHost);
+		if (!x.inbox || typeof(x.inbox) !== 'string') {
+			throw new UnrecoverableError(`invalid Actor ${uri}: missing or invalid inbox ${x.inbox}`);
 		}
 
-		this.apUtilityService.assertApUrl(x.inbox);
-		const inboxHost = this.utilityService.punyHostPSLDomain(x.inbox);
-		if (inboxHost !== expectHost) {
-			throw new UnrecoverableError(`invalid Actor ${uri}: wrong inbox host ${inboxHost}`);
+		// Sanitize sharedInbox
+		this.apUtilityService.sanitizeInlineObject(x, 'sharedInbox', parsedUri, expectHost);
+
+		// Sanitize endpoints object
+		if (typeof(x.endpoints) === 'object') {
+			x.endpoints = {
+				sharedInbox: x.endpoints.sharedInbox,
+			};
+		} else {
+			x.endpoints = undefined;
 		}
 
-		const sharedInboxObject = x.sharedInbox ?? (x.endpoints ? x.endpoints.sharedInbox : undefined);
-		if (sharedInboxObject != null) {
-			const sharedInbox = getApId(sharedInboxObject);
-			this.apUtilityService.assertApUrl(sharedInbox);
-			if (!(typeof sharedInbox === 'string' && sharedInbox.length > 0 && this.utilityService.punyHostPSLDomain(sharedInbox) === expectHost)) {
-				throw new UnrecoverableError(`invalid Actor ${uri}: wrong shared inbox ${sharedInbox}`);
+		// Sanitize endpoints.sharedInbox
+		if (x.endpoints) {
+			this.apUtilityService.sanitizeInlineObject(x.endpoints, 'sharedInbox', parsedUri, expectHost, 'endpoints.');
+
+			if (!x.endpoints.sharedInbox) {
+				x.endpoints = undefined;
 			}
 		}
 
-		for (const collection of ['outbox', 'followers', 'following'] as (keyof IActor)[]) {
-			const xCollection = (x as IActor)[collection];
-			if (xCollection != null) {
-				const collectionUri = getApId(xCollection);
-				if (typeof collectionUri === 'string' && collectionUri.length > 0) {
-					this.apUtilityService.assertApUrl(collectionUri);
-					if (this.utilityService.punyHostPSLDomain(collectionUri) !== expectHost) {
-						throw new UnrecoverableError(`invalid Actor ${uri}: wrong ${collection} host ${collectionUri}`);
-					}
-				} else if (collectionUri != null) {
-					throw new UnrecoverableError(`invalid Actor ${uri}: wrong ${collection} type`);
-				}
-			}
+		// Sanitize collections
+		for (const collection of ['outbox', 'followers', 'following', 'featured'] as const) {
+			this.apUtilityService.sanitizeInlineObject(x, collection, parsedUri, expectHost);
 		}
 
+		// Validate username
 		if (!(typeof x.preferredUsername === 'string' && x.preferredUsername.length > 0 && x.preferredUsername.length <= 128 && /^\w([\w-.]*\w)?$/.test(x.preferredUsername))) {
 			throw new UnrecoverableError(`invalid Actor ${uri}: wrong username`);
 		}
 
+		// Sanitize name
 		// These fields are only informational, and some AP software allows these
 		// fields to be very long. If they are too long, we cut them off. This way
 		// we can at least see these users and their activities.
-		if (x.name) {
-			if (!(typeof x.name === 'string' && x.name.length > 0)) {
-				throw new UnrecoverableError(`invalid Actor ${uri}: wrong name`);
-			}
-			x.name = truncate(x.name, nameLength);
-		} else if (x.name === '') {
-			// Mastodon emits empty string when the name is not set.
+		if (!x.name) {
 			x.name = undefined;
+		} else if (typeof(x.name) !== 'string') {
+			this.logger.warn(`Excluding name from object ${uri}: incorrect type ${typeof(x)}`);
+			x.name = undefined;
+		} else {
+			x.name = truncate(x.name, nameLength);
 		}
-		if (x.summary) {
-			if (!(typeof x.summary === 'string' && x.summary.length > 0)) {
-				throw new UnrecoverableError(`invalid Actor ${uri}: wrong summary`);
-			}
+
+		// Sanitize summary
+		if (!x.summary) {
+			x.summary = undefined;
+		} else if (typeof(x.summary) !== 'string') {
+			this.logger.warn(`Excluding summary from object ${uri}: incorrect type ${typeof(x)}`);
+		} else {
 			x.summary = truncate(x.summary, this.config.maxRemoteBioLength);
 		}
 
-		const idHost = this.utilityService.punyHostPSLDomain(x.id);
-		if (idHost !== expectHost) {
-			throw new UnrecoverableError(`invalid Actor ${uri}: wrong id ${x.id}`);
-		}
-
-		if (x.publicKey) {
-			if (typeof x.publicKey.id !== 'string') {
-				throw new UnrecoverableError(`invalid Actor ${uri}: wrong publicKey.id type`);
-			}
-
-			const publicKeyIdHost = this.utilityService.punyHostPSLDomain(x.publicKey.id);
-			if (publicKeyIdHost !== expectHost) {
-				throw new UnrecoverableError(`invalid Actor ${uri}: wrong publicKey.id ${x.publicKey.id}`);
-			}
-		}
+		// Sanitize publicKey
+		this.apUtilityService.sanitizeInlineObject(x, 'publicKey', parsedUri, expectHost);
 
 		return x;
 	}
@@ -375,7 +374,8 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
 
 		const url = this.apUtilityService.findBestObjectUrl(person);
 
-		const verifiedLinks = url ? await verifyFieldLinks(fields, url, this.httpRequestService) : [];
+		const profileUrls = url ? [url, person.id] : [person.id];
+		const verifiedLinks = await verifyFieldLinks(fields, profileUrls, this.httpRequestService);
 
 		// Create user
 		let user: MiRemoteUser | null = null;
@@ -494,7 +494,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
 				user = u as MiRemoteUser;
 				publicKey = await this.userPublickeysRepository.findOneBy({ userId: user.id });
 			} else {
-				this.logger.error('Error creating Person:', e instanceof Error ? e : new Error(e as string));
+				this.logger.error(`Error creating Person ${uri}: ${renderInlineError(e)}`);
 				throw e;
 			}
 		}
@@ -623,7 +623,8 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
 
 		const url = this.apUtilityService.findBestObjectUrl(person);
 
-		const verifiedLinks = url ? await verifyFieldLinks(fields, url, this.httpRequestService) : [];
+		const profileUrls = url ? [url, person.id] : [person.id];
+		const verifiedLinks = await verifyFieldLinks(fields, profileUrls, this.httpRequestService);
 
 		const updates = {
 			lastFetchedAt: new Date(),
@@ -776,7 +777,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
 					return result;
 				})
 				.catch(e => {
-					this.logger.info(`Processing Move Failed @${updated.username}@${updated.host} (${uri})`, { stack: e });
+					this.logger.info(`Processing Move Failed @${updated.username}@${updated.host} (${uri}): ${renderInlineError(e)}`);
 				});
 		}
 

packages/backend/src/core/activitypub/type.ts

@@ -28,8 +28,9 @@ export interface IObject {
 	inReplyTo?: any;
 	replies?: ICollection | IOrderedCollection | string;
 	content?: string | null;
-	startTime?: Date;
-	endTime?: Date;
+	startTime?: Date; // TODO these are wrong - should be string
+	endTime?: Date; // TODO these are wrong - should be string
+	updated?: string;
 	icon?: any;
 	image?: any;
 	mediaType?: string;
@@ -86,7 +87,7 @@ export function getOneApId(value: ApObject): string {
 /**
  * Get ActivityStreams Object id
  */
-export function getApId(value: string | IObject | [string | IObject], sourceForLogs?: string): string {
+export function getApId(value: unknown | [unknown] | unknown[], sourceForLogs?: string): string {
 	const id = getNullableApId(value);
 
 	if (id == null) {
@@ -102,7 +103,7 @@ export function getApId(value: string | IObject | [string | IObject], sourceForL
 /**
  * Get ActivityStreams Object id, or null if not present
  */
-export function getNullableApId(source: string | IObject | [string | IObject]): string | null {
+export function getNullableApId(source: unknown | [unknown] | unknown[]): string | null {
 	const value: unknown = fromTuple(source);
 
 	if (value != null) {
@@ -216,7 +217,6 @@ export interface IPost extends IObject {
 	quoteUrl?: string;
 	quoteUri?: string;
 	quote?: string;
-	updated?: string;
 }
 
 export interface IQuestion extends IObject {
@@ -276,7 +276,7 @@ export interface IActor extends IObject {
 	followers?: string | ICollection | IOrderedCollection;
 	following?: string | ICollection | IOrderedCollection;
 	featured?: string | IOrderedCollection;
-	outbox: string | IOrderedCollection;
+	outbox?: string | IOrderedCollection;
 	endpoints?: {
 		sharedInbox?: string;
 	};

packages/backend/src/core/entities/MetaEntityService.ts

@@ -75,6 +75,7 @@ export class MetaEntityService {
 			shortName: instance.shortName,
 			uri: this.config.url,
 			description: instance.description,
+			about: instance.about,
 			langs: instance.langs,
 			tosUrl: instance.termsOfServiceUrl,
 			repositoryUrl: instance.repositoryUrl,

packages/backend/src/core/entities/UserEntityService.ts

@@ -546,6 +546,8 @@ export class UserEntityService implements OnModuleInit {
 			avatarBlurhash: (user.avatarId == null ? null : user.avatarBlurhash),
 			description: mastoapi ? mastoapi.description : profile ? profile.description : '',
 			createdAt: this.idService.parse(user.id).date.toISOString(),
+			updatedAt: user.updatedAt ? user.updatedAt.toISOString() : null,
+			lastFetchedAt: user.lastFetchedAt ? user.lastFetchedAt.toISOString() : null,
 			avatarDecorations: user.avatarDecorations.length > 0 ? this.avatarDecorationService.getAll().then(decorations => user.avatarDecorations.filter(ud => decorations.some(d => d.id === ud.id)).map(ud => ({
 				id: ud.id,
 				angle: ud.angle || undefined,
@@ -601,8 +603,6 @@ export class UserEntityService implements OnModuleInit {
 					? Promise.all(user.alsoKnownAs.map(uri => Promise.resolve(opts.userIdsByUri?.get(uri) ?? this.apPersonService.fetchPerson(uri).then(user => user?.id).catch(() => null))))
 						.then(xs => xs.length === 0 ? null : xs.filter(x => x != null))
 					: null,
-				updatedAt: user.updatedAt ? user.updatedAt.toISOString() : null,
-				lastFetchedAt: user.lastFetchedAt ? user.lastFetchedAt.toISOString() : null,
 				bannerUrl: user.bannerId == null ? null : user.bannerUrl,
 				bannerBlurhash: user.bannerId == null ? null : user.bannerBlurhash,
 				backgroundUrl: user.backgroundId == null ? null : user.backgroundUrl,

packages/backend/src/misc/captcha-error.ts

@@ -0,0 +1,18 @@
+/*
+ * SPDX-FileCopyrightText: syuilo and misskey-project
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+import type { CaptchaErrorCode } from '@/core/CaptchaService.js';
+
+export class CaptchaError extends Error {
+	public readonly code: CaptchaErrorCode;
+	public readonly cause?: unknown;
+
+	constructor(code: CaptchaErrorCode, message: string, cause?: unknown) {
+		super(message, cause ? { cause } : undefined);
+		this.code = code;
+		this.cause = cause;
+		this.name = 'CaptchaError';
+	}
+}

packages/backend/src/misc/render-inline-error.ts

@@ -5,7 +5,7 @@
 
 import { IdentifiableError } from '@/misc/identifiable-error.js';
 import { StatusError } from '@/misc/status-error.js';
-import { CaptchaError } from '@/core/CaptchaService.js';
+import { CaptchaError } from '@/misc/captcha-error.js';
 
 export function renderInlineError(err: unknown): string {
 	const parts: string[] = [];

packages/backend/src/misc/verify-field-link.ts

@@ -8,17 +8,18 @@ import type { HttpRequestService } from '@/core/HttpRequestService.js';
 
 type Field = { name: string, value: string };
 
-export async function verifyFieldLinks(fields: Field[], profile_url: string, httpRequestService: HttpRequestService): Promise<string[]> {
+export async function verifyFieldLinks(fields: Field[], profileUrls: string[], httpRequestService: HttpRequestService): Promise<string[]> {
 	const verified_links = [];
-	for (const field_url of fields.filter(x => URL.canParse(x.value) && ['http:', 'https:'].includes((new URL(x.value).protocol)))) {
+	for (const field_url of fields) {
 		try {
+			// getHtml validates the input URL, so we can safely pass in untrusted values
 			const html = await httpRequestService.getHtml(field_url.value);
 
 			const doc = cheerio(html);
 
 			const links = doc('a[rel~="me"][href], link[rel~="me"][href]').toArray();
 
-			const includesProfileLinks = links.some(link => link.attribs.href === profile_url);
+			const includesProfileLinks = links.some(link => profileUrls.includes(link.attribs.href));
 			if (includesProfileLinks) {
 				verified_links.push(field_url.value);
 			}

packages/backend/src/models/Announcement.ts

@@ -6,6 +6,7 @@
 import { Entity, Index, Column, PrimaryColumn, ManyToOne, JoinColumn } from 'typeorm';
 import { id } from './util/id.js';
 import { MiUser } from './User.js';
+import { MiRole } from './Role.js';
 
 @Entity('announcement')
 export class MiAnnouncement {
@@ -66,6 +67,12 @@ export class MiAnnouncement {
 	})
 	public forExistingUsers: boolean;
 
+	@Column('text', {
+		array: true,
+		default: '{}', nullable: false,
+	})
+	public forRoles: MiRole['id'][];
+
 	@Index()
 	@Column('boolean', {
 		default: false,

packages/backend/src/models/ChatMessage.ts

@@ -50,8 +50,8 @@ export class MiChatMessage {
 	@JoinColumn()
 	public toRoom: MiChatRoom | null;
 
-	@Column('varchar', {
-		length: 4096, nullable: true,
+	@Column('text', {
+		nullable: true,
 	})
 	public text: string | null;
 

packages/backend/src/models/Meta.ts

@@ -43,6 +43,11 @@ export class MiMeta {
 	})
 	public description: string | null;
 
+	@Column('text', {
+		nullable: true,
+	})
+	public about: string | null;
+
 	/**
 	 * メンテナの名前
 	 */
@@ -628,8 +633,7 @@ export class MiMeta {
 	})
 	public policies: Record<string, any>;
 
-	@Column('varchar', {
-		length: 280,
+	@Column('text', {
 		array: true,
 		default: '{}',
 	})

packages/backend/src/models/UserPending.ts

@@ -37,4 +37,10 @@ export class MiUserPending {
 		nullable: true,
 	})
 	public reason: string;
+
+	@Column('varchar', {
+		length: 128,
+		nullable: true,
+	})
+	public requestOriginIp: string | null;
 }

packages/backend/src/models/json-schema/meta.ts

@@ -43,6 +43,10 @@ export const packedMetaLiteSchema = {
 			type: 'string',
 			optional: false, nullable: true,
 		},
+		about: {
+			type: 'string',
+			optional: false, nullable: true,
+		},
 		langs: {
 			type: 'array',
 			optional: false, nullable: false,

packages/backend/src/models/json-schema/user.ts

@@ -69,6 +69,16 @@ export const packedUserLiteSchema = {
 			nullable: false, optional: false,
 			format: 'date-time',
 		},
+		updatedAt: {
+			type: 'string',
+			nullable: true, optional: false,
+			format: 'date-time',
+		},
+		lastFetchedAt: {
+			type: 'string',
+			nullable: true, optional: false,
+			format: 'date-time',
+		},
 		approved: {
 			type: 'boolean',
 			nullable: false, optional: false,
@@ -304,16 +314,6 @@ export const packedUserDetailedNotMeOnlySchema = {
 				nullable: false, optional: false,
 			},
 		},
-		updatedAt: {
-			type: 'string',
-			nullable: true, optional: false,
-			format: 'date-time',
-		},
-		lastFetchedAt: {
-			type: 'string',
-			nullable: true, optional: false,
-			format: 'date-time',
-		},
 		bannerUrl: {
 			type: 'string',
 			format: 'url',

packages/backend/src/queue/processors/ImportAntennasProcessorService.ts

@@ -11,8 +11,8 @@ import Logger from '@/logger.js';
 import type { AntennasRepository, UsersRepository } from '@/models/_.js';
 import { DI } from '@/di-symbols.js';
 import { bindThis } from '@/decorators.js';
-import { QueueLoggerService } from '../QueueLoggerService.js';
 import { NotificationService } from '@/core/NotificationService.js';
+import { QueueLoggerService } from '../QueueLoggerService.js';
 import { DBAntennaImportJobData } from '../types.js';
 import type * as Bull from 'bullmq';
 
@@ -79,7 +79,7 @@ export class ImportAntennasProcessorService {
 			return;
 		}
 
-		this.logger.debug(`Importing blocking of ${job.data.user.id} ...`);
+		this.logger.debug(`Importing antennas of ${job.data.user.id} ...`);
 
 		const now = new Date();
 		try {

packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts

@@ -68,9 +68,7 @@ export class ImportCustomEmojisProcessorService {
 			fs.writeFileSync(destPath, '', 'binary');
 			await this.downloadService.downloadUrl(file.url, destPath, { operationTimeout: this.config.import?.downloadTimeout, maxSize: this.config.import?.maxFileSize });
 		} catch (e) { // TODO: 何度か再試行
-			if (e instanceof Error || typeof e === 'string') {
-				this.logger.error('Error importing custom emojis:', e as Error);
-			}
+			this.logger.error(`Error importing custom emojis: ${renderInlineError(e)}`);
 			throw e;
 		}
 

packages/backend/src/queue/types.ts

@@ -110,6 +110,7 @@ export type DbNoteImportJobData = {
 export type DBAntennaImportJobData = {
 	user: ThinUser,
 	antenna: Antenna
+	fileId: MiDriveFile['id'];
 };
 
 export type DbUserImportToDbJobData = {

packages/backend/src/server/SkRateLimiterService.md

@@ -81,7 +81,7 @@ The Atomic Leaky Bucket algorithm is described here, in pseudocode:
 # * Delta Timestamp - Difference between current and expected timestamp value 
 
 # 0 - Calculations
-dripRate = ceil(limit.dripRate ?? 1000);
+dripRate = ceil((limit.dripRate ?? 1000) * factor);
 dripSize = ceil(limit.dripSize ?? 1);
 bucketSize = max(ceil(limit.size / factor), 1);
 maxExpiration = max(ceil((dripRate * ceil(bucketSize / dripSize)) / 1000), 1);;

packages/backend/src/server/SkRateLimiterService.ts

@@ -206,7 +206,7 @@ export class SkRateLimiterService {
 		// 0 - Calculate
 		const now = this.timeService.now;
 		const bucketSize = Math.max(Math.ceil(limit.size / factor), 1);
-		const dripRate = Math.ceil(limit.dripRate ?? 1000);
+		const dripRate = Math.ceil((limit.dripRate ?? 1000) * factor);
 		const dripSize = Math.ceil(limit.dripSize ?? 1);
 		const fullResetMs = dripRate * Math.ceil(bucketSize / dripSize);
 		const fullResetSec = Math.max(Math.ceil(fullResetMs / 1000), 1);

packages/backend/src/server/api/SignupApiService.ts

@@ -7,7 +7,7 @@ import { Inject, Injectable } from '@nestjs/common';
 import * as argon2 from 'argon2';
 import { IsNull } from 'typeorm';
 import { DI } from '@/di-symbols.js';
-import type { RegistrationTicketsRepository, UsedUsernamesRepository, UserPendingsRepository, UserProfilesRepository, UsersRepository, MiRegistrationTicket, MiMeta } from '@/models/_.js';
+import type { RegistrationTicketsRepository, UsedUsernamesRepository, UserPendingsRepository, UserProfilesRepository, UsersRepository, MiRegistrationTicket, MiMeta, UserIpsRepository } from '@/models/_.js';
 import type { Config } from '@/config.js';
 import { CaptchaService } from '@/core/CaptchaService.js';
 import { IdService } from '@/core/IdService.js';
@@ -19,11 +19,14 @@ import { FastifyReplyError } from '@/misc/fastify-reply-error.js';
 import { bindThis } from '@/decorators.js';
 import { L_CHARS, secureRndstr } from '@/misc/secure-rndstr.js';
 import { RoleService } from '@/core/RoleService.js';
+import Logger from '@/logger.js';
+import { LoggerService } from '@/core/LoggerService.js';
 import { SigninService } from './SigninService.js';
 import type { FastifyRequest, FastifyReply } from 'fastify';
 
 @Injectable()
 export class SignupApiService {
+	private logger: Logger;
 	constructor(
 		@Inject(DI.config)
 		private config: Config,
@@ -46,6 +49,9 @@ export class SignupApiService {
 		@Inject(DI.registrationTicketsRepository)
 		private registrationTicketsRepository: RegistrationTicketsRepository,
 
+		@Inject(DI.userIpsRepository)
+		private userIpsRepository: UserIpsRepository,
+
 		private userEntityService: UserEntityService,
 		private idService: IdService,
 		private captchaService: CaptchaService,
@@ -53,7 +59,9 @@ export class SignupApiService {
 		private signinService: SigninService,
 		private emailService: EmailService,
 		private roleService: RoleService,
+		private loggerService: LoggerService,
 	) {
+		this.logger = this.loggerService.getLogger('Signup');
 	}
 
 	@bindThis
@@ -213,6 +221,7 @@ export class SignupApiService {
 				username: username,
 				password: hash,
 				reason: reason,
+				requestOriginIp: this.meta.enableIpLogging ? request.ip : null,
 			});
 
 			const link = `${this.config.url}/signup-complete/${code}`;
@@ -249,6 +258,10 @@ export class SignupApiService {
 				});
 			}
 
+			if (this.meta.enableIpLogging) {
+				this.logIp(request.ip, null, account.id);
+			}
+
 			const moderators = await this.roleService.getModerators();
 
 			for (const moderator of moderators) {
@@ -282,6 +295,10 @@ export class SignupApiService {
 					});
 				}
 
+				if (this.meta.enableIpLogging) {
+					this.logIp(request.ip, null, account.id);
+				}
+
 				return {
 					...res,
 					token: secret,
@@ -332,6 +349,15 @@ export class SignupApiService {
 				});
 			}
 
+			if (pendingUser.requestOriginIp) {
+				this.logIp(pendingUser.requestOriginIp, this.idService.parse(pendingUser.id).date, account.id);
+			}
+
+			// The sign-up request and the confirmation may've come from different addresses: log both
+			if (this.meta.enableIpLogging) {
+				this.logIp(request.ip, null, account.id);
+			}
+
 			if (this.meta.approvalRequiredForSignup) {
 				if (pendingUser.email) {
 					this.emailService.sendEmail(pendingUser.email, 'Approval pending',
@@ -359,4 +385,17 @@ export class SignupApiService {
 			throw new FastifyReplyError(400, String(err), err);
 		}
 	}
+
+	@bindThis
+	private logIp(ip: string, ipDate: Date | null, userId: MiLocalUser['id']) {
+		try {
+			this.userIpsRepository.createQueryBuilder().insert().values({
+				createdAt: ipDate ?? new Date(),
+				userId,
+				ip,
+			}).orIgnore(true).execute();
+		} catch (err) {
+			this.logger.error(err as Error);
+		}
+	}
 }

packages/backend/src/server/api/StreamingApiServerService.ts

@@ -12,6 +12,7 @@ import { DI } from '@/di-symbols.js';
 import type { UsersRepository, MiAccessToken, MiUser, NoteReactionsRepository, NotesRepository, NoteFavoritesRepository } from '@/models/_.js';
 import type { Config } from '@/config.js';
 import type { Keyed, RateLimit } from '@/misc/rate-limit-utils.js';
+import { renderInlineError } from '@/misc/render-inline-error.js';
 import { NotificationService } from '@/core/NotificationService.js';
 import { bindThis } from '@/decorators.js';
 import { CacheService } from '@/core/CacheService.js';
@@ -20,6 +21,7 @@ import { UserService } from '@/core/UserService.js';
 import { ChannelFollowingService } from '@/core/ChannelFollowingService.js';
 import { getIpHash } from '@/misc/get-ip-hash.js';
 import { LoggerService } from '@/core/LoggerService.js';
+import type Logger from '@/logger.js';
 import { SkRateLimiterService } from '@/server/SkRateLimiterService.js';
 import { QueryService } from '@/core/QueryService.js';
 import { AuthenticateService, AuthenticationError } from './AuthenticateService.js';
@@ -38,6 +40,7 @@ export class StreamingApiServerService implements OnApplicationShutdown {
 	#connectionsByClient = new Map<string, Set<WebSocket.WebSocket>>(); // key: IP / user ID -> value: connection
 	#cleanConnectionsIntervalId: NodeJS.Timeout | null = null;
 	readonly #globalEv = new EventEmitter();
+	#logger: Logger;
 
 	constructor(
 		@Inject(DI.redisForSub)
@@ -69,6 +72,7 @@ export class StreamingApiServerService implements OnApplicationShutdown {
 		private config: Config,
 	) {
 		this.redisForSub.on('message', this.onRedis);
+		this.#logger = loggerService.getLogger('streaming', 'coral');
 	}
 
 	@bindThis
@@ -112,6 +116,7 @@ export class StreamingApiServerService implements OnApplicationShutdown {
 
 			let user: MiLocalUser | null = null;
 			let app: MiAccessToken | null = null;
+			let dieInstantly: [number, string] | null = null;
 
 			// https://datatracker.ietf.org/doc/html/rfc6750.html#section-2.1
 			// Note that the standard WHATWG WebSocket API does not support setting any headers,
@@ -128,21 +133,16 @@ export class StreamingApiServerService implements OnApplicationShutdown {
 				}
 			} catch (e) {
 				if (e instanceof AuthenticationError) {
-					socket.write([
-						'HTTP/1.1 401 Unauthorized',
-						'WWW-Authenticate: Bearer realm="Misskey", error="invalid_token", error_description="Failed to authenticate"',
-					].join('\r\n') + '\r\n\r\n');
+					dieInstantly = [4000, 'Failed to authenticate'];
 				} else {
 					socket.write('HTTP/1.1 500 Internal Server Error\r\n\r\n');
+					socket.destroy();
+					return;
 				}
-				socket.destroy();
-				return;
 			}
 
 			if (user?.isSuspended) {
-				socket.write('HTTP/1.1 403 Forbidden\r\n\r\n');
-				socket.destroy();
-				return;
+				dieInstantly = [4001, 'User suspended'];
 			}
 
 			// ServerServices sets `trustProxy: true`, which inside fastify/request.js ends up calling `proxyAddr` in this way, so we do the same.
@@ -220,8 +220,20 @@ export class StreamingApiServerService implements OnApplicationShutdown {
 					if (connectionsForClient.size < 1) {
 						this.#connectionsByClient.delete(limitActor);
 					}
+
+					stream.dispose();
 				});
 
+				ws.once('error', (e) => {
+					this.#logger.error(`Unhandled error in Streaming Api: ${renderInlineError(e)}`);
+					ws.terminate();
+				});
+
+				if (dieInstantly !== null) {
+					ws.close(...dieInstantly);
+					return;
+				}
+
 				this.#wss.emit('connection', ws, request, {
 					stream, user, app,
 				});

packages/backend/src/server/api/endpoints/admin/announcements/create.ts

@@ -69,6 +69,7 @@ export const paramDef = {
 		icon: { type: 'string', enum: ['info', 'warning', 'error', 'success'], default: 'info' },
 		display: { type: 'string', enum: ['normal', 'banner', 'dialog'], default: 'normal' },
 		forExistingUsers: { type: 'boolean', default: false },
+		forRoles: { type: 'array', default: [], items: { type: 'string', nullable: false, format: 'misskey:id' }, },
 		silence: { type: 'boolean', default: false },
 		needConfirmationToRead: { type: 'boolean', default: false },
 		confetti: { type: 'boolean', default: false },
@@ -93,11 +94,13 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 					icon: ps.icon,
 					display: ps.display,
 					forExistingUsers: ps.forExistingUsers,
+					forRoles: ps.forRoles,
 					silence: ps.silence,
 					needConfirmationToRead: ps.needConfirmationToRead,
 					confetti: ps.confetti,
 					userId: ps.userId,
 				}, me);
+
 				return packed;
 			} catch (e) {
 				if (e instanceof IdentifiableError) {

packages/backend/src/server/api/endpoints/admin/announcements/list.ts

@@ -57,6 +57,15 @@ export const meta = {
 					type: 'number',
 					optional: false, nullable: false,
 				},
+				forRoles: {
+					type: 'array',
+					optional: false, nullable: false,
+					items: {
+						type: 'string',
+						optional: false, nullable: false,
+						format: 'misskey:id'
+					}
+				},
 			},
 		},
 	},
@@ -122,6 +131,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				display: announcement.display,
 				isActive: announcement.isActive,
 				forExistingUsers: announcement.forExistingUsers,
+				forRoles: announcement.forRoles,
 				silence: announcement.silence,
 				needConfirmationToRead: announcement.needConfirmationToRead,
 				confetti: announcement.confetti,

packages/backend/src/server/api/endpoints/admin/announcements/update.ts

@@ -42,6 +42,7 @@ export const paramDef = {
 		icon: { type: 'string', enum: ['info', 'warning', 'error', 'success'] },
 		display: { type: 'string', enum: ['normal', 'banner', 'dialog'] },
 		forExistingUsers: { type: 'boolean' },
+		forRoles: { type: 'array', default: [], items: { type: 'string', nullable: false, format: 'misskey:id' }, },
 		silence: { type: 'boolean' },
 		needConfirmationToRead: { type: 'boolean' },
 		confetti: { type: 'boolean' },
@@ -73,6 +74,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 					display: ps.display,
 					icon: ps.icon,
 					forExistingUsers: ps.forExistingUsers,
+					forRoles: ps.forRoles,
 					silence: ps.silence,
 					needConfirmationToRead: ps.needConfirmationToRead,
 					confetti: ps.confetti,

packages/backend/src/server/api/endpoints/admin/decline-user.ts

@@ -61,7 +61,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 					'Your Account has been declined!');
 			}
 
-			await this.usedUsernamesRepository.delete({ username: user.username });
+			await this.usedUsernamesRepository.delete({ username: user.username.toLowerCase() });
 
 			await this.deleteAccountService.deleteAccount(user);
 

packages/backend/src/server/api/endpoints/admin/delete-all-files-of-a-user.ts

@@ -50,7 +50,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 			});
 
 			for (const file of files) {
-				this.driveService.deleteFile(file, false, me);
+				this.driveService.deleteFile(file);
 			}
 		});
 	}

packages/backend/src/server/api/endpoints/admin/meta.ts

@@ -664,6 +664,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				shortName: instance.shortName,
 				uri: this.config.url,
 				description: instance.description,
+				about: instance.about,
 				langs: instance.langs,
 				tosUrl: instance.termsOfServiceUrl,
 				repositoryUrl: instance.repositoryUrl,

packages/backend/src/server/api/endpoints/admin/roles/users.ts

@@ -35,7 +35,7 @@ export const meta = {
 			properties: {
 				id: { type: 'string', format: 'misskey:id' },
 				createdAt: { type: 'string', format: 'date-time' },
-				user: { ref: 'UserDetailed' },
+				user: { ref: 'User' },
 				expiresAt: { type: 'string', format: 'date-time', nullable: true },
 			},
 			required: ['id', 'createdAt', 'user'],
@@ -50,6 +50,11 @@ export const paramDef = {
 		sinceId: { type: 'string', format: 'misskey:id' },
 		untilId: { type: 'string', format: 'misskey:id' },
 		limit: { type: 'integer', minimum: 1, maximum: 100, default: 10 },
+		detail: {
+			type: 'boolean',
+			nullable: false,
+			default: true,
+		},
 	},
 	required: ['roleId'],
 } as const;
@@ -90,12 +95,12 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				.getMany();
 
 			const _users = assigns.map(({ user, userId }) => user ?? userId);
-			const _userMap = await this.userEntityService.packMany(_users, me, { schema: 'UserDetailed' })
+			const _userMap = await this.userEntityService.packMany(_users, me, { schema: ps.detail ? 'UserDetailed' : 'UserLite' })
 				.then(users => new Map(users.map(u => [u.id, u])));
 			return await Promise.all(assigns.map(async assign => ({
 				id: assign.id,
 				createdAt: this.idService.parse(assign.id).date.toISOString(),
-				user: _userMap.get(assign.userId) ?? await this.userEntityService.pack(assign.user!, me, { schema: 'UserDetailed' }),
+				user: _userMap.get(assign.userId) ?? await this.userEntityService.pack(assign.user!, me, { schema: ps.detail ? 'UserDetailed' : 'UserLite' }),
 				expiresAt: assign.expiresAt?.toISOString() ?? null,
 			})));
 		});

packages/backend/src/server/api/endpoints/admin/show-users.ts

@@ -24,7 +24,7 @@ export const meta = {
 		items: {
 			type: 'object',
 			nullable: false, optional: false,
-			ref: 'UserDetailed',
+			ref: 'User',
 		},
 	},
 } as const;
@@ -44,6 +44,11 @@ export const paramDef = {
 			default: null,
 			description: 'The local host is represented with `null`.',
 		},
+		detail: {
+			type: 'boolean',
+			nullable: false,
+			default: true,
+		},
 	},
 	required: [],
 } as const;
@@ -115,7 +120,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 
 			const users = await query.getMany();
 
-			return await this.userEntityService.packMany(users, me, { schema: 'UserDetailed' });
+			return await this.userEntityService.packMany(users, me, { schema: ps.detail ? 'UserDetailed' : 'UserLite' });
 		});
 	}
 }

packages/backend/src/server/api/endpoints/admin/update-meta.ts

@@ -67,6 +67,7 @@ export const paramDef = {
 		name: { type: 'string', nullable: true },
 		shortName: { type: 'string', nullable: true },
 		description: { type: 'string', nullable: true },
+		about: { type: 'string', nullable: true },
 		defaultLightTheme: { type: 'string', nullable: true },
 		defaultDarkTheme: { type: 'string', nullable: true },
 		defaultLike: { type: 'string' },
@@ -340,6 +341,10 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				set.description = ps.description;
 			}
 
+			if (ps.about !== undefined) {
+				set.about = ps.about;
+			}
+
 			if (ps.defaultLightTheme !== undefined) {
 				set.defaultLightTheme = ps.defaultLightTheme;
 			}

packages/backend/src/server/api/endpoints/announcements.ts

@@ -7,6 +7,7 @@ import { Inject, Injectable } from '@nestjs/common';
 import { Brackets } from 'typeorm';
 import { Endpoint } from '@/server/api/endpoint-base.js';
 import { QueryService } from '@/core/QueryService.js';
+import { RoleService } from '@/core/RoleService.js';
 import { AnnouncementEntityService } from '@/core/entities/AnnouncementEntityService.js';
 import { DI } from '@/di-symbols.js';
 import type { AnnouncementsRepository } from '@/models/_.js';
@@ -51,14 +52,20 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 		private announcementsRepository: AnnouncementsRepository,
 
 		private queryService: QueryService,
+		private roleService: RoleService,
 		private announcementEntityService: AnnouncementEntityService,
 	) {
 		super(meta, paramDef, async (ps, me) => {
+			const roles = me ? await this.roleService.getUserRoles(me) : [];
 			const query = this.queryService.makePaginationQuery(this.announcementsRepository.createQueryBuilder('announcement'), ps.sinceId, ps.untilId)
 				.andWhere('announcement.isActive = :isActive', { isActive: ps.isActive })
 				.andWhere(new Brackets(qb => {
 					if (me) qb.orWhere('announcement.userId = :meId', { meId: me.id });
 					qb.orWhere('announcement.userId IS NULL');
+				}))
+				.andWhere(new Brackets(qb => {
+					if (me) qb.orWhere('announcement.forRoles && :roles', { roles: roles.map((r) => r.id) });
+					qb.orWhere('announcement.forRoles = \'{}\'');
 				}));
 
 			const announcements = await query.limit(ps.limit).getMany();

packages/backend/src/server/api/endpoints/chat/messages/create-to-room.ts

@@ -11,6 +11,7 @@ import { DI } from '@/di-symbols.js';
 import { ApiError } from '@/server/api/error.js';
 import { ChatService } from '@/core/ChatService.js';
 import type { DriveFilesRepository, MiUser } from '@/models/_.js';
+import type { Config } from '@/config.js';
 
 export const meta = {
 	tags: ['chat'],
@@ -21,9 +22,11 @@ export const meta = {
 
 	kind: 'write:chat',
 
+	// Up to 10 message burst, then 2/second
 	limit: {
-		duration: ms('1hour'),
-		max: 500,
+		type: 'bucket',
+		size: 10,
+		dripRate: 500,
 	},
 
 	res: {
@@ -50,13 +53,19 @@ export const meta = {
 			code: 'CONTENT_REQUIRED',
 			id: '340517b7-6d04-42c0-bac1-37ee804e3594',
 		},
+
+		maxLength: {
+			message: 'You tried posting a message which is too long.',
+			code: 'MAX_LENGTH',
+			id: '3ac74a84-8fd5-4bb0-870f-01804f82ce16',
+		},
 	},
 } as const;
 
 export const paramDef = {
 	type: 'object',
 	properties: {
-		text: { type: 'string', nullable: true, maxLength: 2000 },
+		text: { type: 'string', nullable: true, minLength: 1 },
 		fileId: { type: 'string', format: 'misskey:id' },
 		toRoomId: { type: 'string', format: 'misskey:id' },
 	},
@@ -69,12 +78,19 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 		@Inject(DI.driveFilesRepository)
 		private driveFilesRepository: DriveFilesRepository,
 
+		@Inject(DI.config)
+		private config: Config,
+
 		private getterService: GetterService,
 		private chatService: ChatService,
 	) {
 		super(meta, paramDef, async (ps, me) => {
 			await this.chatService.checkChatAvailability(me.id, 'write');
 
+			if (ps.text && ps.text.length > this.config.maxNoteLength) {
+				throw new ApiError(meta.errors.maxLength);
+			}
+
 			const room = await this.chatService.findRoomById(ps.toRoomId);
 			if (room == null) {
 				throw new ApiError(meta.errors.noSuchRoom);

packages/backend/src/server/api/endpoints/chat/messages/create-to-user.ts

@@ -11,6 +11,7 @@ import { DI } from '@/di-symbols.js';
 import { ApiError } from '@/server/api/error.js';
 import { ChatService } from '@/core/ChatService.js';
 import type { DriveFilesRepository, MiUser } from '@/models/_.js';
+import type { Config } from '@/config.js';
 
 export const meta = {
 	tags: ['chat'],
@@ -21,9 +22,11 @@ export const meta = {
 
 	kind: 'write:chat',
 
+	// Up to 10 message burst, then 2/second
 	limit: {
-		duration: ms('1hour'),
-		max: 500,
+		type: 'bucket',
+		size: 10,
+		dripRate: 500,
 	},
 
 	res: {
@@ -62,13 +65,19 @@ export const meta = {
 			code: 'YOU_HAVE_BEEN_BLOCKED',
 			id: 'c15a5199-7422-4968-941a-2a462c478f7d',
 		},
+
+		maxLength: {
+			message: 'You tried posting a message which is too long.',
+			code: 'MAX_LENGTH',
+			id: '3ac74a84-8fd5-4bb0-870f-01804f82ce16',
+		},
 	},
 } as const;
 
 export const paramDef = {
 	type: 'object',
 	properties: {
-		text: { type: 'string', nullable: true, maxLength: 2000 },
+		text: { type: 'string', nullable: true, minLength: 1 },
 		fileId: { type: 'string', format: 'misskey:id' },
 		toUserId: { type: 'string', format: 'misskey:id' },
 	},
@@ -81,12 +90,19 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 		@Inject(DI.driveFilesRepository)
 		private driveFilesRepository: DriveFilesRepository,
 
+		@Inject(DI.config)
+		private config: Config,
+
 		private getterService: GetterService,
 		private chatService: ChatService,
 	) {
 		super(meta, paramDef, async (ps, me) => {
 			await this.chatService.checkChatAvailability(me.id, 'write');
 
+			if (ps.text && ps.text.length > this.config.maxNoteLength) {
+				throw new ApiError(meta.errors.maxLength);
+			}
+
 			let file = null;
 			if (ps.fileId != null) {
 				file = await this.driveFilesRepository.findOneBy({

packages/backend/src/server/api/endpoints/federation/update-remote-user.ts

@@ -7,12 +7,22 @@ import { Injectable } from '@nestjs/common';
 import { Endpoint } from '@/server/api/endpoint-base.js';
 import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js';
 import { GetterService } from '@/server/api/GetterService.js';
+import { CacheService } from '@/core/CacheService.js';
+import { ApiError } from '@/server/api/error.js';
 
 export const meta = {
 	tags: ['federation'],
 
 	requireCredential: false,
 
+	errors: {
+		noSuchUser: {
+			message: 'No such user.',
+			code: 'NO_SUCH_USER',
+			id: '558ea170-f653-4700-94d0-5a818371d0df',
+		},
+	},
+
 	// Up to 10 calls, then 4 / second.
 	// This allows for reliable automation.
 	limit: {
@@ -35,9 +45,15 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 	constructor(
 		private getterService: GetterService,
 		private apPersonService: ApPersonService,
+		private readonly cacheService: CacheService,
 	) {
 		super(meta, paramDef, async (ps) => {
-			const user = await this.getterService.getRemoteUser(ps.userId);
+			const user = await this.cacheService.findRemoteUserById(ps.userId);
+
+			if (!user) {
+				throw new ApiError(meta.errors.noSuchUser);
+			}
+
 			await this.apPersonService.updatePerson(user.uri!);
 		});
 	}

packages/backend/src/server/api/endpoints/hashtags/users.ts

@@ -23,7 +23,7 @@ export const meta = {
 		items: {
 			type: 'object',
 			optional: false, nullable: false,
-			ref: 'UserDetailed',
+			ref: 'User',
 		},
 	},
 
@@ -43,6 +43,11 @@ export const paramDef = {
 		state: { type: 'string', enum: ['all', 'alive'], default: 'all' },
 		origin: { type: 'string', enum: ['combined', 'local', 'remote'], default: 'local' },
 		trending: { type: 'boolean', default: false },
+		detail: {
+			type: 'boolean',
+			nullable: false,
+			default: true,
+		},
 	},
 	required: ['tag', 'sort'],
 } as const;
@@ -96,7 +101,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 					.map(([u]) => u);
 			}
 
-			return await this.userEntityService.packMany(users, me, { schema: 'UserDetailed' });
+			return await this.userEntityService.packMany(users, me, { schema: ps.detail ? 'UserDetailed' : 'UserLite' });
 		});
 	}
 }

packages/backend/src/server/api/endpoints/i/import-antennas.ts

@@ -4,7 +4,6 @@
  */
 
 import { Inject, Injectable } from '@nestjs/common';
-import ms from 'ms';
 import { Endpoint } from '@/server/api/endpoint-base.js';
 import { QueueService } from '@/core/QueueService.js';
 import type { AntennasRepository, DriveFilesRepository, UsersRepository, MiAntenna as _Antenna } from '@/models/_.js';
@@ -19,9 +18,11 @@ export const meta = {
 	requiredRolePolicy: 'canImportAntennas',
 	prohibitMoved: true,
 
+	// 1 per minute
 	limit: {
-		duration: ms('1hour'),
-		max: 1,
+		type: 'bucket',
+		size: 1,
+		dripRate: 1000 * 60,
 	},
 	errors: {
 		noSuchFile: {
@@ -82,7 +83,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
 			if (currentAntennasCount + antennas.length >= (await this.roleService.getUserPolicies(me.id)).antennaLimit) {
 				throw new ApiError(meta.errors.tooManyAntennas);
 			}
-			this.queueService.createImportAntennasJob(me, antennas);
+			await this.queueService.createImportAntennasJob(me, antennas, file.id);
 		});
 	}
 }

packages/backend/src/server/api/endpoints/i/import-blocking.ts

@@ -4,7 +4,6 @@
  */
 
 import { Inject, Injectable } from '@nestjs/common';
-import ms from 'ms';
 import { Endpoint } from '@/server/api/endpoint-base.js';
 import { QueueService } from '@/core/QueueService.js';
 import { AccountMoveService } from '@/core/AccountMoveService.js';
@@ -18,9 +17,11 @@ export const meta = {
 	requiredRolePolicy: 'canImportBlocking',
 	prohibitMoved: true,
 
+	// 1 per minute
 	limit: {
-		duration: ms('1hour'),
-		max: 1,
+		type: 'bucket',
+		size: 1,
+		dripRate: 1000 * 60,
 	},
 
 	errors: {

packages/backend/src/server/api/endpoints/i/import-following.ts

@@ -17,9 +17,12 @@ export const meta = {
 	requireCredential: true,
 	requiredRolePolicy: 'canImportFollowing',
 	prohibitMoved: true,
+
+	// 1 per minute
 	limit: {
-		duration: ms('1hour'),
-		max: 1,
+		type: 'bucket',
+		size: 1,
+		dripRate: 1000 * 60,
 	},
 
 	errors: {

packages/backend/src/server/api/endpoints/i/import-muting.ts

@@ -4,7 +4,6 @@
  */
 
 import { Inject, Injectable } from '@nestjs/common';
-import ms from 'ms';
 import { Endpoint } from '@/server/api/endpoint-base.js';
 import { QueueService } from '@/core/QueueService.js';
 import { AccountMoveService } from '@/core/AccountMoveService.js';
@@ -18,9 +17,11 @@ export const meta = {
 	requiredRolePolicy: 'canImportMuting',
 	prohibitMoved: true,
 
+	// 1 per minute
 	limit: {
-		duration: ms('1hour'),
-		max: 1,
+		type: 'bucket',
+		size: 1,
+		dripRate: 1000 * 60,
 	},
 
 	errors: {

packages/backend/src/server/api/endpoints/i/import-notes.ts

@@ -4,7 +4,6 @@
  */
 
 import { Inject, Injectable } from '@nestjs/common';
-import ms from 'ms';
 import { Endpoint } from '@/server/api/endpoint-base.js';
 import { QueueService } from '@/core/QueueService.js';
 import type { DriveFilesRepository } from '@/models/_.js';
@@ -16,9 +15,12 @@ export const meta = {
 	secure: true,
 	requireCredential: true,
 	prohibitMoved: true,
+
+	// 1 per minute
 	limit: {
-		duration: ms('1hour'),
-		max: 2,
+		type: 'bucket',
+		size: 1,
+		dripRate: 1000 * 60,
 	},
 
 	errors: {

packages/backend/src/server/api/endpoints/i/import-user-lists.ts

@@ -4,7 +4,6 @@
  */
 
 import { Inject, Injectable } from '@nestjs/common';
-import ms from 'ms';
 import { Endpoint } from '@/server/api/endpoint-base.js';
 import { QueueService } from '@/core/QueueService.js';
 import { AccountMoveService } from '@/core/AccountMoveService.js';
@@ -17,9 +16,12 @@ export const meta = {
 	requireCredential: true,
 	requiredRolePolicy: 'canImportUserLists',
 	prohibitMoved: true,
+
+	// 1 per minute
 	limit: {
-		duration: ms('1hour'),
-		max: 1,
+		type: 'bucket',
+		size: 1,
+		dripRate: 1000 * 60,
 	},
 
 	errors: {

packages/backend/src/server/api/endpoints/i/notifications-grouped.ts

@@ -116,7 +116,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 			// user has many notifications, the pagination will break the
 			// groups
 
-			// scan `notifications` newest-to-oldest
+			// scan `notifications` newest-to-oldest (unless we have sinceId && !untilId, in which case it's oldest-to-newest)
 			for (let i = 0; i < notifications.length; i++) {
 				const notification = notifications[i];
 
@@ -135,7 +135,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 					if (prevReaction.type !== 'reaction:grouped') {
 						prevReaction = groupedNotifications[reactionIdx] = {
 							type: 'reaction:grouped',
-							id: prevReaction.id, // this will be the newest id in this group
+							id: '',
 							createdAt: prevReaction.createdAt,
 							noteId: prevReaction.noteId!,
 							reactions: [{
@@ -149,6 +149,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 						userId: notification.notifierId!,
 						reaction: notification.reaction!,
 					});
+					prevReaction.id = notification.id; // this will be the *oldest* id in this group (newest if sinceId && !untilId)
 					continue;
 				}
 
@@ -167,7 +168,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 					if (prevRenote.type !== 'renote:grouped') {
 						prevRenote = groupedNotifications[renoteIdx] = {
 							type: 'renote:grouped',
-							id: prevRenote.id, // this will be the newest id in this group
+							id: '',
 							createdAt: prevRenote.createdAt,
 							noteId: prevRenote.noteId!,
 							userIds: [prevRenote.notifierId!],
@@ -175,6 +176,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 					}
 					// add this new renote to the existing group
 					(prevRenote as FilterUnionByProperty<MiGroupedNotification, 'type', 'renote:grouped'>).userIds.push(notification.notifierId!);
+					prevRenote.id = notification.id; // this will be the *oldest* id in this group (newest if sinceId && !untilId)
 					continue;
 				}
 
@@ -182,10 +184,12 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				groupedNotifications.push(notification);
 			}
 
-			// sort the groups by their id, newest first
+			// sort the groups by their id
 			groupedNotifications.sort(
 				(a, b) => a.id < b.id ? 1 : a.id > b.id ? -1 : 0,
 			);
+			// this matches the logic in NotificationService and it's what MkPagination expects
+			if (ps.sinceId && !ps.untilId) groupedNotifications.reverse();
 
 			return await this.notificationEntityService.packGroupedMany(groupedNotifications, me.id);
 		});

packages/backend/src/server/api/endpoints/i/update.ts

@@ -615,11 +615,15 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				this.globalEventService.publishInternalEvent('localUserUpdated', { id: user.id });
 			}
 
-			const verified_links = await verifyFieldLinks(newFields, `${this.config.url}/@${user.username}`, this.httpRequestService);
+			const profileUrls = [
+				this.userEntityService.genLocalUserUri(user.id),
+				`${this.config.url}/@${user.username}`,
+			];
+			const verifiedLinks = await verifyFieldLinks(newFields, profileUrls, this.httpRequestService);
 
 			await this.userProfilesRepository.update(user.id, {
 				...profileUpdates,
-				verifiedLinks: verified_links,
+				verifiedLinks,
 			});
 
 			const iObj = await this.userEntityService.pack(user.id, user, {

packages/backend/src/server/api/endpoints/notes/create.ts

@@ -27,10 +27,11 @@ export const meta = {
 
 	prohibitMoved: true,
 
+	// Up to 10 post burst, then 4/second
 	limit: {
-		duration: ms('1hour'),
-		max: 300,
-		minInterval: ms('1sec'),
+		type: 'bucket',
+		size: 10,
+		dripRate: 250,
 	},
 
 	kind: 'write:notes',

packages/backend/src/server/api/endpoints/notes/edit.ts

@@ -1,4 +1,8 @@
-import ms from 'ms';
+/*
+ * SPDX-FileCopyrightText: marie and other Sharkey contributors
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
 import { In } from 'typeorm';
 import { Inject, Injectable } from '@nestjs/common';
 import type { MiUser } from '@/models/User.js';
@@ -22,9 +26,11 @@ export const meta = {
 
 	prohibitMoved: true,
 
+	// Up to 10 post burst, then 2/second
 	limit: {
-		duration: ms('1hour'),
-		max: 300,
+		type: 'bucket',
+		size: 10,
+		dripRate: 500,
 	},
 
 	kind: 'write:notes',

packages/backend/src/server/api/endpoints/pinned-users.ts

@@ -23,7 +23,7 @@ export const meta = {
 		items: {
 			type: 'object',
 			optional: false, nullable: false,
-			ref: 'UserDetailed',
+			ref: 'User',
 		},
 	},
 
@@ -36,7 +36,13 @@ export const meta = {
 
 export const paramDef = {
 	type: 'object',
-	properties: {},
+	properties: {
+		detail: {
+			type: 'boolean',
+			nullable: false,
+			default: true,
+		},
+	},
 	required: [],
 } as const;
 
@@ -57,7 +63,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				host: acct.host ?? IsNull(),
 			})));
 
-			return await this.userEntityService.packMany(users.filter(x => x != null), me, { schema: 'UserDetailed' });
+			return await this.userEntityService.packMany(users.filter(x => x != null), me, { schema: ps.detail ? 'UserDetailed' : 'UserLite' });
 		});
 	}
 }

packages/backend/src/server/api/endpoints/roles/users.ts

@@ -37,7 +37,7 @@ export const meta = {
 				},
 				user: {
 					type: 'object',
-					ref: 'UserDetailed',
+					ref: 'User',
 				},
 			},
 			required: ['id', 'user'],
@@ -58,6 +58,11 @@ export const paramDef = {
 		sinceId: { type: 'string', format: 'misskey:id' },
 		untilId: { type: 'string', format: 'misskey:id' },
 		limit: { type: 'integer', minimum: 1, maximum: 100, default: 10 },
+		detail: {
+			type: 'boolean',
+			nullable: false,
+			default: true,
+		},
 	},
 	required: ['roleId'],
 } as const;
@@ -99,11 +104,11 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				.getMany();
 
 			const _users = assigns.map(({ user, userId }) => user ?? userId);
-			const _userMap = await this.userEntityService.packMany(_users, me, { schema: 'UserDetailed' })
+			const _userMap = await this.userEntityService.packMany(_users, me, { schema: ps.detail ? 'UserDetailed' : 'UserLite' })
 				.then(users => new Map(users.map(u => [u.id, u])));
 			return await Promise.all(assigns.map(async assign => ({
 				id: assign.id,
-				user: _userMap.get(assign.userId) ?? await this.userEntityService.pack(assign.user!, me, { schema: 'UserDetailed' }),
+				user: _userMap.get(assign.userId) ?? await this.userEntityService.pack(assign.user!, me, { schema: ps.detail ? 'UserDetailed' : 'UserLite' }),
 			})));
 		});
 	}

packages/backend/src/server/api/endpoints/users.ts

@@ -24,7 +24,7 @@ export const meta = {
 		items: {
 			type: 'object',
 			optional: false, nullable: false,
-			ref: 'UserDetailed',
+			ref: 'User',
 		},
 	},
 
@@ -50,6 +50,11 @@ export const paramDef = {
 			default: null,
 			description: 'The local host is represented with `null`.',
 		},
+		detail: {
+			type: 'boolean',
+			nullable: false,
+			default: true,
+		},
 	},
 	required: [],
 } as const;
@@ -111,7 +116,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				.filter(([,p]) => p.canTrend)
 				.map(([u]) => u);
 
-			return await this.userEntityService.packMany(users, me, { schema: 'UserDetailed' });
+			return await this.userEntityService.packMany(users, me, { schema: ps.detail ? 'UserDetailed' : 'UserLite' });
 		});
 	}
 

packages/backend/src/server/api/endpoints/users/get-frequently-replied-users.ts

@@ -30,7 +30,7 @@ export const meta = {
 				user: {
 					type: 'object',
 					optional: false, nullable: false,
-					ref: 'UserDetailed',
+					ref: 'User',
 				},
 				weight: {
 					type: 'number',
@@ -60,6 +60,11 @@ export const paramDef = {
 	properties: {
 		userId: { type: 'string', format: 'misskey:id' },
 		limit: { type: 'integer', minimum: 1, maximum: 100, default: 10 },
+		detail: {
+			type: 'boolean',
+			nullable: false,
+			default: true,
+		},
 	},
 	required: ['userId'],
 } as const;
@@ -127,10 +132,10 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 			const topRepliedUserIds = repliedUsersSorted.slice(0, ps.limit);
 
 			// Make replies object (includes weights)
-			const _userMap = await this.userEntityService.packMany(topRepliedUserIds, me, { schema: 'UserDetailed' })
+			const _userMap = await this.userEntityService.packMany(topRepliedUserIds, me, { schema: ps.detail ? 'UserDetailed' : 'UserLite' })
 				.then(users => new Map(users.map(u => [u.id, u])));
 			const repliesObj = await Promise.all(topRepliedUserIds.map(async (userId) => ({
-				user: _userMap.get(userId) ?? await this.userEntityService.pack(userId, me, { schema: 'UserDetailed' }),
+				user: _userMap.get(userId) ?? await this.userEntityService.pack(userId, me, { schema: ps.detail ? 'UserDetailed' : 'UserLite' }),
 				weight: repliedUsers[userId] / peak,
 			})));
 

packages/backend/src/server/api/endpoints/users/recommendation.ts

@@ -26,7 +26,7 @@ export const meta = {
 		items: {
 			type: 'object',
 			optional: false, nullable: false,
-			ref: 'UserDetailed',
+			ref: 'User',
 		},
 	},
 
@@ -42,6 +42,11 @@ export const paramDef = {
 	properties: {
 		limit: { type: 'integer', minimum: 1, maximum: 100, default: 10 },
 		offset: { type: 'integer', default: 0 },
+		detail: {
+			type: 'boolean',
+			nullable: false,
+			default: true,
+		},
 	},
 	required: [],
 } as const;
@@ -83,7 +88,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 
 			const users = await query.limit(ps.limit).offset(ps.offset).getMany();
 
-			return await this.userEntityService.packMany(users, me, { schema: 'UserDetailed' });
+			return await this.userEntityService.packMany(users, me, { schema: ps.detail ? 'UserDetailed' : 'UserLite' });
 		});
 	}
 }

packages/backend/src/server/api/endpoints/users/report-abuse.ts

@@ -9,6 +9,7 @@ import { GetterService } from '@/server/api/GetterService.js';
 import { RoleService } from '@/core/RoleService.js';
 import { AbuseReportService } from '@/core/AbuseReportService.js';
 import { ApiError } from '../../error.js';
+import { CacheService } from '@/core/CacheService.js';
 
 export const meta = {
 	tags: ['users'],
@@ -60,13 +61,14 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 		private getterService: GetterService,
 		private roleService: RoleService,
 		private abuseReportService: AbuseReportService,
+		private readonly cacheService: CacheService,
 	) {
 		super(meta, paramDef, async (ps, me) => {
 			// Lookup user
-			const targetUser = await this.getterService.getUser(ps.userId).catch(err => {
-				if (err.id === '15348ddd-432d-49c2-8a5a-8069753becff') throw new ApiError(meta.errors.noSuchUser);
-				throw err;
-			});
+			const targetUser = await this.cacheService.findOptionalUserById(ps.userId);
+			if (!targetUser) {
+				throw new ApiError(meta.errors.noSuchUser);
+			}
 
 			if (targetUser.id === me.id) {
 				throw new ApiError(meta.errors.cannotReportYourself);

packages/backend/src/server/api/endpoints/users/show.ts

@@ -30,13 +30,13 @@ export const meta = {
 		oneOf: [
 			{
 				type: 'object',
-				ref: 'UserDetailed',
+				ref: 'User',
 			},
 			{
 				type: 'array',
 				items: {
 					type: 'object',
-					ref: 'UserDetailed',
+					ref: 'User',
 				},
 			},
 		],
@@ -79,6 +79,11 @@ export const paramDef = {
 			nullable: true,
 			description: 'The local host is represented with `null`.',
 		},
+		detail: {
+			type: 'boolean',
+			nullable: false,
+			default: true,
+		},
 	},
 	anyOf: [
 		{ required: ['userId'] },
@@ -125,7 +130,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 					if (user != null) _users.push(user);
 				}
 
-				const _userMap = await this.userEntityService.packMany(_users, me, { schema: 'UserDetailed' })
+				const _userMap = await this.userEntityService.packMany(_users, me, { schema: ps.detail ? 'UserDetailed' : 'UserLite' })
 					.then(users => new Map(users.map(u => [u.id, u])));
 				return _users.map(u => _userMap.get(u.id)!);
 			} else {
@@ -156,7 +161,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				}
 
 				return await this.userEntityService.pack(user, me, {
-					schema: 'UserDetailed',
+					schema: ps.detail ? 'UserDetailed' : 'UserLite',
 				});
 			}
 		});

packages/backend/src/server/api/mastodon/MastodonDataService.ts

@@ -59,7 +59,7 @@ export class MastodonDataService {
 				if (typeof(relations.reply) === 'object') {
 					if (relations.reply.reply) query.leftJoinAndSelect('reply.reply', 'replyReply');
 					if (relations.reply.renote) query.leftJoinAndSelect('reply.renote', 'replyRenote');
-					if (relations.reply.user) query.innerJoinAndSelect('reply.user', 'replyUser');
+					if (relations.reply.user) query.leftJoinAndSelect('reply.user', 'replyUser');
 					if (relations.reply.channel) query.leftJoinAndSelect('reply.channel', 'replyChannel');
 				}
 			}
@@ -68,7 +68,7 @@ export class MastodonDataService {
 				if (typeof(relations.renote) === 'object') {
 					if (relations.renote.reply) query.leftJoinAndSelect('renote.reply', 'renoteReply');
 					if (relations.renote.renote) query.leftJoinAndSelect('renote.renote', 'renoteRenote');
-					if (relations.renote.user) query.innerJoinAndSelect('renote.user', 'renoteUser');
+					if (relations.renote.user) query.leftJoinAndSelect('renote.user', 'renoteUser');
 					if (relations.renote.channel) query.leftJoinAndSelect('renote.channel', 'renoteChannel');
 				}
 			}

packages/backend/src/server/api/mastodon/endpoints/instance.ts

@@ -41,7 +41,8 @@ export class ApiInstanceMastodon {
 			const response: MastodonEntity.Instance = {
 				uri: this.config.host,
 				title: this.meta.name || 'Sharkey',
-				description: this.meta.description || 'This is a vanilla Sharkey Instance. It doesn\'t seem to have a description.',
+				shortDescription: this.meta.description || 'This is a vanilla Sharkey Instance. It doesn\'t seem to have a description.',
+				description: this.meta.about || 'This is a vanilla Sharkey Instance.',
 				email: instance.email || '',
 				version: `3.0.0 (compatible; Sharkey ${this.config.version}; like Akkoma)`,
 				urls: instance.urls,

packages/backend/src/server/web/UrlPreviewService.ts

@@ -33,6 +33,7 @@ import { getIpHash } from '@/misc/get-ip-hash.js';
 import { isRetryableError } from '@/misc/is-retryable-error.js';
 import * as Acct from '@/misc/acct.js';
 import { isNote } from '@/core/activitypub/type.js';
+import { renderInlineError } from '@/misc/render-inline-error.js';
 import type { FastifyRequest, FastifyReply } from 'fastify';
 
 export type LocalSummalyResult = SummalyResult & {
@@ -260,7 +261,7 @@ export class UrlPreviewService {
 
 			return reply.code(200).send(summary);
 		} catch (err) {
-			this.logger.warn(`Failed to get preview of ${url} for ${lang}: ${err}`);
+			this.logger.warn(`Failed to get preview of ${url} for ${lang}: ${renderInlineError(err)}`);
 
 			reply.header('Cache-Control', 'max-age=3600');
 			return reply.code(422).send({

packages/backend/src/server/web/boot.js

@@ -129,7 +129,12 @@
 
 	const fontSize = localStorage.getItem('fontSize');
 	if (fontSize) {
-		document.documentElement.classList.add('f-' + fontSize);
+		if (fontSize === "custom") {
+			const customFontSize = localStorage.getItem('customFontSize');
+			document.documentElement.style.setProperty('font-size', `${customFontSize}px`);
+		} else {
+			document.documentElement.classList.add('f-' + fontSize);
+		}
 	}
 
 	const cornerRadius = localStorage.getItem('cornerRadius');

packages/backend/src/server/web/views/note.pug

@@ -21,6 +21,7 @@ block og
 	meta(property='og:url'         content= url)
 	if videos.length
 		each video in videos
+			meta(property='og:video'            content= video.url)
 			meta(property='og:video:url'        content= video.url)
 			meta(property='og:video:secure_url' content= video.url)
 			meta(property='og:video:type'       content= video.type)