misty.cafe/mistykey
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Merge branch 'develop' into merge/2024-02-03

Browse source 
# Conflicts:
#	packages/backend/src/server/ActivityPubServerService.ts
#	pnpm-lock.yaml
This commit is contained in:
Hazelnoot 2025-02-08 13:16:17 -05:00
commit 7e1b4b259a
18 changed files with 727 additions and 689 deletions
Download patch file Download diff file Expand all files Collapse all files

39
packages/backend/src/server/api/ApiCallService.ts
View file

@ -19,7 +19,7 @@ import { bindThis } from '@/decorators.js';
import { RoleService } from '@/core/RoleService.js';
import type { Config } from '@/config.js';
import { sendRateLimitHeaders } from '@/misc/rate-limit-utils.js';
import { SkRateLimiterService } from '@/server/api/SkRateLimiterService.js';
import { SkRateLimiterService } from '@/server/SkRateLimiterService.js';
import { ApiError } from './error.js';
import { ApiLoggerService } from './ApiLoggerService.js';
import { AuthenticateService, AuthenticationError } from './AuthenticateService.js';
@ -313,35 +313,30 @@ export class ApiCallService implements OnApplicationShutdown {
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
if (endpointLimit) {
// koa will automatically load the `X-Forwarded-For` header if `proxy: true` is configured in the app.
let limitActor: string;
let limitActor: string | MiLocalUser;
if (user) {
limitActor = user.id;
limitActor = user;
} else {
limitActor = getIpHash(request.ip);
}
// TODO: 毎リクエスト計算するのもあれだしキャッシュしたい
const factor = user ? (await this.roleService.getUserPolicies(user.id)).rateLimitFactor : 1;
const limit = {
key: ep.name,
...endpointLimit,
};
if (factor > 0) {
const limit = {
key: ep.name,
...endpointLimit,
};
// Rate limit
const info = await this.rateLimiterService.limit(limit, limitActor);
// Rate limit
const info = await this.rateLimiterService.limit(limit, limitActor, factor);
sendRateLimitHeaders(reply, info);
sendRateLimitHeaders(reply, info);
if (info.blocked) {
throw new ApiError({
message: 'Rate limit exceeded. Please try again later.',
code: 'RATE_LIMIT_EXCEEDED',
id: 'd5826d14-3982-4d2e-8011-b9e9f02499ef',
httpStatusCode: 429,
}, info);
}
if (info.blocked) {
throw new ApiError({
message: 'Rate limit exceeded. Please try again later.',
code: 'RATE_LIMIT_EXCEEDED',
id: 'd5826d14-3982-4d2e-8011-b9e9f02499ef',
httpStatusCode: 429,
}, info);
}
}

13
packages/backend/src/server/api/SigninApiService.ts
View file

@ -26,12 +26,19 @@ import { UserAuthService } from '@/core/UserAuthService.js';
import { CaptchaService } from '@/core/CaptchaService.js';
import { FastifyReplyError } from '@/misc/fastify-reply-error.js';
import { isSystemAccount } from '@/misc/is-system-account.js';
import { SkRateLimiterService } from '@/server/api/SkRateLimiterService.js';
import { sendRateLimitHeaders } from '@/misc/rate-limit-utils.js';
import { SkRateLimiterService } from '@/server/SkRateLimiterService.js';
import { Keyed, RateLimit, sendRateLimitHeaders } from '@/misc/rate-limit-utils.js';
import { SigninService } from './SigninService.js';
import type { AuthenticationResponseJSON } from '@simplewebauthn/types';
import type { FastifyReply, FastifyRequest } from 'fastify';
// Up to 10 attempts, then 1 per minute
const signinRateLimit: Keyed<RateLimit> = {
key: 'signin',
max: 10,
dripRate: 1000 * 60,
};
@Injectable()
export class SigninApiService {
constructor(
@ -94,7 +101,7 @@ export class SigninApiService {
}
// not more than 1 attempt per second and not more than 10 attempts per hour
const rateLimit = await this.rateLimiterService.limit({ key: 'signin', duration: 60 * 60 * 1000, max: 10, minInterval: 1000 }, getIpHash(request.ip));
const rateLimit = await this.rateLimiterService.limit(signinRateLimit, getIpHash(request.ip));
sendRateLimitHeaders(reply, rateLimit);

2
packages/backend/src/server/api/SigninWithPasskeyApiService.ts
View file

@ -21,7 +21,7 @@ import { WebAuthnService } from '@/core/WebAuthnService.js';
import Logger from '@/logger.js';
import { LoggerService } from '@/core/LoggerService.js';
import type { IdentifiableError } from '@/misc/identifiable-error.js';
import { SkRateLimiterService } from '@/server/api/SkRateLimiterService.js';
import { SkRateLimiterService } from '@/server/SkRateLimiterService.js';
import { sendRateLimitHeaders } from '@/misc/rate-limit-utils.js';
import { SigninService } from './SigninService.js';
import type { AuthenticationResponseJSON } from '@simplewebauthn/types';

253
packages/backend/src/server/api/SkRateLimiterService.ts
View file

@ -1,253 +0,0 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Inject, Injectable } from '@nestjs/common';
import Redis from 'ioredis';
import { TimeService } from '@/core/TimeService.js';
import { EnvService } from '@/core/EnvService.js';
import { BucketRateLimit, LegacyRateLimit, LimitInfo, RateLimit, hasMinLimit, isLegacyRateLimit, Keyed, hasMaxLimit, disabledLimitInfo, MaxLegacyLimit, MinLegacyLimit } from '@/misc/rate-limit-utils.js';
import { DI } from '@/di-symbols.js';
@Injectable()
export class SkRateLimiterService {
private readonly disabled: boolean;
constructor(
@Inject(TimeService)
private readonly timeService: TimeService,
@Inject(DI.redis)
private readonly redisClient: Redis.Redis,
@Inject(EnvService)
envService: EnvService,
) {
this.disabled = envService.env.NODE_ENV === 'test';
}
/**
* Check & increment a rate limit
* @param limit The limit definition
* @param actor Client who is calling this limit
* @param factor Scaling factor - smaller = larger limit (less restrictive)
*/
public async limit(limit: Keyed<RateLimit>, actor: string, factor = 1): Promise<LimitInfo> {
if (this.disabled || factor === 0) {
return disabledLimitInfo;
}
if (factor < 0) {
throw new Error(`Rate limit factor is zero or negative: ${factor}`);
}
return await this.tryLimit(limit, actor, factor);
}
private async tryLimit(limit: Keyed<RateLimit>, actor: string, factor: number): Promise<LimitInfo> {
if (isLegacyRateLimit(limit)) {
return await this.limitLegacy(limit, actor, factor);
} else {
return await this.limitBucket(limit, actor, factor);
}
}
private async limitLegacy(limit: Keyed<LegacyRateLimit>, actor: string, factor: number): Promise<LimitInfo> {
if (hasMaxLimit(limit)) {
return await this.limitLegacyMinMax(limit, actor, factor);
} else if (hasMinLimit(limit)) {
return await this.limitLegacyMinOnly(limit, actor, factor);
} else {
return disabledLimitInfo;
}
}
private async limitLegacyMinMax(limit: Keyed<MaxLegacyLimit>, actor: string, factor: number): Promise<LimitInfo> {
if (limit.duration === 0) return disabledLimitInfo;
if (limit.duration < 0) throw new Error(`Invalid rate limit ${limit.key}: duration is negative (${limit.duration})`);
if (limit.max < 1) throw new Error(`Invalid rate limit ${limit.key}: max is less than 1 (${limit.max})`);
// Derive initial dripRate from minInterval OR duration/max.
const initialDripRate = Math.max(limit.minInterval ?? Math.round(limit.duration / limit.max), 1);
// Calculate dripSize to reach max at exactly duration
const dripSize = Math.max(Math.round(limit.max / (limit.duration / initialDripRate)), 1);
// Calculate final dripRate from dripSize and duration/max
const dripRate = Math.max(Math.round(limit.duration / (limit.max / dripSize)), 1);
const bucketLimit: Keyed<BucketRateLimit> = {
type: 'bucket',
key: limit.key,
size: limit.max,
dripRate,
dripSize,
};
return await this.limitBucket(bucketLimit, actor, factor);
}
private async limitLegacyMinOnly(limit: Keyed<MinLegacyLimit>, actor: string, factor: number): Promise<LimitInfo> {
if (limit.minInterval === 0) return disabledLimitInfo;
if (limit.minInterval < 0) throw new Error(`Invalid rate limit ${limit.key}: minInterval is negative (${limit.minInterval})`);
const dripRate = Math.max(Math.round(limit.minInterval), 1);
const bucketLimit: Keyed<BucketRateLimit> = {
type: 'bucket',
key: limit.key,
size: 1,
dripRate,
dripSize: 1,
};
return await this.limitBucket(bucketLimit, actor, factor);
}
/**
* Implementation of Leaky Bucket rate limiting - see SkRateLimiterService.md for details.
*/
private async limitBucket(limit: Keyed<BucketRateLimit>, actor: string, factor: number): Promise<LimitInfo> {
if (limit.size < 1) throw new Error(`Invalid rate limit ${limit.key}: size is less than 1 (${limit.size})`);
if (limit.dripRate != null && limit.dripRate < 1) throw new Error(`Invalid rate limit ${limit.key}: dripRate is less than 1 (${limit.dripRate})`);
if (limit.dripSize != null && limit.dripSize < 1) throw new Error(`Invalid rate limit ${limit.key}: dripSize is less than 1 (${limit.dripSize})`);
// 0 - Calculate
const now = this.timeService.now;
const bucketSize = Math.max(Math.ceil(limit.size / factor), 1);
const dripRate = Math.ceil(limit.dripRate ?? 1000);
const dripSize = Math.ceil(limit.dripSize ?? 1);
const expirationSec = Math.max(Math.ceil((dripRate * Math.ceil(bucketSize / dripSize)) / 1000), 1);
// 1 - Read
const counterKey = createLimitKey(limit, actor, 'c');
const timestampKey = createLimitKey(limit, actor, 't');
const counter = await this.getLimitCounter(counterKey, timestampKey);
// 2 - Drip
const dripsSinceLastTick = Math.floor((now - counter.timestamp) / dripRate) * dripSize;
const deltaCounter = Math.min(dripsSinceLastTick, counter.counter);
const deltaTimestamp = dripsSinceLastTick * dripRate;
if (deltaCounter > 0) {
// Execute the next drip(s)
const results = await this.executeRedisMulti(
['get', timestampKey],
['incrby', timestampKey, deltaTimestamp],
['expire', timestampKey, expirationSec],
['get', timestampKey],
['decrby', counterKey, deltaCounter],
['expire', counterKey, expirationSec],
['get', counterKey],
);
const expectedTimestamp = counter.timestamp;
const canaryTimestamp = results[0] ? parseInt(results[0]) : 0;
counter.timestamp = results[3] ? parseInt(results[3]) : 0;
counter.counter = results[6] ? parseInt(results[6]) : 0;
// Check for a data collision and rollback
if (canaryTimestamp !== expectedTimestamp) {
const rollbackResults = await this.executeRedisMulti(
['decrby', timestampKey, deltaTimestamp],
['get', timestampKey],
['incrby', counterKey, deltaCounter],
['get', counterKey],
);
counter.timestamp = rollbackResults[1] ? parseInt(rollbackResults[1]) : 0;
counter.counter = rollbackResults[3] ? parseInt(rollbackResults[3]) : 0;
}
}
// 3 - Check
const blocked = counter.counter >= bucketSize;
if (!blocked) {
if (counter.timestamp === 0) {
const results = await this.executeRedisMulti(
['set', timestampKey, now],
['expire', timestampKey, expirationSec],
['incr', counterKey],
['expire', counterKey, expirationSec],
['get', counterKey],
);
counter.timestamp = now;
counter.counter = results[4] ? parseInt(results[4]) : 0;
} else {
const results = await this.executeRedisMulti(
['incr', counterKey],
['expire', counterKey, expirationSec],
['get', counterKey],
);
counter.counter = results[2] ? parseInt(results[2]) : 0;
}
}
// Calculate how much time is needed to free up a bucket slot
const overflow = Math.max((counter.counter + 1) - bucketSize, 0);
const dripsNeeded = Math.ceil(overflow / dripSize);
const timeNeeded = Math.max((dripRate * dripsNeeded) - (this.timeService.now - counter.timestamp), 0);
// Calculate limit status
const remaining = Math.max(bucketSize - counter.counter, 0);
const resetMs = timeNeeded;
const resetSec = Math.ceil(resetMs / 1000);
const fullResetMs = Math.ceil(counter.counter / dripSize) * dripRate;
const fullResetSec = Math.ceil(fullResetMs / 1000);
return { blocked, remaining, resetSec, resetMs, fullResetSec, fullResetMs };
}
private async getLimitCounter(counterKey: string, timestampKey: string): Promise<LimitCounter> {
const [counter, timestamp] = await this.executeRedisMulti(
['get', counterKey],
['get', timestampKey],
);
return {
counter: counter ? parseInt(counter) : 0,
timestamp: timestamp ? parseInt(timestamp) : 0,
};
}
private async executeRedisMulti(...batch: RedisCommand[]): Promise<RedisResult[]> {
const results = await this.redisClient.multi(batch).exec();
// Transaction conflict (retryable)
if (!results) {
throw new ConflictError('Redis error: transaction conflict');
}
// Transaction failed (fatal)
if (results.length !== batch.length) {
throw new Error('Redis error: failed to execute batch');
}
// Map responses
const errors: Error[] = [];
const responses: RedisResult[] = [];
for (const [error, response] of results) {
if (error) errors.push(error);
responses.push(response as RedisResult);
}
// Command failed (fatal)
if (errors.length > 0) {
const errorMessages = errors
.map((e, i) => `Error in command ${i}: ${e}`)
.join('\', \'');
throw new AggregateError(errors, `Redis error: failed to execute command(s): '${errorMessages}'`);
}
return responses;
}
}
// Not correct, but good enough for the basic commands we use.
type RedisResult = string | null;
type RedisCommand = [command: string, ...args: unknown[]];
function createLimitKey(limit: Keyed<RateLimit>, actor: string, value: string): string {
return `rl_${actor}_${limit.key}_${value}`;
}
class ConflictError extends Error {}
interface LimitCounter {
timestamp: number;
counter: number;
}

18
packages/backend/src/server/api/StreamingApiServerService.ts
View file

@ -18,10 +18,9 @@ import { CacheService } from '@/core/CacheService.js';
import { MiLocalUser } from '@/models/User.js';
import { UserService } from '@/core/UserService.js';
import { ChannelFollowingService } from '@/core/ChannelFollowingService.js';
import { RoleService } from '@/core/RoleService.js';
import { getIpHash } from '@/misc/get-ip-hash.js';
import { LoggerService } from '@/core/LoggerService.js';
import { SkRateLimiterService } from '@/server/api/SkRateLimiterService.js';
import { SkRateLimiterService } from '@/server/SkRateLimiterService.js';
import { AuthenticateService, AuthenticationError } from './AuthenticateService.js';
import MainStreamConnection from './stream/Connection.js';
import { ChannelsService } from './stream/ChannelsService.js';
@ -49,7 +48,6 @@ export class StreamingApiServerService {
private usersService: UserService,
private channelFollowingService: ChannelFollowingService,
private rateLimiterService: SkRateLimiterService,
private roleService: RoleService,
private loggerService: LoggerService,
) {
}
@ -57,22 +55,18 @@ export class StreamingApiServerService {
@bindThis
private async rateLimitThis(
user: MiLocalUser | null | undefined,
requestIp: string | undefined,
requestIp: string,
limit: IEndpointMeta['limit'] & { key: NonNullable<string> },
) : Promise<boolean> {
let limitActor: string;
let limitActor: string | MiLocalUser;
if (user) {
limitActor = user.id;
limitActor = user;
} else {
limitActor = getIpHash(requestIp || 'wtf');
limitActor = getIpHash(requestIp);
}
const factor = user ? (await this.roleService.getUserPolicies(user.id)).rateLimitFactor : 1;
if (factor <= 0) return false;
// Rate limit
const rateLimit = await this.rateLimiterService.limit(limit, limitActor, factor);
const rateLimit = await this.rateLimiterService.limit(limit, limitActor);
return rateLimit.blocked;
}