From ece57345d7a6d66bf58536a7c7b012ddd7971b26 Mon Sep 17 00:00:00 2001 From: dakkar Date: Wed, 5 Mar 2025 14:12:51 +0000 Subject: [PATCH 01/72] when creating a note as a side-effect, make it silent - fixes #986 --- packages/backend/src/core/activitypub/ApInboxService.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/backend/src/core/activitypub/ApInboxService.ts b/packages/backend/src/core/activitypub/ApInboxService.ts index 402d5ab2a4..3995eeea67 100644 --- a/packages/backend/src/core/activitypub/ApInboxService.ts +++ b/packages/backend/src/core/activitypub/ApInboxService.ts @@ -470,7 +470,7 @@ export class ApInboxService { } @bindThis - private async create(actor: MiRemoteUser, activity: ICreate | IUpdate, resolver?: Resolver): Promise { + private async create(actor: MiRemoteUser, activity: ICreate | IUpdate, resolver?: Resolver, silent = false): Promise { const uri = getApId(activity); this.logger.info(`Create: ${uri}`); @@ -505,7 +505,7 @@ export class ApInboxService { }); if (isPost(object)) { - await this.createNote(resolver, actor, object, false); + await this.createNote(resolver, actor, object, silent); } else { return `skip: Unsupported type for Create: ${getApType(object)} ${getNullableApId(object)}`; } @@ -896,7 +896,7 @@ export class ApInboxService { } else if (getApType(object) === 'Question') { // If we get an Update(Question) for a note that doesn't exist, then create it instead if (!await this.apNoteService.hasNote(object)) { - return await this.create(actor, activity, resolver); + return await this.create(actor, activity, resolver, true); } await this.apQuestionService.updateQuestion(object, actor, resolver); @@ -904,7 +904,7 @@ export class ApInboxService { } else if (isPost(object)) { // If we get an Update(Note) for a note that doesn't exist, then create it instead if (!await this.apNoteService.hasNote(object)) { - return await this.create(actor, activity, resolver); + return await this.create(actor, activity, resolver, true); } await this.apNoteService.updateNote(object, actor, resolver); From 239bfd3b625091f12f2908f3d5c117e8c4f229d4 Mon Sep 17 00:00:00 2001 From: Marie Date: Thu, 8 May 2025 11:45:36 +0200 Subject: [PATCH 02/72] add missing state ref: https://github.com/misskey-dev/misskey/issues/15992 --- packages/backend/src/server/api/endpoints/admin/queue/jobs.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/server/api/endpoints/admin/queue/jobs.ts b/packages/backend/src/server/api/endpoints/admin/queue/jobs.ts index 79731c9786..aba68376ad 100644 --- a/packages/backend/src/server/api/endpoints/admin/queue/jobs.ts +++ b/packages/backend/src/server/api/endpoints/admin/queue/jobs.ts @@ -20,7 +20,7 @@ export const paramDef = { type: 'object', properties: { queue: { type: 'string', enum: QUEUE_TYPES }, - state: { type: 'array', items: { type: 'string', enum: ['active', 'wait', 'delayed', 'completed', 'failed'] } }, + state: { type: 'array', items: { type: 'string', enum: ['active', 'paused', 'wait', 'delayed', 'completed', 'failed'] } }, search: { type: 'string' }, }, required: ['queue', 'state'], From 8635365b8f475c5febdff981ee12777f1e421110 Mon Sep 17 00:00:00 2001 From: Marie Date: Thu, 8 May 2025 11:46:21 +0200 Subject: [PATCH 03/72] add missing state to types --- packages/misskey-js/src/autogen/types.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/misskey-js/src/autogen/types.ts b/packages/misskey-js/src/autogen/types.ts index 1868ba44d5..57e98f2f88 100644 --- a/packages/misskey-js/src/autogen/types.ts +++ b/packages/misskey-js/src/autogen/types.ts @@ -9578,7 +9578,7 @@ export type operations = { 'application/json': { /** @enum {string} */ queue: 'system' | 'endedPollNotification' | 'deliver' | 'inbox' | 'db' | 'relationship' | 'objectStorage' | 'userWebhookDeliver' | 'systemWebhookDeliver'; - state: ('active' | 'wait' | 'delayed' | 'completed' | 'failed')[]; + state: ('active' | 'paused' | 'wait' | 'delayed' | 'completed' | 'failed')[]; search?: string; }; }; @@ -17529,6 +17529,8 @@ export type operations = { sort?: '+createdAt' | '-createdAt' | '+name' | '-name' | '+size' | '-size' | null; /** @default */ searchQuery?: string; + /** @default false */ + showAll?: boolean; }; }; }; From 1a94437ac060eb5f1f8f704270684423b1ca6bda Mon Sep 17 00:00:00 2001 From: dakkar Date: Thu, 8 May 2025 11:37:06 +0100 Subject: [PATCH 04/72] bump version to be a `-rc` --- package.json | 2 +- packages/misskey-js/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 46a4e63aad..ffd07c5950 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sharkey", - "version": "2025.4.1", + "version": "2025.4.2-rc", "codename": "shonk", "repository": { "type": "git", diff --git a/packages/misskey-js/package.json b/packages/misskey-js/package.json index 5949b0e2f6..d6073c9861 100644 --- a/packages/misskey-js/package.json +++ b/packages/misskey-js/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "misskey-js", - "version": "2025.4.1", + "version": "2025.4.2-rc", "description": "Misskey SDK for JavaScript", "license": "MIT", "main": "./built/index.js", From 938e094a1a7bf0f156883e46d538f3fb176dcc67 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 5 May 2025 08:44:42 -0400 Subject: [PATCH 05/72] set summary.haveNoteLocally before caching summary --- packages/backend/src/server/web/UrlPreviewService.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index aa8fcd0c2a..6c4e95cabc 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -137,12 +137,12 @@ export class UrlPreviewService { summary.icon = this.wrap(summary.icon); summary.thumbnail = this.wrap(summary.thumbnail); - this.previewCache.set(key, summary); - if (summary.activityPub) { - summary.haveNoteLocally = !! await this.apDbResolverService.getNoteFromApId(summary.activityPub); + summary.haveNoteLocally = !!await this.apDbResolverService.getNoteFromApId(summary.activityPub); } + this.previewCache.set(key, summary); + // Cache 7days reply.header('Cache-Control', 'max-age=604800, immutable'); From 129dfa964946a079aba08a5fa21379cfcbec0d70 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 5 May 2025 09:06:27 -0400 Subject: [PATCH 06/72] extract LocalSummalyResult type --- .../src/server/web/UrlPreviewService.ts | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index 6c4e95cabc..a9be094ce7 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -18,31 +18,35 @@ import { ApiError } from '@/server/api/error.js'; import { MiMeta } from '@/models/Meta.js'; import { RedisKVCache } from '@/misc/cache.js'; import { UtilityService } from '@/core/UtilityService.js'; -import type { FastifyRequest, FastifyReply } from 'fastify'; import { ApDbResolverService } from '@/core/activitypub/ApDbResolverService.js'; +import type { FastifyRequest, FastifyReply } from 'fastify'; + +export type LocalSummalyResult = SummalyResult & { + haveNoteLocally?: boolean; +}; @Injectable() export class UrlPreviewService { private logger: Logger; - private previewCache: RedisKVCache; + private previewCache: RedisKVCache; constructor( @Inject(DI.config) private config: Config, @Inject(DI.redis) - private redisClient: Redis.Redis, + private readonly redisClient: Redis.Redis, @Inject(DI.meta) - private meta: MiMeta, + private readonly meta: MiMeta, private httpRequestService: HttpRequestService, private loggerService: LoggerService, - private utilityService: UtilityService, - private apDbResolverService: ApDbResolverService, + private readonly utilityService: UtilityService, + private readonly apDbResolverService: ApDbResolverService, ) { this.logger = this.loggerService.getLogger('url-preview'); - this.previewCache = new RedisKVCache(this.redisClient, 'summaly', { + this.previewCache = new RedisKVCache(this.redisClient, 'summaly', { lifetime: 1000 * 60 * 60 * 24, // 1d memoryCacheLifetime: 1000 * 60 * 10, // 10m fetcher: () => { throw new Error('the UrlPreview cache should never fetch'); }, @@ -102,7 +106,7 @@ export class UrlPreviewService { } const key = `${url}@${lang}`; - const cached = await this.previewCache.get(key) as SummalyResult & { haveNoteLocally?: boolean }; + const cached = await this.previewCache.get(key); if (cached !== undefined) { this.logger.info(`Returning cache preview of ${key}`); // Cache 7days @@ -120,7 +124,7 @@ export class UrlPreviewService { : `Getting preview of ${key} ...`); try { - const summary: SummalyResult & { haveNoteLocally?: boolean } = this.meta.urlPreviewSummaryProxyUrl + const summary: LocalSummalyResult = this.meta.urlPreviewSummaryProxyUrl ? await this.fetchSummaryFromProxy(url, this.meta, lang) : await this.fetchSummary(url, this.meta, lang); @@ -162,7 +166,7 @@ export class UrlPreviewService { } } - private fetchSummary(url: string, meta: MiMeta, lang?: string): Promise { + private fetchSummary(url: string, meta: MiMeta, lang?: string): Promise { const agent = this.config.proxy ? { http: this.httpRequestService.httpAgent, @@ -181,7 +185,7 @@ export class UrlPreviewService { }); } - private fetchSummaryFromProxy(url: string, meta: MiMeta, lang?: string): Promise { + private fetchSummaryFromProxy(url: string, meta: MiMeta, lang?: string): Promise { const proxy = meta.urlPreviewSummaryProxyUrl!; const queryStr = query({ url: url, @@ -192,6 +196,6 @@ export class UrlPreviewService { contentLengthRequired: meta.urlPreviewRequireContentLength, }); - return this.httpRequestService.getJson(`${proxy}?${queryStr}`, 'application/json, */*', undefined, true); + return this.httpRequestService.getJson(`${proxy}?${queryStr}`, 'application/json, */*', undefined, true); } } From 2fb56bc4ead2196b65b900e915be5bc658e4934e Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 5 May 2025 09:06:45 -0400 Subject: [PATCH 07/72] fix eslint warning in UrlPreviewService.ts --- packages/backend/src/server/web/UrlPreviewService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index a9be094ce7..8c1776568b 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -67,7 +67,7 @@ export class UrlPreviewService { @bindThis public async handle( - request: FastifyRequest<{ Querystring: { url: string; lang?: string; } }>, + request: FastifyRequest<{ Querystring: { url?: string; lang?: string; } }>, reply: FastifyReply, ): Promise { const url = request.query.url; From ab65f4b8b2d0cc15c8279467b6150b4f0e51f704 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 5 May 2025 09:07:26 -0400 Subject: [PATCH 08/72] infer ActivityPub links from local DB --- .../src/server/web/UrlPreviewService.ts | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index 8c1776568b..a38454fafc 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -20,6 +20,9 @@ import { RedisKVCache } from '@/misc/cache.js'; import { UtilityService } from '@/core/UtilityService.js'; import { ApDbResolverService } from '@/core/activitypub/ApDbResolverService.js'; import type { FastifyRequest, FastifyReply } from 'fastify'; +import type { NotesRepository } from '@/models/_.js'; +import { ApUtilityService } from '@/core/activitypub/ApUtilityService.js'; +import { IsNull, Not } from 'typeorm'; export type LocalSummalyResult = SummalyResult & { haveNoteLocally?: boolean; @@ -40,9 +43,13 @@ export class UrlPreviewService { @Inject(DI.meta) private readonly meta: MiMeta, + @Inject(DI.notesRepository) + private readonly notesRepository: NotesRepository, + private httpRequestService: HttpRequestService, private loggerService: LoggerService, private readonly utilityService: UtilityService, + private readonly apUtilityService: ApUtilityService, private readonly apDbResolverService: ApDbResolverService, ) { this.logger = this.loggerService.getLogger('url-preview'); @@ -143,6 +150,9 @@ export class UrlPreviewService { if (summary.activityPub) { summary.haveNoteLocally = !!await this.apDbResolverService.getNoteFromApId(summary.activityPub); + } else { + // Summaly cannot always detect links to a fedi post, so check if it matches anything we already have + await this.inferActivityPubLink(summary); } this.previewCache.set(key, summary); @@ -198,4 +208,33 @@ export class UrlPreviewService { return this.httpRequestService.getJson(`${proxy}?${queryStr}`, 'application/json, */*', undefined, true); } + + private async inferActivityPubLink(summary: LocalSummalyResult) { + // Match canonical URI first. + // This covers local and remote links. + const isCanonicalUri = !!await this.apDbResolverService.getNoteFromApId(summary.url); + if (isCanonicalUri) { + summary.activityPub = summary.url; + summary.haveNoteLocally = true; + } + + // Try public URL next. + // This is necessary for Mastodon and other software with a different public URL. + const urlMatches = await this.notesRepository.find({ + select: { + uri: true, + }, + where: { + url: summary.url, + uri: Not(IsNull()), + }, + }) as { uri: string }[]; + + // Older versions did not validate URL, so do it now to avoid impersonation. + const matchByUrl = urlMatches.find(({ uri }) => this.apUtilityService.haveSameAuthority(uri, summary.url)); + if (matchByUrl) { + summary.activityPub = matchByUrl.uri; + summary.haveNoteLocally = true; + } + } } From 1d2a4c6f5631ffa1155a284b58e41b72ed8b7cc2 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 5 May 2025 09:14:09 -0400 Subject: [PATCH 09/72] infer ActivityPub links from signed GET --- .../src/server/web/UrlPreviewService.ts | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index a38454fafc..8f2ec5be00 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -7,6 +7,7 @@ import { Inject, Injectable } from '@nestjs/common'; import { summaly } from '@misskey-dev/summaly'; import { SummalyResult } from '@misskey-dev/summaly/built/summary.js'; import * as Redis from 'ioredis'; +import { IsNull, Not } from 'typeorm'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import { HttpRequestService } from '@/core/HttpRequestService.js'; @@ -19,10 +20,11 @@ import { MiMeta } from '@/models/Meta.js'; import { RedisKVCache } from '@/misc/cache.js'; import { UtilityService } from '@/core/UtilityService.js'; import { ApDbResolverService } from '@/core/activitypub/ApDbResolverService.js'; -import type { FastifyRequest, FastifyReply } from 'fastify'; import type { NotesRepository } from '@/models/_.js'; import { ApUtilityService } from '@/core/activitypub/ApUtilityService.js'; -import { IsNull, Not } from 'typeorm'; +import { ApRequestService } from '@/core/activitypub/ApRequestService.js'; +import { SystemAccountService } from '@/core/SystemAccountService.js'; +import type { FastifyRequest, FastifyReply } from 'fastify'; export type LocalSummalyResult = SummalyResult & { haveNoteLocally?: boolean; @@ -51,6 +53,8 @@ export class UrlPreviewService { private readonly utilityService: UtilityService, private readonly apUtilityService: ApUtilityService, private readonly apDbResolverService: ApDbResolverService, + private readonly apRequestService: ApRequestService, + private readonly systemAccountService: SystemAccountService, ) { this.logger = this.loggerService.getLogger('url-preview'); this.previewCache = new RedisKVCache(this.redisClient, 'summaly', { @@ -216,6 +220,7 @@ export class UrlPreviewService { if (isCanonicalUri) { summary.activityPub = summary.url; summary.haveNoteLocally = true; + return; } // Try public URL next. @@ -235,6 +240,16 @@ export class UrlPreviewService { if (matchByUrl) { summary.activityPub = matchByUrl.uri; summary.haveNoteLocally = true; + return; + } + + // Finally, attempt a signed GET in case it's a direct link to an instance with authorized fetch. + const instanceActor = await this.systemAccountService.getInstanceActor(); + const remoteObject = await this.apRequestService.signedGet(summary.url, instanceActor).catch(() => null); + if (remoteObject) { + summary.activityPub = remoteObject.id; + summary.haveNoteLocally = false; + return; } } } From 05201f71ccffe7aa1b8faab447c66c598fd2b4e5 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 5 May 2025 09:15:24 -0400 Subject: [PATCH 10/72] allow summaly previews to redirect --- .../src/server/web/UrlPreviewService.ts | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index 8f2ec5be00..876c9a9674 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -104,8 +104,7 @@ export class UrlPreviewService { }; } - const host = new URL(url).host; - if (this.utilityService.isBlockedHost(this.meta.blockedHosts, host)) { + if (this.utilityService.isBlockedHost(this.meta.blockedHosts, new URL(url).host)) { reply.code(403); return { error: new ApiError({ @@ -139,6 +138,18 @@ export class UrlPreviewService { ? await this.fetchSummaryFromProxy(url, this.meta, lang) : await this.fetchSummary(url, this.meta, lang); + // Repeat check, since redirects are allowed. + if (this.utilityService.isBlockedHost(this.meta.blockedHosts, new URL(summary.url).host)) { + reply.code(403); + return { + error: new ApiError({ + message: 'URL is blocked', + code: 'URL_PREVIEW_BLOCKED', + id: '50294652-857b-4b13-9700-8e5c7a8deae8', + }), + }; + } + this.logger.succ(`Got preview of ${url}: ${summary.title}`); if (!(summary.url.startsWith('http://') || summary.url.startsWith('https://'))) { @@ -189,7 +200,7 @@ export class UrlPreviewService { : undefined; return summaly(url, { - followRedirects: false, + followRedirects: true, lang: lang ?? 'ja-JP', agent: agent, userAgent: meta.urlPreviewUserAgent ?? undefined, @@ -202,6 +213,7 @@ export class UrlPreviewService { private fetchSummaryFromProxy(url: string, meta: MiMeta, lang?: string): Promise { const proxy = meta.urlPreviewSummaryProxyUrl!; const queryStr = query({ + followRedirects: true, url: url, lang: lang ?? 'ja-JP', userAgent: meta.urlPreviewUserAgent ?? undefined, From 80819f03e7a9404cf603648abf4581d0352e5997 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 5 May 2025 09:21:53 -0400 Subject: [PATCH 11/72] don't proxy local URLs --- .../backend/src/server/web/UrlPreviewService.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index 876c9a9674..0312dff16d 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -68,12 +68,16 @@ export class UrlPreviewService { @bindThis private wrap(url?: string | null): string | null { - return url != null - ? `${this.config.mediaProxy}/preview.webp?${query({ - url, - preview: '1', - })}` - : null; + if (url == null) return null; + + // Don't proxy our own media + if (this.utilityService.isUriLocal(url)) { + return url; + } + + // But proxy everything else! + const mediaQuery = query({ url, preview: '1' }); + return `${this.config.mediaProxy}/preview.webp?${mediaQuery}`; } @bindThis From 387efac23ffcb5743279ba96798fc5db26ce6807 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 5 May 2025 09:24:28 -0400 Subject: [PATCH 12/72] add version specifier to URL preview cache --- .../backend/src/server/web/UrlPreviewService.ts | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index 0312dff16d..fc09554ce8 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -30,6 +30,9 @@ export type LocalSummalyResult = SummalyResult & { haveNoteLocally?: boolean; }; +// Increment this to invalidate cached previews after a major change. +const cacheFormatVersion = 1; + @Injectable() export class UrlPreviewService { private logger: Logger; @@ -119,10 +122,10 @@ export class UrlPreviewService { }; } - const key = `${url}@${lang}`; - const cached = await this.previewCache.get(key); + const cacheKey = `${url}@${lang}@${cacheFormatVersion}`; + const cached = await this.previewCache.get(cacheKey); if (cached !== undefined) { - this.logger.info(`Returning cache preview of ${key}`); + this.logger.info(`Returning cache preview of ${cacheKey}`); // Cache 7days reply.header('Cache-Control', 'max-age=604800, immutable'); @@ -134,8 +137,8 @@ export class UrlPreviewService { } this.logger.info(this.meta.urlPreviewSummaryProxyUrl - ? `(Proxy) Getting preview of ${key} ...` - : `Getting preview of ${key} ...`); + ? `(Proxy) Getting preview of ${cacheKey} ...` + : `Getting preview of ${cacheKey} ...`); try { const summary: LocalSummalyResult = this.meta.urlPreviewSummaryProxyUrl @@ -174,7 +177,7 @@ export class UrlPreviewService { await this.inferActivityPubLink(summary); } - this.previewCache.set(key, summary); + this.previewCache.set(cacheKey, summary); // Cache 7days reply.header('Cache-Control', 'max-age=604800, immutable'); From 163be8d4a4a978d3fbaad37909f8c8f9be61e08c Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 5 May 2025 09:25:38 -0400 Subject: [PATCH 13/72] match preview cache duration for HTTP and Redis --- packages/backend/src/server/web/UrlPreviewService.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index fc09554ce8..7eeb6535b1 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -126,8 +126,8 @@ export class UrlPreviewService { const cached = await this.previewCache.get(cacheKey); if (cached !== undefined) { this.logger.info(`Returning cache preview of ${cacheKey}`); - // Cache 7days - reply.header('Cache-Control', 'max-age=604800, immutable'); + // Cache 1 day (matching redis) + reply.header('Cache-Control', 'public, max-age=86400'); if (cached.activityPub) { cached.haveNoteLocally = !! await this.apDbResolverService.getNoteFromApId(cached.activityPub); @@ -179,8 +179,8 @@ export class UrlPreviewService { this.previewCache.set(cacheKey, summary); - // Cache 7days - reply.header('Cache-Control', 'max-age=604800, immutable'); + // Cache 1 day (matching redis) + reply.header('Cache-Control', 'public, max-age=86400'); return summary; } catch (err) { From c23b1c3be77be82ef73d3dac47c58e771fea7eff Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 5 May 2025 09:33:32 -0400 Subject: [PATCH 14/72] reduce log spam from UrlPreviewService.ts --- packages/backend/src/server/web/UrlPreviewService.ts | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index 7eeb6535b1..13e103c780 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -125,7 +125,6 @@ export class UrlPreviewService { const cacheKey = `${url}@${lang}@${cacheFormatVersion}`; const cached = await this.previewCache.get(cacheKey); if (cached !== undefined) { - this.logger.info(`Returning cache preview of ${cacheKey}`); // Cache 1 day (matching redis) reply.header('Cache-Control', 'public, max-age=86400'); @@ -136,10 +135,6 @@ export class UrlPreviewService { return cached; } - this.logger.info(this.meta.urlPreviewSummaryProxyUrl - ? `(Proxy) Getting preview of ${cacheKey} ...` - : `Getting preview of ${cacheKey} ...`); - try { const summary: LocalSummalyResult = this.meta.urlPreviewSummaryProxyUrl ? await this.fetchSummaryFromProxy(url, this.meta, lang) @@ -157,7 +152,7 @@ export class UrlPreviewService { }; } - this.logger.succ(`Got preview of ${url}: ${summary.title}`); + this.logger.info(`Got preview of ${url} in ${lang}: ${summary.title}`); if (!(summary.url.startsWith('http://') || summary.url.startsWith('https://'))) { throw new Error('unsupported schema included'); @@ -184,7 +179,7 @@ export class UrlPreviewService { return summary; } catch (err) { - this.logger.warn(`Failed to get preview of ${url}: ${err}`); + this.logger.warn(`Failed to get preview of ${url} for ${lang}: ${err}`); reply.code(422); reply.header('Cache-Control', 'max-age=86400, immutable'); From a1fcf554fa6bf823f0efca1c70a1e4adfd4aaae3 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 5 May 2025 09:35:13 -0400 Subject: [PATCH 15/72] reduce caching for failed previews --- packages/backend/src/server/web/UrlPreviewService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index 13e103c780..e192bae2a8 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -182,7 +182,7 @@ export class UrlPreviewService { this.logger.warn(`Failed to get preview of ${url} for ${lang}: ${err}`); reply.code(422); - reply.header('Cache-Control', 'max-age=86400, immutable'); + reply.header('Cache-Control', 'max-age=3600'); return { error: new ApiError({ message: 'Failed to get preview', From 23267a3a9648e5e713ac021bf4ebd6d8ed067934 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 5 May 2025 09:36:17 -0400 Subject: [PATCH 16/72] await cache update to avoid hammering redis in UrlPreviewService.ts --- packages/backend/src/server/web/UrlPreviewService.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index e192bae2a8..d6151b665a 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -172,7 +172,8 @@ export class UrlPreviewService { await this.inferActivityPubLink(summary); } - this.previewCache.set(cacheKey, summary); + // Await this to avoid hammering redis when a bunch of URLs are fetched at once + await this.previewCache.set(cacheKey, summary); // Cache 1 day (matching redis) reply.header('Cache-Control', 'public, max-age=86400'); From d6c2140821a4595862e063949d2f92530bd16cfd Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 5 May 2025 09:43:40 -0400 Subject: [PATCH 17/72] validate more URLs in UrlPreviewService.ts --- packages/backend/src/core/UtilityService.ts | 10 +++++ .../src/server/web/UrlPreviewService.ts | 45 +++++++++++++++---- 2 files changed, 47 insertions(+), 8 deletions(-) diff --git a/packages/backend/src/core/UtilityService.ts b/packages/backend/src/core/UtilityService.ts index f8d04c0592..170afc72dc 100644 --- a/packages/backend/src/core/UtilityService.ts +++ b/packages/backend/src/core/UtilityService.ts @@ -176,4 +176,14 @@ export class UtilityService { const host = this.extractDbHost(uri); return this.isFederationAllowedHost(host); } + + @bindThis + public getUrlScheme(url: string): string { + try { + // Returns in the format "https:" or an empty string + return new URL(url).protocol; + } catch { + return ''; + } + } } diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index d6151b665a..fe03583270 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -140,6 +140,8 @@ export class UrlPreviewService { ? await this.fetchSummaryFromProxy(url, this.meta, lang) : await this.fetchSummary(url, this.meta, lang); + this.validateUrls(summary); + // Repeat check, since redirects are allowed. if (this.utilityService.isBlockedHost(this.meta.blockedHosts, new URL(summary.url).host)) { reply.code(403); @@ -154,14 +156,6 @@ export class UrlPreviewService { this.logger.info(`Got preview of ${url} in ${lang}: ${summary.title}`); - if (!(summary.url.startsWith('http://') || summary.url.startsWith('https://'))) { - throw new Error('unsupported schema included'); - } - - if (summary.player.url && !(summary.player.url.startsWith('http://') || summary.player.url.startsWith('https://'))) { - throw new Error('unsupported schema included'); - } - summary.icon = this.wrap(summary.icon); summary.thumbnail = this.wrap(summary.thumbnail); @@ -228,6 +222,41 @@ export class UrlPreviewService { return this.httpRequestService.getJson(`${proxy}?${queryStr}`, 'application/json, */*', undefined, true); } + private validateUrls(summary: LocalSummalyResult) { + const urlScheme = this.utilityService.getUrlScheme(summary.url); + if (urlScheme !== 'http:' && urlScheme !== 'https:') { + throw new Error(`unsupported scheme in preview URL: "${urlScheme}"`); + } + + if (summary.player.url) { + const playerScheme = this.utilityService.getUrlScheme(summary.player.url); + if (playerScheme !== 'http:' && playerScheme !== 'https:') { + throw new Error(`unsupported scheme in player URL: "${playerScheme}"`); + } + } + + if (summary.icon) { + const iconScheme = this.utilityService.getUrlScheme(summary.icon); + if (iconScheme !== 'http:' && iconScheme !== 'https:') { + throw new Error(`unsupported scheme in icon URL: "${iconScheme}"`); + } + } + + if (summary.thumbnail) { + const thumbnailScheme = this.utilityService.getUrlScheme(summary.thumbnail); + if (thumbnailScheme !== 'http:' && thumbnailScheme !== 'https:') { + throw new Error(`unsupported scheme in thumbnail URL: "${thumbnailScheme}"`); + } + } + + if (summary.activityPub) { + const activityPubScheme = this.utilityService.getUrlScheme(summary.activityPub); + if (activityPubScheme !== 'http:' && activityPubScheme !== 'https:') { + throw new Error(`unsupported scheme in ActivityPub URL: "${activityPubScheme}"`); + } + } + } + private async inferActivityPubLink(summary: LocalSummalyResult) { // Match canonical URI first. // This covers local and remote links. From c05aa7a28181b64669fb9e23c5e40b162292bef2 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 5 May 2025 09:56:28 -0400 Subject: [PATCH 18/72] softer URL preview validation: remove unsupported URLs instead of rejecting the whole preview --- packages/backend/src/server/web/UrlPreviewService.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index fe03583270..0bd3eff3ba 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -231,28 +231,32 @@ export class UrlPreviewService { if (summary.player.url) { const playerScheme = this.utilityService.getUrlScheme(summary.player.url); if (playerScheme !== 'http:' && playerScheme !== 'https:') { - throw new Error(`unsupported scheme in player URL: "${playerScheme}"`); + this.logger.warn(`Redacting preview for ${summary.url}: player URL has unsupported scheme "${playerScheme}"`); + summary.player.url = null; } } if (summary.icon) { const iconScheme = this.utilityService.getUrlScheme(summary.icon); if (iconScheme !== 'http:' && iconScheme !== 'https:') { - throw new Error(`unsupported scheme in icon URL: "${iconScheme}"`); + this.logger.warn(`Redacting preview for ${summary.url}: icon URL has unsupported scheme "${iconScheme}"`); + summary.icon = null; } } if (summary.thumbnail) { const thumbnailScheme = this.utilityService.getUrlScheme(summary.thumbnail); if (thumbnailScheme !== 'http:' && thumbnailScheme !== 'https:') { - throw new Error(`unsupported scheme in thumbnail URL: "${thumbnailScheme}"`); + this.logger.warn(`Redacting preview for ${summary.url}: thumbnail URL has unsupported scheme "${thumbnailScheme}"`); + summary.thumbnail = null; } } if (summary.activityPub) { const activityPubScheme = this.utilityService.getUrlScheme(summary.activityPub); if (activityPubScheme !== 'http:' && activityPubScheme !== 'https:') { - throw new Error(`unsupported scheme in ActivityPub URL: "${activityPubScheme}"`); + this.logger.warn(`Redacting preview for ${summary.url}: ActivityPub URL has unsupported scheme "${activityPubScheme}"`); + summary.activityPub = null; } } } From 70d75f1d57ccaa751778b13cf3b7c8e2e360c8aa Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 5 May 2025 10:12:21 -0400 Subject: [PATCH 19/72] check summary.haveNoteLocally after setting summary.activityPub to improve support for Akkoma --- .../src/server/web/UrlPreviewService.ts | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index 0bd3eff3ba..2ae626f8b7 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -31,7 +31,7 @@ export type LocalSummalyResult = SummalyResult & { }; // Increment this to invalidate cached previews after a major change. -const cacheFormatVersion = 1; +const cacheFormatVersion = 2; @Injectable() export class UrlPreviewService { @@ -159,11 +159,11 @@ export class UrlPreviewService { summary.icon = this.wrap(summary.icon); summary.thumbnail = this.wrap(summary.thumbnail); + // Summaly cannot always detect links to a fedi post, so do some additional tests to try and find missed cases. + summary.activityPub ??= await this.inferActivityPubLink(summary); + if (summary.activityPub) { summary.haveNoteLocally = !!await this.apDbResolverService.getNoteFromApId(summary.activityPub); - } else { - // Summaly cannot always detect links to a fedi post, so check if it matches anything we already have - await this.inferActivityPubLink(summary); } // Await this to avoid hammering redis when a bunch of URLs are fetched at once @@ -261,14 +261,12 @@ export class UrlPreviewService { } } - private async inferActivityPubLink(summary: LocalSummalyResult) { + private async inferActivityPubLink(summary: LocalSummalyResult): Promise { // Match canonical URI first. // This covers local and remote links. const isCanonicalUri = !!await this.apDbResolverService.getNoteFromApId(summary.url); if (isCanonicalUri) { - summary.activityPub = summary.url; - summary.haveNoteLocally = true; - return; + return summary.url; } // Try public URL next. @@ -286,18 +284,17 @@ export class UrlPreviewService { // Older versions did not validate URL, so do it now to avoid impersonation. const matchByUrl = urlMatches.find(({ uri }) => this.apUtilityService.haveSameAuthority(uri, summary.url)); if (matchByUrl) { - summary.activityPub = matchByUrl.uri; - summary.haveNoteLocally = true; - return; + return matchByUrl.uri; } // Finally, attempt a signed GET in case it's a direct link to an instance with authorized fetch. const instanceActor = await this.systemAccountService.getInstanceActor(); const remoteObject = await this.apRequestService.signedGet(summary.url, instanceActor).catch(() => null); if (remoteObject) { - summary.activityPub = remoteObject.id; - summary.haveNoteLocally = false; - return; + return remoteObject.id; } + + // No match :( + return null; } } From 633718ffe90bb0eacb4013208416cc79a2eed09e Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 5 May 2025 10:32:06 -0400 Subject: [PATCH 20/72] avoid fetching notes twice in UrlPreviewService --- .../src/server/web/UrlPreviewService.ts | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index 2ae626f8b7..4c40496305 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -160,10 +160,13 @@ export class UrlPreviewService { summary.thumbnail = this.wrap(summary.thumbnail); // Summaly cannot always detect links to a fedi post, so do some additional tests to try and find missed cases. - summary.activityPub ??= await this.inferActivityPubLink(summary); + if (!summary.activityPub) { + await this.inferActivityPubLink(summary); + } if (summary.activityPub) { - summary.haveNoteLocally = !!await this.apDbResolverService.getNoteFromApId(summary.activityPub); + // Avoid duplicate checks in case inferActivityPubLink already set this. + summary.haveNoteLocally ||= !!await this.apDbResolverService.getNoteFromApId(summary.activityPub); } // Await this to avoid hammering redis when a bunch of URLs are fetched at once @@ -261,12 +264,14 @@ export class UrlPreviewService { } } - private async inferActivityPubLink(summary: LocalSummalyResult): Promise { + private async inferActivityPubLink(summary: LocalSummalyResult) { // Match canonical URI first. // This covers local and remote links. const isCanonicalUri = !!await this.apDbResolverService.getNoteFromApId(summary.url); if (isCanonicalUri) { - return summary.url; + summary.activityPub = summary.url; + summary.haveNoteLocally = true; + return; } // Try public URL next. @@ -284,17 +289,17 @@ export class UrlPreviewService { // Older versions did not validate URL, so do it now to avoid impersonation. const matchByUrl = urlMatches.find(({ uri }) => this.apUtilityService.haveSameAuthority(uri, summary.url)); if (matchByUrl) { - return matchByUrl.uri; + summary.activityPub = matchByUrl.uri; + summary.haveNoteLocally = true; + return; } // Finally, attempt a signed GET in case it's a direct link to an instance with authorized fetch. const instanceActor = await this.systemAccountService.getInstanceActor(); const remoteObject = await this.apRequestService.signedGet(summary.url, instanceActor).catch(() => null); if (remoteObject) { - return remoteObject.id; + summary.activityPub = remoteObject.id; + return; } - - // No match :( - return null; } } From 1ac9625eea5a33544f2424bac6de6e94ffe0a4ad Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 5 May 2025 10:37:04 -0400 Subject: [PATCH 21/72] add same-authority check between fetched note and summary url --- packages/backend/src/server/web/UrlPreviewService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index 4c40496305..15a4fc946f 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -297,7 +297,7 @@ export class UrlPreviewService { // Finally, attempt a signed GET in case it's a direct link to an instance with authorized fetch. const instanceActor = await this.systemAccountService.getInstanceActor(); const remoteObject = await this.apRequestService.signedGet(summary.url, instanceActor).catch(() => null); - if (remoteObject) { + if (remoteObject && this.apUtilityService.haveSameAuthority(remoteObject.id, summary.url)) { summary.activityPub = remoteObject.id; return; } From 207915856aaf5b3fd8fd797cae5876951cd8566b Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Thu, 8 May 2025 11:06:06 -0400 Subject: [PATCH 22/72] fix return type of fetchSummary and fetchSummaryFromProxy --- packages/backend/src/server/web/UrlPreviewService.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index 15a4fc946f..0cab657c23 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -191,7 +191,7 @@ export class UrlPreviewService { } } - private fetchSummary(url: string, meta: MiMeta, lang?: string): Promise { + private fetchSummary(url: string, meta: MiMeta, lang?: string): Promise { const agent = this.config.proxy ? { http: this.httpRequestService.httpAgent, @@ -210,7 +210,7 @@ export class UrlPreviewService { }); } - private fetchSummaryFromProxy(url: string, meta: MiMeta, lang?: string): Promise { + private fetchSummaryFromProxy(url: string, meta: MiMeta, lang?: string): Promise { const proxy = meta.urlPreviewSummaryProxyUrl!; const queryStr = query({ followRedirects: true, From cd4fbc851b0fc766c93552971cb916e4ccd1ef55 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Tue, 6 May 2025 12:55:51 -0400 Subject: [PATCH 23/72] improve compatibility with multipart/form-data mastodon API requests --- packages/backend/package.json | 1 - packages/backend/src/server/ServerModule.ts | 2 + .../src/server/ServerUtilityService.ts | 141 ++++++++++++++++++ .../api/mastodon/MastodonApiServerService.ts | 132 +++------------- .../server/api/mastodon/endpoints/account.ts | 53 +++---- .../src/server/api/mastodon/endpoints/apps.ts | 5 +- .../server/api/mastodon/endpoints/filter.ts | 7 +- .../api/mastodon/endpoints/notifications.ts | 7 +- .../src/server/oauth/OAuth2ProviderService.ts | 40 +---- packages/megalodon/src/misskey.ts | 6 +- pnpm-lock.yaml | 49 ------ 11 files changed, 202 insertions(+), 241 deletions(-) create mode 100644 packages/backend/src/server/ServerUtilityService.ts diff --git a/packages/backend/package.json b/packages/backend/package.json index 9aa26033d0..4a9560e833 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -115,7 +115,6 @@ "deep-email-validator": "0.1.21", "fast-xml-parser": "4.4.1", "fastify": "5.3.2", - "fastify-multer": "^2.0.3", "fastify-raw-body": "5.0.0", "feed": "4.2.2", "file-type": "19.6.0", diff --git a/packages/backend/src/server/ServerModule.ts b/packages/backend/src/server/ServerModule.ts index 6726d4aa67..8ff8da380a 100644 --- a/packages/backend/src/server/ServerModule.ts +++ b/packages/backend/src/server/ServerModule.ts @@ -16,6 +16,7 @@ import { ApiTimelineMastodon } from '@/server/api/mastodon/endpoints/timeline.js import { ApiAppsMastodon } from '@/server/api/mastodon/endpoints/apps.js'; import { ApiInstanceMastodon } from '@/server/api/mastodon/endpoints/instance.js'; import { ApiStatusMastodon } from '@/server/api/mastodon/endpoints/status.js'; +import { ServerUtilityService } from '@/server/ServerUtilityService.js'; import { ApiCallService } from './api/ApiCallService.js'; import { FileServerService } from './FileServerService.js'; import { HealthServerService } from './HealthServerService.js'; @@ -126,6 +127,7 @@ import { SigninWithPasskeyApiService } from './api/SigninWithPasskeyApiService.j ApiSearchMastodon, ApiStatusMastodon, ApiTimelineMastodon, + ServerUtilityService, ], exports: [ ServerService, diff --git a/packages/backend/src/server/ServerUtilityService.ts b/packages/backend/src/server/ServerUtilityService.ts new file mode 100644 index 0000000000..f2900fad4f --- /dev/null +++ b/packages/backend/src/server/ServerUtilityService.ts @@ -0,0 +1,141 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import querystring from 'querystring'; +import multipart from '@fastify/multipart'; +import { Inject, Injectable } from '@nestjs/common'; +import { FastifyInstance } from 'fastify'; +import { DI } from '@/di-symbols.js'; +import type { Config } from '@/config.js'; + +@Injectable() +export class ServerUtilityService { + constructor( + @Inject(DI.config) + private readonly config: Config, + ) {} + + public addMultipartFormDataContentType(fastify: FastifyInstance): void { + fastify.register(multipart, { + limits: { + fileSize: this.config.maxFileSize, + files: 1, + }, + }); + + // Default behavior saves files to memory - we don't want that! + // Store to temporary file instead, and copy the body fields while we're at it. + fastify.addHook<{ Body?: Record }>('onRequest', async request => { + if (request.isMultipart()) { + const body = request.body ??= {}; + + // Save upload to temp directory. + // These are attached to request.savedRequestFiles + await request.saveRequestFiles(); + + // Copy fields to body + const formData = await request.formData(); + formData.forEach((v, k) => { + // This can be string or File, and we handle files above. + if (typeof(v) === 'string') { + // This is just progressive conversion from undefined -> string -> string[] + if (body[k]) { + if (Array.isArray(body[k])) { + body[k].push(v); + } else { + body[k] = [body[k], v]; + } + } else { + body[k] = v; + } + } + }); + } + }); + } + + public addFormUrlEncodedContentType(fastify: FastifyInstance) { + fastify.addContentTypeParser('application/x-www-form-urlencoded', (_, payload, done) => { + let body = ''; + payload.on('data', (data) => { + body += data; + }); + payload.on('end', () => { + try { + const parsed = querystring.parse(body); + done(null, parsed); + } catch (e) { + done(e as Error); + } + }); + payload.on('error', done); + }); + } + + public addCORS(fastify: FastifyInstance) { + fastify.addHook('onRequest', (_, reply, done) => { + // Allow web-based clients to connect from other origins. + reply.header('Access-Control-Allow-Origin', '*'); + + // Mastodon uses all types of request methods. + reply.header('Access-Control-Allow-Methods', '*'); + + // Allow web-based clients to access Link header - required for mastodon pagination. + // https://stackoverflow.com/a/54928828 + // https://docs.joinmastodon.org/api/guidelines/#pagination + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Access-Control-Expose-Headers + reply.header('Access-Control-Expose-Headers', 'Link'); + + // Cache to avoid extra pre-flight requests + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Access-Control-Max-Age + reply.header('Access-Control-Max-Age', 60 * 60 * 24); // 1 day in seconds + + done(); + }); + } + + public addFlattenedQueryType(fastify: FastifyInstance) { + // Remove trailing "[]" from query params + fastify.addHook<{ Querystring?: Record }>('preValidation', (request, _reply, done) => { + if (!request.query || typeof(request.query) !== 'object') { + return done(); + } + + for (const key of Object.keys(request.query)) { + if (!key.endsWith('[]')) { + continue; + } + if (request.query[key] == null) { + continue; + } + + const newKey = key.substring(0, key.length - 2); + const newValue = request.query[key]; + const oldValue = request.query[newKey]; + + // Move the value to the correct key + if (oldValue != null) { + if (Array.isArray(oldValue)) { + // Works for both array and single values + request.query[newKey] = oldValue.concat(newValue); + } else if (Array.isArray(newValue)) { + // Preserve order + request.query[newKey] = [oldValue, ...newValue]; + } else { + // Preserve order + request.query[newKey] = [oldValue, newValue]; + } + } else { + request.query[newKey] = newValue; + } + + // Remove the invalid key + delete request.query[key]; + } + + return done(); + }); + } +} diff --git a/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts b/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts index 59ab3b71aa..757610450a 100644 --- a/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts +++ b/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts @@ -3,12 +3,8 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import querystring from 'querystring'; -import multer from 'fastify-multer'; -import { Inject, Injectable } from '@nestjs/common'; -import { DI } from '@/di-symbols.js'; +import { Injectable } from '@nestjs/common'; import { bindThis } from '@/decorators.js'; -import type { Config } from '@/config.js'; import { getErrorData, getErrorStatus, MastodonLogger } from '@/server/api/mastodon/MastodonLogger.js'; import { MastodonClientService } from '@/server/api/mastodon/MastodonClientService.js'; import { ApiAccountMastodon } from '@/server/api/mastodon/endpoints/account.js'; @@ -20,6 +16,7 @@ import { ApiNotificationsMastodon } from '@/server/api/mastodon/endpoints/notifi import { ApiTimelineMastodon } from '@/server/api/mastodon/endpoints/timeline.js'; import { ApiSearchMastodon } from '@/server/api/mastodon/endpoints/search.js'; import { ApiError } from '@/server/api/error.js'; +import { ServerUtilityService } from '@/server/ServerUtilityService.js'; import { parseTimelineArgs, TimelineArgs, toBoolean } from './argsUtils.js'; import { convertAnnouncement, convertAttachment, MastodonConverters, convertRelationship } from './MastodonConverters.js'; import type { Entity } from 'megalodon'; @@ -28,9 +25,6 @@ import type { FastifyInstance, FastifyPluginOptions } from 'fastify'; @Injectable() export class MastodonApiServerService { constructor( - @Inject(DI.config) - private readonly config: Config, - private readonly mastoConverters: MastodonConverters, private readonly logger: MastodonLogger, private readonly clientService: MastodonClientService, @@ -42,97 +36,15 @@ export class MastodonApiServerService { private readonly apiSearchMastodon: ApiSearchMastodon, private readonly apiStatusMastodon: ApiStatusMastodon, private readonly apiTimelineMastodon: ApiTimelineMastodon, + private readonly serverUtilityService: ServerUtilityService, ) {} @bindThis public createServer(fastify: FastifyInstance, _options: FastifyPluginOptions, done: (err?: Error) => void) { - const upload = multer({ - storage: multer.diskStorage({}), - limits: { - fileSize: this.config.maxFileSize || 262144000, - files: 1, - }, - }); - - fastify.addHook('onRequest', (_, reply, done) => { - // Allow web-based clients to connect from other origins. - reply.header('Access-Control-Allow-Origin', '*'); - - // Mastodon uses all types of request methods. - reply.header('Access-Control-Allow-Methods', '*'); - - // Allow web-based clients to access Link header - required for mastodon pagination. - // https://stackoverflow.com/a/54928828 - // https://docs.joinmastodon.org/api/guidelines/#pagination - // https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Access-Control-Expose-Headers - reply.header('Access-Control-Expose-Headers', 'Link'); - - // Cache to avoid extra pre-flight requests - // https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Access-Control-Max-Age - reply.header('Access-Control-Max-Age', 60 * 60 * 24); // 1 day in seconds - - done(); - }); - - fastify.addContentTypeParser('application/x-www-form-urlencoded', (_, payload, done) => { - let body = ''; - payload.on('data', (data) => { - body += data; - }); - payload.on('end', () => { - try { - const parsed = querystring.parse(body); - done(null, parsed); - } catch (e) { - done(e as Error); - } - }); - payload.on('error', done); - }); - - // Remove trailing "[]" from query params - fastify.addHook('preValidation', (request, _reply, done) => { - if (!request.query || typeof(request.query) !== 'object') { - return done(); - } - - // Same object aliased with a different type - const query = request.query as Record; - - for (const key of Object.keys(query)) { - if (!key.endsWith('[]')) { - continue; - } - if (query[key] == null) { - continue; - } - - const newKey = key.substring(0, key.length - 2); - const newValue = query[key]; - const oldValue = query[newKey]; - - // Move the value to the correct key - if (oldValue != null) { - if (Array.isArray(oldValue)) { - // Works for both array and single values - query[newKey] = oldValue.concat(newValue); - } else if (Array.isArray(newValue)) { - // Preserve order - query[newKey] = [oldValue, ...newValue]; - } else { - // Preserve order - query[newKey] = [oldValue, newValue]; - } - } else { - query[newKey] = newValue; - } - - // Remove the invalid key - delete query[key]; - } - - return done(); - }); + this.serverUtilityService.addMultipartFormDataContentType(fastify); + this.serverUtilityService.addFormUrlEncodedContentType(fastify); + this.serverUtilityService.addCORS(fastify); + this.serverUtilityService.addFlattenedQueryType(fastify); fastify.setErrorHandler((error, request, reply) => { const data = getErrorData(error); @@ -143,14 +55,12 @@ export class MastodonApiServerService { reply.code(status).send(data); }); - fastify.register(multer.contentParser); - // External endpoints - this.apiAccountMastodon.register(fastify, upload); - this.apiAppsMastodon.register(fastify, upload); - this.apiFilterMastodon.register(fastify, upload); + this.apiAccountMastodon.register(fastify); + this.apiAppsMastodon.register(fastify); + this.apiFilterMastodon.register(fastify); this.apiInstanceMastodon.register(fastify); - this.apiNotificationsMastodon.register(fastify, upload); + this.apiNotificationsMastodon.register(fastify); this.apiSearchMastodon.register(fastify); this.apiStatusMastodon.register(fastify); this.apiTimelineMastodon.register(fastify); @@ -178,11 +88,10 @@ export class MastodonApiServerService { reply.send(data.data); }); - fastify.post('/v1/media', { preHandler: upload.single('file') }, async (_request, reply) => { - const multipartData = await _request.file(); + fastify.post('/v1/media', async (_request, reply) => { + const multipartData = _request.savedRequestFiles?.[0]; if (!multipartData) { - reply.code(401).send({ error: 'No image' }); - return; + return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'No image' }); } const client = this.clientService.getClient(_request); @@ -192,11 +101,10 @@ export class MastodonApiServerService { reply.send(response); }); - fastify.post<{ Body: { description?: string; focus?: string } }>('/v2/media', { preHandler: upload.single('file') }, async (_request, reply) => { - const multipartData = await _request.file(); + fastify.post<{ Body: { description?: string; focus?: string } }>('/v2/media', async (_request, reply) => { + const multipartData = _request.savedRequestFiles?.[0]; if (!multipartData) { - reply.code(401).send({ error: 'No image' }); - return; + return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'No image' }); } const client = this.clientService.getClient(_request); @@ -294,7 +202,7 @@ export class MastodonApiServerService { reply.send(response); }); - fastify.post<{ Querystring: TimelineArgs, Params: { id?: string } }>('/v1/follow_requests/:id/authorize', { preHandler: upload.single('none') }, async (_request, reply) => { + fastify.post<{ Params: { id?: string } }>('/v1/follow_requests/:id/authorize', async (_request, reply) => { if (!_request.params.id) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required parameter "id"' }); const client = this.clientService.getClient(_request); @@ -304,7 +212,7 @@ export class MastodonApiServerService { reply.send(response); }); - fastify.post<{ Querystring: TimelineArgs, Params: { id?: string } }>('/v1/follow_requests/:id/reject', { preHandler: upload.single('none') }, async (_request, reply) => { + fastify.post<{ Params: { id?: string } }>('/v1/follow_requests/:id/reject', async (_request, reply) => { if (!_request.params.id) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required parameter "id"' }); const client = this.clientService.getClient(_request); @@ -325,7 +233,7 @@ export class MastodonApiServerService { focus?: string, is_sensitive?: string, }, - }>('/v1/media/:id', { preHandler: upload.none() }, async (_request, reply) => { + }>('/v1/media/:id', async (_request, reply) => { if (!_request.params.id) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required parameter "id"' }); const options = { diff --git a/packages/backend/src/server/api/mastodon/endpoints/account.ts b/packages/backend/src/server/api/mastodon/endpoints/account.ts index 8bc3c14c15..b4ce56408e 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/account.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/account.ts @@ -11,7 +11,6 @@ import { DI } from '@/di-symbols.js'; import type { AccessTokensRepository, UserProfilesRepository } from '@/models/_.js'; import { attachMinMaxPagination } from '@/server/api/mastodon/pagination.js'; import { MastodonConverters, convertRelationship, convertFeaturedTag, convertList } from '../MastodonConverters.js'; -import type multer from 'fastify-multer'; import type { FastifyInstance } from 'fastify'; interface ApiAccountMastodonRoute { @@ -34,7 +33,7 @@ export class ApiAccountMastodon { private readonly driveService: DriveService, ) {} - public register(fastify: FastifyInstance, upload: ReturnType): void { + public register(fastify: FastifyInstance): void { fastify.get('/v1/accounts/verify_credentials', async (_request, reply) => { const client = this.clientService.getClient(_request); const data = await client.verifyAccountCredentials(); @@ -70,60 +69,50 @@ export class ApiAccountMastodon { value: string, }[], }, - }>('/v1/accounts/update_credentials', { preHandler: upload.any() }, async (_request, reply) => { + }>('/v1/accounts/update_credentials', async (_request, reply) => { const accessTokens = _request.headers.authorization; const client = this.clientService.getClient(_request); // Check if there is a Header or Avatar being uploaded, if there is proceed to upload it to the drive of the user and then set it. - if (_request.files.length > 0 && accessTokens) { + if (_request.savedRequestFiles?.length && accessTokens) { const tokeninfo = await this.accessTokensRepository.findOneBy({ token: accessTokens.replace('Bearer ', '') }); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const avatar = (_request.files as any).find((obj: any) => { + const avatar = _request.savedRequestFiles.find(obj => { return obj.fieldname === 'avatar'; }); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const header = (_request.files as any).find((obj: any) => { + const header = _request.savedRequestFiles.find(obj => { return obj.fieldname === 'header'; }); if (tokeninfo && avatar) { const upload = await this.driveService.addFile({ user: { id: tokeninfo.userId, host: null }, - path: avatar.path, - name: avatar.originalname !== null && avatar.originalname !== 'file' ? avatar.originalname : undefined, + path: avatar.filepath, + name: avatar.filename && avatar.filename !== 'file' ? avatar.filename : undefined, sensitive: false, }); if (upload.type.startsWith('image/')) { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (_request.body as any).avatar = upload.id; + _request.body.avatar = upload.id; } } else if (tokeninfo && header) { const upload = await this.driveService.addFile({ user: { id: tokeninfo.userId, host: null }, - path: header.path, - name: header.originalname !== null && header.originalname !== 'file' ? header.originalname : undefined, + path: header.filepath, + name: header.filename && header.filename !== 'file' ? header.filename : undefined, sensitive: false, }); if (upload.type.startsWith('image/')) { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (_request.body as any).header = upload.id; + _request.body.header = upload.id; } } } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - if ((_request.body as any).fields_attributes) { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const fields = (_request.body as any).fields_attributes.map((field: any) => { + if (_request.body.fields_attributes) { + for (const field of _request.body.fields_attributes) { if (!(field.name.trim() === '' && field.value.trim() === '')) { if (field.name.trim() === '') return reply.code(400).send('Field name can not be empty'); if (field.value.trim() === '') return reply.code(400).send('Field value can not be empty'); } - return { - ...field, - }; - }); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (_request.body as any).fields_attributes = fields.filter((field: any) => field.name.trim().length > 0 && field.value.length > 0); + } + _request.body.fields_attributes = _request.body.fields_attributes.filter(field => field.name.trim().length > 0 && field.value.length > 0); } const options = { @@ -234,7 +223,7 @@ export class ApiAccountMastodon { reply.send(response); }); - fastify.post('/v1/accounts/:id/follow', { preHandler: upload.single('none') }, async (_request, reply) => { + fastify.post('/v1/accounts/:id/follow', async (_request, reply) => { if (!_request.params.id) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required parameter "id"' }); const client = this.clientService.getClient(_request); @@ -245,7 +234,7 @@ export class ApiAccountMastodon { reply.send(acct); }); - fastify.post('/v1/accounts/:id/unfollow', { preHandler: upload.single('none') }, async (_request, reply) => { + fastify.post('/v1/accounts/:id/unfollow', async (_request, reply) => { if (!_request.params.id) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required parameter "id"' }); const client = this.clientService.getClient(_request); @@ -256,7 +245,7 @@ export class ApiAccountMastodon { reply.send(acct); }); - fastify.post('/v1/accounts/:id/block', { preHandler: upload.single('none') }, async (_request, reply) => { + fastify.post('/v1/accounts/:id/block', async (_request, reply) => { if (!_request.params.id) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required parameter "id"' }); const client = this.clientService.getClient(_request); @@ -266,7 +255,7 @@ export class ApiAccountMastodon { reply.send(response); }); - fastify.post('/v1/accounts/:id/unblock', { preHandler: upload.single('none') }, async (_request, reply) => { + fastify.post('/v1/accounts/:id/unblock', async (_request, reply) => { if (!_request.params.id) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required parameter "id"' }); const client = this.clientService.getClient(_request); @@ -276,7 +265,7 @@ export class ApiAccountMastodon { return reply.send(response); }); - fastify.post('/v1/accounts/:id/mute', { preHandler: upload.single('none') }, async (_request, reply) => { + fastify.post('/v1/accounts/:id/mute', async (_request, reply) => { if (!_request.params.id) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required parameter "id"' }); const client = this.clientService.getClient(_request); @@ -289,7 +278,7 @@ export class ApiAccountMastodon { reply.send(response); }); - fastify.post('/v1/accounts/:id/unmute', { preHandler: upload.single('none') }, async (_request, reply) => { + fastify.post('/v1/accounts/:id/unmute', async (_request, reply) => { if (!_request.params.id) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required parameter "id"' }); const client = this.clientService.getClient(_request); diff --git a/packages/backend/src/server/api/mastodon/endpoints/apps.ts b/packages/backend/src/server/api/mastodon/endpoints/apps.ts index dbef3b7d35..ec08600e53 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/apps.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/apps.ts @@ -6,7 +6,6 @@ import { Injectable } from '@nestjs/common'; import { MastodonClientService } from '@/server/api/mastodon/MastodonClientService.js'; import type { FastifyInstance } from 'fastify'; -import type multer from 'fastify-multer'; const readScope = [ 'read:account', @@ -62,8 +61,8 @@ export class ApiAppsMastodon { private readonly clientService: MastodonClientService, ) {} - public register(fastify: FastifyInstance, upload: ReturnType): void { - fastify.post('/v1/apps', { preHandler: upload.single('none') }, async (_request, reply) => { + public register(fastify: FastifyInstance): void { + fastify.post('/v1/apps', async (_request, reply) => { const body = _request.body ?? _request.query; if (!body.scopes) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required payload "scopes"' }); if (!body.redirect_uris) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required payload "redirect_uris"' }); diff --git a/packages/backend/src/server/api/mastodon/endpoints/filter.ts b/packages/backend/src/server/api/mastodon/endpoints/filter.ts index deac1e9aad..242f068b99 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/filter.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/filter.ts @@ -8,7 +8,6 @@ import { toBoolean } from '@/server/api/mastodon/argsUtils.js'; import { MastodonClientService } from '@/server/api/mastodon/MastodonClientService.js'; import { convertFilter } from '../MastodonConverters.js'; import type { FastifyInstance } from 'fastify'; -import type multer from 'fastify-multer'; interface ApiFilterMastodonRoute { Params: { @@ -29,7 +28,7 @@ export class ApiFilterMastodon { private readonly clientService: MastodonClientService, ) {} - public register(fastify: FastifyInstance, upload: ReturnType): void { + public register(fastify: FastifyInstance): void { fastify.get('/v1/filters', async (_request, reply) => { const client = this.clientService.getClient(_request); @@ -49,7 +48,7 @@ export class ApiFilterMastodon { reply.send(response); }); - fastify.post('/v1/filters', { preHandler: upload.single('none') }, async (_request, reply) => { + fastify.post('/v1/filters', async (_request, reply) => { if (!_request.body.phrase) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required payload "phrase"' }); if (!_request.body.context) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required payload "context"' }); @@ -68,7 +67,7 @@ export class ApiFilterMastodon { reply.send(response); }); - fastify.post('/v1/filters/:id', { preHandler: upload.single('none') }, async (_request, reply) => { + fastify.post('/v1/filters/:id', async (_request, reply) => { if (!_request.params.id) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required parameter "id"' }); if (!_request.body.phrase) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required payload "phrase"' }); if (!_request.body.context) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required payload "context"' }); diff --git a/packages/backend/src/server/api/mastodon/endpoints/notifications.ts b/packages/backend/src/server/api/mastodon/endpoints/notifications.ts index ee6c990fd1..75512c2efc 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/notifications.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/notifications.ts @@ -10,7 +10,6 @@ import { MastodonConverters } from '@/server/api/mastodon/MastodonConverters.js' import { attachMinMaxPagination } from '@/server/api/mastodon/pagination.js'; import { MastodonClientService } from '../MastodonClientService.js'; import type { FastifyInstance } from 'fastify'; -import type multer from 'fastify-multer'; interface ApiNotifyMastodonRoute { Params: { @@ -26,7 +25,7 @@ export class ApiNotificationsMastodon { private readonly clientService: MastodonClientService, ) {} - public register(fastify: FastifyInstance, upload: ReturnType): void { + public register(fastify: FastifyInstance): void { fastify.get('/v1/notifications', async (request, reply) => { const { client, me } = await this.clientService.getAuthClient(request); const data = await client.getNotifications(parseTimelineArgs(request.query)); @@ -66,7 +65,7 @@ export class ApiNotificationsMastodon { reply.send(response); }); - fastify.post('/v1/notification/:id/dismiss', { preHandler: upload.single('none') }, async (_request, reply) => { + fastify.post('/v1/notification/:id/dismiss', async (_request, reply) => { if (!_request.params.id) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required parameter "id"' }); const client = this.clientService.getClient(_request); @@ -75,7 +74,7 @@ export class ApiNotificationsMastodon { reply.send(data.data); }); - fastify.post('/v1/notifications/clear', { preHandler: upload.single('none') }, async (_request, reply) => { + fastify.post('/v1/notifications/clear', async (_request, reply) => { const client = this.clientService.getClient(_request); const data = await client.dismissNotifications(); diff --git a/packages/backend/src/server/oauth/OAuth2ProviderService.ts b/packages/backend/src/server/oauth/OAuth2ProviderService.ts index a65acb7c9b..e1f39dd9b6 100644 --- a/packages/backend/src/server/oauth/OAuth2ProviderService.ts +++ b/packages/backend/src/server/oauth/OAuth2ProviderService.ts @@ -3,15 +3,14 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import querystring from 'querystring'; import { Inject, Injectable } from '@nestjs/common'; import { v4 as uuid } from 'uuid'; -import multer from 'fastify-multer'; import { bindThis } from '@/decorators.js'; import type { Config } from '@/config.js'; import { DI } from '@/di-symbols.js'; import { MastodonClientService } from '@/server/api/mastodon/MastodonClientService.js'; import { getErrorData } from '@/server/api/mastodon/MastodonLogger.js'; +import { ServerUtilityService } from '@/server/ServerUtilityService.js'; import type { FastifyInstance } from 'fastify'; const kinds = [ @@ -56,6 +55,7 @@ export class OAuth2ProviderService { private config: Config, private readonly mastodonClientService: MastodonClientService, + private readonly serverUtilityService: ServerUtilityService, ) { } // https://datatracker.ietf.org/doc/html/rfc8414.html @@ -92,36 +92,10 @@ export class OAuth2ProviderService { }); }); */ - const upload = multer({ - storage: multer.diskStorage({}), - limits: { - fileSize: this.config.maxFileSize || 262144000, - files: 1, - }, - }); - - fastify.addHook('onRequest', (request, reply, done) => { - reply.header('Access-Control-Allow-Origin', '*'); - done(); - }); - - fastify.addContentTypeParser('application/x-www-form-urlencoded', (request, payload, done) => { - let body = ''; - payload.on('data', (data) => { - body += data; - }); - payload.on('end', () => { - try { - const parsed = querystring.parse(body); - done(null, parsed); - } catch (e: unknown) { - done(e instanceof Error ? e : new Error(String(e))); - } - }); - payload.on('error', done); - }); - - fastify.register(multer.contentParser); + this.serverUtilityService.addMultipartFormDataContentType(fastify); + this.serverUtilityService.addFormUrlEncodedContentType(fastify); + this.serverUtilityService.addCORS(fastify); + this.serverUtilityService.addFlattenedQueryType(fastify); for (const url of ['/authorize', '/authorize/']) { fastify.get<{ Querystring: Record }>(url, async (request, reply) => { @@ -136,7 +110,7 @@ export class OAuth2ProviderService { }); } - fastify.post<{ Body?: Record, Querystring: Record }>('/token', { preHandler: upload.none() }, async (request, reply) => { + fastify.post<{ Body?: Record, Querystring: Record }>('/token', async (request, reply) => { const body = request.body ?? request.query; if (body.grant_type === 'client_credentials') { diff --git a/packages/megalodon/src/misskey.ts b/packages/megalodon/src/misskey.ts index 670b31e838..a7d604de26 100644 --- a/packages/megalodon/src/misskey.ts +++ b/packages/megalodon/src/misskey.ts @@ -1502,13 +1502,13 @@ export default class Misskey implements MegalodonInterface { /** * POST /api/drive/files/create */ - public async uploadMedia(file: any, _options?: { description?: string; focus?: string }): Promise> { + public async uploadMedia(file: { filepath: fs.PathLike, mimetype: string, filename: string }, _options?: { description?: string; focus?: string }): Promise> { const formData = new FormData() - formData.append('file', fs.createReadStream(file.path), { + formData.append('file', fs.createReadStream(file.filepath), { contentType: file.mimetype, }); - if (file.originalname != null && file.originalname !== "file") formData.append("name", file.originalname); + if (file.filename && file.filename !== "file") formData.append("name", file.filename); if (_options?.description != null) formData.append("comment", _options.description); let headers: { [key: string]: string } = {} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 691069f563..4eb78db1b5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -233,9 +233,6 @@ importers: fastify: specifier: 5.3.2 version: 5.3.2 - fastify-multer: - specifier: ^2.0.3 - version: 2.0.3 fastify-raw-body: specifier: 5.0.0 version: 5.0.0 @@ -2274,10 +2271,6 @@ packages: '@fastify/ajv-compiler@4.0.1': resolution: {integrity: sha512-DxrBdgsjNLP0YM6W5Hd6/Fmj43S8zMKiFJYgi+Ri3htTGAowPVG/tG1wpnWLMjufEnehRivUCKZ1pLDIoZdTuw==} - '@fastify/busboy@1.2.1': - resolution: {integrity: sha512-7PQA7EH43S0CxcOa9OeAnaeA0oQ+e/DHNPZwSQM9CQHW76jle5+OvLdibRp/Aafs9KXbLhxyjOTkRjWUbQEd3Q==} - engines: {node: '>=14'} - '@fastify/busboy@2.1.0': resolution: {integrity: sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA==} engines: {node: '>=14'} @@ -5455,10 +5448,6 @@ packages: resolution: {integrity: sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==} engines: {'0': node >= 0.8} - concat-stream@2.0.0: - resolution: {integrity: sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==} - engines: {'0': node >= 6.0} - config-chain@1.1.13: resolution: {integrity: sha512-qj+f8APARXHrM0hraqXYb2/bOVSV4PvJQlNZ/DVj0QrmNM2q2euizkeuVckQ57J+W0mRH6Hvi+k50M4Jul2VRQ==} @@ -6313,13 +6302,6 @@ packages: resolution: {integrity: sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==} hasBin: true - fastify-multer@2.0.3: - resolution: {integrity: sha512-QnFqrRgxmUwWHTgX9uyQSu0C/hmVCfcxopqjApZ4uaZD5W9MJ+nHUlW4+9q7Yd3BRxDIuHvgiM5mjrh6XG8cAA==} - engines: {node: '>=10.17.0'} - - fastify-plugin@2.3.4: - resolution: {integrity: sha512-I+Oaj6p9oiRozbam30sh39BiuiqBda7yK2nmSPVwDCfIBlKnT8YB3MY+pRQc2Fcd07bf6KPGklHJaQ2Qu81TYQ==} - fastify-plugin@4.5.1: resolution: {integrity: sha512-stRHYGeuqpEZTL1Ef0Ovr2ltazUT9g844X5z/zEBFLG8RYlpDiOCIG+ATvYEp+/zmc7sN29mcIMp8gvYplYPIQ==} @@ -9960,9 +9942,6 @@ packages: resolution: {integrity: sha512-pFYqmTw68LXVjeWJMST4+borgQP2AyMNbg1BpZh9LbyhUeNkeaPF9gzfPGUAnSMV3qPYdWUwDIjjCLiSDOl7vg==} engines: {node: '>=18'} - text-decoding@1.0.0: - resolution: {integrity: sha512-/0TJD42KDnVwKmDK6jj3xP7E2MG7SHAOG4tyTgyUCRPdHwvkquYNLEQltmdMa3owq3TkddCVcTsoctJI8VQNKA==} - textarea-caret@3.1.0: resolution: {integrity: sha512-cXAvzO9pP5CGa6NKx0WYHl+8CHKZs8byMkt3PCJBCmq2a34YA9pO1NrQET5pzeqnBjBdToF5No4rrmkDUgQC2Q==} @@ -12060,10 +12039,6 @@ snapshots: ajv-formats: 3.0.1(ajv@8.17.1) fast-uri: 3.0.1 - '@fastify/busboy@1.2.1': - dependencies: - text-decoding: 1.0.0 - '@fastify/busboy@2.1.0': {} '@fastify/busboy@3.0.0': {} @@ -15990,13 +15965,6 @@ snapshots: readable-stream: 2.3.7 typedarray: 0.0.6 - concat-stream@2.0.0: - dependencies: - buffer-from: 1.1.2 - inherits: 2.0.4 - readable-stream: 3.6.0 - typedarray: 0.0.6 - config-chain@1.1.13: dependencies: ini: 1.3.8 @@ -17122,21 +17090,6 @@ snapshots: dependencies: strnum: 1.0.5 - fastify-multer@2.0.3: - dependencies: - '@fastify/busboy': 1.2.1 - append-field: 1.0.0 - concat-stream: 2.0.0 - fastify-plugin: 2.3.4 - mkdirp: 1.0.4 - on-finished: 2.4.1 - type-is: 1.6.18 - xtend: 4.0.2 - - fastify-plugin@2.3.4: - dependencies: - semver: 7.6.0 - fastify-plugin@4.5.1: {} fastify-plugin@5.0.1: {} @@ -21438,8 +21391,6 @@ snapshots: glob: 10.4.5 minimatch: 9.0.5 - text-decoding@1.0.0: {} - textarea-caret@3.1.0: {} thenify-all@1.6.0: From 317f5602fe2104a25a441319b2e826ca246731df Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Tue, 6 May 2025 13:02:02 -0400 Subject: [PATCH 24/72] temporary: add recursive error handler to MastodonApiServerService.ts --- .../server/api/mastodon/MastodonApiServerService.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts b/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts index 757610450a..5b682df529 100644 --- a/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts +++ b/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts @@ -47,12 +47,16 @@ export class MastodonApiServerService { this.serverUtilityService.addFlattenedQueryType(fastify); fastify.setErrorHandler((error, request, reply) => { - const data = getErrorData(error); - const status = getErrorStatus(error); + try { + const data = getErrorData(error); + const status = getErrorStatus(error); - this.logger.error(request, data, status); + this.logger.error(request, data, status); - reply.code(status).send(data); + reply.code(status).send(data); + } catch (e) { + this.logger.logger.error('Recursive error in mastodon API - this is a bug!', { e }, true); + } }); // External endpoints From 7cd181df71ebac46c1c6a0ffb00ad81f82b62f3a Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Tue, 6 May 2025 13:08:40 -0400 Subject: [PATCH 25/72] improve type checks in POST /api/v1/apps endpoint --- .../src/server/api/mastodon/endpoints/apps.ts | 15 ++++++++------- packages/megalodon/src/misskey.ts | 12 ++++++------ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/packages/backend/src/server/api/mastodon/endpoints/apps.ts b/packages/backend/src/server/api/mastodon/endpoints/apps.ts index ec08600e53..aae6103146 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/apps.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/apps.ts @@ -47,9 +47,9 @@ const writeScope = [ export interface AuthPayload { scopes?: string | string[], - redirect_uris?: string, - client_name?: string, - website?: string, + redirect_uris?: string | string[], + client_name?: string | string[], + website?: string | string[], } // Not entirely right, but it gets TypeScript to work so *shrug* @@ -66,7 +66,10 @@ export class ApiAppsMastodon { const body = _request.body ?? _request.query; if (!body.scopes) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required payload "scopes"' }); if (!body.redirect_uris) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required payload "redirect_uris"' }); + if (Array.isArray(body.redirect_uris)) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Invalid payload "redirect_uris": only one value is allowed' }); if (!body.client_name) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required payload "client_name"' }); + if (Array.isArray(body.client_name)) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Invalid payload "client_name": only one value is allowed' }); + if (Array.isArray(body.website)) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Invalid payload "website": only one value is allowed' }); let scope = body.scopes; if (typeof scope === 'string') { @@ -87,12 +90,10 @@ export class ApiAppsMastodon { } } - const red = body.redirect_uris; - const client = this.clientService.getClient(_request); const appData = await client.registerApp(body.client_name, { scopes: Array.from(pushScope), - redirect_uris: red, + redirect_uri: body.redirect_uris, website: body.website, }); @@ -100,7 +101,7 @@ export class ApiAppsMastodon { id: Math.floor(Math.random() * 100).toString(), name: appData.name, website: body.website, - redirect_uri: red, + redirect_uri: body.redirect_uris, client_id: Buffer.from(appData.url || '').toString('base64'), client_secret: appData.clientSecret, }; diff --git a/packages/megalodon/src/misskey.ts b/packages/megalodon/src/misskey.ts index a7d604de26..cfca2c291c 100644 --- a/packages/megalodon/src/misskey.ts +++ b/packages/megalodon/src/misskey.ts @@ -39,9 +39,9 @@ export default class Misskey implements MegalodonInterface { public async registerApp( client_name: string, - options: Partial<{ scopes: Array; redirect_uris: string; website: string }> = { + options: Partial<{ scopes: Array; redirect_uri: string; website: string }> = { scopes: MisskeyAPI.DEFAULT_SCOPE, - redirect_uris: this.baseUrl + redirect_uri: this.baseUrl } ): Promise { return this.createApp(client_name, options).then(async appData => { @@ -62,12 +62,12 @@ export default class Misskey implements MegalodonInterface { */ public async createApp( client_name: string, - options: Partial<{ scopes: Array; redirect_uris: string; website: string }> = { + options: Partial<{ scopes: Array; redirect_uri: string; website: string }> = { scopes: MisskeyAPI.DEFAULT_SCOPE, - redirect_uris: this.baseUrl + redirect_uri: this.baseUrl } ): Promise { - const redirect_uris = options.redirect_uris || this.baseUrl + const redirect_uri = options.redirect_uri || this.baseUrl const scopes = options.scopes || MisskeyAPI.DEFAULT_SCOPE const params: { @@ -79,7 +79,7 @@ export default class Misskey implements MegalodonInterface { name: client_name, description: '', permission: scopes, - callbackUrl: redirect_uris + callbackUrl: redirect_uri } /** From 7db03f61b1d37a4f61c7b8d4bfcd3c04966b8d7d Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Tue, 6 May 2025 13:09:46 -0400 Subject: [PATCH 26/72] store OAuth "website" in POST /api/v1/apps --- packages/megalodon/src/misskey.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/megalodon/src/misskey.ts b/packages/megalodon/src/misskey.ts index cfca2c291c..669eb0f106 100644 --- a/packages/megalodon/src/misskey.ts +++ b/packages/megalodon/src/misskey.ts @@ -39,7 +39,7 @@ export default class Misskey implements MegalodonInterface { public async registerApp( client_name: string, - options: Partial<{ scopes: Array; redirect_uri: string; website: string }> = { + options: Partial<{ scopes: Array; redirect_uri: string; website?: string }> = { scopes: MisskeyAPI.DEFAULT_SCOPE, redirect_uri: this.baseUrl } @@ -62,13 +62,14 @@ export default class Misskey implements MegalodonInterface { */ public async createApp( client_name: string, - options: Partial<{ scopes: Array; redirect_uri: string; website: string }> = { + options: Partial<{ scopes: Array; redirect_uri: string; website?: string }> = { scopes: MisskeyAPI.DEFAULT_SCOPE, redirect_uri: this.baseUrl } ): Promise { const redirect_uri = options.redirect_uri || this.baseUrl const scopes = options.scopes || MisskeyAPI.DEFAULT_SCOPE + const website = options.website ?? ''; const params: { name: string @@ -77,7 +78,7 @@ export default class Misskey implements MegalodonInterface { callbackUrl: string } = { name: client_name, - description: '', + description: website, permission: scopes, callbackUrl: redirect_uri } From fd5a3eb3f8753b7f1a4c22c25e7b42f380de2a31 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Tue, 6 May 2025 13:34:57 -0400 Subject: [PATCH 27/72] add `logging.verbose` option to enable debug logging in production. (same function as `MK_VERBOSE` environment variable) --- .config/ci.yml | 3 +++ .config/cypress-devcontainer.yml | 3 +++ .config/docker_example.yml | 3 +++ .config/example.yml | 3 +++ packages/backend/src/boot/master.ts | 2 +- packages/backend/src/config.ts | 7 +++++-- packages/backend/src/core/DriveService.ts | 5 ++++- packages/backend/src/core/LoggerService.ts | 10 ++++++++-- packages/backend/src/core/UserFollowingService.ts | 13 ++++++++----- packages/backend/src/logger.ts | 8 +++++--- 10 files changed, 43 insertions(+), 14 deletions(-) diff --git a/.config/ci.yml b/.config/ci.yml index b0b97e9471..4fd32c8a74 100644 --- a/.config/ci.yml +++ b/.config/ci.yml @@ -349,6 +349,9 @@ attachLdSignatureForRelays: true # # Disable query truncation. If set to true, the full text of the query will be output to the log. # # default: false # disableQueryTruncation: false +# # Shows debug log messages after instance startup. To capture earlier debug logs, set the MK_VERBOSE environment variable. +# # default: false in production, true otherwise. +# #verbose: false # Settings for the activity logger, which records inbound activities to the database. # Disabled by default due to the large volume of data it saves. diff --git a/.config/cypress-devcontainer.yml b/.config/cypress-devcontainer.yml index 83be98e429..586678a24e 100644 --- a/.config/cypress-devcontainer.yml +++ b/.config/cypress-devcontainer.yml @@ -295,6 +295,9 @@ allowedPrivateNetworks: [ # # Disable query truncation. If set to true, the full text of the query will be output to the log. # # default: false # disableQueryTruncation: false +# # Shows debug log messages after instance startup. To capture earlier debug logs, set the MK_VERBOSE environment variable. +# # default: false in production, true otherwise. +# #verbose: false # Settings for the activity logger, which records inbound activities to the database. # Disabled by default due to the large volume of data it saves. diff --git a/.config/docker_example.yml b/.config/docker_example.yml index ee57da781f..3aa9935d77 100644 --- a/.config/docker_example.yml +++ b/.config/docker_example.yml @@ -411,6 +411,9 @@ attachLdSignatureForRelays: true # # Disable query truncation. If set to true, the full text of the query will be output to the log. # # default: false # disableQueryTruncation: false +# # Shows debug log messages after instance startup. To capture earlier debug logs, set the MK_VERBOSE environment variable. +# # default: false in production, true otherwise. +# #verbose: false # Settings for the activity logger, which records inbound activities to the database. # Disabled by default due to the large volume of data it saves. diff --git a/.config/example.yml b/.config/example.yml index 704a80d413..e21f8ed501 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -417,6 +417,9 @@ attachLdSignatureForRelays: true # # Disable query truncation. If set to true, the full text of the query will be output to the log. # # default: false # disableQueryTruncation: false +# # Shows debug log messages after instance startup. To capture earlier debug logs, set the MK_VERBOSE environment variable. +# # default: false in production, true otherwise. +# #verbose: false # Settings for the activity logger, which records inbound activities to the database. # Disabled by default due to the large volume of data it saves. diff --git a/packages/backend/src/boot/master.ts b/packages/backend/src/boot/master.ts index cf9e9a9bae..538c529106 100644 --- a/packages/backend/src/boot/master.ts +++ b/packages/backend/src/boot/master.ts @@ -8,6 +8,7 @@ import { fileURLToPath } from 'node:url'; import { dirname } from 'node:path'; import * as os from 'node:os'; import cluster from 'node:cluster'; +import * as net from 'node:net'; import chalk from 'chalk'; import chalkTemplate from 'chalk-template'; import * as Sentry from '@sentry/node'; @@ -18,7 +19,6 @@ import type { Config } from '@/config.js'; import { showMachineInfo } from '@/misc/show-machine-info.js'; import { envOption } from '@/env.js'; import { jobQueue, server } from './common.js'; -import * as net from 'node:net'; const _filename = fileURLToPath(import.meta.url); const _dirname = dirname(_filename); diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 40f154c000..92fc2b8a13 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -135,7 +135,8 @@ type Source = { sql?: { disableQueryTruncation?: boolean, enableQueryParamLogging?: boolean, - } + }; + verbose?: boolean; } activityLogging?: { @@ -220,7 +221,8 @@ export type Config = { sql?: { disableQueryTruncation?: boolean, enableQueryParamLogging?: boolean, - } + }; + verbose?: boolean; } version: string; @@ -585,6 +587,7 @@ function applyEnvOverrides(config: Source) { _apply_top(['import', ['downloadTimeout', 'maxFileSize']]); _apply_top([['signToActivityPubGet', 'checkActivityPubGetSignature', 'setupPassword', 'disallowExternalApRedirect']]); _apply_top(['logging', 'sql', ['disableQueryTruncation', 'enableQueryParamLogging']]); + _apply_top(['logging', ['verbose']]); _apply_top(['activityLogging', ['enabled', 'preSave', 'maxAge']]); _apply_top(['customHtml', ['head']]); } diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts index 8cc7df1a81..82c447baaa 100644 --- a/packages/backend/src/core/DriveService.ts +++ b/packages/backend/src/core/DriveService.ts @@ -45,6 +45,7 @@ import { isMimeImage } from '@/misc/is-mime-image.js'; import { ModerationLogService } from '@/core/ModerationLogService.js'; import { UtilityService } from '@/core/UtilityService.js'; import { BunnyService } from '@/core/BunnyService.js'; +import { LoggerService } from './LoggerService.js'; type AddFileArgs = { /** User who wish to add file */ @@ -133,8 +134,10 @@ export class DriveService { private perUserDriveChart: PerUserDriveChart, private instanceChart: InstanceChart, private utilityService: UtilityService, + + loggerService: LoggerService, ) { - const logger = new Logger('drive', 'blue'); + const logger = loggerService.getLogger('drive', 'blue'); this.registerLogger = logger.createSubLogger('register', 'yellow'); this.downloaderLogger = logger.createSubLogger('downloader'); this.deleteLogger = logger.createSubLogger('delete'); diff --git a/packages/backend/src/core/LoggerService.ts b/packages/backend/src/core/LoggerService.ts index f102461a50..25721f0630 100644 --- a/packages/backend/src/core/LoggerService.ts +++ b/packages/backend/src/core/LoggerService.ts @@ -3,19 +3,25 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { Injectable } from '@nestjs/common'; +import { Inject, Injectable } from '@nestjs/common'; import Logger from '@/logger.js'; import { bindThis } from '@/decorators.js'; import type { KEYWORD } from 'color-convert/conversions.js'; +import { envOption } from '@/env.js'; +import { DI } from '@/di-symbols.js'; +import type { Config } from '@/config.js'; @Injectable() export class LoggerService { constructor( + @Inject(DI.config) + private config: Config, ) { } @bindThis public getLogger(domain: string, color?: KEYWORD | undefined) { - return new Logger(domain, color); + const verbose = this.config.logging?.verbose || envOption.verbose; + return new Logger(domain, color, verbose); } } diff --git a/packages/backend/src/core/UserFollowingService.ts b/packages/backend/src/core/UserFollowingService.ts index e7a6be99fb..897b950022 100644 --- a/packages/backend/src/core/UserFollowingService.ts +++ b/packages/backend/src/core/UserFollowingService.ts @@ -28,9 +28,8 @@ import type { Config } from '@/config.js'; import { AccountMoveService } from '@/core/AccountMoveService.js'; import { UtilityService } from '@/core/UtilityService.js'; import type { ThinUser } from '@/queue/types.js'; -import Logger from '../logger.js'; - -const logger = new Logger('following/create'); +import { LoggerService } from '@/core/LoggerService.js'; +import type Logger from '../logger.js'; type Local = MiLocalUser | { id: MiLocalUser['id']; @@ -48,6 +47,7 @@ type Both = Local | Remote; @Injectable() export class UserFollowingService implements OnModuleInit { private userBlockingService: UserBlockingService; + private readonly logger: Logger; constructor( private moduleRef: ModuleRef, @@ -86,7 +86,10 @@ export class UserFollowingService implements OnModuleInit { private accountMoveService: AccountMoveService, private perUserFollowingChart: PerUserFollowingChart, private instanceChart: InstanceChart, + + loggerService: LoggerService, ) { + this.logger = loggerService.getLogger('following/create'); } onModuleInit() { @@ -254,7 +257,7 @@ export class UserFollowingService implements OnModuleInit { followeeSharedInbox: this.userEntityService.isRemoteUser(followee) ? followee.sharedInbox : null, }).catch(err => { if (isDuplicateKeyValueError(err) && this.userEntityService.isRemoteUser(follower) && this.userEntityService.isLocalUser(followee)) { - logger.info(`Insert duplicated ignore. ${follower.id} => ${followee.id}`); + this.logger.info(`Insert duplicated ignore. ${follower.id} => ${followee.id}`); alreadyFollowed = true; } else { throw err; @@ -372,7 +375,7 @@ export class UserFollowingService implements OnModuleInit { }); if (following === null || !following.follower || !following.followee) { - logger.warn('フォロー解除がリクエストされましたがフォローしていませんでした'); + this.logger.warn('フォロー解除がリクエストされましたがフォローしていませんでした'); return; } diff --git a/packages/backend/src/logger.ts b/packages/backend/src/logger.ts index 79623768a8..46edac8666 100644 --- a/packages/backend/src/logger.ts +++ b/packages/backend/src/logger.ts @@ -27,17 +27,19 @@ export type DataObject = Record | (object & { length?: never; } export default class Logger { private context: Context; private parentLogger: Logger | null = null; + private readonly verbose: boolean; - constructor(context: string, color?: KEYWORD) { + constructor(context: string, color?: KEYWORD, verbose?: boolean) { this.context = { name: context, color: color, }; + this.verbose = verbose ?? envOption.verbose; } @bindThis public createSubLogger(context: string, color?: KEYWORD): Logger { - const logger = new Logger(context, color); + const logger = new Logger(context, color, this.verbose); logger.parentLogger = this; return logger; } @@ -110,7 +112,7 @@ export default class Logger { @bindThis public debug(message: string, data?: Data, important = false): void { // デバッグ用に使う(開発者に必要だが利用者に不要な情報) - if (process.env.NODE_ENV !== 'production' || envOption.verbose) { + if (process.env.NODE_ENV !== 'production' || this.verbose) { this.log('debug', message, data, important); } } From e9ae78c0b757060895b843bc3cb7f951a39dbc4d Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Tue, 6 May 2025 13:36:05 -0400 Subject: [PATCH 28/72] enable debug logging for Mastodon API --- packages/backend/src/server/api/mastodon/MastodonLogger.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/server/api/mastodon/MastodonLogger.ts b/packages/backend/src/server/api/mastodon/MastodonLogger.ts index 81d3e8f03d..228f9a631b 100644 --- a/packages/backend/src/server/api/mastodon/MastodonLogger.ts +++ b/packages/backend/src/server/api/mastodon/MastodonLogger.ts @@ -25,8 +25,10 @@ export class MastodonLogger { } public error(request: FastifyRequest, error: MastodonError, status: number): void { - if ((status < 400 && status > 499) || this.envService.env.NODE_ENV === 'development') { - const path = new URL(request.url, getBaseUrl(request)).pathname; + const path = new URL(request.url, getBaseUrl(request)).pathname; + if (status >= 400 && status <= 499) { // Client errors + this.logger.debug(`Error in mastodon endpoint ${request.method} ${path}:`, error); + } else { // Server errors this.logger.error(`Error in mastodon endpoint ${request.method} ${path}:`, error); } } From d9d0adbc6fb2efd5b5b14dfec665adc101bb94a8 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Tue, 6 May 2025 13:40:09 -0400 Subject: [PATCH 29/72] fix indent of `logging.verbose` option --- .config/ci.yml | 6 +++--- .config/cypress-devcontainer.yml | 6 +++--- .config/docker_example.yml | 6 +++--- .config/example.yml | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.config/ci.yml b/.config/ci.yml index 4fd32c8a74..fefa45643c 100644 --- a/.config/ci.yml +++ b/.config/ci.yml @@ -349,9 +349,9 @@ attachLdSignatureForRelays: true # # Disable query truncation. If set to true, the full text of the query will be output to the log. # # default: false # disableQueryTruncation: false -# # Shows debug log messages after instance startup. To capture earlier debug logs, set the MK_VERBOSE environment variable. -# # default: false in production, true otherwise. -# #verbose: false +# # Shows debug log messages after instance startup. To capture earlier debug logs, set the MK_VERBOSE environment variable. +# # default: false in production, true otherwise. +# #verbose: false # Settings for the activity logger, which records inbound activities to the database. # Disabled by default due to the large volume of data it saves. diff --git a/.config/cypress-devcontainer.yml b/.config/cypress-devcontainer.yml index 586678a24e..e4eb8cc805 100644 --- a/.config/cypress-devcontainer.yml +++ b/.config/cypress-devcontainer.yml @@ -295,9 +295,9 @@ allowedPrivateNetworks: [ # # Disable query truncation. If set to true, the full text of the query will be output to the log. # # default: false # disableQueryTruncation: false -# # Shows debug log messages after instance startup. To capture earlier debug logs, set the MK_VERBOSE environment variable. -# # default: false in production, true otherwise. -# #verbose: false +# # Shows debug log messages after instance startup. To capture earlier debug logs, set the MK_VERBOSE environment variable. +# # default: false in production, true otherwise. +# #verbose: false # Settings for the activity logger, which records inbound activities to the database. # Disabled by default due to the large volume of data it saves. diff --git a/.config/docker_example.yml b/.config/docker_example.yml index 3aa9935d77..7968a7d1f4 100644 --- a/.config/docker_example.yml +++ b/.config/docker_example.yml @@ -411,9 +411,9 @@ attachLdSignatureForRelays: true # # Disable query truncation. If set to true, the full text of the query will be output to the log. # # default: false # disableQueryTruncation: false -# # Shows debug log messages after instance startup. To capture earlier debug logs, set the MK_VERBOSE environment variable. -# # default: false in production, true otherwise. -# #verbose: false +# # Shows debug log messages after instance startup. To capture earlier debug logs, set the MK_VERBOSE environment variable. +# # default: false in production, true otherwise. +# #verbose: false # Settings for the activity logger, which records inbound activities to the database. # Disabled by default due to the large volume of data it saves. diff --git a/.config/example.yml b/.config/example.yml index e21f8ed501..d0ed4defaa 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -417,9 +417,9 @@ attachLdSignatureForRelays: true # # Disable query truncation. If set to true, the full text of the query will be output to the log. # # default: false # disableQueryTruncation: false -# # Shows debug log messages after instance startup. To capture earlier debug logs, set the MK_VERBOSE environment variable. -# # default: false in production, true otherwise. -# #verbose: false +# # Shows debug log messages after instance startup. To capture earlier debug logs, set the MK_VERBOSE environment variable. +# # default: false in production, true otherwise. +# #verbose: false # Settings for the activity logger, which records inbound activities to the database. # Disabled by default due to the large volume of data it saves. From 2aa3cf27311218f2a1ba2e630f4d352968dd87d5 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Tue, 6 May 2025 17:27:39 -0400 Subject: [PATCH 30/72] debug-log mastodon error responses --- .../api/mastodon/MastodonApiServerService.ts | 20 +++++++++++-------- .../src/server/api/mastodon/MastodonLogger.ts | 7 +++++++ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts b/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts index 5b682df529..359408d882 100644 --- a/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts +++ b/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts @@ -46,17 +46,21 @@ export class MastodonApiServerService { this.serverUtilityService.addCORS(fastify); this.serverUtilityService.addFlattenedQueryType(fastify); - fastify.setErrorHandler((error, request, reply) => { - try { - const data = getErrorData(error); - const status = getErrorStatus(error); + // Convert JS exceptions into error responses + fastify.setErrorHandler((error, _, reply) => { + const data = getErrorData(error); + const status = getErrorStatus(error); - this.logger.error(request, data, status); + reply.code(status).send(data); + }); - reply.code(status).send(data); - } catch (e) { - this.logger.logger.error('Recursive error in mastodon API - this is a bug!', { e }, true); + // Log error responses (including converted JSON exceptions) + fastify.addHook('onSend', (request, reply, payload, done) => { + if (reply.statusCode >= 400) { + const data = getErrorData(payload); + this.logger.error(request, data, reply.statusCode); } + done(); }); // External endpoints diff --git a/packages/backend/src/server/api/mastodon/MastodonLogger.ts b/packages/backend/src/server/api/mastodon/MastodonLogger.ts index 228f9a631b..096a3521a7 100644 --- a/packages/backend/src/server/api/mastodon/MastodonLogger.ts +++ b/packages/backend/src/server/api/mastodon/MastodonLogger.ts @@ -65,6 +65,13 @@ export function getErrorData(error: unknown): MastodonError { return convertGenericError(error); } + if ('error' in error && typeof(error.error) === 'string') { + // "error_description" is string, undefined, or not present. + if (!('error_description' in error) || typeof(error.error_description) === 'string' || typeof(error.error_description) === 'undefined') { + return error as MastodonError; + } + } + return convertUnknownError(error); } From c0f24eaf5de2fc0650abb2911abb747234e58236 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Tue, 6 May 2025 17:42:23 -0400 Subject: [PATCH 31/72] correctly parse response errors for logging --- packages/backend/src/logger.ts | 2 +- .../server/api/mastodon/MastodonApiServerService.ts | 9 ++++++--- .../backend/src/server/api/mastodon/MastodonLogger.ts | 10 +++++----- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/packages/backend/src/logger.ts b/packages/backend/src/logger.ts index 46edac8666..b3735200eb 100644 --- a/packages/backend/src/logger.ts +++ b/packages/backend/src/logger.ts @@ -27,7 +27,7 @@ export type DataObject = Record | (object & { length?: never; } export default class Logger { private context: Context; private parentLogger: Logger | null = null; - private readonly verbose: boolean; + public readonly verbose: boolean; constructor(context: string, color?: KEYWORD, verbose?: boolean) { this.context = { diff --git a/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts b/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts index 359408d882..d95d75f12f 100644 --- a/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts +++ b/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts @@ -56,9 +56,12 @@ export class MastodonApiServerService { // Log error responses (including converted JSON exceptions) fastify.addHook('onSend', (request, reply, payload, done) => { - if (reply.statusCode >= 400) { - const data = getErrorData(payload); - this.logger.error(request, data, reply.statusCode); + if (reply.statusCode >= 500 || (reply.statusCode >= 400 && this.logger.verbose)) { + if (typeof(payload) === 'string' && String(reply.getHeader('content-type')).toLowerCase().includes('application/json')) { + const body = JSON.parse(payload); + const data = getErrorData(body); + this.logger.error(request, data, reply.statusCode); + } } done(); }); diff --git a/packages/backend/src/server/api/mastodon/MastodonLogger.ts b/packages/backend/src/server/api/mastodon/MastodonLogger.ts index 096a3521a7..f43e2fc764 100644 --- a/packages/backend/src/server/api/mastodon/MastodonLogger.ts +++ b/packages/backend/src/server/api/mastodon/MastodonLogger.ts @@ -3,22 +3,22 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { Inject, Injectable } from '@nestjs/common'; +import { Injectable } from '@nestjs/common'; import { FastifyRequest } from 'fastify'; import Logger from '@/logger.js'; import { LoggerService } from '@/core/LoggerService.js'; import { ApiError } from '@/server/api/error.js'; -import { EnvService } from '@/core/EnvService.js'; import { getBaseUrl } from '@/server/api/mastodon/MastodonClientService.js'; @Injectable() export class MastodonLogger { public readonly logger: Logger; - constructor( - @Inject(EnvService) - private readonly envService: EnvService, + public get verbose() { + return this.logger.verbose; + } + constructor( loggerService: LoggerService, ) { this.logger = loggerService.getLogger('masto-api'); From 89cab66898f4edb238eee2321564337034dfa2bc Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Tue, 6 May 2025 18:26:33 -0400 Subject: [PATCH 32/72] fix multipart/form-data decoding --- packages/backend/src/misc/create-temp.ts | 13 +++++++ .../src/server/ServerUtilityService.ts | 39 ++++++++++++++----- 2 files changed, 42 insertions(+), 10 deletions(-) diff --git a/packages/backend/src/misc/create-temp.ts b/packages/backend/src/misc/create-temp.ts index 6cc896046f..fda63c7a9d 100644 --- a/packages/backend/src/misc/create-temp.ts +++ b/packages/backend/src/misc/create-temp.ts @@ -3,6 +3,8 @@ * SPDX-License-Identifier: AGPL-3.0-only */ +import { pipeline } from 'node:stream/promises'; +import fs from 'node:fs'; import * as tmp from 'tmp'; export function createTemp(): Promise<[string, () => void]> { @@ -27,3 +29,14 @@ export function createTempDir(): Promise<[string, () => void]> { ); }); } + +export async function saveToTempFile(stream: NodeJS.ReadableStream): Promise { + const [filepath, cleanup] = await createTemp(); + try { + await pipeline(stream, fs.createWriteStream(filepath)); + return filepath; + } catch (e) { + cleanup(); + throw e; + } +} diff --git a/packages/backend/src/server/ServerUtilityService.ts b/packages/backend/src/server/ServerUtilityService.ts index f2900fad4f..d90b37ca50 100644 --- a/packages/backend/src/server/ServerUtilityService.ts +++ b/packages/backend/src/server/ServerUtilityService.ts @@ -9,6 +9,7 @@ import { Inject, Injectable } from '@nestjs/common'; import { FastifyInstance } from 'fastify'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; +import { saveToTempFile } from '@/misc/create-temp.js'; @Injectable() export class ServerUtilityService { @@ -29,17 +30,16 @@ export class ServerUtilityService { // Store to temporary file instead, and copy the body fields while we're at it. fastify.addHook<{ Body?: Record }>('onRequest', async request => { if (request.isMultipart()) { - const body = request.body ??= {}; + // We can't use saveRequestFiles() because it erases all the data fields. + // Instead, recreate it manually. + // https://github.com/fastify/fastify-multipart/issues/549 - // Save upload to temp directory. - // These are attached to request.savedRequestFiles - await request.saveRequestFiles(); + for await (const part of request.parts()) { + if (part.type === 'field') { + const k = part.fieldname; + const v = String(part.value); + const body = request.body ??= {}; - // Copy fields to body - const formData = await request.formData(); - formData.forEach((v, k) => { - // This can be string or File, and we handle files above. - if (typeof(v) === 'string') { // This is just progressive conversion from undefined -> string -> string[] if (body[k]) { if (Array.isArray(body[k])) { @@ -50,8 +50,27 @@ export class ServerUtilityService { } else { body[k] = v; } + } else { // Otherwise it's a file + try { + const [filepath] = await saveToTempFile(part.file); + + const tmpUploads = (request.tmpUploads ??= []); + tmpUploads.push(filepath); + + const requestSavedFiles = (request.savedRequestFiles ??= []); + requestSavedFiles.push({ + ...part, + filepath, + }); + } catch (e) { + // Cleanup to avoid file leak in case of errors + await request.cleanRequestFiles(); + request.tmpUploads = null; + request.savedRequestFiles = null; + throw e; + } } - }); + } } }); } From cbe88122b9c1446a61d69082df8f0ba729837a08 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Tue, 6 May 2025 18:46:08 -0400 Subject: [PATCH 33/72] fix hook targets --- packages/backend/src/server/ServerUtilityService.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/server/ServerUtilityService.ts b/packages/backend/src/server/ServerUtilityService.ts index d90b37ca50..115717534f 100644 --- a/packages/backend/src/server/ServerUtilityService.ts +++ b/packages/backend/src/server/ServerUtilityService.ts @@ -28,7 +28,7 @@ export class ServerUtilityService { // Default behavior saves files to memory - we don't want that! // Store to temporary file instead, and copy the body fields while we're at it. - fastify.addHook<{ Body?: Record }>('onRequest', async request => { + fastify.addHook<{ Body?: Record }>('preValidation', async request => { if (request.isMultipart()) { // We can't use saveRequestFiles() because it erases all the data fields. // Instead, recreate it manually. @@ -94,7 +94,7 @@ export class ServerUtilityService { } public addCORS(fastify: FastifyInstance) { - fastify.addHook('onRequest', (_, reply, done) => { + fastify.addHook('preHandler', (_, reply, done) => { // Allow web-based clients to connect from other origins. reply.header('Access-Control-Allow-Origin', '*'); From 6757c227a9509f9b92a9e7a0e9f606a89f3e9727 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Tue, 6 May 2025 18:48:22 -0400 Subject: [PATCH 34/72] check type of field values --- packages/backend/src/server/ServerUtilityService.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/server/ServerUtilityService.ts b/packages/backend/src/server/ServerUtilityService.ts index 115717534f..8cecc5df58 100644 --- a/packages/backend/src/server/ServerUtilityService.ts +++ b/packages/backend/src/server/ServerUtilityService.ts @@ -37,9 +37,13 @@ export class ServerUtilityService { for await (const part of request.parts()) { if (part.type === 'field') { const k = part.fieldname; - const v = String(part.value); + const v = part.value; const body = request.body ??= {}; + // Value can be string, buffer, or undefined. + // We only support the first one. + if (typeof(v) !== 'string') continue; + // This is just progressive conversion from undefined -> string -> string[] if (body[k]) { if (Array.isArray(body[k])) { From 3b44e1179903600e1711b5fdbdd356ec1d35ee70 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Tue, 6 May 2025 20:17:54 -0400 Subject: [PATCH 35/72] improvements to Mastodon error conversion --- .../src/server/api/mastodon/MastodonLogger.ts | 46 +++++++++++++++++-- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/packages/backend/src/server/api/mastodon/MastodonLogger.ts b/packages/backend/src/server/api/mastodon/MastodonLogger.ts index f43e2fc764..8d8f62bc0d 100644 --- a/packages/backend/src/server/api/mastodon/MastodonLogger.ts +++ b/packages/backend/src/server/api/mastodon/MastodonLogger.ts @@ -61,6 +61,12 @@ export function getErrorData(error: unknown): MastodonError { } } + if ('error' in error && typeof (error.error) === 'string') { + if ('message' in error && typeof (error.message) === 'string') { + return convertErrorMessageError(error as { error: string, message: string }); + } + } + if (error instanceof Error) { return convertGenericError(error); } @@ -95,38 +101,64 @@ function unpackAxiosError(error: unknown): unknown { } function convertApiError(apiError: ApiError): MastodonError { - const mastoError: MastodonError & Partial = { + const mastoError: MastodonError & Partial & { stack?: unknown, statusCode?: number } = { + ...apiError, error: apiError.code, error_description: apiError.message, - ...apiError, }; delete mastoError.code; + delete mastoError.stack; delete mastoError.message; delete mastoError.httpStatusCode; return mastoError; } +function convertErrorMessageError(error: { error: string, message: string }): MastodonError { + const mastoError: MastodonError & { stack?: unknown, message?: string, statusCode?: number } = { + ...error, + error: error.error, + error_description: error.message, + }; + + delete mastoError.stack; + delete mastoError.message; + delete mastoError.statusCode; + + return mastoError; +} + function convertUnknownError(data: object = {}): MastodonError { - return Object.assign({}, data, { + const mastoError = Object.assign({}, data, { error: 'INTERNAL_ERROR', error_description: 'Internal error occurred. Please contact us if the error persists.', id: '5d37dbcb-891e-41ca-a3d6-e690c97775ac', kind: 'server', }); + + if ('statusCode' in mastoError) { + delete mastoError.statusCode; + } + + if ('stack' in mastoError) { + delete mastoError.stack; + } + + return mastoError; } function convertGenericError(error: Error): MastodonError { - const mastoError: MastodonError & Partial = { + const mastoError: MastodonError & Partial & { statusCode?: number } = { + ...error, error: 'INTERNAL_ERROR', error_description: String(error), - ...error, }; delete mastoError.name; delete mastoError.message; delete mastoError.stack; + delete mastoError.statusCode; return mastoError; } @@ -143,6 +175,10 @@ export function getErrorStatus(error: unknown): number { if ('httpStatusCode' in error && typeof(error.httpStatusCode) === 'number') { return error.httpStatusCode; } + + if ('statusCode' in error && typeof(error.statusCode) === 'number') { + return error.statusCode; + } } return 500; From f7ca7a2cc04bd686ede95ac39524bc6f3fefe5ea Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Tue, 6 May 2025 20:24:14 -0400 Subject: [PATCH 36/72] minor refactor to ServerUtilityService.addMultipartFormDataContentType --- packages/backend/src/server/ServerUtilityService.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/packages/backend/src/server/ServerUtilityService.ts b/packages/backend/src/server/ServerUtilityService.ts index 8cecc5df58..00eb97f679 100644 --- a/packages/backend/src/server/ServerUtilityService.ts +++ b/packages/backend/src/server/ServerUtilityService.ts @@ -45,14 +45,12 @@ export class ServerUtilityService { if (typeof(v) !== 'string') continue; // This is just progressive conversion from undefined -> string -> string[] - if (body[k]) { - if (Array.isArray(body[k])) { - body[k].push(v); - } else { - body[k] = [body[k], v]; - } - } else { + if (!body[k]) { body[k] = v; + } else if (Array.isArray(body[k])) { + body[k].push(v); + } else { + body[k] = [body[k], v]; } } else { // Otherwise it's a file try { From 282ef9e6734cf80f6a726345b1781054269b338b Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Tue, 6 May 2025 20:43:32 -0400 Subject: [PATCH 37/72] split exception logging from error logging to simplify and improve mastodon errors --- .../api/mastodon/MastodonApiServerService.ts | 10 ++- .../src/server/api/mastodon/MastodonLogger.ts | 81 ++++++++----------- 2 files changed, 42 insertions(+), 49 deletions(-) diff --git a/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts b/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts index d95d75f12f..1b0ed2ce4e 100644 --- a/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts +++ b/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts @@ -5,7 +5,7 @@ import { Injectable } from '@nestjs/common'; import { bindThis } from '@/decorators.js'; -import { getErrorData, getErrorStatus, MastodonLogger } from '@/server/api/mastodon/MastodonLogger.js'; +import { getErrorData, getErrorException, getErrorStatus, MastodonLogger } from '@/server/api/mastodon/MastodonLogger.js'; import { MastodonClientService } from '@/server/api/mastodon/MastodonClientService.js'; import { ApiAccountMastodon } from '@/server/api/mastodon/endpoints/account.js'; import { ApiAppsMastodon } from '@/server/api/mastodon/endpoints/apps.js'; @@ -47,16 +47,20 @@ export class MastodonApiServerService { this.serverUtilityService.addFlattenedQueryType(fastify); // Convert JS exceptions into error responses - fastify.setErrorHandler((error, _, reply) => { + fastify.setErrorHandler((error, request, reply) => { const data = getErrorData(error); const status = getErrorStatus(error); + const exception = getErrorException(error); + if (exception) { + this.logger.exception(request, exception); + } reply.code(status).send(data); }); // Log error responses (including converted JSON exceptions) fastify.addHook('onSend', (request, reply, payload, done) => { - if (reply.statusCode >= 500 || (reply.statusCode >= 400 && this.logger.verbose)) { + if (reply.statusCode >= 400 && reply.statusCode <= 500) { if (typeof(payload) === 'string' && String(reply.getHeader('content-type')).toLowerCase().includes('application/json')) { const body = JSON.parse(payload); const data = getErrorData(body); diff --git a/packages/backend/src/server/api/mastodon/MastodonLogger.ts b/packages/backend/src/server/api/mastodon/MastodonLogger.ts index 8d8f62bc0d..29d44918a3 100644 --- a/packages/backend/src/server/api/mastodon/MastodonLogger.ts +++ b/packages/backend/src/server/api/mastodon/MastodonLogger.ts @@ -25,13 +25,29 @@ export class MastodonLogger { } public error(request: FastifyRequest, error: MastodonError, status: number): void { - const path = new URL(request.url, getBaseUrl(request)).pathname; + const path = getPath(request); + if (status >= 400 && status <= 499) { // Client errors this.logger.debug(`Error in mastodon endpoint ${request.method} ${path}:`, error); } else { // Server errors this.logger.error(`Error in mastodon endpoint ${request.method} ${path}:`, error); } } + + public exception(request: FastifyRequest, ex: Error): void { + const path = getPath(request); + + // Exceptions are always server errors, and should therefore always be logged. + this.logger.error(`Error in mastodon endpoint ${request.method} ${path}:`, ex); + } +} + +function getPath(request: FastifyRequest): string { + try { + return new URL(request.url, getBaseUrl(request)).pathname; + } catch { + return request.url; + } } // TODO move elsewhere @@ -40,6 +56,17 @@ export interface MastodonError { error_description?: string; } +export function getErrorException(error: unknown): Error | null { + if (error instanceof Error) { + // Axios errors are not exceptions - they're from the remote + if (!('response' in error) || !error.response || typeof (error.response) !== 'object') { + return error; + } + } + + return null; +} + export function getErrorData(error: unknown): MastodonError { // Axios wraps errors from the backend error = unpackAxiosError(error); @@ -78,7 +105,10 @@ export function getErrorData(error: unknown): MastodonError { } } - return convertUnknownError(error); + return { + error: 'INTERNAL_ERROR', + error_description: 'Internal error occurred. Please contact us if the error persists.', + }; } function unpackAxiosError(error: unknown): unknown { @@ -101,66 +131,25 @@ function unpackAxiosError(error: unknown): unknown { } function convertApiError(apiError: ApiError): MastodonError { - const mastoError: MastodonError & Partial & { stack?: unknown, statusCode?: number } = { - ...apiError, + return { error: apiError.code, error_description: apiError.message, }; - - delete mastoError.code; - delete mastoError.stack; - delete mastoError.message; - delete mastoError.httpStatusCode; - - return mastoError; } function convertErrorMessageError(error: { error: string, message: string }): MastodonError { - const mastoError: MastodonError & { stack?: unknown, message?: string, statusCode?: number } = { - ...error, + return { error: error.error, error_description: error.message, }; - - delete mastoError.stack; - delete mastoError.message; - delete mastoError.statusCode; - - return mastoError; -} - -function convertUnknownError(data: object = {}): MastodonError { - const mastoError = Object.assign({}, data, { - error: 'INTERNAL_ERROR', - error_description: 'Internal error occurred. Please contact us if the error persists.', - id: '5d37dbcb-891e-41ca-a3d6-e690c97775ac', - kind: 'server', - }); - - if ('statusCode' in mastoError) { - delete mastoError.statusCode; - } - - if ('stack' in mastoError) { - delete mastoError.stack; - } - - return mastoError; } function convertGenericError(error: Error): MastodonError { - const mastoError: MastodonError & Partial & { statusCode?: number } = { + return { ...error, error: 'INTERNAL_ERROR', error_description: String(error), }; - - delete mastoError.name; - delete mastoError.message; - delete mastoError.stack; - delete mastoError.statusCode; - - return mastoError; } export function getErrorStatus(error: unknown): number { From b6f4fda80da4f08241b294656603de8f5a64179d Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Tue, 6 May 2025 20:55:54 -0400 Subject: [PATCH 38/72] handle AxiosErrors without a response --- .../src/server/api/mastodon/MastodonLogger.ts | 33 +++++++++++++++++-- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/server/api/mastodon/MastodonLogger.ts b/packages/backend/src/server/api/mastodon/MastodonLogger.ts index 29d44918a3..54966f6236 100644 --- a/packages/backend/src/server/api/mastodon/MastodonLogger.ts +++ b/packages/backend/src/server/api/mastodon/MastodonLogger.ts @@ -57,10 +57,21 @@ export interface MastodonError { } export function getErrorException(error: unknown): Error | null { - if (error instanceof Error) { - // Axios errors are not exceptions - they're from the remote + if (error instanceof Error && error.name === 'AxiosError') { + // Axios errors with a response are from the remote if (!('response' in error) || !error.response || typeof (error.response) !== 'object') { - return error; + // This is the inner exception, basically + if ('cause' in error && error.cause instanceof Error) { + return error.cause; + } + + // Horrible hack to "recreate" the error without calling a constructor (since we want to re-use the stack). + return Object.assign(Object.create(Error), { + name: error.name, + stack: error.stack, + message: error.message, + cause: error.cause, + }); } } @@ -125,6 +136,22 @@ function unpackAxiosError(error: unknown): unknown { // No data - this is a fallback to avoid leaking request/response details in the error return undefined; } + + if (error instanceof Error && error.name === 'AxiosError') { + if ('cause' in error) { + return error.cause; + } + + if ('code' in error) { + return { + code: error.code, + message: String(error), + }; + } + + // No data - this is a fallback to avoid leaking request/response details in the error + return String(error); + } } return error; From 9db39d449fc6842a17919a45bb370569f0b50949 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Tue, 6 May 2025 21:06:33 -0400 Subject: [PATCH 39/72] more fixes to Mastodon logging --- .../api/mastodon/MastodonApiServerService.ts | 3 ++- .../src/server/api/mastodon/MastodonLogger.ts | 24 +++++-------------- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts b/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts index 1b0ed2ce4e..478c8f5cf2 100644 --- a/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts +++ b/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts @@ -51,6 +51,7 @@ export class MastodonApiServerService { const data = getErrorData(error); const status = getErrorStatus(error); const exception = getErrorException(error); + if (exception) { this.logger.exception(request, exception); } @@ -60,7 +61,7 @@ export class MastodonApiServerService { // Log error responses (including converted JSON exceptions) fastify.addHook('onSend', (request, reply, payload, done) => { - if (reply.statusCode >= 400 && reply.statusCode <= 500) { + if (reply.statusCode >= 400) { if (typeof(payload) === 'string' && String(reply.getHeader('content-type')).toLowerCase().includes('application/json')) { const body = JSON.parse(payload); const data = getErrorData(body); diff --git a/packages/backend/src/server/api/mastodon/MastodonLogger.ts b/packages/backend/src/server/api/mastodon/MastodonLogger.ts index 54966f6236..8581e30a9a 100644 --- a/packages/backend/src/server/api/mastodon/MastodonLogger.ts +++ b/packages/backend/src/server/api/mastodon/MastodonLogger.ts @@ -14,10 +14,6 @@ import { getBaseUrl } from '@/server/api/mastodon/MastodonClientService.js'; export class MastodonLogger { public readonly logger: Logger; - public get verbose() { - return this.logger.verbose; - } - constructor( loggerService: LoggerService, ) { @@ -65,13 +61,12 @@ export function getErrorException(error: unknown): Error | null { return error.cause; } - // Horrible hack to "recreate" the error without calling a constructor (since we want to re-use the stack). - return Object.assign(Object.create(Error), { - name: error.name, - stack: error.stack, - message: error.message, - cause: error.cause, - }); + const ex = new Error(); + ex.name = error.name; + ex.stack = error.stack; + ex.message = error.message; + ex.cause = error.cause; + return ex; } } @@ -142,13 +137,6 @@ function unpackAxiosError(error: unknown): unknown { return error.cause; } - if ('code' in error) { - return { - code: error.code, - message: String(error), - }; - } - // No data - this is a fallback to avoid leaking request/response details in the error return String(error); } From 2c5fb36e7f07fde634bcc91f6489c0c5b4885fcd Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Tue, 6 May 2025 21:15:56 -0400 Subject: [PATCH 40/72] add missing "return reply" calls to async fastify routes Required, according to docs: https://fastify.dev/docs/latest/Reference/Routes/#async-await --- .../api/mastodon/MastodonApiServerService.ts | 38 +++++++-------- .../server/api/mastodon/endpoints/account.ts | 30 ++++++------ .../src/server/api/mastodon/endpoints/apps.ts | 2 +- .../server/api/mastodon/endpoints/filter.ts | 10 ++-- .../server/api/mastodon/endpoints/instance.ts | 2 +- .../api/mastodon/endpoints/notifications.ts | 8 ++-- .../server/api/mastodon/endpoints/search.ts | 8 ++-- .../server/api/mastodon/endpoints/status.ts | 48 +++++++++---------- .../server/api/mastodon/endpoints/timeline.ts | 26 +++++----- .../src/server/oauth/OAuth2ProviderService.ts | 8 ++-- 10 files changed, 90 insertions(+), 90 deletions(-) diff --git a/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts b/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts index 478c8f5cf2..74fd9d7d59 100644 --- a/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts +++ b/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts @@ -56,7 +56,7 @@ export class MastodonApiServerService { this.logger.exception(request, exception); } - reply.code(status).send(data); + return reply.code(status).send(data); }); // Log error responses (including converted JSON exceptions) @@ -84,7 +84,7 @@ export class MastodonApiServerService { fastify.get('/v1/custom_emojis', async (_request, reply) => { const client = this.clientService.getClient(_request); const data = await client.getInstanceCustomEmojis(); - reply.send(data.data); + return reply.send(data.data); }); fastify.get('/v1/announcements', async (_request, reply) => { @@ -92,7 +92,7 @@ export class MastodonApiServerService { const data = await client.getInstanceAnnouncements(); const response = data.data.map((announcement) => convertAnnouncement(announcement)); - reply.send(response); + return reply.send(response); }); fastify.post<{ Body: { id?: string } }>('/v1/announcements/:id/dismiss', async (_request, reply) => { @@ -101,7 +101,7 @@ export class MastodonApiServerService { const client = this.clientService.getClient(_request); const data = await client.dismissInstanceAnnouncement(_request.body.id); - reply.send(data.data); + return reply.send(data.data); }); fastify.post('/v1/media', async (_request, reply) => { @@ -114,7 +114,7 @@ export class MastodonApiServerService { const data = await client.uploadMedia(multipartData); const response = convertAttachment(data.data as Entity.Attachment); - reply.send(response); + return reply.send(response); }); fastify.post<{ Body: { description?: string; focus?: string } }>('/v2/media', async (_request, reply) => { @@ -127,36 +127,36 @@ export class MastodonApiServerService { const data = await client.uploadMedia(multipartData, _request.body); const response = convertAttachment(data.data as Entity.Attachment); - reply.send(response); + return reply.send(response); }); fastify.get('/v1/trends', async (_request, reply) => { const client = this.clientService.getClient(_request); const data = await client.getInstanceTrends(); - reply.send(data.data); + return reply.send(data.data); }); fastify.get('/v1/trends/tags', async (_request, reply) => { const client = this.clientService.getClient(_request); const data = await client.getInstanceTrends(); - reply.send(data.data); + return reply.send(data.data); }); fastify.get('/v1/trends/links', async (_request, reply) => { // As we do not have any system for news/links this will just return empty - reply.send([]); + return reply.send([]); }); fastify.get('/v1/preferences', async (_request, reply) => { const client = this.clientService.getClient(_request); const data = await client.getPreferences(); - reply.send(data.data); + return reply.send(data.data); }); fastify.get('/v1/followed_tags', async (_request, reply) => { const client = this.clientService.getClient(_request); const data = await client.getFollowedTags(); - reply.send(data.data); + return reply.send(data.data); }); fastify.get<{ Querystring: TimelineArgs }>('/v1/bookmarks', async (_request, reply) => { @@ -165,7 +165,7 @@ export class MastodonApiServerService { const data = await client.getBookmarks(parseTimelineArgs(_request.query)); const response = await Promise.all(data.data.map((status) => this.mastoConverters.convertStatus(status, me))); - reply.send(response); + return reply.send(response); }); fastify.get<{ Querystring: TimelineArgs }>('/v1/favourites', async (_request, reply) => { @@ -187,7 +187,7 @@ export class MastodonApiServerService { const data = await client.getFavourites(args); const response = await Promise.all(data.data.map((status) => this.mastoConverters.convertStatus(status, me))); - reply.send(response); + return reply.send(response); }); fastify.get<{ Querystring: TimelineArgs }>('/v1/mutes', async (_request, reply) => { @@ -196,7 +196,7 @@ export class MastodonApiServerService { const data = await client.getMutes(parseTimelineArgs(_request.query)); const response = await Promise.all(data.data.map((account) => this.mastoConverters.convertAccount(account))); - reply.send(response); + return reply.send(response); }); fastify.get<{ Querystring: TimelineArgs }>('/v1/blocks', async (_request, reply) => { @@ -205,7 +205,7 @@ export class MastodonApiServerService { const data = await client.getBlocks(parseTimelineArgs(_request.query)); const response = await Promise.all(data.data.map((account) => this.mastoConverters.convertAccount(account))); - reply.send(response); + return reply.send(response); }); fastify.get<{ Querystring: { limit?: string } }>('/v1/follow_requests', async (_request, reply) => { @@ -215,7 +215,7 @@ export class MastodonApiServerService { const data = await client.getFollowRequests(limit); const response = await Promise.all(data.data.map((account) => this.mastoConverters.convertAccount(account as Entity.Account))); - reply.send(response); + return reply.send(response); }); fastify.post<{ Params: { id?: string } }>('/v1/follow_requests/:id/authorize', async (_request, reply) => { @@ -225,7 +225,7 @@ export class MastodonApiServerService { const data = await client.acceptFollowRequest(_request.params.id); const response = convertRelationship(data.data); - reply.send(response); + return reply.send(response); }); fastify.post<{ Params: { id?: string } }>('/v1/follow_requests/:id/reject', async (_request, reply) => { @@ -235,7 +235,7 @@ export class MastodonApiServerService { const data = await client.rejectFollowRequest(_request.params.id); const response = convertRelationship(data.data); - reply.send(response); + return reply.send(response); }); //#endregion @@ -260,7 +260,7 @@ export class MastodonApiServerService { const data = await client.updateMedia(_request.params.id, options); const response = convertAttachment(data.data); - reply.send(response); + return reply.send(response); }); done(); diff --git a/packages/backend/src/server/api/mastodon/endpoints/account.ts b/packages/backend/src/server/api/mastodon/endpoints/account.ts index b4ce56408e..6a1af62be7 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/account.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/account.ts @@ -47,7 +47,7 @@ export class ApiAccountMastodon { language: '', }, }); - reply.send(response); + return reply.send(response); }); fastify.patch<{ @@ -128,7 +128,7 @@ export class ApiAccountMastodon { const data = await client.updateCredentials(options); const response = await this.mastoConverters.convertAccount(data.data); - reply.send(response); + return reply.send(response); }); fastify.get<{ Querystring: { acct?: string } }>('/v1/accounts/lookup', async (_request, reply) => { @@ -140,7 +140,7 @@ export class ApiAccountMastodon { data.data.accounts[0].fields = profile?.fields.map(f => ({ ...f, verified_at: null })) ?? []; const response = await this.mastoConverters.convertAccount(data.data.accounts[0]); - reply.send(response); + return reply.send(response); }); fastify.get('/v1/accounts/relationships', async (_request, reply) => { @@ -150,7 +150,7 @@ export class ApiAccountMastodon { const data = await client.getRelationships(_request.query.id); const response = data.data.map(relationship => convertRelationship(relationship)); - reply.send(response); + return reply.send(response); }); fastify.get<{ Params: { id?: string } }>('/v1/accounts/:id', async (_request, reply) => { @@ -160,7 +160,7 @@ export class ApiAccountMastodon { const data = await client.getAccount(_request.params.id); const account = await this.mastoConverters.convertAccount(data.data); - reply.send(account); + return reply.send(account); }); fastify.get('/v1/accounts/:id/statuses', async (request, reply) => { @@ -172,7 +172,7 @@ export class ApiAccountMastodon { const response = await Promise.all(data.data.map(async (status) => await this.mastoConverters.convertStatus(status, me))); attachMinMaxPagination(request, reply, response); - reply.send(response); + return reply.send(response); }); fastify.get<{ Params: { id?: string } }>('/v1/accounts/:id/featured_tags', async (_request, reply) => { @@ -182,7 +182,7 @@ export class ApiAccountMastodon { const data = await client.getFeaturedTags(); const response = data.data.map((tag) => convertFeaturedTag(tag)); - reply.send(response); + return reply.send(response); }); fastify.get('/v1/accounts/:id/followers', async (request, reply) => { @@ -196,7 +196,7 @@ export class ApiAccountMastodon { const response = await Promise.all(data.data.map(async (account) => await this.mastoConverters.convertAccount(account))); attachMinMaxPagination(request, reply, response); - reply.send(response); + return reply.send(response); }); fastify.get('/v1/accounts/:id/following', async (request, reply) => { @@ -210,7 +210,7 @@ export class ApiAccountMastodon { const response = await Promise.all(data.data.map(async (account) => await this.mastoConverters.convertAccount(account))); attachMinMaxPagination(request, reply, response); - reply.send(response); + return reply.send(response); }); fastify.get<{ Params: { id?: string } }>('/v1/accounts/:id/lists', async (_request, reply) => { @@ -220,7 +220,7 @@ export class ApiAccountMastodon { const data = await client.getAccountLists(_request.params.id); const response = data.data.map((list) => convertList(list)); - reply.send(response); + return reply.send(response); }); fastify.post('/v1/accounts/:id/follow', async (_request, reply) => { @@ -231,7 +231,7 @@ export class ApiAccountMastodon { const acct = convertRelationship(data.data); acct.following = true; // TODO this is wrong, follow may not have processed immediately - reply.send(acct); + return reply.send(acct); }); fastify.post('/v1/accounts/:id/unfollow', async (_request, reply) => { @@ -242,7 +242,7 @@ export class ApiAccountMastodon { const acct = convertRelationship(data.data); acct.following = false; - reply.send(acct); + return reply.send(acct); }); fastify.post('/v1/accounts/:id/block', async (_request, reply) => { @@ -252,7 +252,7 @@ export class ApiAccountMastodon { const data = await client.blockAccount(_request.params.id); const response = convertRelationship(data.data); - reply.send(response); + return reply.send(response); }); fastify.post('/v1/accounts/:id/unblock', async (_request, reply) => { @@ -275,7 +275,7 @@ export class ApiAccountMastodon { ); const response = convertRelationship(data.data); - reply.send(response); + return reply.send(response); }); fastify.post('/v1/accounts/:id/unmute', async (_request, reply) => { @@ -285,7 +285,7 @@ export class ApiAccountMastodon { const data = await client.unmuteAccount(_request.params.id); const response = convertRelationship(data.data); - reply.send(response); + return reply.send(response); }); } } diff --git a/packages/backend/src/server/api/mastodon/endpoints/apps.ts b/packages/backend/src/server/api/mastodon/endpoints/apps.ts index aae6103146..5fce838f47 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/apps.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/apps.ts @@ -106,7 +106,7 @@ export class ApiAppsMastodon { client_secret: appData.clientSecret, }; - reply.send(response); + return reply.send(response); }); } } diff --git a/packages/backend/src/server/api/mastodon/endpoints/filter.ts b/packages/backend/src/server/api/mastodon/endpoints/filter.ts index 242f068b99..f2bd0052d5 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/filter.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/filter.ts @@ -35,7 +35,7 @@ export class ApiFilterMastodon { const data = await client.getFilters(); const response = data.data.map((filter) => convertFilter(filter)); - reply.send(response); + return reply.send(response); }); fastify.get('/v1/filters/:id', async (_request, reply) => { @@ -45,7 +45,7 @@ export class ApiFilterMastodon { const data = await client.getFilter(_request.params.id); const response = convertFilter(data.data); - reply.send(response); + return reply.send(response); }); fastify.post('/v1/filters', async (_request, reply) => { @@ -64,7 +64,7 @@ export class ApiFilterMastodon { const data = await client.createFilter(_request.body.phrase, _request.body.context, options); const response = convertFilter(data.data); - reply.send(response); + return reply.send(response); }); fastify.post('/v1/filters/:id', async (_request, reply) => { @@ -84,7 +84,7 @@ export class ApiFilterMastodon { const data = await client.updateFilter(_request.params.id, _request.body.phrase, _request.body.context, options); const response = convertFilter(data.data); - reply.send(response); + return reply.send(response); }); fastify.delete('/v1/filters/:id', async (_request, reply) => { @@ -93,7 +93,7 @@ export class ApiFilterMastodon { const client = this.clientService.getClient(_request); const data = await client.deleteFilter(_request.params.id); - reply.send(data.data); + return reply.send(data.data); }); } } diff --git a/packages/backend/src/server/api/mastodon/endpoints/instance.ts b/packages/backend/src/server/api/mastodon/endpoints/instance.ts index a168339ac6..cfca5b1350 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/instance.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/instance.ts @@ -87,7 +87,7 @@ export class ApiInstanceMastodon { rules: instance.rules ?? [], }; - reply.send(response); + return reply.send(response); }); } } diff --git a/packages/backend/src/server/api/mastodon/endpoints/notifications.ts b/packages/backend/src/server/api/mastodon/endpoints/notifications.ts index 75512c2efc..f6cc59e782 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/notifications.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/notifications.ts @@ -45,7 +45,7 @@ export class ApiNotificationsMastodon { } attachMinMaxPagination(request, reply, response); - reply.send(response); + return reply.send(response); }); fastify.get('/v1/notification/:id', async (_request, reply) => { @@ -62,7 +62,7 @@ export class ApiNotificationsMastodon { }); } - reply.send(response); + return reply.send(response); }); fastify.post('/v1/notification/:id/dismiss', async (_request, reply) => { @@ -71,14 +71,14 @@ export class ApiNotificationsMastodon { const client = this.clientService.getClient(_request); const data = await client.dismissNotification(_request.params.id); - reply.send(data.data); + return reply.send(data.data); }); fastify.post('/v1/notifications/clear', async (_request, reply) => { const client = this.clientService.getClient(_request); const data = await client.dismissNotifications(); - reply.send(data.data); + return reply.send(data.data); }); } } diff --git a/packages/backend/src/server/api/mastodon/endpoints/search.ts b/packages/backend/src/server/api/mastodon/endpoints/search.ts index 33bfa87e5f..f58f21966c 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/search.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/search.ts @@ -62,7 +62,7 @@ export class ApiSearchMastodon { attachMinMaxPagination(request, reply, response[type]); } - reply.send(response); + return reply.send(response); }); fastify.get('/v2/search', async (request, reply) => { @@ -103,7 +103,7 @@ export class ApiSearchMastodon { // Offset pagination is the only possible option attachOffsetPagination(request, reply, longestResult); - reply.send(response); + return reply.send(response); }); fastify.get('/v1/trends/statuses', async (request, reply) => { @@ -126,7 +126,7 @@ export class ApiSearchMastodon { const response = await Promise.all(data.map(status => this.mastoConverters.convertStatus(status, me))); attachMinMaxPagination(request, reply, response); - reply.send(response); + return reply.send(response); }); fastify.get('/v2/suggestions', async (request, reply) => { @@ -158,7 +158,7 @@ export class ApiSearchMastodon { })); attachOffsetPagination(request, reply, response); - reply.send(response); + return reply.send(response); }); } } diff --git a/packages/backend/src/server/api/mastodon/endpoints/status.ts b/packages/backend/src/server/api/mastodon/endpoints/status.ts index ec31e0cc46..22b8a911ca 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/status.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/status.ts @@ -38,7 +38,7 @@ export class ApiStatusMastodon { response.media_attachments = []; } - reply.send(response); + return reply.send(response); }); fastify.get<{ Params: { id?: string } }>('/v1/statuses/:id/source', async (_request, reply) => { @@ -47,7 +47,7 @@ export class ApiStatusMastodon { const client = this.clientService.getClient(_request); const data = await client.getStatusSource(_request.params.id); - reply.send(data.data); + return reply.send(data.data); }); fastify.get<{ Params: { id?: string }, Querystring: TimelineArgs }>('/v1/statuses/:id/context', async (_request, reply) => { @@ -59,7 +59,7 @@ export class ApiStatusMastodon { const descendants = await Promise.all(data.descendants.map(async (status: Entity.Status) => await this.mastoConverters.convertStatus(status, me))); const response = { ancestors, descendants }; - reply.send(response); + return reply.send(response); }); fastify.get<{ Params: { id?: string } }>('/v1/statuses/:id/history', async (_request, reply) => { @@ -68,7 +68,7 @@ export class ApiStatusMastodon { const user = await this.clientService.getAuth(_request); const edits = await this.mastoConverters.getEdits(_request.params.id, user); - reply.send(edits); + return reply.send(edits); }); fastify.get<{ Params: { id?: string } }>('/v1/statuses/:id/reblogged_by', async (_request, reply) => { @@ -78,7 +78,7 @@ export class ApiStatusMastodon { const data = await client.getStatusRebloggedBy(_request.params.id); const response = await Promise.all(data.data.map((account: Entity.Account) => this.mastoConverters.convertAccount(account))); - reply.send(response); + return reply.send(response); }); fastify.get<{ Params: { id?: string } }>('/v1/statuses/:id/favourited_by', async (_request, reply) => { @@ -88,7 +88,7 @@ export class ApiStatusMastodon { const data = await client.getStatusFavouritedBy(_request.params.id); const response = await Promise.all(data.data.map((account: Entity.Account) => this.mastoConverters.convertAccount(account))); - reply.send(response); + return reply.send(response); }); fastify.get<{ Params: { id?: string } }>('/v1/media/:id', async (_request, reply) => { @@ -98,7 +98,7 @@ export class ApiStatusMastodon { const data = await client.getMedia(_request.params.id); const response = convertAttachment(data.data); - reply.send(response); + return reply.send(response); }); fastify.get<{ Params: { id?: string } }>('/v1/polls/:id', async (_request, reply) => { @@ -108,7 +108,7 @@ export class ApiStatusMastodon { const data = await client.getPoll(_request.params.id); const response = convertPoll(data.data); - reply.send(response); + return reply.send(response); }); fastify.post<{ Params: { id?: string }, Body: { choices?: number[] } }>('/v1/polls/:id/votes', async (_request, reply) => { @@ -119,7 +119,7 @@ export class ApiStatusMastodon { const data = await client.votePoll(_request.params.id, _request.body.choices); const response = convertPoll(data.data); - reply.send(response); + return reply.send(response); }); fastify.post<{ @@ -161,14 +161,14 @@ export class ApiStatusMastodon { body.in_reply_to_id, removed, ); - reply.send(a.data); + return reply.send(a.data); } if (body.in_reply_to_id && removed === '/unreact') { const id = body.in_reply_to_id; const post = await client.getStatus(id); const react = post.data.emoji_reactions.filter((e: Entity.Emoji) => e.me)[0].name; const data = await client.deleteEmojiReaction(id, react); - reply.send(data.data); + return reply.send(data.data); } if (!body.media_ids) body.media_ids = undefined; if (body.media_ids && !body.media_ids.length) body.media_ids = undefined; @@ -194,7 +194,7 @@ export class ApiStatusMastodon { const data = await client.postStatus(text, options); const response = await this.mastoConverters.convertStatus(data.data as Entity.Status, me); - reply.send(response); + return reply.send(response); }); fastify.put<{ @@ -233,7 +233,7 @@ export class ApiStatusMastodon { const data = await client.editStatus(_request.params.id, options); const response = await this.mastoConverters.convertStatus(data.data, me); - reply.send(response); + return reply.send(response); }); fastify.post<{ Params: { id?: string } }>('/v1/statuses/:id/favourite', async (_request, reply) => { @@ -243,7 +243,7 @@ export class ApiStatusMastodon { const data = await client.createEmojiReaction(_request.params.id, '❤'); const response = await this.mastoConverters.convertStatus(data.data, me); - reply.send(response); + return reply.send(response); }); fastify.post<{ Params: { id?: string } }>('/v1/statuses/:id/unfavourite', async (_request, reply) => { @@ -253,7 +253,7 @@ export class ApiStatusMastodon { const data = await client.deleteEmojiReaction(_request.params.id, '❤'); const response = await this.mastoConverters.convertStatus(data.data, me); - reply.send(response); + return reply.send(response); }); fastify.post<{ Params: { id?: string } }>('/v1/statuses/:id/reblog', async (_request, reply) => { @@ -263,7 +263,7 @@ export class ApiStatusMastodon { const data = await client.reblogStatus(_request.params.id); const response = await this.mastoConverters.convertStatus(data.data, me); - reply.send(response); + return reply.send(response); }); fastify.post<{ Params: { id?: string } }>('/v1/statuses/:id/unreblog', async (_request, reply) => { @@ -273,7 +273,7 @@ export class ApiStatusMastodon { const data = await client.unreblogStatus(_request.params.id); const response = await this.mastoConverters.convertStatus(data.data, me); - reply.send(response); + return reply.send(response); }); fastify.post<{ Params: { id?: string } }>('/v1/statuses/:id/bookmark', async (_request, reply) => { @@ -283,7 +283,7 @@ export class ApiStatusMastodon { const data = await client.bookmarkStatus(_request.params.id); const response = await this.mastoConverters.convertStatus(data.data, me); - reply.send(response); + return reply.send(response); }); fastify.post<{ Params: { id?: string } }>('/v1/statuses/:id/unbookmark', async (_request, reply) => { @@ -293,7 +293,7 @@ export class ApiStatusMastodon { const data = await client.unbookmarkStatus(_request.params.id); const response = await this.mastoConverters.convertStatus(data.data, me); - reply.send(response); + return reply.send(response); }); fastify.post<{ Params: { id?: string } }>('/v1/statuses/:id/pin', async (_request, reply) => { if (!_request.params.id) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required parameter "id"' }); @@ -302,7 +302,7 @@ export class ApiStatusMastodon { const data = await client.pinStatus(_request.params.id); const response = await this.mastoConverters.convertStatus(data.data, me); - reply.send(response); + return reply.send(response); }); fastify.post<{ Params: { id?: string } }>('/v1/statuses/:id/unpin', async (_request, reply) => { @@ -312,7 +312,7 @@ export class ApiStatusMastodon { const data = await client.unpinStatus(_request.params.id); const response = await this.mastoConverters.convertStatus(data.data, me); - reply.send(response); + return reply.send(response); }); fastify.post<{ Params: { id?: string, name?: string } }>('/v1/statuses/:id/react/:name', async (_request, reply) => { @@ -323,7 +323,7 @@ export class ApiStatusMastodon { const data = await client.createEmojiReaction(_request.params.id, _request.params.name); const response = await this.mastoConverters.convertStatus(data.data, me); - reply.send(response); + return reply.send(response); }); fastify.post<{ Params: { id?: string, name?: string } }>('/v1/statuses/:id/unreact/:name', async (_request, reply) => { @@ -334,7 +334,7 @@ export class ApiStatusMastodon { const data = await client.deleteEmojiReaction(_request.params.id, _request.params.name); const response = await this.mastoConverters.convertStatus(data.data, me); - reply.send(response); + return reply.send(response); }); fastify.delete<{ Params: { id?: string } }>('/v1/statuses/:id', async (_request, reply) => { @@ -343,7 +343,7 @@ export class ApiStatusMastodon { const client = this.clientService.getClient(_request); const data = await client.deleteStatus(_request.params.id); - reply.send(data.data); + return reply.send(data.data); }); } } diff --git a/packages/backend/src/server/api/mastodon/endpoints/timeline.ts b/packages/backend/src/server/api/mastodon/endpoints/timeline.ts index b6162d9eb2..b2f7b18dc9 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/timeline.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/timeline.ts @@ -28,7 +28,7 @@ export class ApiTimelineMastodon { const response = await Promise.all(data.data.map((status: Entity.Status) => this.mastoConverters.convertStatus(status, me))); attachMinMaxPagination(request, reply, response); - reply.send(response); + return reply.send(response); }); fastify.get<{ Querystring: TimelineArgs }>('/v1/timelines/home', async (request, reply) => { @@ -38,7 +38,7 @@ export class ApiTimelineMastodon { const response = await Promise.all(data.data.map((status: Entity.Status) => this.mastoConverters.convertStatus(status, me))); attachMinMaxPagination(request, reply, response); - reply.send(response); + return reply.send(response); }); fastify.get<{ Params: { hashtag?: string }, Querystring: TimelineArgs }>('/v1/timelines/tag/:hashtag', async (request, reply) => { @@ -50,7 +50,7 @@ export class ApiTimelineMastodon { const response = await Promise.all(data.data.map((status: Entity.Status) => this.mastoConverters.convertStatus(status, me))); attachMinMaxPagination(request, reply, response); - reply.send(response); + return reply.send(response); }); fastify.get<{ Params: { id?: string }, Querystring: TimelineArgs }>('/v1/timelines/list/:id', async (request, reply) => { @@ -62,7 +62,7 @@ export class ApiTimelineMastodon { const response = await Promise.all(data.data.map(async (status: Entity.Status) => await this.mastoConverters.convertStatus(status, me))); attachMinMaxPagination(request, reply, response); - reply.send(response); + return reply.send(response); }); fastify.get<{ Querystring: TimelineArgs }>('/v1/conversations', async (request, reply) => { @@ -72,7 +72,7 @@ export class ApiTimelineMastodon { const response = await Promise.all(data.data.map((conversation: Entity.Conversation) => this.mastoConverters.convertConversation(conversation, me))); attachMinMaxPagination(request, reply, response); - reply.send(response); + return reply.send(response); }); fastify.get<{ Params: { id?: string } }>('/v1/lists/:id', async (_request, reply) => { @@ -82,7 +82,7 @@ export class ApiTimelineMastodon { const data = await client.getList(_request.params.id); const response = convertList(data.data); - reply.send(response); + return reply.send(response); }); fastify.get('/v1/lists', async (request, reply) => { @@ -91,7 +91,7 @@ export class ApiTimelineMastodon { const response = data.data.map((list: Entity.List) => convertList(list)); attachMinMaxPagination(request, reply, response); - reply.send(response); + return reply.send(response); }); fastify.get<{ Params: { id?: string }, Querystring: TimelineArgs }>('/v1/lists/:id/accounts', async (request, reply) => { @@ -102,7 +102,7 @@ export class ApiTimelineMastodon { const response = await Promise.all(data.data.map((account: Entity.Account) => this.mastoConverters.convertAccount(account))); attachMinMaxPagination(request, reply, response); - reply.send(response); + return reply.send(response); }); fastify.post<{ Params: { id?: string }, Querystring: { accounts_id?: string[] } }>('/v1/lists/:id/accounts', async (_request, reply) => { @@ -112,7 +112,7 @@ export class ApiTimelineMastodon { const client = this.clientService.getClient(_request); const data = await client.addAccountsToList(_request.params.id, _request.query.accounts_id); - reply.send(data.data); + return reply.send(data.data); }); fastify.delete<{ Params: { id?: string }, Querystring: { accounts_id?: string[] } }>('/v1/lists/:id/accounts', async (_request, reply) => { @@ -122,7 +122,7 @@ export class ApiTimelineMastodon { const client = this.clientService.getClient(_request); const data = await client.deleteAccountsFromList(_request.params.id, _request.query.accounts_id); - reply.send(data.data); + return reply.send(data.data); }); fastify.post<{ Body: { title?: string } }>('/v1/lists', async (_request, reply) => { @@ -132,7 +132,7 @@ export class ApiTimelineMastodon { const data = await client.createList(_request.body.title); const response = convertList(data.data); - reply.send(response); + return reply.send(response); }); fastify.put<{ Params: { id?: string }, Body: { title?: string } }>('/v1/lists/:id', async (_request, reply) => { @@ -143,7 +143,7 @@ export class ApiTimelineMastodon { const data = await client.updateList(_request.params.id, _request.body.title); const response = convertList(data.data); - reply.send(response); + return reply.send(response); }); fastify.delete<{ Params: { id?: string } }>('/v1/lists/:id', async (_request, reply) => { @@ -152,7 +152,7 @@ export class ApiTimelineMastodon { const client = this.clientService.getClient(_request); await client.deleteList(_request.params.id); - reply.send({}); + return reply.send({}); }); } } diff --git a/packages/backend/src/server/oauth/OAuth2ProviderService.ts b/packages/backend/src/server/oauth/OAuth2ProviderService.ts index e1f39dd9b6..18f585ea28 100644 --- a/packages/backend/src/server/oauth/OAuth2ProviderService.ts +++ b/packages/backend/src/server/oauth/OAuth2ProviderService.ts @@ -106,7 +106,7 @@ export class OAuth2ProviderService { if (request.query.state) redirectUri.searchParams.set('state', String(request.query.state)); if (request.query.redirect_uri) redirectUri.searchParams.set('redirect_uri', String(request.query.redirect_uri)); - reply.redirect(redirectUri.toString()); + return reply.redirect(redirectUri.toString()); }); } @@ -120,7 +120,7 @@ export class OAuth2ProviderService { scope: 'read', created_at: Math.floor(new Date().getTime() / 1000), }; - reply.send(ret); + return reply.send(ret); } try { @@ -140,10 +140,10 @@ export class OAuth2ProviderService { scope: body.scope || 'read write follow push', created_at: Math.floor(new Date().getTime() / 1000), }; - reply.send(ret); + return reply.send(ret); } catch (e: unknown) { const data = getErrorData(e); - reply.code(401).send(data); + return reply.code(401).send(data); } }); } From 34e0d73a60686d25fc496366c2fdb13c6288b63a Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Tue, 6 May 2025 21:20:31 -0400 Subject: [PATCH 41/72] differentiate between "error" and "exception" in mastodon API --- packages/backend/src/server/api/mastodon/MastodonLogger.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/server/api/mastodon/MastodonLogger.ts b/packages/backend/src/server/api/mastodon/MastodonLogger.ts index 8581e30a9a..ef8788f9b7 100644 --- a/packages/backend/src/server/api/mastodon/MastodonLogger.ts +++ b/packages/backend/src/server/api/mastodon/MastodonLogger.ts @@ -34,7 +34,7 @@ export class MastodonLogger { const path = getPath(request); // Exceptions are always server errors, and should therefore always be logged. - this.logger.error(`Error in mastodon endpoint ${request.method} ${path}:`, ex); + this.logger.error(`Exception in mastodon endpoint ${request.method} ${path}:`, ex); } } From fefe2f6db82f2e0aaa84c761c613e46a1b6f6b18 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Tue, 6 May 2025 21:31:53 -0400 Subject: [PATCH 42/72] more improvements to Mastodon error logging --- .../src/server/api/mastodon/MastodonLogger.ts | 59 ++++++++++++------- 1 file changed, 39 insertions(+), 20 deletions(-) diff --git a/packages/backend/src/server/api/mastodon/MastodonLogger.ts b/packages/backend/src/server/api/mastodon/MastodonLogger.ts index ef8788f9b7..5b4070eae9 100644 --- a/packages/backend/src/server/api/mastodon/MastodonLogger.ts +++ b/packages/backend/src/server/api/mastodon/MastodonLogger.ts @@ -4,11 +4,12 @@ */ import { Injectable } from '@nestjs/common'; -import { FastifyRequest } from 'fastify'; -import Logger from '@/logger.js'; +import type Logger from '@/logger.js'; import { LoggerService } from '@/core/LoggerService.js'; import { ApiError } from '@/server/api/error.js'; import { getBaseUrl } from '@/server/api/mastodon/MastodonClientService.js'; +import { AuthenticationError } from '@/server/api/AuthenticateService.js'; +import type { FastifyRequest } from 'fastify'; @Injectable() export class MastodonLogger { @@ -53,24 +54,36 @@ export interface MastodonError { } export function getErrorException(error: unknown): Error | null { - if (error instanceof Error && error.name === 'AxiosError') { - // Axios errors with a response are from the remote - if (!('response' in error) || !error.response || typeof (error.response) !== 'object') { - // This is the inner exception, basically - if ('cause' in error && error.cause instanceof Error) { - return error.cause; - } - - const ex = new Error(); - ex.name = error.name; - ex.stack = error.stack; - ex.message = error.message; - ex.cause = error.cause; - return ex; - } + if (!(error instanceof Error)) { + return null; } - return null; + // AxiosErrors need special decoding + if (error.name === 'AxiosError') { + // Axios errors with a response are from the remote + if ('response' in error && error.response && typeof (error.response) === 'object') { + return null; + } + + // This is the inner exception, basically + if ('cause' in error && error.cause instanceof Error) { + return error.cause; + } + + const ex = new Error(); + ex.name = error.name; + ex.stack = error.stack; + ex.message = error.message; + ex.cause = error.cause; + return ex; + } + + // AuthenticationError is a client error + if (error instanceof AuthenticationError) { + return null; + } + + return error; } export function getErrorData(error: unknown): MastodonError { @@ -107,7 +120,7 @@ export function getErrorData(error: unknown): MastodonError { if ('error' in error && typeof(error.error) === 'string') { // "error_description" is string, undefined, or not present. if (!('error_description' in error) || typeof(error.error_description) === 'string' || typeof(error.error_description) === 'undefined') { - return error as MastodonError; + return convertMastodonError(error as MastodonError); } } @@ -161,12 +174,18 @@ function convertErrorMessageError(error: { error: string, message: string }): Ma function convertGenericError(error: Error): MastodonError { return { - ...error, error: 'INTERNAL_ERROR', error_description: String(error), }; } +function convertMastodonError(error: MastodonError): MastodonError { + return { + error: error.error, + error_description: error.error_description, + }; +} + export function getErrorStatus(error: unknown): number { if (error && typeof(error) === 'object') { // Axios wraps errors from the backend From 38b1e1225c0b39ce5c0cca172fa656a007eb1cd6 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Tue, 6 May 2025 21:39:45 -0400 Subject: [PATCH 43/72] use isAxiosError to improve type detection --- packages/backend/package.json | 1 + .../src/server/api/mastodon/MastodonLogger.ts | 25 +++++++++---------- pnpm-lock.yaml | 16 +++--------- 3 files changed, 17 insertions(+), 25 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index 4a9560e833..bb60eff05b 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -97,6 +97,7 @@ "ajv": "8.17.1", "archiver": "7.0.1", "argon2": "^0.40.1", + "axios": "1.7.4", "async-mutex": "0.5.0", "bcryptjs": "2.4.3", "blurhash": "2.0.5", diff --git a/packages/backend/src/server/api/mastodon/MastodonLogger.ts b/packages/backend/src/server/api/mastodon/MastodonLogger.ts index 5b4070eae9..85df66a23c 100644 --- a/packages/backend/src/server/api/mastodon/MastodonLogger.ts +++ b/packages/backend/src/server/api/mastodon/MastodonLogger.ts @@ -4,6 +4,7 @@ */ import { Injectable } from '@nestjs/common'; +import { isAxiosError } from 'axios'; import type Logger from '@/logger.js'; import { LoggerService } from '@/core/LoggerService.js'; import { ApiError } from '@/server/api/error.js'; @@ -59,14 +60,14 @@ export function getErrorException(error: unknown): Error | null { } // AxiosErrors need special decoding - if (error.name === 'AxiosError') { + if (isAxiosError(error)) { // Axios errors with a response are from the remote - if ('response' in error && error.response && typeof (error.response) === 'object') { + if (error.response) { return null; } // This is the inner exception, basically - if ('cause' in error && error.cause instanceof Error) { + if (error.cause && !isAxiosError(error.cause)) { return error.cause; } @@ -131,9 +132,9 @@ export function getErrorData(error: unknown): MastodonError { } function unpackAxiosError(error: unknown): unknown { - if (error && typeof(error) === 'object') { - if ('response' in error && error.response && typeof (error.response) === 'object') { - if ('data' in error.response && error.response.data && typeof (error.response.data) === 'object') { + if (isAxiosError(error)) { + if (error.response) { + if (error.response.data) { if ('error' in error.response.data && error.response.data.error && typeof(error.response.data.error) === 'object') { return error.response.data.error; } @@ -145,14 +146,12 @@ function unpackAxiosError(error: unknown): unknown { return undefined; } - if (error instanceof Error && error.name === 'AxiosError') { - if ('cause' in error) { - return error.cause; - } - - // No data - this is a fallback to avoid leaking request/response details in the error - return String(error); + if (error.cause && !isAxiosError(error.cause)) { + return error.cause; } + + // No data - this is a fallback to avoid leaking request/response details in the error + return String(error); } return error; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4eb78db1b5..ed31b44e1e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -182,6 +182,9 @@ importers: async-mutex: specifier: 0.5.0 version: 0.5.0 + axios: + specifier: 1.7.4 + version: 1.7.4 bcryptjs: specifier: 2.4.3 version: 2.4.3 @@ -6417,15 +6420,6 @@ packages: resolution: {integrity: sha512-Be3narBNt2s6bsaqP6Jzq91heDgOEaDCJAXcE3qcma/EJBSy5FB4cvO31XBInuAuKBx8Kptf8dkhjK0IOru39Q==} engines: {node: '>=18'} - follow-redirects@1.15.2: - resolution: {integrity: sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==} - engines: {node: '>=4.0'} - peerDependencies: - debug: '*' - peerDependenciesMeta: - debug: - optional: true - follow-redirects@1.15.9: resolution: {integrity: sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==} engines: {node: '>=4.0'} @@ -15329,7 +15323,7 @@ snapshots: axios@0.24.0: dependencies: - follow-redirects: 1.15.2 + follow-redirects: 1.15.9(debug@4.4.0) transitivePeerDependencies: - debug @@ -17251,8 +17245,6 @@ snapshots: async: 0.2.10 which: 1.3.1 - follow-redirects@1.15.2: {} - follow-redirects@1.15.9(debug@4.4.0): optionalDependencies: debug: 4.4.0(supports-color@8.1.1) From 347edb107b0bd22c023f0b115fdc16f7fc8681e0 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Tue, 6 May 2025 21:41:38 -0400 Subject: [PATCH 44/72] copy stack to AxiosError causes --- .../backend/src/server/api/mastodon/MastodonLogger.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/backend/src/server/api/mastodon/MastodonLogger.ts b/packages/backend/src/server/api/mastodon/MastodonLogger.ts index 85df66a23c..555151a605 100644 --- a/packages/backend/src/server/api/mastodon/MastodonLogger.ts +++ b/packages/backend/src/server/api/mastodon/MastodonLogger.ts @@ -68,6 +68,10 @@ export function getErrorException(error: unknown): Error | null { // This is the inner exception, basically if (error.cause && !isAxiosError(error.cause)) { + if (!error.cause.stack) { + error.cause.stack = error.stack; + } + return error.cause; } @@ -147,6 +151,10 @@ function unpackAxiosError(error: unknown): unknown { } if (error.cause && !isAxiosError(error.cause)) { + if (!error.cause.stack) { + error.cause.stack = error.stack; + } + return error.cause; } From 22bba7fe6d787b1341340e331f3b0a706b00a383 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Tue, 6 May 2025 21:52:19 -0400 Subject: [PATCH 45/72] fix media upload error caused by extraneous array brackets --- packages/backend/src/server/ServerUtilityService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/server/ServerUtilityService.ts b/packages/backend/src/server/ServerUtilityService.ts index 00eb97f679..c2a3132489 100644 --- a/packages/backend/src/server/ServerUtilityService.ts +++ b/packages/backend/src/server/ServerUtilityService.ts @@ -54,7 +54,7 @@ export class ServerUtilityService { } } else { // Otherwise it's a file try { - const [filepath] = await saveToTempFile(part.file); + const filepath = await saveToTempFile(part.file); const tmpUploads = (request.tmpUploads ??= []); tmpUploads.push(filepath); From 5815d2f537139b6748c8dbd5e003a360de66ca7c Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Tue, 6 May 2025 22:53:43 -0400 Subject: [PATCH 46/72] fix user-agent / authorization passing in megalodon --- packages/megalodon/src/misskey/api_client.ts | 42 ++++++++++---------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/packages/megalodon/src/misskey/api_client.ts b/packages/megalodon/src/misskey/api_client.ts index 550897b669..659184d156 100644 --- a/packages/megalodon/src/misskey/api_client.ts +++ b/packages/megalodon/src/misskey/api_client.ts @@ -1,6 +1,5 @@ import axios, { AxiosResponse, AxiosRequestConfig } from 'axios' import dayjs from 'dayjs' -import FormData from 'form-data' import { DEFAULT_UA } from '../default' import Response from '../response' @@ -575,22 +574,26 @@ namespace MisskeyAPI { this.accessToken = accessToken this.baseUrl = baseUrl this.userAgent = userAgent - this.abortController = new AbortController() - axios.defaults.signal = this.abortController.signal + this.abortController = new AbortController(); } /** * GET request to misskey API. **/ public async get(path: string, params: any = {}, headers: { [key: string]: string } = {}): Promise> { + if (!headers['Authorization'] && this.accessToken) { + headers['Authorization'] = `Bearer ${this.accessToken}`; + } + if (!headers['User-Agent']) { + headers['User-Agent'] = this.userAgent; + } + let options: AxiosRequestConfig = { params: params, - headers: { - 'User-Agent': this.userAgent, - ...headers, - }, + headers, maxContentLength: Infinity, - maxBodyLength: Infinity + maxBodyLength: Infinity, + signal: this.abortController.signal, } return axios.get(this.baseUrl + path, options).then((resp: AxiosResponse) => { const res: Response = { @@ -610,22 +613,21 @@ namespace MisskeyAPI { * @param headers Request header object */ public async post(path: string, params: any = {}, headers: { [key: string]: string } = {}): Promise> { + if (!headers['Authorization'] && this.accessToken) { + headers['Authorization'] = `Bearer ${this.accessToken}`; + } + if (!headers['User-Agent']) { + headers['User-Agent'] = this.userAgent; + } + let options: AxiosRequestConfig = { headers: headers, maxContentLength: Infinity, - maxBodyLength: Infinity + maxBodyLength: Infinity, + signal: this.abortController.signal, } - let bodyParams = params - if (this.accessToken) { - if (params instanceof FormData) { - bodyParams.append('i', this.accessToken) - } else { - bodyParams = Object.assign(params, { - i: this.accessToken - }) - } - } - return axios.post(this.baseUrl + path, bodyParams, options).then((resp: AxiosResponse) => { + + return axios.post(this.baseUrl + path, params, options).then((resp: AxiosResponse) => { const res: Response = { data: resp.data, status: resp.status, From 5ec9be0b8c8f28aa6111a2fc2a0411545fed844b Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Tue, 6 May 2025 23:03:23 -0400 Subject: [PATCH 47/72] fix "cannot use 'in' operator" error --- packages/backend/src/server/api/mastodon/MastodonLogger.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/server/api/mastodon/MastodonLogger.ts b/packages/backend/src/server/api/mastodon/MastodonLogger.ts index 555151a605..5ea69ed151 100644 --- a/packages/backend/src/server/api/mastodon/MastodonLogger.ts +++ b/packages/backend/src/server/api/mastodon/MastodonLogger.ts @@ -138,7 +138,7 @@ export function getErrorData(error: unknown): MastodonError { function unpackAxiosError(error: unknown): unknown { if (isAxiosError(error)) { if (error.response) { - if (error.response.data) { + if (error.response.data && typeof(error.response.data) === 'object') { if ('error' in error.response.data && error.response.data.error && typeof(error.response.data.error) === 'object') { return error.response.data.error; } From b2ea03383cd53ac213c4dee6dbd086ab6f54daa7 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Tue, 6 May 2025 23:19:23 -0400 Subject: [PATCH 48/72] implement '/v1/apps/verify_credentials' --- .../src/server/api/AuthenticateService.ts | 2 + .../backend/src/server/api/endpoint-list.ts | 1 + .../src/server/api/endpoints/app/current.ts | 73 +++++++++++++++++++ .../server/api/mastodon/MastodonConverters.ts | 12 ++- .../src/server/api/mastodon/endpoints/apps.ts | 9 +++ packages/megalodon/src/index.ts | 2 + .../src/mastodon/entities/application.ts | 3 + packages/megalodon/src/misskey.ts | 9 +-- .../megalodon/src/misskey/entities/app.ts | 2 +- packages/misskey-js/etc/misskey-js.api.md | 4 + .../misskey-js/src/autogen/apiClientJSDoc.ts | 11 +++ packages/misskey-js/src/autogen/endpoint.ts | 2 + packages/misskey-js/src/autogen/entities.ts | 1 + packages/misskey-js/src/autogen/types.ts | 61 ++++++++++++++++ 14 files changed, 183 insertions(+), 9 deletions(-) create mode 100644 packages/backend/src/server/api/endpoints/app/current.ts diff --git a/packages/backend/src/server/api/AuthenticateService.ts b/packages/backend/src/server/api/AuthenticateService.ts index 601618553e..397626c49d 100644 --- a/packages/backend/src/server/api/AuthenticateService.ts +++ b/packages/backend/src/server/api/AuthenticateService.ts @@ -84,6 +84,8 @@ export class AuthenticateService implements OnApplicationShutdown { return [user, { id: accessToken.id, permission: app.permission, + appId: app.id, + app, } as MiAccessToken]; } else { return [user, accessToken]; diff --git a/packages/backend/src/server/api/endpoint-list.ts b/packages/backend/src/server/api/endpoint-list.ts index 1c5a781fd9..a78c3e9ae6 100644 --- a/packages/backend/src/server/api/endpoint-list.ts +++ b/packages/backend/src/server/api/endpoint-list.ts @@ -128,6 +128,7 @@ export * as 'antennas/update' from './endpoints/antennas/update.js'; export * as 'ap/get' from './endpoints/ap/get.js'; export * as 'ap/show' from './endpoints/ap/show.js'; export * as 'app/create' from './endpoints/app/create.js'; +export * as 'app/current' from './endpoints/app/current.js'; export * as 'app/show' from './endpoints/app/show.js'; export * as 'auth/accept' from './endpoints/auth/accept.js'; export * as 'auth/session/generate' from './endpoints/auth/session/generate.js'; diff --git a/packages/backend/src/server/api/endpoints/app/current.ts b/packages/backend/src/server/api/endpoints/app/current.ts new file mode 100644 index 0000000000..39b5ef347c --- /dev/null +++ b/packages/backend/src/server/api/endpoints/app/current.ts @@ -0,0 +1,73 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Inject, Injectable } from '@nestjs/common'; +import { Endpoint } from '@/server/api/endpoint-base.js'; +import type { AppsRepository } from '@/models/_.js'; +import { AppEntityService } from '@/core/entities/AppEntityService.js'; +import { DI } from '@/di-symbols.js'; +import { ApiError } from '../../error.js'; + +export const meta = { + tags: ['app'], + + errors: { + credentialRequired: { + message: 'Credential required.', + code: 'CREDENTIAL_REQUIRED', + id: '1384574d-a912-4b81-8601-c7b1c4085df1', + httpStatusCode: 401, + }, + noAppLogin: { + message: 'Not logged in with an app.', + code: 'NO_APP_LOGIN', + id: '339a4ad2-48c3-47fc-bd9d-2408f05120f8', + }, + }, + + res: { + type: 'object', + optional: false, nullable: false, + ref: 'App', + }, + + // 10 calls per 5 seconds + limit: { + duration: 1000 * 5, + max: 10, + }, +} as const; + +export const paramDef = { + type: 'object', + properties: {}, + required: [], +} as const; + +@Injectable() +export default class extends Endpoint { // eslint-disable-line import/no-default-export + constructor( + @Inject(DI.appsRepository) + private appsRepository: AppsRepository, + + private appEntityService: AppEntityService, + ) { + super(meta, paramDef, async (_, user, token) => { + if (!user) { + throw new ApiError(meta.errors.credentialRequired); + } + if (!token || !token.appId) { + throw new ApiError(meta.errors.noAppLogin); + } + + const app = token.app ?? await this.appsRepository.findOneByOrFail({ id: token.appId }); + + return await this.appEntityService.pack(app, user, { + detail: true, + includeSecret: false, + }); + }); + } +} diff --git a/packages/backend/src/server/api/mastodon/MastodonConverters.ts b/packages/backend/src/server/api/mastodon/MastodonConverters.ts index cf625d6e94..375ea1ef08 100644 --- a/packages/backend/src/server/api/mastodon/MastodonConverters.ts +++ b/packages/backend/src/server/api/mastodon/MastodonConverters.ts @@ -4,7 +4,7 @@ */ import { Inject, Injectable } from '@nestjs/common'; -import { Entity, MastodonEntity } from 'megalodon'; +import { Entity, MastodonEntity, MisskeyEntity } from 'megalodon'; import mfm from '@transfem-org/sfm-js'; import { MastodonNotificationType } from 'megalodon/lib/src/mastodon/notification.js'; import { NotificationType } from 'megalodon/lib/src/notification.js'; @@ -369,6 +369,15 @@ export class MastodonConverters { type: convertNotificationType(notification.type as NotificationType), }; } + + public convertApplication(app: MisskeyEntity.App): MastodonEntity.Application { + return { + name: app.name, + scopes: app.permission, + redirect_uri: app.callbackUrl, + redirect_uris: [app.callbackUrl], + }; + } } function simpleConvert(data: T): T { @@ -459,4 +468,3 @@ export function convertRelationship(relationship: Partial & note: relationship.note ?? '', }; } - diff --git a/packages/backend/src/server/api/mastodon/endpoints/apps.ts b/packages/backend/src/server/api/mastodon/endpoints/apps.ts index 5fce838f47..72b520c74a 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/apps.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/apps.ts @@ -5,6 +5,7 @@ import { Injectable } from '@nestjs/common'; import { MastodonClientService } from '@/server/api/mastodon/MastodonClientService.js'; +import { MastodonConverters } from '@/server/api/mastodon/MastodonConverters.js'; import type { FastifyInstance } from 'fastify'; const readScope = [ @@ -59,6 +60,7 @@ type AuthMastodonRoute = { Body?: AuthPayload, Querystring: AuthPayload }; export class ApiAppsMastodon { constructor( private readonly clientService: MastodonClientService, + private readonly mastoConverters: MastodonConverters, ) {} public register(fastify: FastifyInstance): void { @@ -108,6 +110,13 @@ export class ApiAppsMastodon { return reply.send(response); }); + + fastify.get('/v1/apps/verify_credentials', async (_request, reply) => { + const client = this.clientService.getClient(_request); + const data = await client.verifyAppCredentials(); + const response = this.mastoConverters.convertApplication(data.data); + return reply.send(response); + }); } } diff --git a/packages/megalodon/src/index.ts b/packages/megalodon/src/index.ts index 50663c3ce5..bacd0574d4 100644 --- a/packages/megalodon/src/index.ts +++ b/packages/megalodon/src/index.ts @@ -9,6 +9,7 @@ import * as NotificationType from './notification' import FilterContext from './filter_context' import Converter from './converter' import MastodonEntity from './mastodon/entity'; +import MisskeyEntity from './misskey/entity'; export { Response, @@ -23,4 +24,5 @@ export { Entity, Converter, MastodonEntity, + MisskeyEntity, } diff --git a/packages/megalodon/src/mastodon/entities/application.ts b/packages/megalodon/src/mastodon/entities/application.ts index a3f07997ee..f402152bf6 100644 --- a/packages/megalodon/src/mastodon/entities/application.ts +++ b/packages/megalodon/src/mastodon/entities/application.ts @@ -3,5 +3,8 @@ namespace MastodonEntity { name: string website?: string | null vapid_key?: string | null + scopes: string[] + redirect_uris: string[] + redirect_uri?: string } } diff --git a/packages/megalodon/src/misskey.ts b/packages/megalodon/src/misskey.ts index 669eb0f106..bc38e27ce5 100644 --- a/packages/megalodon/src/misskey.ts +++ b/packages/megalodon/src/misskey.ts @@ -102,7 +102,7 @@ export default class Misskey implements MegalodonInterface { website: null, redirect_uri: res.data.callbackUrl, client_id: '', - client_secret: res.data.secret + client_secret: res.data.secret! } return OAuth.AppData.from(appData) }) @@ -122,11 +122,8 @@ export default class Misskey implements MegalodonInterface { // ====================================== // apps // ====================================== - public async verifyAppCredentials(): Promise> { - return new Promise((_, reject) => { - const err = new NoImplementedError('misskey does not support') - reject(err) - }) + public async verifyAppCredentials(): Promise> { + return await this.client.post('/api/app/current'); } // ====================================== diff --git a/packages/megalodon/src/misskey/entities/app.ts b/packages/megalodon/src/misskey/entities/app.ts index 40a704b944..49c431596f 100644 --- a/packages/megalodon/src/misskey/entities/app.ts +++ b/packages/megalodon/src/misskey/entities/app.ts @@ -4,6 +4,6 @@ namespace MisskeyEntity { name: string callbackUrl: string permission: Array - secret: string + secret?: string } } diff --git a/packages/misskey-js/etc/misskey-js.api.md b/packages/misskey-js/etc/misskey-js.api.md index 18cb070af5..44700add31 100644 --- a/packages/misskey-js/etc/misskey-js.api.md +++ b/packages/misskey-js/etc/misskey-js.api.md @@ -552,6 +552,9 @@ type AppCreateRequest = operations['app___create']['requestBody']['content']['ap // @public (undocumented) type AppCreateResponse = operations['app___create']['responses']['200']['content']['application/json']; +// @public (undocumented) +type AppCurrentResponse = operations['app___current']['responses']['200']['content']['application/json']; + // @public (undocumented) type AppShowRequest = operations['app___show']['requestBody']['content']['application/json']; @@ -1643,6 +1646,7 @@ declare namespace entities { ApShowResponse, AppCreateRequest, AppCreateResponse, + AppCurrentResponse, AppShowRequest, AppShowResponse, AuthAcceptRequest, diff --git a/packages/misskey-js/src/autogen/apiClientJSDoc.ts b/packages/misskey-js/src/autogen/apiClientJSDoc.ts index 75b3c5769e..0dfe042811 100644 --- a/packages/misskey-js/src/autogen/apiClientJSDoc.ts +++ b/packages/misskey-js/src/autogen/apiClientJSDoc.ts @@ -1313,6 +1313,17 @@ declare module '../api.js' { credential?: string | null, ): Promise>; + /** + * No description provided. + * + * **Credential required**: *No* + */ + request( + endpoint: E, + params: P, + credential?: string | null, + ): Promise>; + /** * No description provided. * diff --git a/packages/misskey-js/src/autogen/endpoint.ts b/packages/misskey-js/src/autogen/endpoint.ts index 9293a5e950..b424927316 100644 --- a/packages/misskey-js/src/autogen/endpoint.ts +++ b/packages/misskey-js/src/autogen/endpoint.ts @@ -159,6 +159,7 @@ import type { ApShowResponse, AppCreateRequest, AppCreateResponse, + AppCurrentResponse, AppShowRequest, AppShowResponse, AuthAcceptRequest, @@ -778,6 +779,7 @@ export type Endpoints = { 'ap/get': { req: ApGetRequest; res: ApGetResponse }; 'ap/show': { req: ApShowRequest; res: ApShowResponse }; 'app/create': { req: AppCreateRequest; res: AppCreateResponse }; + 'app/current': { req: EmptyRequest; res: AppCurrentResponse }; 'app/show': { req: AppShowRequest; res: AppShowResponse }; 'auth/accept': { req: AuthAcceptRequest; res: EmptyResponse }; 'auth/session/generate': { req: AuthSessionGenerateRequest; res: AuthSessionGenerateResponse }; diff --git a/packages/misskey-js/src/autogen/entities.ts b/packages/misskey-js/src/autogen/entities.ts index f71407a6ae..39359e3cfa 100644 --- a/packages/misskey-js/src/autogen/entities.ts +++ b/packages/misskey-js/src/autogen/entities.ts @@ -162,6 +162,7 @@ export type ApShowRequest = operations['ap___show']['requestBody']['content']['a export type ApShowResponse = operations['ap___show']['responses']['200']['content']['application/json']; export type AppCreateRequest = operations['app___create']['requestBody']['content']['application/json']; export type AppCreateResponse = operations['app___create']['responses']['200']['content']['application/json']; +export type AppCurrentResponse = operations['app___current']['responses']['200']['content']['application/json']; export type AppShowRequest = operations['app___show']['requestBody']['content']['application/json']; export type AppShowResponse = operations['app___show']['responses']['200']['content']['application/json']; export type AuthAcceptRequest = operations['auth___accept']['requestBody']['content']['application/json']; diff --git a/packages/misskey-js/src/autogen/types.ts b/packages/misskey-js/src/autogen/types.ts index 57e98f2f88..077ea35729 100644 --- a/packages/misskey-js/src/autogen/types.ts +++ b/packages/misskey-js/src/autogen/types.ts @@ -1086,6 +1086,15 @@ export type paths = { */ post: operations['app___create']; }; + '/app/current': { + /** + * app/current + * @description No description provided. + * + * **Credential required**: *No* + */ + post: operations['app___current']; + }; '/app/show': { /** * app/show @@ -13071,6 +13080,58 @@ export type operations = { }; }; }; + /** + * app/current + * @description No description provided. + * + * **Credential required**: *No* + */ + app___current: { + responses: { + /** @description OK (with results) */ + 200: { + content: { + 'application/json': components['schemas']['App']; + }; + }; + /** @description Client error */ + 400: { + content: { + 'application/json': components['schemas']['Error']; + }; + }; + /** @description Authentication error */ + 401: { + content: { + 'application/json': components['schemas']['Error']; + }; + }; + /** @description Forbidden error */ + 403: { + content: { + 'application/json': components['schemas']['Error']; + }; + }; + /** @description I'm Ai */ + 418: { + content: { + 'application/json': components['schemas']['Error']; + }; + }; + /** @description Too many requests */ + 429: { + content: { + 'application/json': components['schemas']['Error']; + }; + }; + /** @description Internal server error */ + 500: { + content: { + 'application/json': components['schemas']['Error']; + }; + }; + }; + }; /** * app/show * @description No description provided. From a40cc825380a03cae4922ad05b958ac6a6d86755 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Wed, 7 May 2025 00:41:51 -0400 Subject: [PATCH 49/72] fix oauth data --- packages/backend/src/server/oauth/OAuth2ProviderService.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/server/oauth/OAuth2ProviderService.ts b/packages/backend/src/server/oauth/OAuth2ProviderService.ts index 18f585ea28..01ee451297 100644 --- a/packages/backend/src/server/oauth/OAuth2ProviderService.ts +++ b/packages/backend/src/server/oauth/OAuth2ProviderService.ts @@ -137,8 +137,8 @@ export class OAuth2ProviderService { const ret = { access_token: atData.accessToken, token_type: 'Bearer', - scope: body.scope || 'read write follow push', - created_at: Math.floor(new Date().getTime() / 1000), + scope: atData.scope || body.scope || 'read write follow push', + created_at: atData.createdAt || Math.floor(new Date().getTime() / 1000), }; return reply.send(ret); } catch (e: unknown) { From 231ef297b5b650048b1daaaafa9fd78ca917eeec Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Wed, 30 Apr 2025 11:12:54 -0400 Subject: [PATCH 50/72] replace JSDOM with cheerio --- packages/backend/package.json | 7 +- .../src/core/FetchInstanceMetadataService.ts | 44 +++-- .../backend/src/misc/verify-field-link.ts | 23 +-- .../src/server/api/endpoints/i/update.ts | 3 +- packages/backend/test/e2e/oauth.ts | 10 +- packages/backend/test/utils.ts | 6 +- pnpm-lock.yaml | 187 +++++------------- 7 files changed, 94 insertions(+), 186 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index 9aa26033d0..4bd940c957 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -91,8 +91,6 @@ "@swc/core": "1.11.24", "@transfem-org/sfm-js": "0.24.6", "@twemoji/parser": "15.1.1", - "@types/redis-info": "3.0.3", - "@types/psl": "^1.1.3", "accepts": "1.3.8", "ajv": "8.17.1", "archiver": "7.0.1", @@ -107,6 +105,7 @@ "cbor": "9.0.2", "chalk": "5.4.1", "chalk-template": "1.1.0", + "cheerio": "1.0.0", "chokidar": "3.6.0", "cli-highlight": "2.1.11", "color-convert": "2.0.1", @@ -132,7 +131,6 @@ "ipaddr.js": "2.2.0", "is-svg": "5.1.0", "js-yaml": "4.1.0", - "jsdom": "26.1.0", "json5": "2.2.3", "jsonld": "8.3.3", "jsrsasign": "11.1.0", @@ -209,7 +207,6 @@ "@types/http-link-header": "1.0.7", "@types/jest": "29.5.14", "@types/js-yaml": "4.0.9", - "@types/jsdom": "21.1.7", "@types/jsonld": "1.5.15", "@types/jsrsasign": "10.5.15", "@types/mime-types": "2.1.4", @@ -221,10 +218,12 @@ "@types/oauth2orize-pkce": "0.1.2", "@types/pg": "8.11.14", "@types/proxy-addr": "^2.0.3", + "@types/psl": "^1.1.3", "@types/pug": "2.0.10", "@types/qrcode": "1.5.5", "@types/random-seed": "0.3.5", "@types/ratelimiter": "3.4.6", + "@types/redis-info": "3.0.3", "@types/rename": "1.0.7", "@types/sanitize-html": "2.15.0", "@types/semver": "7.7.0", diff --git a/packages/backend/src/core/FetchInstanceMetadataService.ts b/packages/backend/src/core/FetchInstanceMetadataService.ts index ce3af7c774..5bfcfc5c98 100644 --- a/packages/backend/src/core/FetchInstanceMetadataService.ts +++ b/packages/backend/src/core/FetchInstanceMetadataService.ts @@ -5,9 +5,9 @@ import { URL } from 'node:url'; import { Inject, Injectable } from '@nestjs/common'; -import { JSDOM } from 'jsdom'; import tinycolor from 'tinycolor2'; import * as Redis from 'ioredis'; +import { load as cheerio, CheerioAPI } from 'cheerio'; import type { MiInstance } from '@/models/Instance.js'; import type Logger from '@/logger.js'; import { DI } from '@/di-symbols.js'; @@ -15,7 +15,6 @@ import { LoggerService } from '@/core/LoggerService.js'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { bindThis } from '@/decorators.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; -import type { DOMWindow } from 'jsdom'; type NodeInfo = { openRegistrations?: unknown; @@ -181,17 +180,14 @@ export class FetchInstanceMetadataService { } @bindThis - private async fetchDom(instance: MiInstance): Promise { + private async fetchDom(instance: MiInstance): Promise { this.logger.info(`Fetching HTML of ${instance.host} ...`); const url = 'https://' + instance.host; const html = await this.httpRequestService.getHtml(url); - const { window } = new JSDOM(html); - const doc = window.document; - - return doc; + return cheerio(html); } @bindThis @@ -206,12 +202,15 @@ export class FetchInstanceMetadataService { } @bindThis - private async fetchFaviconUrl(instance: MiInstance, doc: Document | null): Promise { + private async fetchFaviconUrl(instance: MiInstance, doc: CheerioAPI | null): Promise { const url = 'https://' + instance.host; if (doc) { // https://github.com/misskey-dev/misskey/pull/8220#issuecomment-1025104043 - const href = Array.from(doc.getElementsByTagName('link')).reverse().find(link => link.relList.contains('icon'))?.href; + const href = doc('link[rel][href]') + .filter((_, link) => link.attribs.rel.split(' ').includes('icon')) + .last() + .attr('href'); if (href) { return (new URL(href, url)).href; @@ -232,7 +231,7 @@ export class FetchInstanceMetadataService { } @bindThis - private async fetchIconUrl(instance: MiInstance, doc: Document | null, manifest: Record | null): Promise { + private async fetchIconUrl(instance: MiInstance, doc: CheerioAPI | null, manifest: Record | null): Promise { if (manifest && manifest.icons && manifest.icons.length > 0 && manifest.icons[0].src) { const url = 'https://' + instance.host; return (new URL(manifest.icons[0].src, url)).href; @@ -242,13 +241,16 @@ export class FetchInstanceMetadataService { const url = 'https://' + instance.host; // https://github.com/misskey-dev/misskey/pull/8220#issuecomment-1025104043 - const links = Array.from(doc.getElementsByTagName('link')).reverse(); + const links = Array.from(doc('link[rel][href]')).reverse().map(link => ({ + rel: link.attribs.rel.split(' '), + href: link.attribs.href, + })); // https://github.com/misskey-dev/misskey/pull/8220/files/0ec4eba22a914e31b86874f12448f88b3e58dd5a#r796487559 const href = [ - links.find(link => link.relList.contains('apple-touch-icon-precomposed'))?.href, - links.find(link => link.relList.contains('apple-touch-icon'))?.href, - links.find(link => link.relList.contains('icon'))?.href, + links.find(link => link.rel.includes('apple-touch-icon-precomposed'))?.href, + links.find(link => link.rel.includes('apple-touch-icon'))?.href, + links.find(link => link.rel.includes('icon'))?.href, ] .find(href => href); @@ -261,8 +263,8 @@ export class FetchInstanceMetadataService { } @bindThis - private async getThemeColor(info: NodeInfo | null, doc: Document | null, manifest: Record | null): Promise { - const themeColor = info?.metadata?.themeColor ?? doc?.querySelector('meta[name="theme-color"]')?.getAttribute('content') ?? manifest?.theme_color; + private async getThemeColor(info: NodeInfo | null, doc: CheerioAPI | null, manifest: Record | null): Promise { + const themeColor = info?.metadata?.themeColor ?? doc?.('meta[name="theme-color"][content]').attr('content') ?? manifest?.theme_color; if (themeColor) { const color = new tinycolor(themeColor); @@ -273,7 +275,7 @@ export class FetchInstanceMetadataService { } @bindThis - private async getSiteName(info: NodeInfo | null, doc: Document | null, manifest: Record | null): Promise { + private async getSiteName(info: NodeInfo | null, doc: CheerioAPI | null, manifest: Record | null): Promise { if (info && info.metadata) { if (typeof info.metadata.nodeName === 'string') { return info.metadata.nodeName; @@ -283,7 +285,7 @@ export class FetchInstanceMetadataService { } if (doc) { - const og = doc.querySelector('meta[property="og:title"]')?.getAttribute('content'); + const og = doc('meta[property="og:title"][content]').attr('content'); if (og) { return og; @@ -298,7 +300,7 @@ export class FetchInstanceMetadataService { } @bindThis - private async getDescription(info: NodeInfo | null, doc: Document | null, manifest: Record | null): Promise { + private async getDescription(info: NodeInfo | null, doc: CheerioAPI | null, manifest: Record | null): Promise { if (info && info.metadata) { if (typeof info.metadata.nodeDescription === 'string') { return info.metadata.nodeDescription; @@ -308,12 +310,12 @@ export class FetchInstanceMetadataService { } if (doc) { - const meta = doc.querySelector('meta[name="description"]')?.getAttribute('content'); + const meta = doc('meta[name="description"][content]').attr('content'); if (meta) { return meta; } - const og = doc.querySelector('meta[property="og:description"]')?.getAttribute('content'); + const og = doc('meta[property="og:description"][content]').attr('content'); if (og) { return og; } diff --git a/packages/backend/src/misc/verify-field-link.ts b/packages/backend/src/misc/verify-field-link.ts index f519acfba0..62542eaaa0 100644 --- a/packages/backend/src/misc/verify-field-link.ts +++ b/packages/backend/src/misc/verify-field-link.ts @@ -3,32 +3,29 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { JSDOM } from 'jsdom'; +import { load as cheerio } from 'cheerio'; import type { HttpRequestService } from '@/core/HttpRequestService.js'; type Field = { name: string, value: string }; export async function verifyFieldLinks(fields: Field[], profile_url: string, httpRequestService: HttpRequestService): Promise { const verified_links = []; - for (const field_url of fields - .filter(x => URL.canParse(x.value) && ['http:', 'https:'].includes((new URL(x.value).protocol)))) { + for (const field_url of fields.filter(x => URL.canParse(x.value) && ['http:', 'https:'].includes((new URL(x.value).protocol)))) { try { const html = await httpRequestService.getHtml(field_url.value); - const { window } = new JSDOM(html); - const doc: Document = window.document; + const doc = cheerio(html); - const aEls = Array.from(doc.getElementsByTagName('a')); - const linkEls = Array.from(doc.getElementsByTagName('link')); + const links = doc('a[rel~="me"][href], link[rel~="me"][href]').toArray(); - const includesProfileLinks = [...aEls, ...linkEls].some(link => link.rel === 'me' && link.href === profile_url); - if (includesProfileLinks) { verified_links.push(field_url.value); } - - window.close(); - } catch (err) { + const includesProfileLinks = links.some(link => link.attribs.href === profile_url); + if (includesProfileLinks) { + verified_links.push(field_url.value); + } + } catch { // don't do anything. - continue; } } + return verified_links; } diff --git a/packages/backend/src/server/api/endpoints/i/update.ts b/packages/backend/src/server/api/endpoints/i/update.ts index 5f93597fd7..ad8f38703b 100644 --- a/packages/backend/src/server/api/endpoints/i/update.ts +++ b/packages/backend/src/server/api/endpoints/i/update.ts @@ -6,7 +6,6 @@ import * as mfm from '@transfem-org/sfm-js'; import { Inject, Injectable } from '@nestjs/common'; import ms from 'ms'; -import { JSDOM } from 'jsdom'; import { extractCustomEmojisFromMfm } from '@/misc/extract-custom-emojis-from-mfm.js'; import { extractHashtags } from '@/misc/extract-hashtags.js'; import * as Acct from '@/misc/acct.js'; @@ -622,6 +621,7 @@ export default class extends Endpoint { // eslint- } // this function is superseded by '@/misc/verify-field-link.ts' + /* private async verifyLink(url: string, user: MiLocalUser) { if (!safeForSql(url)) return; @@ -653,6 +653,7 @@ export default class extends Endpoint { // eslint- // なにもしない } } + */ // these two methods need to be kept in sync with // `ApRendererService.renderPerson` diff --git a/packages/backend/test/e2e/oauth.ts b/packages/backend/test/e2e/oauth.ts index 7434701e67..47851e9474 100644 --- a/packages/backend/test/e2e/oauth.ts +++ b/packages/backend/test/e2e/oauth.ts @@ -19,7 +19,7 @@ import { ResourceOwnerPassword, } from 'simple-oauth2'; import pkceChallenge from 'pkce-challenge'; -import { JSDOM } from 'jsdom'; +import { load as cheerio } from 'cheerio'; import Fastify, { type FastifyInstance, type FastifyReply } from 'fastify'; import { api, port, sendEnvUpdateRequest, signup } from '../utils.js'; import type * as misskey from 'misskey-js'; @@ -73,11 +73,11 @@ const clientConfig: ModuleOptions<'client_id'> = { }; function getMeta(html: string): { transactionId: string | undefined, clientName: string | undefined, clientLogo: string | undefined } { - const fragment = JSDOM.fragment(html); + const fragment = cheerio(html); return { - transactionId: fragment.querySelector('meta[name="misskey:oauth:transaction-id"]')?.content, - clientName: fragment.querySelector('meta[name="misskey:oauth:client-name"]')?.content, - clientLogo: fragment.querySelector('meta[name="misskey:oauth:client-logo"]')?.content, + transactionId: fragment('meta[name="misskey:oauth:transaction-id"][content]').attr('content'), + clientName: fragment('meta[name="misskey:oauth:client-name"][content]').attr('content'), + clientLogo: fragment('meta[name="misskey:oauth:client-logo"][content]').attr('content'), }; } diff --git a/packages/backend/test/utils.ts b/packages/backend/test/utils.ts index 7b69cb04f4..70deff2e2d 100644 --- a/packages/backend/test/utils.ts +++ b/packages/backend/test/utils.ts @@ -11,7 +11,7 @@ import { inspect } from 'node:util'; import WebSocket, { ClientOptions } from 'ws'; import fetch, { File, RequestInit, type Headers } from 'node-fetch'; import { DataSource } from 'typeorm'; -import { JSDOM } from 'jsdom'; +import { load as cheerio, CheerioAPI } from 'cheerio'; import { type Response } from 'node-fetch'; import Fastify from 'fastify'; import { entities } from '../src/postgres.js'; @@ -464,7 +464,7 @@ export function makeStreamCatcher( export type SimpleGetResponse = { status: number, - body: any | JSDOM | null, + body: any | CheerioAPI | null, type: string | null, location: string | null }; @@ -495,7 +495,7 @@ export const simpleGet = async (path: string, accept = '*/*', cookie: any = unde const body = jsonTypes.includes(res.headers.get('content-type') ?? '') ? await res.json() : - htmlTypes.includes(res.headers.get('content-type') ?? '') ? new JSDOM(await res.text()) : + htmlTypes.includes(res.headers.get('content-type') ?? '') ? cheerio(await res.text()) : await bodyExtractor(res); return { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 691069f563..69c37b12fe 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -161,12 +161,6 @@ importers: '@twemoji/parser': specifier: 15.1.1 version: 15.1.1 - '@types/psl': - specifier: ^1.1.3 - version: 1.1.3 - '@types/redis-info': - specifier: 3.0.3 - version: 3.0.3 accepts: specifier: 1.3.8 version: 1.3.8 @@ -209,6 +203,9 @@ importers: chalk-template: specifier: 1.1.0 version: 1.1.0 + cheerio: + specifier: 1.0.0 + version: 1.0.0 chokidar: specifier: 4.0.3 version: 4.0.3 @@ -284,9 +281,6 @@ importers: js-yaml: specifier: 4.1.0 version: 4.1.0 - jsdom: - specifier: 26.1.0 - version: 26.1.0(bufferutil@4.0.9)(canvas@3.1.0)(utf-8-validate@6.0.5) json5: specifier: 2.2.3 version: 2.2.3 @@ -592,9 +586,6 @@ importers: '@types/js-yaml': specifier: 4.0.9 version: 4.0.9 - '@types/jsdom': - specifier: 21.1.7 - version: 21.1.7 '@types/jsonld': specifier: 1.5.15 version: 1.5.15 @@ -628,6 +619,9 @@ importers: '@types/proxy-addr': specifier: ^2.0.3 version: 2.0.3 + '@types/psl': + specifier: ^1.1.3 + version: 1.1.3 '@types/pug': specifier: 2.0.10 version: 2.0.10 @@ -640,6 +634,9 @@ importers: '@types/ratelimiter': specifier: 3.4.6 version: 3.4.6 + '@types/redis-info': + specifier: 3.0.3 + version: 3.0.3 '@types/rename': specifier: 1.0.7 version: 1.0.7 @@ -1016,7 +1013,7 @@ importers: version: 8.31.0(eslint@9.25.1)(typescript@5.8.3) '@vitest/coverage-v8': specifier: 3.1.2 - version: 3.1.2(vitest@3.1.2(@types/debug@4.1.12)(@types/node@22.15.2)(happy-dom@17.4.4)(jsdom@26.1.0(bufferutil@4.0.9)(utf-8-validate@6.0.5))(msw@2.7.5(@types/node@22.15.2)(typescript@5.8.3))(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3)) + version: 3.1.2(vitest@3.1.2(@types/debug@4.1.12)(@types/node@22.15.2)(happy-dom@17.4.4)(jsdom@26.1.0)(msw@2.7.5(@types/node@22.15.2)(typescript@5.8.3))(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3)) '@vue/compiler-core': specifier: 3.5.13 version: 3.5.13 @@ -1085,7 +1082,7 @@ importers: version: 1.0.3 vitest: specifier: 3.1.2 - version: 3.1.2(@types/debug@4.1.12)(@types/node@22.15.2)(happy-dom@17.4.4)(jsdom@26.1.0(bufferutil@4.0.9)(utf-8-validate@6.0.5))(msw@2.7.5(@types/node@22.15.2)(typescript@5.8.3))(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3) + version: 3.1.2(@types/debug@4.1.12)(@types/node@22.15.2)(happy-dom@17.4.4)(jsdom@26.1.0)(msw@2.7.5(@types/node@22.15.2)(typescript@5.8.3))(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3) vitest-fetch-mock: specifier: 0.4.5 version: 0.4.5(vitest@3.1.2(@types/debug@4.1.12)(@types/node@22.15.2)(happy-dom@17.4.4)(jsdom@26.1.0(bufferutil@4.0.9)(utf-8-validate@6.0.5))(msw@2.7.5(@types/node@22.15.2)(typescript@5.8.3))(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3)) @@ -4220,9 +4217,6 @@ packages: '@types/js-yaml@4.0.9': resolution: {integrity: sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==} - '@types/jsdom@21.1.7': - resolution: {integrity: sha512-yOriVnggzrnQ3a9OKOCxaVuSug3w3/SbOj5i7VwXWZEyUNl3bLF9V3MfxGbZKuwqJOQyRfqXyROBB1CoZLFWzA==} - '@types/json-schema@7.0.15': resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==} @@ -4400,9 +4394,6 @@ packages: '@types/tmp@0.2.6': resolution: {integrity: sha512-chhaNf2oKHlRkDGt+tiKE2Z5aJ6qalm7Z9rlLdBwmOiAAf09YQvvoLXjWK4HWPF1xU/fqvMgfNfpVoBscA/tKA==} - '@types/tough-cookie@4.0.2': - resolution: {integrity: sha512-Q5vtl1W5ue16D+nIaW8JWebSSraJVlK+EthKn7e7UcD4KWsaSJ8BqGPXNaPghgtcn/fhvrN17Tv8ksUsQpiplw==} - '@types/tough-cookie@4.0.5': resolution: {integrity: sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==} @@ -10327,10 +10318,6 @@ packages: undici-types@6.21.0: resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} - undici@5.28.2: - resolution: {integrity: sha512-wh1pHJHnUeQV5Xa8/kyQhO7WFa8M34l026L5P/+2TYiakvGy5Rdc8jWZVyG7ieht/0WgJLEd3kcU5gKx+6GC8w==} - engines: {node: '>=14.0'} - undici@5.29.0: resolution: {integrity: sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg==} engines: {node: '>=14.0'} @@ -10918,6 +10905,7 @@ snapshots: '@csstools/css-parser-algorithms': 3.0.4(@csstools/css-tokenizer@3.0.3) '@csstools/css-tokenizer': 3.0.3 lru-cache: 10.4.3 + optional: true '@aws-crypto/crc32@5.2.0': dependencies: @@ -11851,12 +11839,14 @@ snapshots: '@cropper/utils@2.0.0': {} - '@csstools/color-helpers@5.0.2': {} + '@csstools/color-helpers@5.0.2': + optional: true '@csstools/css-calc@2.1.2(@csstools/css-parser-algorithms@3.0.4(@csstools/css-tokenizer@3.0.3))(@csstools/css-tokenizer@3.0.3)': dependencies: '@csstools/css-parser-algorithms': 3.0.4(@csstools/css-tokenizer@3.0.3) '@csstools/css-tokenizer': 3.0.3 + optional: true '@csstools/css-color-parser@3.0.8(@csstools/css-parser-algorithms@3.0.4(@csstools/css-tokenizer@3.0.3))(@csstools/css-tokenizer@3.0.3)': dependencies: @@ -11864,12 +11854,15 @@ snapshots: '@csstools/css-calc': 2.1.2(@csstools/css-parser-algorithms@3.0.4(@csstools/css-tokenizer@3.0.3))(@csstools/css-tokenizer@3.0.3) '@csstools/css-parser-algorithms': 3.0.4(@csstools/css-tokenizer@3.0.3) '@csstools/css-tokenizer': 3.0.3 + optional: true '@csstools/css-parser-algorithms@3.0.4(@csstools/css-tokenizer@3.0.3)': dependencies: '@csstools/css-tokenizer': 3.0.3 + optional: true - '@csstools/css-tokenizer@3.0.3': {} + '@csstools/css-tokenizer@3.0.3': + optional: true '@cypress/request@3.0.8': dependencies: @@ -11905,7 +11898,7 @@ snapshots: dependencies: ky: 0.33.3 ky-universal: 0.11.0(ky@0.33.3)(web-streams-polyfill@4.0.0) - undici: 5.28.2 + undici: 5.29.0 transitivePeerDependencies: - web-streams-polyfill @@ -14454,12 +14447,6 @@ snapshots: '@types/js-yaml@4.0.9': {} - '@types/jsdom@21.1.7': - dependencies: - '@types/node': 22.15.2 - '@types/tough-cookie': 4.0.2 - parse5: 7.3.0 - '@types/json-schema@7.0.15': {} '@types/json5@0.0.29': {} @@ -14634,8 +14621,6 @@ snapshots: '@types/tmp@0.2.6': {} - '@types/tough-cookie@4.0.2': {} - '@types/tough-cookie@4.0.5': {} '@types/unist@3.0.2': {} @@ -14749,24 +14734,6 @@ snapshots: vite: 6.3.3(@types/node@22.15.2)(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3) vue: 3.5.13(typescript@5.8.3) - '@vitest/coverage-v8@3.1.2(vitest@3.1.2(@types/debug@4.1.12)(@types/node@22.15.2)(happy-dom@17.4.4)(jsdom@26.1.0(bufferutil@4.0.9)(utf-8-validate@6.0.5))(msw@2.7.5(@types/node@22.15.2)(typescript@5.8.3))(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3))': - dependencies: - '@ampproject/remapping': 2.3.0 - '@bcoe/v8-coverage': 1.0.2 - debug: 4.4.0(supports-color@8.1.1) - istanbul-lib-coverage: 3.2.2 - istanbul-lib-report: 3.0.1 - istanbul-lib-source-maps: 5.0.6 - istanbul-reports: 3.1.7 - magic-string: 0.30.17 - magicast: 0.3.5 - std-env: 3.9.0 - test-exclude: 7.0.1 - tinyrainbow: 2.0.0 - vitest: 3.1.2(@types/debug@4.1.12)(@types/node@22.15.2)(happy-dom@17.4.4)(jsdom@26.1.0(bufferutil@4.0.9)(utf-8-validate@6.0.5))(msw@2.7.5(@types/node@22.15.2)(typescript@5.8.3))(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3) - transitivePeerDependencies: - - supports-color - '@vitest/coverage-v8@3.1.2(vitest@3.1.2(@types/debug@4.1.12)(@types/node@22.15.2)(happy-dom@17.4.4)(jsdom@26.1.0)(msw@2.7.5(@types/node@22.15.2)(typescript@5.8.3))(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3))': dependencies: '@ampproject/remapping': 2.3.0 @@ -16181,6 +16148,7 @@ snapshots: dependencies: '@asamuzakjp/css-color': 3.1.1 rrweb-cssom: 0.8.0 + optional: true csstype@3.1.3: {} @@ -16241,6 +16209,7 @@ snapshots: dependencies: whatwg-mimetype: 4.0.0 whatwg-url: 14.2.0 + optional: true data-view-buffer@1.0.1: dependencies: @@ -16311,7 +16280,8 @@ snapshots: decamelize@1.2.0: {} - decimal.js@10.5.0: {} + decimal.js@10.5.0: + optional: true decode-bmp@0.2.1: dependencies: @@ -17680,6 +17650,7 @@ snapshots: html-encoding-sniffer@4.0.0: dependencies: whatwg-encoding: 3.1.1 + optional: true html-entities@2.5.2: {} @@ -17983,7 +17954,8 @@ snapshots: is-plain-object@5.0.0: {} - is-potential-custom-element-name@1.0.1: {} + is-potential-custom-element-name@1.0.1: + optional: true is-promise@2.2.2: {} @@ -18496,7 +18468,7 @@ snapshots: jsdoc-type-pratt-parser@4.1.0: {} - jsdom@26.1.0: + jsdom@26.1.0(bufferutil@4.0.9)(utf-8-validate@6.0.5): dependencies: cssstyle: 4.3.0 data-urls: 5.0.0 @@ -18524,35 +18496,6 @@ snapshots: - utf-8-validate optional: true - jsdom@26.1.0(bufferutil@4.0.9)(canvas@3.1.0)(utf-8-validate@6.0.5): - dependencies: - cssstyle: 4.3.0 - data-urls: 5.0.0 - decimal.js: 10.5.0 - html-encoding-sniffer: 4.0.0 - http-proxy-agent: 7.0.2 - https-proxy-agent: 7.0.6 - is-potential-custom-element-name: 1.0.1 - nwsapi: 2.2.19 - parse5: 7.3.0 - rrweb-cssom: 0.8.0 - saxes: 6.0.0 - symbol-tree: 3.2.4 - tough-cookie: 5.1.2 - w3c-xmlserializer: 5.0.0 - webidl-conversions: 7.0.0 - whatwg-encoding: 3.1.1 - whatwg-mimetype: 4.0.0 - whatwg-url: 14.2.0 - ws: 8.18.1(bufferutil@4.0.9)(utf-8-validate@6.0.5) - xml-name-validator: 5.0.0 - optionalDependencies: - canvas: 3.1.0 - transitivePeerDependencies: - - bufferutil - - supports-color - - utf-8-validate - jsesc@2.5.2: {} json-buffer@3.0.1: {} @@ -19561,7 +19504,8 @@ snapshots: dependencies: boolbase: 1.0.0 - nwsapi@2.2.19: {} + nwsapi@2.2.19: + optional: true oauth2orize-pkce@0.1.2: {} @@ -20650,7 +20594,8 @@ snapshots: transitivePeerDependencies: - supports-color - rrweb-cssom@0.8.0: {} + rrweb-cssom@0.8.0: + optional: true rss-parser@3.13.0: dependencies: @@ -20712,6 +20657,7 @@ snapshots: saxes@6.0.0: dependencies: xmlchars: 2.2.0 + optional: true scheduler@0.26.0: {} @@ -21366,7 +21312,8 @@ snapshots: csso: 5.0.5 picocolors: 1.1.1 - symbol-tree@3.2.4: {} + symbol-tree@3.2.4: + optional: true systeminformation@5.25.11: {} @@ -21484,11 +21431,13 @@ snapshots: tinyspy@3.0.2: {} - tldts-core@6.1.63: {} + tldts-core@6.1.63: + optional: true tldts@6.1.63: dependencies: tldts-core: 6.1.63 + optional: true tmp@0.2.3: {} @@ -21532,12 +21481,14 @@ snapshots: tough-cookie@5.1.2: dependencies: tldts: 6.1.63 + optional: true tr46@0.0.3: {} tr46@5.1.0: dependencies: punycode: 2.3.1 + optional: true tree-kill@1.2.2: optional: true @@ -21743,10 +21694,6 @@ snapshots: undici-types@6.21.0: {} - undici@5.28.2: - dependencies: - '@fastify/busboy': 2.1.0 - undici@5.29.0: dependencies: '@fastify/busboy': 2.1.0 @@ -21941,49 +21888,7 @@ snapshots: vitest-fetch-mock@0.4.5(vitest@3.1.2(@types/debug@4.1.12)(@types/node@22.15.2)(happy-dom@17.4.4)(jsdom@26.1.0(bufferutil@4.0.9)(utf-8-validate@6.0.5))(msw@2.7.5(@types/node@22.15.2)(typescript@5.8.3))(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3)): dependencies: - vitest: 3.1.2(@types/debug@4.1.12)(@types/node@22.15.2)(happy-dom@17.4.4)(jsdom@26.1.0(bufferutil@4.0.9)(utf-8-validate@6.0.5))(msw@2.7.5(@types/node@22.15.2)(typescript@5.8.3))(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3) - - vitest@3.1.2(@types/debug@4.1.12)(@types/node@22.15.2)(happy-dom@17.4.4)(jsdom@26.1.0(bufferutil@4.0.9)(utf-8-validate@6.0.5))(msw@2.7.5(@types/node@22.15.2)(typescript@5.8.3))(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3): - dependencies: - '@vitest/expect': 3.1.2 - '@vitest/mocker': 3.1.2(msw@2.7.5(@types/node@22.15.2)(typescript@5.8.3))(vite@6.3.3(@types/node@22.15.2)(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3)) - '@vitest/pretty-format': 3.1.2 - '@vitest/runner': 3.1.2 - '@vitest/snapshot': 3.1.2 - '@vitest/spy': 3.1.2 - '@vitest/utils': 3.1.2 - chai: 5.2.0 - debug: 4.4.0(supports-color@8.1.1) - expect-type: 1.2.1 - magic-string: 0.30.17 - pathe: 2.0.3 - std-env: 3.9.0 - tinybench: 2.9.0 - tinyexec: 0.3.2 - tinyglobby: 0.2.13 - tinypool: 1.0.2 - tinyrainbow: 2.0.0 - vite: 6.3.3(@types/node@22.15.2)(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3) - vite-node: 3.1.2(@types/node@22.15.2)(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3) - why-is-node-running: 2.3.0 - optionalDependencies: - '@types/debug': 4.1.12 - '@types/node': 22.15.2 - happy-dom: 17.4.4 - jsdom: 26.1.0(bufferutil@4.0.9)(canvas@3.1.0)(utf-8-validate@6.0.5) - transitivePeerDependencies: - - jiti - - less - - lightningcss - - msw - - sass - - sass-embedded - - stylus - - sugarss - - supports-color - - terser - - tsx - - yaml + vitest: 3.1.2(@types/debug@4.1.12)(@types/node@22.15.2)(happy-dom@17.4.4)(jsdom@26.1.0)(msw@2.7.5(@types/node@22.15.2)(typescript@5.8.3))(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3) vitest@3.1.2(@types/debug@4.1.12)(@types/node@22.15.2)(happy-dom@17.4.4)(jsdom@26.1.0)(msw@2.7.5(@types/node@22.15.2)(typescript@5.8.3))(sass@1.87.0)(terser@5.39.0)(tsx@4.19.3): dependencies: @@ -22012,7 +21917,7 @@ snapshots: '@types/debug': 4.1.12 '@types/node': 22.15.2 happy-dom: 17.4.4 - jsdom: 26.1.0 + jsdom: 26.1.0(bufferutil@4.0.9)(utf-8-validate@6.0.5) transitivePeerDependencies: - jiti - less @@ -22132,6 +22037,7 @@ snapshots: w3c-xmlserializer@5.0.0: dependencies: xml-name-validator: 5.0.0 + optional: true wait-on@8.0.3(debug@4.4.0): dependencies: @@ -22192,6 +22098,7 @@ snapshots: dependencies: tr46: 5.1.0 webidl-conversions: 7.0.0 + optional: true whatwg-url@5.0.0: dependencies: @@ -22295,7 +22202,8 @@ snapshots: xml-name-validator@4.0.0: {} - xml-name-validator@5.0.0: {} + xml-name-validator@5.0.0: + optional: true xml2js@0.5.0: dependencies: @@ -22304,7 +22212,8 @@ snapshots: xmlbuilder@11.0.1: {} - xmlchars@2.2.0: {} + xmlchars@2.2.0: + optional: true xtend@4.0.2: {} From 261a7e3ab3063ec0955a9ea4242dce81eaad4544 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sat, 3 May 2025 22:25:33 -0400 Subject: [PATCH 51/72] fix type errors --- packages/backend/src/server/api/mastodon/endpoints/search.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/server/api/mastodon/endpoints/search.ts b/packages/backend/src/server/api/mastodon/endpoints/search.ts index 33bfa87e5f..78672639e5 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/search.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/search.ts @@ -112,7 +112,7 @@ export class ApiSearchMastodon { { method: 'POST', headers: { - ...request.headers as HeadersInit, + ...request.headers, 'Accept': 'application/json', 'Content-Type': 'application/json', }, @@ -135,7 +135,7 @@ export class ApiSearchMastodon { { method: 'POST', headers: { - ...request.headers as HeadersInit, + ...request.headers, 'Accept': 'application/json', 'Content-Type': 'application/json', }, From c43ac87df27e0b88194b315b077ccb43df11ef9c Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Thu, 8 May 2025 11:31:36 -0400 Subject: [PATCH 52/72] separate type imports for Cheerio --- packages/backend/src/core/FetchInstanceMetadataService.ts | 3 ++- packages/backend/test/utils.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/core/FetchInstanceMetadataService.ts b/packages/backend/src/core/FetchInstanceMetadataService.ts index 5bfcfc5c98..980f1fcacf 100644 --- a/packages/backend/src/core/FetchInstanceMetadataService.ts +++ b/packages/backend/src/core/FetchInstanceMetadataService.ts @@ -7,7 +7,7 @@ import { URL } from 'node:url'; import { Inject, Injectable } from '@nestjs/common'; import tinycolor from 'tinycolor2'; import * as Redis from 'ioredis'; -import { load as cheerio, CheerioAPI } from 'cheerio'; +import { load as cheerio } from 'cheerio'; import type { MiInstance } from '@/models/Instance.js'; import type Logger from '@/logger.js'; import { DI } from '@/di-symbols.js'; @@ -15,6 +15,7 @@ import { LoggerService } from '@/core/LoggerService.js'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { bindThis } from '@/decorators.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; +import type { CheerioAPI } from 'cheerio'; type NodeInfo = { openRegistrations?: unknown; diff --git a/packages/backend/test/utils.ts b/packages/backend/test/utils.ts index 70deff2e2d..7f2768488f 100644 --- a/packages/backend/test/utils.ts +++ b/packages/backend/test/utils.ts @@ -11,11 +11,12 @@ import { inspect } from 'node:util'; import WebSocket, { ClientOptions } from 'ws'; import fetch, { File, RequestInit, type Headers } from 'node-fetch'; import { DataSource } from 'typeorm'; -import { load as cheerio, CheerioAPI } from 'cheerio'; +import { load as cheerio } from 'cheerio'; import { type Response } from 'node-fetch'; import Fastify from 'fastify'; import { entities } from '../src/postgres.js'; import { loadConfig } from '../src/config.js'; +import type { CheerioAPI } from 'cheerio'; import type * as misskey from 'misskey-js'; import { DEFAULT_POLICIES } from '@/core/RoleService.js'; import { validateContentTypeSetAsActivityPub } from '@/core/activitypub/misc/validator.js'; From 42f3976b1699533b1cca66eed35de310a6883012 Mon Sep 17 00:00:00 2001 From: dakkar Date: Thu, 8 May 2025 16:43:52 +0100 Subject: [PATCH 53/72] add `scheduleNotePost` queue to dashboard --- packages/backend/src/core/QueueService.ts | 2 ++ packages/frontend/src/pages/admin/job-queue.vue | 1 + packages/misskey-js/src/autogen/types.ts | 14 +++++++------- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index fb0fa8f28d..361e636662 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -52,6 +52,7 @@ export const QUEUE_TYPES = [ 'objectStorage', 'userWebhookDeliver', 'systemWebhookDeliver', + 'scheduleNotePost', ] as const; @Injectable() @@ -783,6 +784,7 @@ export class QueueService { case 'objectStorage': return this.objectStorageQueue; case 'userWebhookDeliver': return this.userWebhookDeliverQueue; case 'systemWebhookDeliver': return this.systemWebhookDeliverQueue; + case 'scheduleNotePost': return this.ScheduleNotePostQueue; default: throw new Error(`Unrecognized queue type: ${type}`); } } diff --git a/packages/frontend/src/pages/admin/job-queue.vue b/packages/frontend/src/pages/admin/job-queue.vue index 3d405c566f..155277c976 100644 --- a/packages/frontend/src/pages/admin/job-queue.vue +++ b/packages/frontend/src/pages/admin/job-queue.vue @@ -203,6 +203,7 @@ const QUEUE_TYPES = [ 'objectStorage', 'userWebhookDeliver', 'systemWebhookDeliver', + 'scheduleNotePost', ] as const; const tab: Ref = ref('-'); diff --git a/packages/misskey-js/src/autogen/types.ts b/packages/misskey-js/src/autogen/types.ts index 57e98f2f88..a060ea148e 100644 --- a/packages/misskey-js/src/autogen/types.ts +++ b/packages/misskey-js/src/autogen/types.ts @@ -9431,7 +9431,7 @@ export type operations = { content: { 'application/json': { /** @enum {string} */ - queue: 'system' | 'endedPollNotification' | 'deliver' | 'inbox' | 'db' | 'relationship' | 'objectStorage' | 'userWebhookDeliver' | 'systemWebhookDeliver'; + queue: 'system' | 'endedPollNotification' | 'deliver' | 'inbox' | 'db' | 'relationship' | 'objectStorage' | 'userWebhookDeliver' | 'systemWebhookDeliver' | 'scheduleNotePost'; /** @enum {string} */ state: '*' | 'completed' | 'wait' | 'active' | 'paused' | 'prioritized' | 'delayed' | 'failed'; }; @@ -9577,7 +9577,7 @@ export type operations = { content: { 'application/json': { /** @enum {string} */ - queue: 'system' | 'endedPollNotification' | 'deliver' | 'inbox' | 'db' | 'relationship' | 'objectStorage' | 'userWebhookDeliver' | 'systemWebhookDeliver'; + queue: 'system' | 'endedPollNotification' | 'deliver' | 'inbox' | 'db' | 'relationship' | 'objectStorage' | 'userWebhookDeliver' | 'systemWebhookDeliver' | 'scheduleNotePost'; state: ('active' | 'paused' | 'wait' | 'delayed' | 'completed' | 'failed')[]; search?: string; }; @@ -9631,7 +9631,7 @@ export type operations = { content: { 'application/json': { /** @enum {string} */ - queue: 'system' | 'endedPollNotification' | 'deliver' | 'inbox' | 'db' | 'relationship' | 'objectStorage' | 'userWebhookDeliver' | 'systemWebhookDeliver'; + queue: 'system' | 'endedPollNotification' | 'deliver' | 'inbox' | 'db' | 'relationship' | 'objectStorage' | 'userWebhookDeliver' | 'systemWebhookDeliver' | 'scheduleNotePost'; }; }; }; @@ -9683,7 +9683,7 @@ export type operations = { content: { 'application/json': { /** @enum {string} */ - queue: 'system' | 'endedPollNotification' | 'deliver' | 'inbox' | 'db' | 'relationship' | 'objectStorage' | 'userWebhookDeliver' | 'systemWebhookDeliver'; + queue: 'system' | 'endedPollNotification' | 'deliver' | 'inbox' | 'db' | 'relationship' | 'objectStorage' | 'userWebhookDeliver' | 'systemWebhookDeliver' | 'scheduleNotePost'; }; }; }; @@ -9779,7 +9779,7 @@ export type operations = { content: { 'application/json': { /** @enum {string} */ - queue: 'system' | 'endedPollNotification' | 'deliver' | 'inbox' | 'db' | 'relationship' | 'objectStorage' | 'userWebhookDeliver' | 'systemWebhookDeliver'; + queue: 'system' | 'endedPollNotification' | 'deliver' | 'inbox' | 'db' | 'relationship' | 'objectStorage' | 'userWebhookDeliver' | 'systemWebhookDeliver' | 'scheduleNotePost'; jobId: string; }; }; @@ -9832,7 +9832,7 @@ export type operations = { content: { 'application/json': { /** @enum {string} */ - queue: 'system' | 'endedPollNotification' | 'deliver' | 'inbox' | 'db' | 'relationship' | 'objectStorage' | 'userWebhookDeliver' | 'systemWebhookDeliver'; + queue: 'system' | 'endedPollNotification' | 'deliver' | 'inbox' | 'db' | 'relationship' | 'objectStorage' | 'userWebhookDeliver' | 'systemWebhookDeliver' | 'scheduleNotePost'; jobId: string; }; }; @@ -9885,7 +9885,7 @@ export type operations = { content: { 'application/json': { /** @enum {string} */ - queue: 'system' | 'endedPollNotification' | 'deliver' | 'inbox' | 'db' | 'relationship' | 'objectStorage' | 'userWebhookDeliver' | 'systemWebhookDeliver'; + queue: 'system' | 'endedPollNotification' | 'deliver' | 'inbox' | 'db' | 'relationship' | 'objectStorage' | 'userWebhookDeliver' | 'systemWebhookDeliver' | 'scheduleNotePost'; jobId: string; }; }; From 2de4b093ffd15d53d67a45d1a00d3d101a8d4dac Mon Sep 17 00:00:00 2001 From: dakkar Date: Thu, 8 May 2025 16:57:37 +0100 Subject: [PATCH 54/72] merge the two post-form menus --- .../frontend/src/components/MkPostForm.vue | 47 ++++++++----------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/packages/frontend/src/components/MkPostForm.vue b/packages/frontend/src/components/MkPostForm.vue index 8a0bbe348a..5f4e40d513 100644 --- a/packages/frontend/src/components/MkPostForm.vue +++ b/packages/frontend/src/components/MkPostForm.vue @@ -92,7 +92,6 @@ SPDX-License-Identifier: AGPL-3.0-only -
@@ -629,6 +628,26 @@ function showOtherSettings() { }, }] satisfies MenuItem[]; + if ($i.policies.scheduleNoteMax > 0) { + menuItems.push({ type: 'divider' }, { + type: 'button', + text: i18n.ts.schedulePost, + icon: 'ti ti-calendar-time', + action: toggleScheduleNote, + }, { + type: 'button', + text: i18n.ts.schedulePostList, + icon: 'ti ti-calendar-event', + action: () => { + const { dispose } = os.popup(defineAsyncComponent(() => import('@/components/MkSchedulePostListDialog.vue')), {}, { + closed: () => { + dispose(); + }, + }); + }, + }); + } + os.popupMenu(menuItems, otherSettingsButton.value); } //#endregion @@ -1121,32 +1140,6 @@ function toggleScheduleNote() { } } -function showOtherMenu(ev: MouseEvent) { - const menuItems: MenuItem[] = []; - - if ($i.policies.scheduleNoteMax > 0) { - menuItems.push({ - type: 'button', - text: i18n.ts.schedulePost, - icon: 'ti ti-calendar-time', - action: toggleScheduleNote, - }, { - type: 'button', - text: i18n.ts.schedulePostList, - icon: 'ti ti-calendar-event', - action: () => { - const { dispose } = os.popup(defineAsyncComponent(() => import('@/components/MkSchedulePostListDialog.vue')), {}, { - closed: () => { - dispose(); - }, - }); - }, - }); - } - - os.popupMenu(menuItems, ev.currentTarget ?? ev.target); -} - onMounted(() => { if (props.autofocus) { focus(); From b753b2ea3aa34f60a0e38a4e7cf29ba3ec6db690 Mon Sep 17 00:00:00 2001 From: Marie Date: Thu, 8 May 2025 19:47:18 +0200 Subject: [PATCH 55/72] add poll edit warning --- locales/index.d.ts | 10 ++++++++++ packages/frontend/src/components/MkPostForm.vue | 11 +++++++++++ sharkey-locales/en-US.yml | 4 ++++ 3 files changed, 25 insertions(+) diff --git a/locales/index.d.ts b/locales/index.d.ts index 977fb43be4..2c9e3e3890 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -12947,6 +12947,16 @@ export interface Locale extends ILocale { * If disabled, then the proxy account will not be used. User lists will only include notes from local or followed users. */ "enableProxyAccountDescription": string; + "_confirmPollEdit": { + /** + * Are you sure you want to edit this poll? + */ + "title": string; + /** + * Editing this poll will cause it to lose all previous votes + */ + "text": string; + }; } declare const locales: { [lang: string]: Locale; diff --git a/packages/frontend/src/components/MkPostForm.vue b/packages/frontend/src/components/MkPostForm.vue index 5f4e40d513..ef3dafb593 100644 --- a/packages/frontend/src/components/MkPostForm.vue +++ b/packages/frontend/src/components/MkPostForm.vue @@ -186,6 +186,7 @@ const posted = ref(false); const text = ref(props.initialText ?? ''); const files = ref(props.initialFiles ?? []); const poll = ref(null); +const initialPoll = ref(null); const useCw = ref(!!props.initialCw); const showPreview = ref(store.s.showPreview); watch(showPreview, () => store.set('showPreview', showPreview.value)); @@ -978,6 +979,15 @@ async function post(ev?: MouseEvent) { token = storedAccounts.find(x => x.id === postAccount.value?.id)?.token; } + if (postData.editId && postData.poll !== initialPoll.value) { + const { canceled } = await os.confirm({ + type: 'warning', + title: i18n.ts._confirmPollEdit.title, + text: i18n.ts._confirmPollEdit.text, + }); + if (canceled) return; + } + posting.value = true; misskeyApi(postData.editId ? 'notes/edit' : (postData.scheduleNote ? 'notes/schedule/create' : 'notes/create'), postData, token).then(() => { if (props.freezeAfterPosted) { @@ -1201,6 +1211,7 @@ onMounted(() => { expiresAt: init.poll.expiresAt ? (new Date(init.poll.expiresAt)).getTime() : null, expiredAfter: null, }; + if (props.editId) initialPoll.value = poll.value; } if (init.visibleUserIds) { misskeyApi('users/show', { userIds: init.visibleUserIds }).then(users => { diff --git a/sharkey-locales/en-US.yml b/sharkey-locales/en-US.yml index 635684596b..8f217a1d3a 100644 --- a/sharkey-locales/en-US.yml +++ b/sharkey-locales/en-US.yml @@ -536,3 +536,7 @@ deleted: "Deleted" enableProxyAccount: "Enable the proxy account." enableProxyAccountDescription: "If disabled, then the proxy account will not be used. User lists will only include notes from local or followed users." + +_confirmPollEdit: + title: Are you sure you want to edit this poll? + text: Editing this poll will cause it to lose all previous votes From d244158db6c9c269e33b3717bc4101d13b5d259d Mon Sep 17 00:00:00 2001 From: Marie Date: Thu, 8 May 2025 17:51:31 +0000 Subject: [PATCH 56/72] remove question mark in title --- sharkey-locales/en-US.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sharkey-locales/en-US.yml b/sharkey-locales/en-US.yml index 8f217a1d3a..a5c3d5f892 100644 --- a/sharkey-locales/en-US.yml +++ b/sharkey-locales/en-US.yml @@ -538,5 +538,5 @@ enableProxyAccount: "Enable the proxy account." enableProxyAccountDescription: "If disabled, then the proxy account will not be used. User lists will only include notes from local or followed users." _confirmPollEdit: - title: Are you sure you want to edit this poll? + title: Are you sure you want to edit this poll text: Editing this poll will cause it to lose all previous votes From b91a67d74ec18b80ceaa8522e07bf28a628284d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=8A=E3=81=95=E3=82=80=E3=81=AE=E3=81=B2=E3=81=A8?= <46447427+samunohito@users.noreply.github.com> Date: Sat, 3 May 2025 16:23:06 +0900 Subject: [PATCH 57/72] =?UTF-8?q?Revert=20"fix:=20=E6=B7=BB=E4=BB=98?= =?UTF-8?q?=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E3=81=AE=E3=81=82=E3=82=8B?= =?UTF-8?q?=E3=83=AA=E3=82=AF=E3=82=A8=E3=82=B9=E3=83=88=E3=82=92=E5=8F=97?= =?UTF-8?q?=E3=81=91=E3=81=9F=E3=81=A8=E3=81=8D=E3=81=AE=E5=88=9D=E5=8B=95?= =?UTF-8?q?=E3=82=92=E6=94=B9=E5=96=84=20(#15896)"=20(#15927)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Revert "fix: 添付ファイルのあるリクエストを受けたときの初動を改善 (#15896)" This reverts commit 7e8cc4d7c0a86ad0bf71a727fb16132e8bc180a8. * fix CHANGELOG.md --- packages/backend/package.json | 4 +- packages/backend/src/server/ServerService.ts | 7 +- .../backend/src/server/api/ApiCallService.ts | 166 ++++++------------ .../backend/src/server/api/endpoint-base.ts | 10 +- packages/backend/test/e2e/api.ts | 4 +- .../unit/server/api/drive/files/create.ts | 108 ------------ pnpm-lock.yaml | 98 ----------- 7 files changed, 65 insertions(+), 332 deletions(-) delete mode 100644 packages/backend/test/unit/server/api/drive/files/create.ts diff --git a/packages/backend/package.json b/packages/backend/package.json index b9cb0002ab..14ac79ae1b 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -229,7 +229,6 @@ "@types/semver": "7.7.0", "@types/simple-oauth2": "5.0.7", "@types/sinonjs__fake-timers": "8.1.5", - "@types/supertest": "6.0.3", "@types/tinycolor2": "1.4.6", "@types/tmp": "0.2.6", "@types/uuid": "^9.0.4", @@ -247,7 +246,6 @@ "jest-mock": "29.7.0", "nodemon": "3.1.10", "pid-port": "1.0.2", - "simple-oauth2": "5.1.0", - "supertest": "7.1.0" + "simple-oauth2": "5.1.0" } } diff --git a/packages/backend/src/server/ServerService.ts b/packages/backend/src/server/ServerService.ts index 5857b3059e..dce47e2290 100644 --- a/packages/backend/src/server/ServerService.ts +++ b/packages/backend/src/server/ServerService.ts @@ -75,7 +75,7 @@ export class ServerService implements OnApplicationShutdown { } @bindThis - public async launch() { + public async launch(): Promise { const fastify = Fastify({ trustProxy: true, logger: false, @@ -135,8 +135,8 @@ export class ServerService implements OnApplicationShutdown { reply.header('content-type', 'text/plain; charset=utf-8'); reply.header('link', `<${encodeURI(location)}>; rel="canonical"`); done(null, [ - 'Refusing to relay remote ActivityPub object lookup.', - '', + "Refusing to relay remote ActivityPub object lookup.", + "", `Please remove 'application/activity+json' and 'application/ld+json' from the Accept header or fetch using the authoritative URL at ${location}.`, ].join('\n')); }); @@ -304,7 +304,6 @@ export class ServerService implements OnApplicationShutdown { } await fastify.ready(); - return fastify; } @bindThis diff --git a/packages/backend/src/server/api/ApiCallService.ts b/packages/backend/src/server/api/ApiCallService.ts index 1b8d33f9c9..b22a8c1837 100644 --- a/packages/backend/src/server/api/ApiCallService.ts +++ b/packages/backend/src/server/api/ApiCallService.ts @@ -6,11 +6,8 @@ import { randomUUID } from 'node:crypto'; import * as fs from 'node:fs'; import * as stream from 'node:stream/promises'; -import { Transform } from 'node:stream'; -import { type MultipartFile } from '@fastify/multipart'; import { Inject, Injectable } from '@nestjs/common'; import * as Sentry from '@sentry/node'; -import { AttachmentFile } from '@/server/api/endpoint-base.js'; import { DI } from '@/di-symbols.js'; import { getIpHash } from '@/misc/get-ip-hash.js'; import type { MiLocalUser, MiUser } from '@/models/User.js'; @@ -19,7 +16,7 @@ import type Logger from '@/logger.js'; import type { MiMeta, UserIpsRepository } from '@/models/_.js'; import { createTemp } from '@/misc/create-temp.js'; import { bindThis } from '@/decorators.js'; -import { type RolePolicies, RoleService } from '@/core/RoleService.js'; +import { RoleService } from '@/core/RoleService.js'; import type { Config } from '@/config.js'; import { sendRateLimitHeaders } from '@/misc/rate-limit-utils.js'; import { SkRateLimiterService } from '@/server/SkRateLimiterService.js'; @@ -194,6 +191,18 @@ export class ApiCallService implements OnApplicationShutdown { return; } + const [path, cleanup] = await createTemp(); + await stream.pipeline(multipartData.file, fs.createWriteStream(path)); + + // ファイルサイズが制限を超えていた場合 + // なお truncated はストリームを読み切ってからでないと機能しないため、stream.pipeline より後にある必要がある + if (multipartData.file.truncated) { + cleanup(); + reply.code(413); + reply.send(); + return; + } + const fields = {} as Record; for (const [k, v] of Object.entries(multipartData.fields)) { fields[k] = typeof v === 'object' && 'value' in v ? v.value : undefined; @@ -208,7 +217,10 @@ export class ApiCallService implements OnApplicationShutdown { return; } this.authenticateService.authenticate(token).then(([user, app]) => { - this.call(endpoint, user, app, fields, multipartData, request, reply).then((res) => { + this.call(endpoint, user, app, fields, { + name: multipartData.filename, + path: path, + }, request, reply).then((res) => { this.send(reply, res); }).catch((err: ApiError) => { this.#sendApiError(reply, err); @@ -278,7 +290,10 @@ export class ApiCallService implements OnApplicationShutdown { user: MiLocalUser | null | undefined, token: MiAccessToken | null | undefined, data: any, - multipartFile: MultipartFile | null, + file: { + name: string; + path: string; + } | null, request: FastifyRequest<{ Body: Record | undefined, Querystring: Record }>, reply: FastifyReply, ) { @@ -354,37 +369,6 @@ export class ApiCallService implements OnApplicationShutdown { } } - // Cast non JSON input - if ((ep.meta.requireFile || request.method === 'GET') && ep.params.properties) { - for (const k of Object.keys(ep.params.properties)) { - const param = ep.params.properties![k]; - if (['boolean', 'number', 'integer'].includes(param.type ?? '') && typeof data[k] === 'string') { - try { - data[k] = JSON.parse(data[k]); - } catch (e) { - throw new ApiError({ - message: 'Invalid param.', - code: 'INVALID_PARAM', - id: '0b5f1631-7c1a-41a6-b399-cce335f34d85', - }, { - param: k, - reason: `cannot cast to ${param.type}`, - }); - } - } - } - } - - if (token && ((ep.meta.kind && !token.permission.some(p => p === ep.meta.kind)) - || (!ep.meta.kind && (ep.meta.requireCredential || ep.meta.requireModerator || ep.meta.requireAdmin)))) { - throw new ApiError({ - message: 'Your app does not have the necessary permissions to use this endpoint.', - code: 'PERMISSION_DENIED', - kind: 'permission', - id: '1370e5b7-d4eb-4566-bb1d-7748ee6a1838', - }); - } - if ((ep.meta.requireModerator || ep.meta.requireAdmin) && (this.meta.rootUserId !== user!.id)) { const myRoles = await this.roleService.getUserRoles(user!.id); if (ep.meta.requireModerator && !myRoles.some(r => r.isModerator || r.isAdministrator)) { @@ -418,91 +402,49 @@ export class ApiCallService implements OnApplicationShutdown { } } - let attachmentFile: AttachmentFile | null = null; - let cleanup = () => {}; - if (ep.meta.requireFile && request.method === 'POST' && multipartFile) { - const policies = await this.roleService.getUserPolicies(user!.id); - const result = await this.handleAttachmentFile( - Math.min((policies.maxFileSizeMb * 1024 * 1024), this.config.maxFileSize), - multipartFile, - ); - attachmentFile = result.attachmentFile; - cleanup = result.cleanup; + if (token && ((ep.meta.kind && !token.permission.some(p => p === ep.meta.kind)) + || (!ep.meta.kind && (ep.meta.requireCredential || ep.meta.requireModerator || ep.meta.requireAdmin)))) { + throw new ApiError({ + message: 'Your app does not have the necessary permissions to use this endpoint.', + code: 'PERMISSION_DENIED', + kind: 'permission', + id: '1370e5b7-d4eb-4566-bb1d-7748ee6a1838', + }); + } + + // Cast non JSON input + if ((ep.meta.requireFile || request.method === 'GET') && ep.params.properties) { + for (const k of Object.keys(ep.params.properties)) { + const param = ep.params.properties![k]; + if (['boolean', 'number', 'integer'].includes(param.type ?? '') && typeof data[k] === 'string') { + try { + data[k] = JSON.parse(data[k]); + } catch (e) { + throw new ApiError({ + message: 'Invalid param.', + code: 'INVALID_PARAM', + id: '0b5f1631-7c1a-41a6-b399-cce335f34d85', + }, { + param: k, + reason: `cannot cast to ${param.type}`, + }); + } + } + } } // API invoking if (this.config.sentryForBackend) { return await Sentry.startSpan({ name: 'API: ' + ep.name, - }, () => { - return ep.exec(data, user, token, attachmentFile, request.ip, request.headers) - .catch((err: Error) => this.#onExecError(ep, data, err, user?.id)) - .finally(() => cleanup()); - }); + }, () => ep.exec(data, user, token, file, request.ip, request.headers) + .catch((err: Error) => this.#onExecError(ep, data, err, user?.id))); } else { - return await ep.exec(data, user, token, attachmentFile, request.ip, request.headers) - .catch((err: Error) => this.#onExecError(ep, data, err, user?.id)) - .finally(() => cleanup()); + return await ep.exec(data, user, token, file, request.ip, request.headers) + .catch((err: Error) => this.#onExecError(ep, data, err, user?.id)); } } - @bindThis - private async handleAttachmentFile( - fileSizeLimit: number, - multipartFile: MultipartFile, - ) { - function createTooLongError() { - return new ApiError({ - httpStatusCode: 413, - kind: 'client', - message: 'File size is too large.', - code: 'FILE_SIZE_TOO_LARGE', - id: 'ff827ce8-9b4b-4808-8511-422222a3362f', - }); - } - - function createLimitStream(limit: number) { - let total = 0; - - return new Transform({ - transform(chunk, _, callback) { - total += chunk.length; - if (total > limit) { - callback(createTooLongError()); - } else { - callback(null, chunk); - } - }, - }); - } - - const [path, cleanup] = await createTemp(); - try { - await stream.pipeline( - multipartFile.file, - createLimitStream(fileSizeLimit), - fs.createWriteStream(path), - ); - - // ファイルサイズが制限を超えていた場合 - // なお truncated はストリームを読み切ってからでないと機能しないため、stream.pipeline より後にある必要がある - if (multipartFile.file.truncated) { - throw createTooLongError(); - } - } catch (err) { - cleanup(); - throw err; - } - - return { - attachmentFile: { - name: multipartFile.filename, - path, - }, - cleanup, - }; - } - @bindThis public dispose(): void { clearInterval(this.userIpHistoriesClearIntervalId); diff --git a/packages/backend/src/server/api/endpoint-base.ts b/packages/backend/src/server/api/endpoint-base.ts index b063487305..e061aa3a8e 100644 --- a/packages/backend/src/server/api/endpoint-base.ts +++ b/packages/backend/src/server/api/endpoint-base.ts @@ -21,23 +21,23 @@ ajv.addFormat('misskey:id', /^[a-zA-Z0-9]+$/); export type Response = Record | void; -export type AttachmentFile = { +type File = { name: string | null; path: string; }; // TODO: paramsの型をT['params']のスキーマ定義から推論する type Executor = - (params: SchemaType, user: T['requireCredential'] extends true ? MiLocalUser : MiLocalUser | null, token: MiAccessToken | null, file?: AttachmentFile, cleanup?: () => any, ip?: string | null, headers?: Record | null) => - Promise>>; + (params: SchemaType, user: T['requireCredential'] extends true ? MiLocalUser : MiLocalUser | null, token: MiAccessToken | null, file?: File, cleanup?: () => any, ip?: string | null, headers?: Record | null) => + Promise>>; export abstract class Endpoint { - public exec: (params: any, user: T['requireCredential'] extends true ? MiLocalUser : MiLocalUser | null, token: MiAccessToken | null, file?: AttachmentFile, ip?: string | null, headers?: Record | null) => Promise; + public exec: (params: any, user: T['requireCredential'] extends true ? MiLocalUser : MiLocalUser | null, token: MiAccessToken | null, file?: File, ip?: string | null, headers?: Record | null) => Promise; constructor(meta: T, paramDef: Ps, cb: Executor) { const validate = ajv.compile(paramDef); - this.exec = (params: any, user: T['requireCredential'] extends true ? MiLocalUser : MiLocalUser | null, token: MiAccessToken | null, file?: AttachmentFile, ip?: string | null, headers?: Record | null) => { + this.exec = (params: any, user: T['requireCredential'] extends true ? MiLocalUser : MiLocalUser | null, token: MiAccessToken | null, file?: File, ip?: string | null, headers?: Record | null) => { let cleanup: undefined | (() => void) = undefined; if (meta.requireFile) { diff --git a/packages/backend/test/e2e/api.ts b/packages/backend/test/e2e/api.ts index f9e65aaa84..49c6a0636b 100644 --- a/packages/backend/test/e2e/api.ts +++ b/packages/backend/test/e2e/api.ts @@ -159,8 +159,8 @@ describe('API', () => { user: { token: application3 }, }, { status: 403, - code: 'PERMISSION_DENIED', - id: '1370e5b7-d4eb-4566-bb1d-7748ee6a1838', + code: 'ROLE_PERMISSION_DENIED', + id: 'c3d38592-54c0-429d-be96-5636b0431a61', }); await failedApiCall({ diff --git a/packages/backend/test/unit/server/api/drive/files/create.ts b/packages/backend/test/unit/server/api/drive/files/create.ts deleted file mode 100644 index b98892fa03..0000000000 --- a/packages/backend/test/unit/server/api/drive/files/create.ts +++ /dev/null @@ -1,108 +0,0 @@ -/* - * SPDX-FileCopyrightText: syuilo and misskey-project - * SPDX-License-Identifier: AGPL-3.0-only - */ - -import { S3Client } from '@aws-sdk/client-s3'; -import { Test, TestingModule } from '@nestjs/testing'; -import { mockClient } from 'aws-sdk-client-mock'; -import { FastifyInstance } from 'fastify'; -import request from 'supertest'; -import { CoreModule } from '@/core/CoreModule.js'; -import { RoleService } from '@/core/RoleService.js'; -import { DI } from '@/di-symbols.js'; -import { GlobalModule } from '@/GlobalModule.js'; -import { MiRole, UserProfilesRepository, UsersRepository } from '@/models/_.js'; -import { MiUser } from '@/models/User.js'; -import { ServerModule } from '@/server/ServerModule.js'; -import { ServerService } from '@/server/ServerService.js'; - -describe('/drive/files/create', () => { - let module: TestingModule; - let server: FastifyInstance; - const s3Mock = mockClient(S3Client); - let roleService: RoleService; - - let root: MiUser; - let role_tinyAttachment: MiRole; - - beforeAll(async () => { - module = await Test.createTestingModule({ - imports: [GlobalModule, CoreModule, ServerModule], - }).compile(); - module.enableShutdownHooks(); - - const serverService = module.get(ServerService); - server = await serverService.launch(); - - const usersRepository = module.get(DI.usersRepository); - root = await usersRepository.insert({ - id: 'root', - username: 'root', - usernameLower: 'root', - token: '1234567890123456', - }).then(x => usersRepository.findOneByOrFail(x.identifiers[0])); - - const userProfilesRepository = module.get(DI.userProfilesRepository); - await userProfilesRepository.insert({ - userId: root.id, - }); - - roleService = module.get(RoleService); - role_tinyAttachment = await roleService.create({ - name: 'test-role001', - description: 'Test role001 description', - target: 'manual', - policies: { - maxFileSizeMb: { - useDefault: false, - priority: 1, - // 10byte - value: 10 / 1024 / 1024, - }, - }, - }); - }); - - beforeEach(async () => { - s3Mock.reset(); - await roleService.unassign(root.id, role_tinyAttachment.id).catch(() => {}); - }); - - afterAll(async () => { - await server.close(); - await module.close(); - }); - - test('200 ok', async () => { - const result = await request(server.server) - .post('/api/drive/files/create') - .set('Content-Type', 'multipart/form-data') - .set('Authorization', `Bearer ${root.token}`) - .attach('file', Buffer.from('a'.repeat(1024 * 1024))); - expect(result.statusCode).toBe(200); - }); - - test('200 ok(with role)', async () => { - await roleService.assign(root.id, role_tinyAttachment.id); - - const result = await request(server.server) - .post('/api/drive/files/create') - .set('Content-Type', 'multipart/form-data') - .set('Authorization', `Bearer ${root.token}`) - .attach('file', Buffer.from('a'.repeat(10))); - expect(result.statusCode).toBe(200); - }); - - test('413 too large', async () => { - await roleService.assign(root.id, role_tinyAttachment.id); - - const result = await request(server.server) - .post('/api/drive/files/create') - .set('Content-Type', 'multipart/form-data') - .set('Authorization', `Bearer ${root.token}`) - .attach('file', Buffer.from('a'.repeat(11))); - expect(result.statusCode).toBe(413); - expect(result.body.error.code).toBe('FILE_SIZE_TOO_LARGE'); - }); -}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fe009057fa..d4836a8a91 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -652,9 +652,6 @@ importers: '@types/sinonjs__fake-timers': specifier: 8.1.5 version: 8.1.5 - '@types/supertest': - specifier: 6.0.3 - version: 6.0.3 '@types/tinycolor2': specifier: 1.4.6 version: 1.4.6 @@ -709,9 +706,6 @@ importers: simple-oauth2: specifier: 5.1.0 version: 5.1.0 - supertest: - specifier: 7.1.0 - version: 7.1.0 packages/frontend: dependencies: @@ -3034,9 +3028,6 @@ packages: peerDependencies: '@opentelemetry/api': ^1.1.0 - '@paralleldrive/cuid2@2.2.2': - resolution: {integrity: sha512-ZOBkgDwEdoYVlSeRbYYXs0S9MejQofiVYoTbKzy/6GQa39/q5tQU2IX46+shYnUkpEl3wc+J6wRlar7r2EK2xA==} - '@parcel/watcher-android-arm64@2.5.1': resolution: {integrity: sha512-KF8+j9nNbUN8vzOFDpRMsaKBHZ/mcjEjMToVMJOhTozkDonQFFrRcfdLWn6yWKCmJKmdVxSgHiYvTCef4/qcBA==} engines: {node: '>= 10.0.0'} @@ -4150,9 +4141,6 @@ packages: '@types/cookie@0.6.0': resolution: {integrity: sha512-4Kh9a6B2bQciAhf7FSuMRRkUWecJgJu9nPnx3yzpsfXX/c50REIqpHY4C82bXP90qrLtXtkDxTZosYO3UpOwlA==} - '@types/cookiejar@2.1.5': - resolution: {integrity: sha512-he+DHOWReW0nghN24E1WUqM0efK4kI9oTqDm6XmK8ZPe2djZ90BSNdGnIyCLzCPw7/pogPlGbzI2wHGGmi4O/Q==} - '@types/debug@4.1.12': resolution: {integrity: sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==} @@ -4237,9 +4225,6 @@ packages: '@types/mdx@2.0.3': resolution: {integrity: sha512-IgHxcT3RC8LzFLhKwP3gbMPeaK7BM9eBH46OdapPA7yvuIUJ8H6zHZV53J8hGZcTSnt95jANt+rTBNUUc22ACQ==} - '@types/methods@1.1.4': - resolution: {integrity: sha512-ymXWVrDiCxTBE3+RIrrP533E70eA+9qu7zdWoHuOmGujkYtzf4HQF96b8nwHLqhuf4ykX61IGRIB38CC6/sImQ==} - '@types/micromatch@4.0.9': resolution: {integrity: sha512-7V+8ncr22h4UoYRLnLXSpTxjQrNUXtWHGeMPRJt1nULXI57G9bIcpyrHlmrQ7QK24EyyuXvYcSSWAM8GA9nqCg==} @@ -4372,12 +4357,6 @@ packages: '@types/statuses@2.0.4': resolution: {integrity: sha512-eqNDvZsCNY49OAXB0Firg/Sc2BgoWsntsLUdybGFOhAfCD6QJ2n9HXUIHGqt5qjrxmMv4wS8WLAw43ZkKcJ8Pw==} - '@types/superagent@8.1.9': - resolution: {integrity: sha512-pTVjI73witn+9ILmoJdajHGW2jkSaOzhiFYF1Rd3EQ94kymLqB9PjD9ISg7WaALC7+dCHT0FGe9T2LktLq/3GQ==} - - '@types/supertest@6.0.3': - resolution: {integrity: sha512-8WzXq62EXFhJ7QsH3Ocb/iKQ/Ty9ZVWnVzoTKc9tyyFRRF3a74Tk2+TLFgaFFw364Ere+npzHKEJ6ga2LzIL7w==} - '@types/tedious@4.0.14': resolution: {integrity: sha512-KHPsfX/FoVbUGbyYvk1q9MMQHLPeRZhRJZdO45Q4YjvFkv4hMNghCWTvy7rdKessBsmtz4euWCWAB6/tVpI1Iw==} @@ -5425,9 +5404,6 @@ packages: compare-versions@6.1.1: resolution: {integrity: sha512-4hm4VPpIecmlg59CHXnRDnqGplJFrbLG4aFEl5vl6cK1u76ws3LLvX7ikFnTDl5vo39sjWD6AaDPYodJp/NNHg==} - component-emitter@1.3.1: - resolution: {integrity: sha512-T0+barUSQRTUQASh8bx02dl+DhF54GtIDY13Y3m9oWTklKbb3Wv974meRpeZ3lp1JpLVECWWNHC4vaG2XHXouQ==} - compress-commons@6.0.2: resolution: {integrity: sha512-6FqVXeETqWPoGcfzrXb37E50NP0LXT8kAMu5ooZayhWWdgEY4lBEEcbQNXtkuKQsGduxiIcI4gOTsxTmuq/bSg==} engines: {node: '>= 14'} @@ -5486,9 +5462,6 @@ packages: resolution: {integrity: sha512-9Kr/j4O16ISv8zBBhJoi4bXOYNTkFLOqSL3UDB0njXxCXNezjeyVrJyGOWtgfs/q2km1gwBcfH8q1yEGoMYunA==} engines: {node: '>=18'} - cookiejar@2.1.4: - resolution: {integrity: sha512-LDx6oHrK+PhzLKJU9j5S7/Y3jM/mUHvD/DeI1WQmJn652iPC5Y4TBzC9l+5OMOXlyTTA+SmVUPm0HQUwpD5Jqw==} - core-util-is@1.0.2: resolution: {integrity: sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==} @@ -5800,9 +5773,6 @@ packages: devlop@1.1.0: resolution: {integrity: sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==} - dezalgo@1.0.4: - resolution: {integrity: sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==} - diff-match-patch@1.0.5: resolution: {integrity: sha512-IayShXAgj/QMXgB0IWmKx+rOPuGMhqm5w6jvFxmVenXKIzRqTAAsbBPT3kWQeGANj3jGgvcvv4yK6SxqYmikgw==} @@ -6446,10 +6416,6 @@ packages: resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} engines: {node: '>=12.20.0'} - formidable@3.5.4: - resolution: {integrity: sha512-YikH+7CUTOtP44ZTnUhR7Ic2UASBPOqmaRkRKxRbywPTe5VxF7RRCck4af9wutiZ/QKM5nME9Bie2fFaPz5Gug==} - engines: {node: '>=14.0.0'} - forwarded-parse@2.1.2: resolution: {integrity: sha512-alTFZZQDKMporBH77856pXgzhEzaUVmLCDk+egLgIgHst3Tpndzz8MnKe+GzRJRfvVdn69HhpW7cmXzvtLvJAw==} @@ -9843,14 +9809,6 @@ packages: peerDependencies: postcss: ^8.4.31 - superagent@9.0.2: - resolution: {integrity: sha512-xuW7dzkUpcJq7QnhOsnNUgtYp3xRwpt2F7abdRYIpCsAt0hhUqia0EdxyXZQQpNmGtsCzYHryaKSV3q3GJnq7w==} - engines: {node: '>=14.18.0'} - - supertest@7.1.0: - resolution: {integrity: sha512-5QeSO8hSrKghtcWEoPiO036fxH0Ii2wVQfFZSP0oqQhmjk8bOLhDFXr4JrvaFmPuEWUoq4znY3uSi8UzLKxGqw==} - engines: {node: '>=14.18.0'} - supports-color@5.5.0: resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==} engines: {node: '>=4'} @@ -12980,10 +12938,6 @@ snapshots: '@opentelemetry/api': 1.9.0 '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.0) - '@paralleldrive/cuid2@2.2.2': - dependencies: - '@noble/hashes': 1.7.1 - '@parcel/watcher-android-arm64@2.5.1': optional: true @@ -14342,8 +14296,6 @@ snapshots: '@types/cookie@0.6.0': {} - '@types/cookiejar@2.1.5': {} - '@types/debug@4.1.12': dependencies: '@types/ms': 0.7.34 @@ -14434,8 +14386,6 @@ snapshots: '@types/mdx@2.0.3': {} - '@types/methods@1.1.4': {} - '@types/micromatch@4.0.9': dependencies: '@types/braces': 3.0.1 @@ -14568,18 +14518,6 @@ snapshots: '@types/statuses@2.0.4': {} - '@types/superagent@8.1.9': - dependencies: - '@types/cookiejar': 2.1.5 - '@types/methods': 1.1.4 - '@types/node': 22.15.2 - form-data: 4.0.2 - - '@types/supertest@6.0.3': - dependencies: - '@types/methods': 1.1.4 - '@types/superagent': 8.1.9 - '@types/tedious@4.0.14': dependencies: '@types/node': 22.15.2 @@ -15905,8 +15843,6 @@ snapshots: compare-versions@6.1.1: {} - component-emitter@1.3.1: {} - compress-commons@6.0.2: dependencies: crc-32: 1.2.2 @@ -15960,8 +15896,6 @@ snapshots: cookie@1.0.2: {} - cookiejar@2.1.4: {} - core-util-is@1.0.2: {} core-util-is@1.0.3: {} @@ -16347,11 +16281,6 @@ snapshots: dependencies: dequal: 2.0.3 - dezalgo@1.0.4: - dependencies: - asap: 2.0.6 - wrappy: 1.0.2 - diff-match-patch@1.0.5: {} diff-sequences@29.6.3: {} @@ -17246,12 +17175,6 @@ snapshots: dependencies: fetch-blob: 3.2.0 - formidable@3.5.4: - dependencies: - '@paralleldrive/cuid2': 2.2.2 - dezalgo: 1.0.4 - once: 1.4.0 - forwarded-parse@2.1.2: {} forwarded@0.2.0: {} @@ -21205,27 +21128,6 @@ snapshots: postcss: 8.5.3 postcss-selector-parser: 6.1.2 - superagent@9.0.2: - dependencies: - component-emitter: 1.3.1 - cookiejar: 2.1.4 - debug: 4.4.0(supports-color@8.1.1) - fast-safe-stringify: 2.1.1 - form-data: 4.0.2 - formidable: 3.5.4 - methods: 1.1.2 - mime: 2.6.0 - qs: 6.13.0 - transitivePeerDependencies: - - supports-color - - supertest@7.1.0: - dependencies: - methods: 1.1.2 - superagent: 9.0.2 - transitivePeerDependencies: - - supports-color - supports-color@5.5.0: dependencies: has-flag: 3.0.0 From e40f3917f39da7e14cd11b89e6506862e7b92d11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=8A=E3=81=95=E3=82=80=E3=81=AE=E3=81=B2=E3=81=A8?= <46447427+samunohito@users.noreply.github.com> Date: Sun, 4 May 2025 09:38:35 +0900 Subject: [PATCH 58/72] =?UTF-8?q?refactor:=20=E3=83=95=E3=82=A1=E3=82=A4?= =?UTF-8?q?=E3=83=AB=E3=82=A2=E3=83=83=E3=83=97=E3=83=AD=E3=83=BC=E3=83=89?= =?UTF-8?q?=E6=99=82=E3=81=AE=E3=83=86=E3=82=B9=E3=83=88=E3=82=92=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0=20(#15928)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor: ファイルアップロード時のテストを追加 * なぜかsemverが消えてた --- packages/backend/package.json | 4 +- packages/backend/src/server/ServerService.ts | 9 +- .../api/endpoints/drive/files/create.ts | 1 + .../unit/server/api/drive/files/create.ts | 164 ++++++++++++++++++ pnpm-lock.yaml | 98 +++++++++++ 5 files changed, 274 insertions(+), 2 deletions(-) create mode 100644 packages/backend/test/unit/server/api/drive/files/create.ts diff --git a/packages/backend/package.json b/packages/backend/package.json index 14ac79ae1b..b9cb0002ab 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -229,6 +229,7 @@ "@types/semver": "7.7.0", "@types/simple-oauth2": "5.0.7", "@types/sinonjs__fake-timers": "8.1.5", + "@types/supertest": "6.0.3", "@types/tinycolor2": "1.4.6", "@types/tmp": "0.2.6", "@types/uuid": "^9.0.4", @@ -246,6 +247,7 @@ "jest-mock": "29.7.0", "nodemon": "3.1.10", "pid-port": "1.0.2", - "simple-oauth2": "5.1.0" + "simple-oauth2": "5.1.0", + "supertest": "7.1.0" } } diff --git a/packages/backend/src/server/ServerService.ts b/packages/backend/src/server/ServerService.ts index dce47e2290..2d20aa1222 100644 --- a/packages/backend/src/server/ServerService.ts +++ b/packages/backend/src/server/ServerService.ts @@ -7,7 +7,7 @@ import cluster from 'node:cluster'; import * as fs from 'node:fs'; import { fileURLToPath } from 'node:url'; import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; -import Fastify, { FastifyInstance } from 'fastify'; +import Fastify, { type FastifyInstance } from 'fastify'; import fastifyStatic from '@fastify/static'; import fastifyRawBody from 'fastify-raw-body'; import { IsNull } from 'typeorm'; @@ -312,6 +312,13 @@ export class ServerService implements OnApplicationShutdown { await this.#fastify.close(); } + /** + * Get the Fastify instance for testing. + */ + public get fastify(): FastifyInstance { + return this.#fastify; + } + @bindThis async onApplicationShutdown(signal: string): Promise { await this.dispose(); diff --git a/packages/backend/src/server/api/endpoints/drive/files/create.ts b/packages/backend/src/server/api/endpoints/drive/files/create.ts index 7043f4883a..f4c47d71bf 100644 --- a/packages/backend/src/server/api/endpoints/drive/files/create.ts +++ b/packages/backend/src/server/api/endpoints/drive/files/create.ts @@ -67,6 +67,7 @@ export const meta = { message: 'Cannot upload the file because it exceeds the maximum file size.', code: 'MAX_FILE_SIZE_EXCEEDED', id: 'b9d8c348-33f0-4673-b9a9-5d4da058977a', + httpStatusCode: 413, }, }, } as const; diff --git a/packages/backend/test/unit/server/api/drive/files/create.ts b/packages/backend/test/unit/server/api/drive/files/create.ts new file mode 100644 index 0000000000..9b38f4d744 --- /dev/null +++ b/packages/backend/test/unit/server/api/drive/files/create.ts @@ -0,0 +1,164 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Test, TestingModule } from '@nestjs/testing'; +import { FastifyInstance } from 'fastify'; +import request from 'supertest'; +import { randomString } from '../../../../../utils.js'; +import { CoreModule } from '@/core/CoreModule.js'; +import { RoleService } from '@/core/RoleService.js'; +import { DI } from '@/di-symbols.js'; +import { GlobalModule } from '@/GlobalModule.js'; +import { DriveFoldersRepository, MiDriveFolder, MiRole, UserProfilesRepository, UsersRepository } from '@/models/_.js'; +import { MiUser } from '@/models/User.js'; +import { ServerModule } from '@/server/ServerModule.js'; +import { ServerService } from '@/server/ServerService.js'; +import { IdService } from '@/core/IdService.js'; + +describe('/drive/files/create', () => { + let module: TestingModule; + let server: FastifyInstance; + let roleService: RoleService; + let idService: IdService; + + let root: MiUser; + let role_tinyAttachment: MiRole; + + let folder: MiDriveFolder; + + beforeAll(async () => { + module = await Test.createTestingModule({ + imports: [GlobalModule, CoreModule, ServerModule], + }).compile(); + module.enableShutdownHooks(); + + const serverService = module.get(ServerService); + await serverService.launch(); + server = serverService.fastify; + + idService = module.get(IdService); + + const usersRepository = module.get(DI.usersRepository); + await usersRepository.delete({}); + root = await usersRepository.insert({ + id: idService.gen(), + username: 'root', + usernameLower: 'root', + token: '1234567890123456', + }).then(x => usersRepository.findOneByOrFail(x.identifiers[0])); + + const userProfilesRepository = module.get(DI.userProfilesRepository); + await userProfilesRepository.delete({}); + await userProfilesRepository.insert({ + userId: root.id, + }); + + const driveFoldersRepository = module.get(DI.driveFoldersRepository); + folder = await driveFoldersRepository.insertOne({ + id: idService.gen(), + name: 'root-folder', + parentId: null, + userId: root.id, + }); + + roleService = module.get(RoleService); + role_tinyAttachment = await roleService.create({ + name: 'test-role001', + description: 'Test role001 description', + target: 'manual', + policies: { + maxFileSizeMb: { + useDefault: false, + priority: 1, + // 10byte + value: 10 / 1024 / 1024, + }, + }, + }); + }); + + beforeEach(async () => { + await roleService.unassign(root.id, role_tinyAttachment.id).catch(() => { + }); + }); + + afterAll(async () => { + await server.close(); + await module.close(); + }); + + async function postFile(props: { + name: string, + comment: string, + isSensitive: boolean, + force: boolean, + fileContent: Buffer | string, + }) { + const { name, comment, isSensitive, force, fileContent } = props; + + return await request(server.server) + .post('/api/drive/files/create') + .set('Content-Type', 'multipart/form-data') + .attach('file', fileContent) + .field('name', name) + .field('comment', comment) + .field('isSensitive', isSensitive) + .field('force', force) + .field('folderId', folder.id) + .field('i', root.token ?? ''); + } + + test('200 ok', async () => { + const name = randomString(); + const comment = randomString(); + const result = await postFile({ + name: name, + comment: comment, + isSensitive: true, + force: true, + fileContent: Buffer.from('a'.repeat(1000 * 1000)), + }); + expect(result.statusCode).toBe(200); + expect(result.body.name).toBe(name + '.unknown'); + expect(result.body.comment).toBe(comment); + expect(result.body.isSensitive).toBe(true); + expect(result.body.folderId).toBe(folder.id); + }); + + test('200 ok(with role)', async () => { + await roleService.assign(root.id, role_tinyAttachment.id); + + const name = randomString(); + const comment = randomString(); + const result = await postFile({ + name: name, + comment: comment, + isSensitive: true, + force: true, + fileContent: Buffer.from('a'.repeat(10)), + }); + expect(result.statusCode).toBe(200); + expect(result.body.name).toBe(name + '.unknown'); + expect(result.body.comment).toBe(comment); + expect(result.body.isSensitive).toBe(true); + expect(result.body.folderId).toBe(folder.id); + }); + + test('413 too large', async () => { + await roleService.assign(root.id, role_tinyAttachment.id); + + const name = randomString(); + const comment = randomString(); + const result = await postFile({ + name: name, + comment: comment, + isSensitive: true, + force: true, + fileContent: Buffer.from('a'.repeat(11)), + }); + expect(result.statusCode).toBe(413); + expect(result.body.error.code).toBe('MAX_FILE_SIZE_EXCEEDED'); + }); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d4836a8a91..cce229ef5a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -652,6 +652,9 @@ importers: '@types/sinonjs__fake-timers': specifier: 8.1.5 version: 8.1.5 + '@types/supertest': + specifier: 6.0.3 + version: 6.0.3 '@types/tinycolor2': specifier: 1.4.6 version: 1.4.6 @@ -706,6 +709,9 @@ importers: simple-oauth2: specifier: 5.1.0 version: 5.1.0 + supertest: + specifier: 7.1.0 + version: 7.1.0 packages/frontend: dependencies: @@ -3028,6 +3034,9 @@ packages: peerDependencies: '@opentelemetry/api': ^1.1.0 + '@paralleldrive/cuid2@2.2.2': + resolution: {integrity: sha512-ZOBkgDwEdoYVlSeRbYYXs0S9MejQofiVYoTbKzy/6GQa39/q5tQU2IX46+shYnUkpEl3wc+J6wRlar7r2EK2xA==} + '@parcel/watcher-android-arm64@2.5.1': resolution: {integrity: sha512-KF8+j9nNbUN8vzOFDpRMsaKBHZ/mcjEjMToVMJOhTozkDonQFFrRcfdLWn6yWKCmJKmdVxSgHiYvTCef4/qcBA==} engines: {node: '>= 10.0.0'} @@ -4141,6 +4150,9 @@ packages: '@types/cookie@0.6.0': resolution: {integrity: sha512-4Kh9a6B2bQciAhf7FSuMRRkUWecJgJu9nPnx3yzpsfXX/c50REIqpHY4C82bXP90qrLtXtkDxTZosYO3UpOwlA==} + '@types/cookiejar@2.1.5': + resolution: {integrity: sha512-he+DHOWReW0nghN24E1WUqM0efK4kI9oTqDm6XmK8ZPe2djZ90BSNdGnIyCLzCPw7/pogPlGbzI2wHGGmi4O/Q==} + '@types/debug@4.1.12': resolution: {integrity: sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==} @@ -4225,6 +4237,9 @@ packages: '@types/mdx@2.0.3': resolution: {integrity: sha512-IgHxcT3RC8LzFLhKwP3gbMPeaK7BM9eBH46OdapPA7yvuIUJ8H6zHZV53J8hGZcTSnt95jANt+rTBNUUc22ACQ==} + '@types/methods@1.1.4': + resolution: {integrity: sha512-ymXWVrDiCxTBE3+RIrrP533E70eA+9qu7zdWoHuOmGujkYtzf4HQF96b8nwHLqhuf4ykX61IGRIB38CC6/sImQ==} + '@types/micromatch@4.0.9': resolution: {integrity: sha512-7V+8ncr22h4UoYRLnLXSpTxjQrNUXtWHGeMPRJt1nULXI57G9bIcpyrHlmrQ7QK24EyyuXvYcSSWAM8GA9nqCg==} @@ -4357,6 +4372,12 @@ packages: '@types/statuses@2.0.4': resolution: {integrity: sha512-eqNDvZsCNY49OAXB0Firg/Sc2BgoWsntsLUdybGFOhAfCD6QJ2n9HXUIHGqt5qjrxmMv4wS8WLAw43ZkKcJ8Pw==} + '@types/superagent@8.1.9': + resolution: {integrity: sha512-pTVjI73witn+9ILmoJdajHGW2jkSaOzhiFYF1Rd3EQ94kymLqB9PjD9ISg7WaALC7+dCHT0FGe9T2LktLq/3GQ==} + + '@types/supertest@6.0.3': + resolution: {integrity: sha512-8WzXq62EXFhJ7QsH3Ocb/iKQ/Ty9ZVWnVzoTKc9tyyFRRF3a74Tk2+TLFgaFFw364Ere+npzHKEJ6ga2LzIL7w==} + '@types/tedious@4.0.14': resolution: {integrity: sha512-KHPsfX/FoVbUGbyYvk1q9MMQHLPeRZhRJZdO45Q4YjvFkv4hMNghCWTvy7rdKessBsmtz4euWCWAB6/tVpI1Iw==} @@ -5404,6 +5425,9 @@ packages: compare-versions@6.1.1: resolution: {integrity: sha512-4hm4VPpIecmlg59CHXnRDnqGplJFrbLG4aFEl5vl6cK1u76ws3LLvX7ikFnTDl5vo39sjWD6AaDPYodJp/NNHg==} + component-emitter@1.3.1: + resolution: {integrity: sha512-T0+barUSQRTUQASh8bx02dl+DhF54GtIDY13Y3m9oWTklKbb3Wv974meRpeZ3lp1JpLVECWWNHC4vaG2XHXouQ==} + compress-commons@6.0.2: resolution: {integrity: sha512-6FqVXeETqWPoGcfzrXb37E50NP0LXT8kAMu5ooZayhWWdgEY4lBEEcbQNXtkuKQsGduxiIcI4gOTsxTmuq/bSg==} engines: {node: '>= 14'} @@ -5462,6 +5486,9 @@ packages: resolution: {integrity: sha512-9Kr/j4O16ISv8zBBhJoi4bXOYNTkFLOqSL3UDB0njXxCXNezjeyVrJyGOWtgfs/q2km1gwBcfH8q1yEGoMYunA==} engines: {node: '>=18'} + cookiejar@2.1.4: + resolution: {integrity: sha512-LDx6oHrK+PhzLKJU9j5S7/Y3jM/mUHvD/DeI1WQmJn652iPC5Y4TBzC9l+5OMOXlyTTA+SmVUPm0HQUwpD5Jqw==} + core-util-is@1.0.2: resolution: {integrity: sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==} @@ -5773,6 +5800,9 @@ packages: devlop@1.1.0: resolution: {integrity: sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==} + dezalgo@1.0.4: + resolution: {integrity: sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==} + diff-match-patch@1.0.5: resolution: {integrity: sha512-IayShXAgj/QMXgB0IWmKx+rOPuGMhqm5w6jvFxmVenXKIzRqTAAsbBPT3kWQeGANj3jGgvcvv4yK6SxqYmikgw==} @@ -6416,6 +6446,10 @@ packages: resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} engines: {node: '>=12.20.0'} + formidable@3.5.4: + resolution: {integrity: sha512-YikH+7CUTOtP44ZTnUhR7Ic2UASBPOqmaRkRKxRbywPTe5VxF7RRCck4af9wutiZ/QKM5nME9Bie2fFaPz5Gug==} + engines: {node: '>=14.0.0'} + forwarded-parse@2.1.2: resolution: {integrity: sha512-alTFZZQDKMporBH77856pXgzhEzaUVmLCDk+egLgIgHst3Tpndzz8MnKe+GzRJRfvVdn69HhpW7cmXzvtLvJAw==} @@ -9809,6 +9843,14 @@ packages: peerDependencies: postcss: ^8.4.31 + superagent@9.0.2: + resolution: {integrity: sha512-xuW7dzkUpcJq7QnhOsnNUgtYp3xRwpt2F7abdRYIpCsAt0hhUqia0EdxyXZQQpNmGtsCzYHryaKSV3q3GJnq7w==} + engines: {node: '>=14.18.0'} + + supertest@7.1.0: + resolution: {integrity: sha512-5QeSO8hSrKghtcWEoPiO036fxH0Ii2wVQfFZSP0oqQhmjk8bOLhDFXr4JrvaFmPuEWUoq4znY3uSi8UzLKxGqw==} + engines: {node: '>=14.18.0'} + supports-color@5.5.0: resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==} engines: {node: '>=4'} @@ -12938,6 +12980,10 @@ snapshots: '@opentelemetry/api': 1.9.0 '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.0) + '@paralleldrive/cuid2@2.2.2': + dependencies: + '@noble/hashes': 1.7.1 + '@parcel/watcher-android-arm64@2.5.1': optional: true @@ -14296,6 +14342,8 @@ snapshots: '@types/cookie@0.6.0': {} + '@types/cookiejar@2.1.5': {} + '@types/debug@4.1.12': dependencies: '@types/ms': 0.7.34 @@ -14386,6 +14434,8 @@ snapshots: '@types/mdx@2.0.3': {} + '@types/methods@1.1.4': {} + '@types/micromatch@4.0.9': dependencies: '@types/braces': 3.0.1 @@ -14518,6 +14568,18 @@ snapshots: '@types/statuses@2.0.4': {} + '@types/superagent@8.1.9': + dependencies: + '@types/cookiejar': 2.1.5 + '@types/methods': 1.1.4 + '@types/node': 22.15.2 + form-data: 4.0.2 + + '@types/supertest@6.0.3': + dependencies: + '@types/methods': 1.1.4 + '@types/superagent': 8.1.9 + '@types/tedious@4.0.14': dependencies: '@types/node': 22.15.2 @@ -15843,6 +15905,8 @@ snapshots: compare-versions@6.1.1: {} + component-emitter@1.3.1: {} + compress-commons@6.0.2: dependencies: crc-32: 1.2.2 @@ -15896,6 +15960,8 @@ snapshots: cookie@1.0.2: {} + cookiejar@2.1.4: {} + core-util-is@1.0.2: {} core-util-is@1.0.3: {} @@ -16281,6 +16347,11 @@ snapshots: dependencies: dequal: 2.0.3 + dezalgo@1.0.4: + dependencies: + asap: 2.0.6 + wrappy: 1.0.2 + diff-match-patch@1.0.5: {} diff-sequences@29.6.3: {} @@ -17175,6 +17246,12 @@ snapshots: dependencies: fetch-blob: 3.2.0 + formidable@3.5.4: + dependencies: + '@paralleldrive/cuid2': 2.2.2 + dezalgo: 1.0.4 + once: 1.4.0 + forwarded-parse@2.1.2: {} forwarded@0.2.0: {} @@ -21128,6 +21205,27 @@ snapshots: postcss: 8.5.3 postcss-selector-parser: 6.1.2 + superagent@9.0.2: + dependencies: + component-emitter: 1.3.1 + cookiejar: 2.1.4 + debug: 4.4.0(supports-color@8.1.1) + fast-safe-stringify: 2.1.1 + form-data: 4.0.2 + formidable: 3.5.4 + methods: 1.1.2 + mime: 2.6.0 + qs: 6.14.0 + transitivePeerDependencies: + - supports-color + + supertest@7.1.0: + dependencies: + methods: 1.1.2 + superagent: 9.0.2 + transitivePeerDependencies: + - supports-color + supports-color@5.5.0: dependencies: has-flag: 3.0.0 From b218251b94e7e0961145c57298f48102f4342818 Mon Sep 17 00:00:00 2001 From: Marie Date: Thu, 8 May 2025 19:46:42 +0000 Subject: [PATCH 59/72] added cleanup to more sections --- packages/backend/src/server/api/ApiCallService.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/backend/src/server/api/ApiCallService.ts b/packages/backend/src/server/api/ApiCallService.ts index b22a8c1837..0d2dafd556 100644 --- a/packages/backend/src/server/api/ApiCallService.ts +++ b/packages/backend/src/server/api/ApiCallService.ts @@ -213,6 +213,7 @@ export class ApiCallService implements OnApplicationShutdown { ? request.headers.authorization.slice(7) : fields['i']; if (token != null && typeof token !== 'string') { + cleanup(); reply.code(400); return; } @@ -223,6 +224,7 @@ export class ApiCallService implements OnApplicationShutdown { }, request, reply).then((res) => { this.send(reply, res); }).catch((err: ApiError) => { + cleanup(); this.#sendApiError(reply, err); }); @@ -230,6 +232,7 @@ export class ApiCallService implements OnApplicationShutdown { this.logIp(request, user); } }).catch(err => { + cleanup(); this.#sendAuthenticationError(reply, err); }); } From 6c9dcb84abc1a11e21d03b6e82b46d1743fe0c03 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sun, 4 May 2025 10:08:41 -0400 Subject: [PATCH 60/72] resolve mentioned user handles on the backend --- .../src/core/entities/NoteEntityService.ts | 58 ++++++++++++++++++- .../backend/src/models/json-schema/note.ts | 10 ++++ packages/misskey-js/src/autogen/types.ts | 3 + 3 files changed, 69 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/core/entities/NoteEntityService.ts b/packages/backend/src/core/entities/NoteEntityService.ts index eceb972a91..94fe9e97ef 100644 --- a/packages/backend/src/core/entities/NoteEntityService.ts +++ b/packages/backend/src/core/entities/NoteEntityService.ts @@ -402,7 +402,8 @@ export class NoteEntityService implements OnModuleInit { bufferedReactions: Map; pairs: ([MiUser['id'], string])[] }> | null; myReactions: Map; packedFiles: Map | null>; - packedUsers: Map> + packedUsers: Map>; + mentionHandles: Record; }; }, ): Promise> { @@ -443,6 +444,9 @@ export class NoteEntityService implements OnModuleInit { const packedFiles = options?._hint_?.packedFiles; const packedUsers = options?._hint_?.packedUsers; + // Do not await - defer until the awaitAll below + const mentionHandles = this.getUserHandles(note.mentions, options?._hint_?.mentionHandles); + const packed: Packed<'Note'> = await awaitAll({ id: note.id, createdAt: this.idService.parse(note.id).date.toISOString(), @@ -476,7 +480,8 @@ export class NoteEntityService implements OnModuleInit { allowRenoteToExternal: channel.allowRenoteToExternal, userId: channel.userId, } : undefined, - mentions: note.mentions && note.mentions.length > 0 ? note.mentions : undefined, + mentions: note.mentions.length > 0 ? note.mentions : undefined, + mentionHandles: note.mentions.length > 0 ? mentionHandles : undefined, uri: note.uri ?? undefined, url: note.url ?? undefined, poll: note.hasPoll ? this.populatePoll(note, meId) : undefined, @@ -594,6 +599,22 @@ export class NoteEntityService implements OnModuleInit { const packedUsers = await this.userEntityService.packMany(users, me) .then(users => new Map(users.map(u => [u.id, u]))); + // Recursively add all mentioned users from all notes + replies + renotes + const allMentionedUsers = notes.reduce((users, note) => { + function add(n: MiNote) { + for (const user of n.mentions) { + users.add(user); + } + + if (n.reply) add(n.reply); + if (n.renote) add(n.renote); + } + + add(note); + return users; + }, new Set()); + const mentionHandles = await this.getUserHandles(Array.from(allMentionedUsers)); + return await Promise.all(notes.map(n => this.pack(n, me, { ...options, _hint_: { @@ -601,6 +622,7 @@ export class NoteEntityService implements OnModuleInit { myReactions: myReactionsMap, packedFiles, packedUsers, + mentionHandles, }, }))); } @@ -636,4 +658,36 @@ export class NoteEntityService implements OnModuleInit { relations: ['user'], }); } + + private async getUserHandles(userIds: string[], hint?: Record): Promise> { + if (userIds.length < 1) return {}; + + // Hint is provided by packMany to avoid N+1 queries. + // It should already include all existing mentioned users. + if (hint) { + const handles = {} as Record; + for (const id of userIds) { + handles[id] = hint[id]; + } + return handles; + } + + const users = await this.usersRepository.find({ + select: { + id: true, + username: true, + host: true, + }, + where: { + id: In(userIds), + }, + }); + + return users.reduce((map, user) => { + map[user.id] = user.host + ? `@${user.username}@${user.host}` + : `@${user.username}`; + return map; + }, {} as Record); + } } diff --git a/packages/backend/src/models/json-schema/note.ts b/packages/backend/src/models/json-schema/note.ts index 16e240ab11..b19c8f7c06 100644 --- a/packages/backend/src/models/json-schema/note.ts +++ b/packages/backend/src/models/json-schema/note.ts @@ -85,6 +85,16 @@ export const packedNoteSchema = { format: 'id', }, }, + mentionHandles: { + type: 'object', + optional: true, nullable: false, + additionalProperties: { + anyOf: [{ + type: 'string', + }], + optional: true, nullable: false, + }, + }, visibleUserIds: { type: 'array', optional: true, nullable: false, diff --git a/packages/misskey-js/src/autogen/types.ts b/packages/misskey-js/src/autogen/types.ts index 5ac3fb26f5..2b4c39c280 100644 --- a/packages/misskey-js/src/autogen/types.ts +++ b/packages/misskey-js/src/autogen/types.ts @@ -4668,6 +4668,9 @@ export type components = { /** @enum {string} */ visibility: 'public' | 'home' | 'followers' | 'specified'; mentions?: string[]; + mentionHandles?: { + [key: string]: string; + }; visibleUserIds?: string[]; fileIds?: string[]; files?: components['schemas']['DriveFile'][]; From a4c7f3affdd05bd994223d7d10bd3d45edcbc08e Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sun, 4 May 2025 10:19:48 -0400 Subject: [PATCH 61/72] when replying to a note, auto-fill mentions based on the backend data instead of parsing the OP text --- packages/frontend/src/components/MkPostForm.vue | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/packages/frontend/src/components/MkPostForm.vue b/packages/frontend/src/components/MkPostForm.vue index 5f4e40d513..59c23b090e 100644 --- a/packages/frontend/src/components/MkPostForm.vue +++ b/packages/frontend/src/components/MkPostForm.vue @@ -316,20 +316,8 @@ if (props.reply && (props.reply.user.username !== $i.username || (props.reply.us text.value = `@${props.reply.user.username}${props.reply.user.host != null ? '@' + toASCII(props.reply.user.host) : ''} `; } -if (props.reply && props.reply.text != null) { - const ast = mfm.parse(props.reply.text); - const otherHost = props.reply.user.host; - - for (const x of extractMentions(ast)) { - const mention = x.host ? - `@${x.username}@${toASCII(x.host)}` : - (otherHost == null || otherHost === host) ? - `@${x.username}` : - `@${x.username}@${toASCII(otherHost)}`; - - // 自分は除外 - if ($i.username === x.username && (x.host == null || x.host === host)) continue; - +if (props.reply && props.reply.mentionHandles) { + for (const mention of Object.values(props.reply.mentionHandles)) { // 重複は除外 if (text.value.includes(`${mention} `)) continue; From df0d8045d5dc289102455315b9930cf82edaf3ad Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sun, 4 May 2025 10:21:16 -0400 Subject: [PATCH 62/72] fix duplicate mentions and spurious "user is not mentioned" warnings when replying to a DM thread including a user with a capitalized username --- packages/frontend/src/components/MkPostForm.vue | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/frontend/src/components/MkPostForm.vue b/packages/frontend/src/components/MkPostForm.vue index 59c23b090e..78e94d48b0 100644 --- a/packages/frontend/src/components/MkPostForm.vue +++ b/packages/frontend/src/components/MkPostForm.vue @@ -412,7 +412,7 @@ function checkMissingMention() { const ast = mfm.parse(text.value); for (const x of extractMentions(ast)) { - if (!visibleUsers.value.some(u => (u.username === x.username) && (u.host === x.host))) { + if (!visibleUsers.value.some(u => (u.username.toLowerCase() === x.username.toLowerCase()) && (u.host === x.host))) { hasNotSpecifiedMentions.value = true; return; } @@ -425,7 +425,7 @@ function addMissingMention() { const ast = mfm.parse(text.value); for (const x of extractMentions(ast)) { - if (!visibleUsers.value.some(u => (u.username === x.username) && (u.host === x.host))) { + if (!visibleUsers.value.some(u => (u.username.toLowerCase() === x.username.toLowerCase()) && (u.host === x.host))) { misskeyApi('users/show', { username: x.username, host: x.host }).then(user => { pushVisibleUser(user); }); @@ -641,7 +641,7 @@ function showOtherSettings() { //#endregion function pushVisibleUser(user: Misskey.entities.UserDetailed) { - if (!visibleUsers.value.some(u => u.username === user.username && u.host === user.host)) { + if (!visibleUsers.value.some(u => u.username.toLowerCase() === user.username.toLowerCase() && u.host === user.host)) { visibleUsers.value.push(user); } } From d06e1e308004a53f5b842e25433717c5dbc3c8c5 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sun, 4 May 2025 11:17:20 -0400 Subject: [PATCH 63/72] don't insert mentions for the current user --- packages/frontend/src/components/MkPostForm.vue | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/frontend/src/components/MkPostForm.vue b/packages/frontend/src/components/MkPostForm.vue index 78e94d48b0..052ac7cf6e 100644 --- a/packages/frontend/src/components/MkPostForm.vue +++ b/packages/frontend/src/components/MkPostForm.vue @@ -317,7 +317,10 @@ if (props.reply && (props.reply.user.username !== $i.username || (props.reply.us } if (props.reply && props.reply.mentionHandles) { - for (const mention of Object.values(props.reply.mentionHandles)) { + for (const [user, mention] of Object.entries(props.reply.mentionHandles)) { + // Don't mention ourself + if (user === $i.id) continue; + // 重複は除外 if (text.value.includes(`${mention} `)) continue; From 5e2cc8eb855157f01d67d45e751837aaebffe65c Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 5 May 2025 10:50:58 -0400 Subject: [PATCH 64/72] avoid error when editing notes without any mentions --- packages/backend/src/core/entities/NoteEntityService.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/backend/src/core/entities/NoteEntityService.ts b/packages/backend/src/core/entities/NoteEntityService.ts index 94fe9e97ef..45d9491e36 100644 --- a/packages/backend/src/core/entities/NoteEntityService.ts +++ b/packages/backend/src/core/entities/NoteEntityService.ts @@ -444,9 +444,6 @@ export class NoteEntityService implements OnModuleInit { const packedFiles = options?._hint_?.packedFiles; const packedUsers = options?._hint_?.packedUsers; - // Do not await - defer until the awaitAll below - const mentionHandles = this.getUserHandles(note.mentions, options?._hint_?.mentionHandles); - const packed: Packed<'Note'> = await awaitAll({ id: note.id, createdAt: this.idService.parse(note.id).date.toISOString(), @@ -481,7 +478,7 @@ export class NoteEntityService implements OnModuleInit { userId: channel.userId, } : undefined, mentions: note.mentions.length > 0 ? note.mentions : undefined, - mentionHandles: note.mentions.length > 0 ? mentionHandles : undefined, + mentionHandles: note.mentions.length > 0 ? this.getUserHandles(note.mentions, options?._hint_?.mentionHandles) : undefined, uri: note.uri ?? undefined, url: note.url ?? undefined, poll: note.hasPoll ? this.populatePoll(note, meId) : undefined, From 58d2c4af6b110458c6912b2ca07d4c2e31f41c64 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Thu, 8 May 2025 11:29:42 -0400 Subject: [PATCH 65/72] use targetNotes to reduce duplicate code --- .../src/core/entities/NoteEntityService.ts | 51 ++++++++----------- 1 file changed, 22 insertions(+), 29 deletions(-) diff --git a/packages/backend/src/core/entities/NoteEntityService.ts b/packages/backend/src/core/entities/NoteEntityService.ts index 45d9491e36..6ada5463a3 100644 --- a/packages/backend/src/core/entities/NoteEntityService.ts +++ b/packages/backend/src/core/entities/NoteEntityService.ts @@ -531,6 +531,25 @@ export class NoteEntityService implements OnModuleInit { ) { if (notes.length === 0) return []; + const targetNotes: MiNote[] = []; + for (const note of notes) { + if (isPureRenote(note)) { + // we may need to fetch 'my reaction' for renote target. + targetNotes.push(note.renote); + if (note.renote.reply) { + // idem if the renote is also a reply. + targetNotes.push(note.renote.reply); + } + } else { + if (note.reply) { + // idem for OP of a regular reply. + targetNotes.push(note.reply); + } + + targetNotes.push(note); + } + } + const bufferedReactions = this.meta.enableReactionsBuffering ? await this.reactionsBufferingService.getMany([...getAppearNoteIds(notes)]) : null; const meId = me ? me.id : null; @@ -538,25 +557,6 @@ export class NoteEntityService implements OnModuleInit { if (meId) { const idsNeedFetchMyReaction = new Set(); - const targetNotes: MiNote[] = []; - for (const note of notes) { - if (isPureRenote(note)) { - // we may need to fetch 'my reaction' for renote target. - targetNotes.push(note.renote); - if (note.renote.reply) { - // idem if the renote is also a reply. - targetNotes.push(note.renote.reply); - } - } else { - if (note.reply) { - // idem for OP of a regular reply. - targetNotes.push(note.reply); - } - - targetNotes.push(note); - } - } - for (const note of targetNotes) { const reactionsCount = Object.values(this.reactionsBufferingService.mergeReactions(note.reactions, bufferedReactions?.get(note.id)?.deltas ?? {})).reduce((a, b) => a + b, 0); if (reactionsCount === 0) { @@ -597,17 +597,10 @@ export class NoteEntityService implements OnModuleInit { .then(users => new Map(users.map(u => [u.id, u]))); // Recursively add all mentioned users from all notes + replies + renotes - const allMentionedUsers = notes.reduce((users, note) => { - function add(n: MiNote) { - for (const user of n.mentions) { - users.add(user); - } - - if (n.reply) add(n.reply); - if (n.renote) add(n.renote); + const allMentionedUsers = targetNotes.reduce((users, note) => { + for (const user of note.mentions) { + users.add(user); } - - add(note); return users; }, new Set()); const mentionHandles = await this.getUserHandles(Array.from(allMentionedUsers)); From 1fe39ed4327b64e8aff4bee7c9ffe507f04d23a7 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Thu, 8 May 2025 16:34:40 -0400 Subject: [PATCH 66/72] re-fetch notes after create/edit to ensure they have all fields populated --- packages/backend/src/core/NoteCreateService.ts | 5 +++-- packages/backend/src/core/NoteEditService.ts | 7 +++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index ed97908f66..a342bc1912 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -227,7 +227,7 @@ export class NoteCreateService implements OnApplicationShutdown { } @bindThis - public async create(user: MiUser & { + public async create(user: MiUser & { id: MiUser['id']; username: MiUser['username']; host: MiUser['host']; @@ -539,7 +539,8 @@ export class NoteCreateService implements OnApplicationShutdown { await this.notesRepository.insert(insert); } - return insert; + // Re-fetch note to get the default values of null / unset fields. + return await this.notesRepository.findOneByOrFail({ id: insert.id }); } catch (e) { // duplicate key error if (isDuplicateKeyValueError(e)) { diff --git a/packages/backend/src/core/NoteEditService.ts b/packages/backend/src/core/NoteEditService.ts index 332560154d..e9637c56c7 100644 --- a/packages/backend/src/core/NoteEditService.ts +++ b/packages/backend/src/core/NoteEditService.ts @@ -574,12 +574,15 @@ export class NoteEditService implements OnApplicationShutdown { await this.notesRepository.update(oldnote.id, note); } + // Re-fetch note to get the default values of null / unset fields. + const edited = await this.notesRepository.findOneByOrFail({ id: note.id }); + setImmediate('post edited', { signal: this.#shutdownController.signal }).then( - () => this.postNoteEdited(note, oldnote, user, data, silent, tags!, mentionedUsers!), + () => this.postNoteEdited(edited, oldnote, user, data, silent, tags!, mentionedUsers!), () => { /* aborted, ignore this */ }, ); - return note; + return edited; } else { return oldnote; } From e75e4f11a20e989038f2a5ab8c3cd2a1c3b44e40 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Thu, 8 May 2025 16:42:16 -0400 Subject: [PATCH 67/72] match saveToTempFile return type with other create-temp function --- packages/backend/src/misc/create-temp.ts | 4 ++-- packages/backend/src/server/ServerUtilityService.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/misc/create-temp.ts b/packages/backend/src/misc/create-temp.ts index fda63c7a9d..f2138abf66 100644 --- a/packages/backend/src/misc/create-temp.ts +++ b/packages/backend/src/misc/create-temp.ts @@ -30,11 +30,11 @@ export function createTempDir(): Promise<[string, () => void]> { }); } -export async function saveToTempFile(stream: NodeJS.ReadableStream): Promise { +export async function saveToTempFile(stream: NodeJS.ReadableStream): Promise<[string, () => void]> { const [filepath, cleanup] = await createTemp(); try { await pipeline(stream, fs.createWriteStream(filepath)); - return filepath; + return [filepath, cleanup]; } catch (e) { cleanup(); throw e; diff --git a/packages/backend/src/server/ServerUtilityService.ts b/packages/backend/src/server/ServerUtilityService.ts index c2a3132489..00eb97f679 100644 --- a/packages/backend/src/server/ServerUtilityService.ts +++ b/packages/backend/src/server/ServerUtilityService.ts @@ -54,7 +54,7 @@ export class ServerUtilityService { } } else { // Otherwise it's a file try { - const filepath = await saveToTempFile(part.file); + const [filepath] = await saveToTempFile(part.file); const tmpUploads = (request.tmpUploads ??= []); tmpUploads.push(filepath); From 7bfe16cbb0ba0562586b471a8d7ae24bfe3dacfa Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Thu, 8 May 2025 16:43:52 -0400 Subject: [PATCH 68/72] check for stream truncation in saveToTempFile --- packages/backend/src/misc/create-temp.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/misc/create-temp.ts b/packages/backend/src/misc/create-temp.ts index f2138abf66..9ba95cff42 100644 --- a/packages/backend/src/misc/create-temp.ts +++ b/packages/backend/src/misc/create-temp.ts @@ -30,13 +30,20 @@ export function createTempDir(): Promise<[string, () => void]> { }); } -export async function saveToTempFile(stream: NodeJS.ReadableStream): Promise<[string, () => void]> { +export async function saveToTempFile(stream: NodeJS.ReadableStream & { truncated?: boolean }): Promise<[string, () => void]> { const [filepath, cleanup] = await createTemp(); + try { await pipeline(stream, fs.createWriteStream(filepath)); - return [filepath, cleanup]; } catch (e) { cleanup(); throw e; } + + if (stream.truncated) { + cleanup(); + throw new Error('Read failed: input stream truncated'); + } + + return [filepath, cleanup]; } From 164c85067fa5664498affadbac48466a1c9ecc6e Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Thu, 8 May 2025 22:43:02 -0400 Subject: [PATCH 69/72] remove extra space in NoteCreateService.ts --- packages/backend/src/core/NoteCreateService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index a342bc1912..e961d4236c 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -227,7 +227,7 @@ export class NoteCreateService implements OnApplicationShutdown { } @bindThis - public async create(user: MiUser & { + public async create(user: MiUser & { id: MiUser['id']; username: MiUser['username']; host: MiUser['host']; From 1adf7ffe3144edfc6aa32262db5b57b1371a99a2 Mon Sep 17 00:00:00 2001 From: Ruben Date: Fri, 9 May 2025 02:32:17 -0500 Subject: [PATCH 70/72] update sharkey logo documentation --- packages/frontend/assets/sharkey-text-src/README.md | 11 +++++++---- .../sharkey-text-src/{sharkey_v2.svg => sharkey.svg} | 0 2 files changed, 7 insertions(+), 4 deletions(-) rename packages/frontend/assets/sharkey-text-src/{sharkey_v2.svg => sharkey.svg} (100%) diff --git a/packages/frontend/assets/sharkey-text-src/README.md b/packages/frontend/assets/sharkey-text-src/README.md index 64b4557ccc..d88aeddb6f 100644 --- a/packages/frontend/assets/sharkey-text-src/README.md +++ b/packages/frontend/assets/sharkey-text-src/README.md @@ -1,7 +1,10 @@ -This logo text was made in Inkscape by @sneexy@booping.synth.download +# Sharkey Logo -If you edit it, you can use -[svgo](https://jakearchibald.github.io/svgomg/) to generate a -browser-compatible file, to replace `../sharkey.svg` +This logo was made in Inkscape by [@sneexy@booping.synth.download](https://booping.synth.download/@sneexy). + +The font used is Lilita One ([Google Fonts](https://fonts.google.com/specimen/Lilita+One)). + +The logo should preferably be modified using Inkscape, but should work in any other software. +If modifications were made using Inkscape, you should use [svgo](https://github.com/svg/svgo) ([Web UI](https://jakearchibald.github.io/svgomg), ensure `Multipass` is enabled) to produce an SVG compatible to be viewed in web browsers, which will replace then [`../sharkey.svg`](../sharkey.svg). This logo is distributed under the same licence as Sharkey (AGPL ≥3). diff --git a/packages/frontend/assets/sharkey-text-src/sharkey_v2.svg b/packages/frontend/assets/sharkey-text-src/sharkey.svg similarity index 100% rename from packages/frontend/assets/sharkey-text-src/sharkey_v2.svg rename to packages/frontend/assets/sharkey-text-src/sharkey.svg From 951cef82a775090694712cc7a3492949d6b5c912 Mon Sep 17 00:00:00 2001 From: Ruben Date: Fri, 9 May 2025 02:38:20 -0500 Subject: [PATCH 71/72] expand inkscape instructions --- packages/frontend/assets/sharkey-text-src/README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/frontend/assets/sharkey-text-src/README.md b/packages/frontend/assets/sharkey-text-src/README.md index d88aeddb6f..061185bed6 100644 --- a/packages/frontend/assets/sharkey-text-src/README.md +++ b/packages/frontend/assets/sharkey-text-src/README.md @@ -4,7 +4,12 @@ This logo was made in Inkscape by [@sneexy@booping.synth.download](https://boopi The font used is Lilita One ([Google Fonts](https://fonts.google.com/specimen/Lilita+One)). -The logo should preferably be modified using Inkscape, but should work in any other software. -If modifications were made using Inkscape, you should use [svgo](https://github.com/svg/svgo) ([Web UI](https://jakearchibald.github.io/svgomg), ensure `Multipass` is enabled) to produce an SVG compatible to be viewed in web browsers, which will replace then [`../sharkey.svg`](../sharkey.svg). +## Using Inkscape + +Preferably, you should use Inkscape when wanting to make modifications to the logo. Once you've made your edits, save the SVG here as is from Inkscape directly, then use [svgo](https://github.com/svg/svgo) ([Web UI](https://jakearchibald.github.io/svgomg), ensure `Multipass` is enabled) to produce an SVG compatible to be viewed in web browsers, which that file will then replace [`../sharkey.svg`](../sharkey.svg). + +## Not using Inkscape + +The logo should preferably be modified using Inkscape, but should work in any other software. If any weird issues appear, try opening it with Inkscape first or try using the version under [`../sharkey.svg`](../sharkey.svg) if you prefer to work with your software. Follow the instructions for Using Inkscape. This logo is distributed under the same licence as Sharkey (AGPL ≥3). From d5253eb2a394673bac9652541d1096b4be723089 Mon Sep 17 00:00:00 2001 From: Ruben Date: Fri, 9 May 2025 02:39:49 -0500 Subject: [PATCH 72/72] licence should be it's own header --- packages/frontend/assets/sharkey-text-src/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/frontend/assets/sharkey-text-src/README.md b/packages/frontend/assets/sharkey-text-src/README.md index 061185bed6..f153ec42e3 100644 --- a/packages/frontend/assets/sharkey-text-src/README.md +++ b/packages/frontend/assets/sharkey-text-src/README.md @@ -12,4 +12,6 @@ Preferably, you should use Inkscape when wanting to make modifications to the lo The logo should preferably be modified using Inkscape, but should work in any other software. If any weird issues appear, try opening it with Inkscape first or try using the version under [`../sharkey.svg`](../sharkey.svg) if you prefer to work with your software. Follow the instructions for Using Inkscape. +## Licence + This logo is distributed under the same licence as Sharkey (AGPL ≥3).