misty.cafe/mistykey
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Merge branch 'develop' into 'nodeinfostats'

Browse source 
# Conflicts:
#   packages/backend/src/server/NodeinfoServerService.ts
This commit is contained in:
Marie 2025-08-16 08:18:54 +00:00
commit 239a4a7a7b
1760 changed files with 69477 additions and 49515 deletions
Download patch file Download diff file Expand all files Collapse all files

2
packages/backend/.swcrc
View file

@ -1,5 +1,5 @@
{
"$schema": "https://json.schemastore.org/swcrc",
"$schema": "https://swc.rs/schema.json",
"jsc": {
"parser": {
"syntax": "typescript",

16
packages/backend/eslint.config.js
View file

@ -1,12 +1,19 @@
import tsParser from '@typescript-eslint/parser';
import sharedConfig from '../shared/eslint.config.js';
import globals from 'globals';
import sharedConfig from '../shared/eslint.config.js';
export default [
...sharedConfig,
{
ignores: ['**/node_modules', 'built', '@types/**/*', 'migration'],
},
{
languageOptions: {
globals: {
...globals.node,
},
},
},
{
files: ['**/*.ts', '**/*.tsx'],
languageOptions: {
@ -42,13 +49,6 @@ export default [
name: '__filename',
message: 'Not in ESModule. Use `import.meta.url` instead.',
}],
// https://typescript-eslint.io/rules/prefer-nullish-coalescing/
'@typescript-eslint/prefer-nullish-coalescing': ['warn', {
ignorePrimitives: {
// Without this, the rule breaks for nullable booleans
boolean: true,
},
}],
},
},
{

31
packages/backend/jest.js Normal file
View file

@ -0,0 +1,31 @@
#!/usr/bin/env node
import child_process from 'node:child_process';
import path from 'node:path';
import url from 'node:url';
import semver from 'semver';
const __filename = url.fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const args = [];
args.push(...[
...semver.satisfies(process.version, '^20.17.0 || ^22.0.0') ? ['--no-experimental-require-module'] : [],
'--experimental-vm-modules',
'--experimental-import-meta-resolve',
path.join(__dirname, 'node_modules/jest/bin/jest.js'),
...process.argv.slice(2),
]);
const child = child_process.spawn(process.execPath, args, { stdio: 'inherit' });
child.on('error', (err) => {
console.error('Failed to start Jest:', err);
process.exit(1);
});
child.on('exit', (code, signal) => {
if (code === null) {
process.exit(128 + signal);
} else {
process.exit(code);
}
});

16
packages/backend/migration/1736230492103-addAntennaHideNotesInSensitiveChannel.js Normal file
View file

@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class AddAntennaHideNotesInSensitiveChannel1736230492103 {
name = 'AddAntennaHideNotesInSensitiveChannel1736230492103'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "antenna" ADD "hideNotesInSensitiveChannel" boolean NOT NULL DEFAULT false`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "antenna" DROP COLUMN "hideNotesInSensitiveChannel"`);
}
}

37
packages/backend/migration/1740121393164-system-accounts.js Normal file
View file

@ -0,0 +1,37 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class SystemAccounts1740121393164 {
name = 'SystemAccounts1740121393164'
async up(queryRunner) {
await queryRunner.query(`CREATE TABLE "system_account" ("id" character varying(32) NOT NULL, "userId" character varying(32) NOT NULL, "type" character varying(256) NOT NULL, CONSTRAINT "PK_edb56f4aaf9ddd50ee556da97ba" PRIMARY KEY ("id"))`);
await queryRunner.query(`CREATE INDEX "IDX_41a3c87a37aea616ee459369e1" ON "system_account" ("userId") `);
await queryRunner.query(`CREATE UNIQUE INDEX "IDX_c362033aee0ea51011386a5a7e" ON "system_account" ("type") `);
await queryRunner.query(`ALTER TABLE "system_account" ADD CONSTRAINT "FK_41a3c87a37aea616ee459369e12" FOREIGN KEY ("userId") REFERENCES "user"("id") ON DELETE CASCADE ON UPDATE NO ACTION`);
const instanceActor = await queryRunner.query(`SELECT "id" FROM "user" WHERE "username" = 'instance.actor'`);
if (instanceActor.length > 0) {
await queryRunner.query(`INSERT INTO "system_account" ("id", "userId", "type") VALUES ('${instanceActor[0].id}', '${instanceActor[0].id}', 'actor')`);
}
const relayActor = await queryRunner.query(`SELECT "id" FROM "user" WHERE "username" = 'relay.actor'`);
if (relayActor.length > 0) {
await queryRunner.query(`INSERT INTO "system_account" ("id", "userId", "type") VALUES ('${relayActor[0].id}', '${relayActor[0].id}', 'relay')`);
}
const meta = await queryRunner.query(`SELECT "proxyAccountId" FROM "meta" ORDER BY "id" DESC LIMIT 1`);
if (!meta && meta.length >= 1 && meta[0].proxyAccountId) {
await queryRunner.query(`INSERT INTO "system_account" ("id", "userId", "type") VALUES ('${meta[0].proxyAccountId}', '${meta[0].proxyAccountId}', 'proxy')`);
}
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "system_account" DROP CONSTRAINT "FK_41a3c87a37aea616ee459369e12"`);
await queryRunner.query(`DROP INDEX "public"."IDX_c362033aee0ea51011386a5a7e"`);
await queryRunner.query(`DROP INDEX "public"."IDX_41a3c87a37aea616ee459369e1"`);
await queryRunner.query(`DROP TABLE "system_account"`);
}
}

22
packages/backend/migration/1740129169650-system-accounts-2.js Normal file
View file

@ -0,0 +1,22 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class SystemAccounts21740129169650 {
name = 'SystemAccounts21740129169650'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" DROP CONSTRAINT "FK_ab1bc0c1e209daa77b8e8d212ad"`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "proxyAccountId"`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" ADD "proxyAccountId" character varying(32)`);
const proxyAccountId = await queryRunner.query(`SELECT "userId" FROM "system_account" WHERE "type" = 'proxy' ORDER BY "id" DESC LIMIT 1`);
if (proxyAccountId && proxyAccountId.length >= 1) {
await queryRunner.query(`UPDATE "meta" SET "proxyAccountId" = '${proxyAccountId[0].userId}'`);
}
await queryRunner.query(`ALTER TABLE "meta" ADD CONSTRAINT "FK_ab1bc0c1e209daa77b8e8d212ad" FOREIGN KEY ("proxyAccountId") REFERENCES "user"("id") ON DELETE SET NULL ON UPDATE NO ACTION`);
}
}

23
packages/backend/migration/1740133121105-system-accounts-3.js Normal file
View file

@ -0,0 +1,23 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class SystemAccounts31740133121105 {
name = 'SystemAccounts31740133121105'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" ADD "rootUserId" character varying(32)`);
await queryRunner.query(`ALTER TABLE "meta" ADD CONSTRAINT "FK_c80e4079d632f95eac06a9d28cc" FOREIGN KEY ("rootUserId") REFERENCES "user"("id") ON DELETE SET NULL ON UPDATE NO ACTION`);
const users = await queryRunner.query(`SELECT "id" FROM "user" WHERE "isRoot" = true LIMIT 1`);
if (users.length > 0) {
await queryRunner.query(`UPDATE "meta" SET "rootUserId" = $1`, [users[0].id]);
}
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" DROP CONSTRAINT "FK_c80e4079d632f95eac06a9d28cc"`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "rootUserId"`);
}
}

35
packages/backend/migration/1740162088574-add_unsignedFetch.js Normal file
View file

@ -0,0 +1,35 @@
import { loadConfig } from '../built/config.js';
export class AddUnsignedFetch1740162088574 {
name = 'AddUnsignedFetch1740162088574'
async up(queryRunner) {
// meta.allowUnsignedFetch
await queryRunner.query(`CREATE TYPE "public"."meta_allowunsignedfetch_enum" AS ENUM('never', 'always', 'essential')`);
await queryRunner.query(`ALTER TABLE "meta" ADD "allowUnsignedFetch" "public"."meta_allowunsignedfetch_enum" NOT NULL DEFAULT 'always'`);
// user.allowUnsignedFetch
await queryRunner.query(`CREATE TYPE "public"."user_allowunsignedfetch_enum" AS ENUM('never', 'always', 'essential', 'staff')`);
await queryRunner.query(`ALTER TABLE "user" ADD "allowUnsignedFetch" "public"."user_allowunsignedfetch_enum" NOT NULL DEFAULT 'staff'`);
// Special one-time migration: allow unauthorized fetch for system accounts
await queryRunner.query(`UPDATE "user" SET "allowUnsignedFetch" = 'always' WHERE "username" LIKE '%.%' AND "host" IS null`);
// Special one-time migration: convert legacy config "" to meta setting ""
const config = await loadConfig();
if (config.checkActivityPubGetSignature) {
// noinspection SqlWithoutWhere
await queryRunner.query(`UPDATE "meta" SET "allowUnsignedFetch" = 'never'`);
}
}
async down(queryRunner) {
// user.allowUnsignedFetch
await queryRunner.query(`ALTER TABLE "user" DROP COLUMN "allowUnsignedFetch"`);
await queryRunner.query(`DROP TYPE "public"."user_allowunsignedfetch_enum"`);
// meta.allowUnsignedFetch
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "allowUnsignedFetch"`);
await queryRunner.query(`DROP TYPE "public"."meta_allowunsignedfetch_enum"`);
}
}

17
packages/backend/migration/1740993126937-system-accounts-4.js Normal file
View file

@ -0,0 +1,17 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class SystemAccounts41740993126937 {
name = 'SystemAccounts41740993126937'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "user" DROP COLUMN "isRoot"`);
}
async down(queryRunner) {
// down 実行時は isRoot = true のユーザーが存在しなくなるため手動で対応する必要あり
await queryRunner.query(`ALTER TABLE "user" ADD "isRoot" boolean NOT NULL DEFAULT false`);
}
}

18
packages/backend/migration/1741215877000-libetranslate.js Normal file
View file

@ -0,0 +1,18 @@
/*
* SPDX-FileCopyrightText: marie and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class Libetranslate1741215877000 {
name = 'Libretranslate1741215877000';
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" ADD "libreTranslateURL" character varying(1024)`);
await queryRunner.query(`ALTER TABLE "meta" ADD "libreTranslateKey" character varying(1024)`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "libreTranslateURL"`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "libreTranslateKey"`);
}
}

26
packages/backend/migration/1741279404074-system-accounts-fixup.js Normal file
View file

@ -0,0 +1,26 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class SystemAccounts1741279404074 {
name = 'SystemAccounts1741279404074'
async up(queryRunner) {
const instanceActor = await queryRunner.query(`SELECT "id" FROM "user" WHERE "username" = 'instance.actor' AND "host" IS NULL AND "id" NOT IN (SELECT "userId" FROM "system_account" WHERE "type" = 'actor')`);
if (instanceActor.length > 0) {
console.warn('instance.actor was incorrect, updating...');
await queryRunner.query(`UPDATE "system_account" SET "id" = '${instanceActor[0].id}', "userId" = '${instanceActor[0].id}' WHERE "type" = 'actor'`);
}
const relayActor = await queryRunner.query(`SELECT "id" FROM "user" WHERE "username" = 'relay.actor' AND "host" IS NULL AND "id" NOT IN (SELECT "userId" FROM "system_account" WHERE "type" = 'relay')`);
if (relayActor.length > 0) {
console.warn('relay.actor was incorrect, updating...');
await queryRunner.query(`UPDATE "system_account" SET "id" = '${relayActor[0].id}', "userId" = '${relayActor[0].id}' WHERE "type" = 'relay'`);
}
}
async down(queryRunner) {
// fixup migration, no down migration
}
}

26
packages/backend/migration/1741424411879-user-featured-fixup.js Normal file
View file

@ -0,0 +1,26 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class UserFeaturedFixup1741424411879 {
name = 'UserFeaturedFixup1741424411879'
async up(queryRunner) {
await queryRunner.query(`CREATE OR REPLACE FUNCTION pg_temp.extract_ap_id(text) RETURNS text AS $$
SELECT
CASE
WHEN $1 ~ '^https?://' THEN $1
WHEN $1 LIKE '{%' THEN COALESCE(jsonb_extract_path_text($1::jsonb, 'id'), null)
ELSE null
END;
$$ LANGUAGE sql IMMUTABLE;`);
// "host" is NOT NULL is not needed but just in case add it to prevent overwriting irreplaceable data
await queryRunner.query(`UPDATE "user" SET "featured" = pg_temp.extract_ap_id("featured") WHERE "host" IS NOT NULL`);
}
async down(queryRunner) {
// fixup migration, no down migration
}
}

46
packages/backend/migration/1742203321812-chat.js Normal file
View file

@ -0,0 +1,46 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class Chat1742203321812 {
name = 'Chat1742203321812'
async up(queryRunner) {
await queryRunner.query(`CREATE TABLE "chat_room" ("id" character varying(32) NOT NULL, "name" character varying(256) NOT NULL, "ownerId" character varying(32) NOT NULL, CONSTRAINT "PK_8aa3a52cf74c96469f0ef9fbe3e" PRIMARY KEY ("id"))`);
await queryRunner.query(`CREATE INDEX "IDX_f0d8ad64243fa2ca2800da0dfd" ON "chat_room" ("ownerId") `);
await queryRunner.query(`CREATE TABLE "chat_message" ("id" character varying(32) NOT NULL, "fromUserId" character varying(32) NOT NULL, "toUserId" character varying(32), "toRoomId" character varying(32), "text" character varying(4096), "uri" character varying(512), "reads" character varying(32) array NOT NULL DEFAULT '{}', "fileId" character varying(32), "reactions" character varying(1024) array NOT NULL DEFAULT '{}', CONSTRAINT "PK_3cc0d85193aade457d3077dd06b" PRIMARY KEY ("id"))`);
await queryRunner.query(`CREATE INDEX "IDX_79a26e7a4d9afa5e4fc05f134e" ON "chat_message" ("fromUserId") `);
await queryRunner.query(`CREATE INDEX "IDX_25e097b51d7622c249452c6f75" ON "chat_message" ("toUserId") `);
await queryRunner.query(`CREATE INDEX "IDX_f006b8a76efd1abf9f221c175c" ON "chat_message" ("toRoomId") `);
await queryRunner.query(`CREATE TABLE "chat_room_membership" ("id" character varying(32) NOT NULL, "userId" character varying(32) NOT NULL, "roomId" character varying(32) NOT NULL, CONSTRAINT "PK_2bd59c741e571b283c048beb69a" PRIMARY KEY ("id"))`);
await queryRunner.query(`CREATE INDEX "IDX_d99c5279460fb77ef58c596ce5" ON "chat_room_membership" ("userId") `);
await queryRunner.query(`CREATE INDEX "IDX_c25143ebab714e930aeca1c0e8" ON "chat_room_membership" ("roomId") `);
await queryRunner.query(`ALTER TABLE "chat_room" ADD CONSTRAINT "FK_f0d8ad64243fa2ca2800da0dfd6" FOREIGN KEY ("ownerId") REFERENCES "user"("id") ON DELETE CASCADE ON UPDATE NO ACTION`);
await queryRunner.query(`ALTER TABLE "chat_message" ADD CONSTRAINT "FK_79a26e7a4d9afa5e4fc05f134ed" FOREIGN KEY ("fromUserId") REFERENCES "user"("id") ON DELETE CASCADE ON UPDATE NO ACTION`);
await queryRunner.query(`ALTER TABLE "chat_message" ADD CONSTRAINT "FK_25e097b51d7622c249452c6f757" FOREIGN KEY ("toUserId") REFERENCES "user"("id") ON DELETE CASCADE ON UPDATE NO ACTION`);
await queryRunner.query(`ALTER TABLE "chat_message" ADD CONSTRAINT "FK_f006b8a76efd1abf9f221c175ce" FOREIGN KEY ("toRoomId") REFERENCES "chat_room"("id") ON DELETE CASCADE ON UPDATE NO ACTION`);
await queryRunner.query(`ALTER TABLE "chat_message" ADD CONSTRAINT "FK_fd0f9a4879430239715ad4f8e2a" FOREIGN KEY ("fileId") REFERENCES "drive_file"("id") ON DELETE SET NULL ON UPDATE NO ACTION`);
await queryRunner.query(`ALTER TABLE "chat_room_membership" ADD CONSTRAINT "FK_d99c5279460fb77ef58c596ce51" FOREIGN KEY ("userId") REFERENCES "user"("id") ON DELETE CASCADE ON UPDATE NO ACTION`);
await queryRunner.query(`ALTER TABLE "chat_room_membership" ADD CONSTRAINT "FK_c25143ebab714e930aeca1c0e8d" FOREIGN KEY ("roomId") REFERENCES "chat_room"("id") ON DELETE CASCADE ON UPDATE NO ACTION`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "chat_room_membership" DROP CONSTRAINT "FK_c25143ebab714e930aeca1c0e8d"`);
await queryRunner.query(`ALTER TABLE "chat_room_membership" DROP CONSTRAINT "FK_d99c5279460fb77ef58c596ce51"`);
await queryRunner.query(`ALTER TABLE "chat_message" DROP CONSTRAINT "FK_fd0f9a4879430239715ad4f8e2a"`);
await queryRunner.query(`ALTER TABLE "chat_message" DROP CONSTRAINT "FK_f006b8a76efd1abf9f221c175ce"`);
await queryRunner.query(`ALTER TABLE "chat_message" DROP CONSTRAINT "FK_25e097b51d7622c249452c6f757"`);
await queryRunner.query(`ALTER TABLE "chat_message" DROP CONSTRAINT "FK_79a26e7a4d9afa5e4fc05f134ed"`);
await queryRunner.query(`ALTER TABLE "chat_room" DROP CONSTRAINT "FK_f0d8ad64243fa2ca2800da0dfd6"`);
await queryRunner.query(`DROP INDEX "public"."IDX_c25143ebab714e930aeca1c0e8"`);
await queryRunner.query(`DROP INDEX "public"."IDX_d99c5279460fb77ef58c596ce5"`);
await queryRunner.query(`DROP TABLE "chat_room_membership"`);
await queryRunner.query(`DROP INDEX "public"."IDX_f006b8a76efd1abf9f221c175c"`);
await queryRunner.query(`DROP INDEX "public"."IDX_25e097b51d7622c249452c6f75"`);
await queryRunner.query(`DROP INDEX "public"."IDX_79a26e7a4d9afa5e4fc05f134e"`);
await queryRunner.query(`DROP TABLE "chat_message"`);
await queryRunner.query(`DROP INDEX "public"."IDX_f0d8ad64243fa2ca2800da0dfd"`);
await queryRunner.query(`DROP TABLE "chat_room"`);
}
}

18
packages/backend/migration/1742608337548-chat-2.js Normal file
View file

@ -0,0 +1,18 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class Chat21742608337548 {
name = 'Chat21742608337548'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "user" ADD "chatScope" character varying(128) NOT NULL DEFAULT 'mutual'`);
await queryRunner.query(`CREATE UNIQUE INDEX "IDX_185b6b5afa707b5d36d1ce3144" ON "chat_room_membership" ("userId", "roomId") `);
}
async down(queryRunner) {
await queryRunner.query(`DROP INDEX "public"."IDX_185b6b5afa707b5d36d1ce3144"`);
await queryRunner.query(`ALTER TABLE "user" DROP COLUMN "chatScope"`);
}
}

26
packages/backend/migration/1742617546147-chat-3.js Normal file
View file

@ -0,0 +1,26 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class Chat31742617546147 {
name = 'Chat31742617546147'
async up(queryRunner) {
await queryRunner.query(`CREATE TABLE "chat_approval" ("id" character varying(32) NOT NULL, "userId" character varying(32) NOT NULL, "otherId" character varying(32) NOT NULL, CONSTRAINT "PK_fbbb95d60acf5c85388345b5f5d" PRIMARY KEY ("id"))`);
await queryRunner.query(`CREATE INDEX "IDX_530257863e1381a7f2f1d3282f" ON "chat_approval" ("userId") `);
await queryRunner.query(`CREATE INDEX "IDX_b1d46037f23d170da5c05fdf75" ON "chat_approval" ("otherId") `);
await queryRunner.query(`CREATE UNIQUE INDEX "IDX_12c4768a2f706fc267f2078903" ON "chat_approval" ("userId", "otherId") `);
await queryRunner.query(`ALTER TABLE "chat_approval" ADD CONSTRAINT "FK_530257863e1381a7f2f1d3282fe" FOREIGN KEY ("userId") REFERENCES "user"("id") ON DELETE CASCADE ON UPDATE NO ACTION`);
await queryRunner.query(`ALTER TABLE "chat_approval" ADD CONSTRAINT "FK_b1d46037f23d170da5c05fdf755" FOREIGN KEY ("otherId") REFERENCES "user"("id") ON DELETE CASCADE ON UPDATE NO ACTION`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "chat_approval" DROP CONSTRAINT "FK_b1d46037f23d170da5c05fdf755"`);
await queryRunner.query(`ALTER TABLE "chat_approval" DROP CONSTRAINT "FK_530257863e1381a7f2f1d3282fe"`);
await queryRunner.query(`DROP INDEX "public"."IDX_12c4768a2f706fc267f2078903"`);
await queryRunner.query(`DROP INDEX "public"."IDX_b1d46037f23d170da5c05fdf75"`);
await queryRunner.query(`DROP INDEX "public"."IDX_530257863e1381a7f2f1d3282f"`);
await queryRunner.query(`DROP TABLE "chat_approval"`);
}
}

26
packages/backend/migration/1742707840715-chat-4.js Normal file
View file

@ -0,0 +1,26 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class Chat41742707840715 {
name = 'Chat41742707840715'
async up(queryRunner) {
await queryRunner.query(`CREATE TABLE "chat_room_invitation" ("id" character varying(32) NOT NULL, "userId" character varying(32) NOT NULL, "roomId" character varying(32) NOT NULL, CONSTRAINT "PK_9d489521a312dd28225672de2dc" PRIMARY KEY ("id"))`);
await queryRunner.query(`CREATE INDEX "IDX_8552bb38e7ed038c5bdd398a38" ON "chat_room_invitation" ("userId") `);
await queryRunner.query(`CREATE INDEX "IDX_5f265075b215fc390a57523b12" ON "chat_room_invitation" ("roomId") `);
await queryRunner.query(`CREATE UNIQUE INDEX "IDX_044f2a7962b8ee5bbfaa02e8a3" ON "chat_room_invitation" ("userId", "roomId") `);
await queryRunner.query(`ALTER TABLE "chat_room_invitation" ADD CONSTRAINT "FK_8552bb38e7ed038c5bdd398a384" FOREIGN KEY ("userId") REFERENCES "user"("id") ON DELETE CASCADE ON UPDATE NO ACTION`);
await queryRunner.query(`ALTER TABLE "chat_room_invitation" ADD CONSTRAINT "FK_5f265075b215fc390a57523b12a" FOREIGN KEY ("roomId") REFERENCES "chat_room"("id") ON DELETE CASCADE ON UPDATE NO ACTION`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "chat_room_invitation" DROP CONSTRAINT "FK_5f265075b215fc390a57523b12a"`);
await queryRunner.query(`ALTER TABLE "chat_room_invitation" DROP CONSTRAINT "FK_8552bb38e7ed038c5bdd398a384"`);
await queryRunner.query(`DROP INDEX "public"."IDX_044f2a7962b8ee5bbfaa02e8a3"`);
await queryRunner.query(`DROP INDEX "public"."IDX_5f265075b215fc390a57523b12"`);
await queryRunner.query(`DROP INDEX "public"."IDX_8552bb38e7ed038c5bdd398a38"`);
await queryRunner.query(`DROP TABLE "chat_room_invitation"`);
}
}

16
packages/backend/migration/1742721896936-chat-5.js Normal file
View file

@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class Chat51742721896936 {
name = 'Chat51742721896936'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "chat_room_invitation" ADD "ignored" boolean NOT NULL DEFAULT false`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "chat_room_invitation" DROP COLUMN "ignored"`);
}
}

20
packages/backend/migration/1742795111958-chat-6.js Normal file
View file

@ -0,0 +1,20 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class Chat61742795111958 {
name = 'Chat61742795111958'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "chat_room" ADD "description" character varying(2048) NOT NULL DEFAULT ''`);
await queryRunner.query(`ALTER TABLE "chat_room" ADD "isArchived" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`ALTER TABLE "chat_room_membership" ADD "isMuted" boolean NOT NULL DEFAULT false`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "chat_room_membership" DROP COLUMN "isMuted"`);
await queryRunner.query(`ALTER TABLE "chat_room" DROP COLUMN "isArchived"`);
await queryRunner.query(`ALTER TABLE "chat_room" DROP COLUMN "description"`);
}
}

16
packages/backend/migration/1743403874305-DeliverSuspendedSoftware.js Normal file
View file

@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class DeliverSuspendedSoftware1743403874305 {
name = 'DeliverSuspendedSoftware1743403874305'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" ADD "deliverSuspendedSoftware" jsonb NOT NULL DEFAULT '[]'`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "deliverSuspendedSoftware"`);
}
}

16
packages/backend/migration/1743558299182-RoleCopyOnMoveAccount.js Normal file
View file

@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class RoleCopyOnMoveAccount1743558299182 {
name = 'RoleCopyOnMoveAccount1743558299182'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "role" ADD "preserveAssignmentOnMoveAccount" boolean NOT NULL DEFAULT false`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "role" DROP COLUMN "preserveAssignmentOnMoveAccount"`);
}
}

16
packages/backend/migration/1744075766000-excludeNotesInSensitiveChannel.js Normal file
View file

@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class ExcludeNotesInSensitiveChannel1744075766000 {
name = 'ExcludeNotesInSensitiveChannel1744075766000'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "antenna" RENAME COLUMN "hideNotesInSensitiveChannel" TO "excludeNotesInSensitiveChannel"`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "antenna" RENAME COLUMN "excludeNotesInSensitiveChannel" TO "hideNotesInSensitiveChannel"`);
}
}

36
packages/backend/migration/1745378064470-composite-note-index.js Normal file
View file

@ -0,0 +1,36 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { isConcurrentIndexMigrationEnabled } from "./js/migration-config.js";
export class CompositeNoteIndex1745378064470 {
name = 'CompositeNoteIndex1745378064470';
transaction = isConcurrentIndexMigrationEnabled() ? false : undefined;
async up(queryRunner) {
const concurrently = isConcurrentIndexMigrationEnabled();
if (concurrently) {
const hasValidIndex = await queryRunner.query(`SELECT indisvalid FROM pg_index INNER JOIN pg_class ON pg_index.indexrelid = pg_class.oid WHERE pg_class.relname = 'IDX_724b311e6f883751f261ebe378'`);
if (hasValidIndex.length === 0 || hasValidIndex[0].indisvalid !== true) {
await queryRunner.query(`DROP INDEX IF EXISTS "IDX_724b311e6f883751f261ebe378"`);
await queryRunner.query(`CREATE INDEX CONCURRENTLY "IDX_724b311e6f883751f261ebe378" ON "note" ("userId", "id" DESC)`);
}
} else {
await queryRunner.query(`CREATE INDEX IF NOT EXISTS "IDX_724b311e6f883751f261ebe378" ON "note" ("userId", "id" DESC)`);
}
await queryRunner.query(`DROP INDEX IF EXISTS "IDX_5b87d9d19127bd5d92026017a7"`);
// Flush all cached Linear Scan Plans and redo statistics for composite index
// this is important for Postgres to learn that even in highly complex queries, using this index first can reduce the result set significantly
await queryRunner.query(`ANALYZE "user", "note"`);
}
async down(queryRunner) {
const mayConcurrently = isConcurrentIndexMigrationEnabled() ? 'CONCURRENTLY' : '';
await queryRunner.query(`DROP INDEX IF EXISTS "IDX_724b311e6f883751f261ebe378"`);
await queryRunner.query(`CREATE INDEX ${mayConcurrently} "IDX_5b87d9d19127bd5d92026017a7" ON "note" ("userId")`);
}
}

16
packages/backend/migration/1746029830779-add_meta-enableProxyAccount.js Normal file
View file

@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class AddMetaEnableProxyAccount1746029830779 {
name = 'AddMetaEnableProxyAccount1746029830779'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" ADD "enableProxyAccount" boolean NOT NULL DEFAULT false`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "enableProxyAccount"`);
}
}

13
packages/backend/migration/1746813431123756-user-distinct-null.js Normal file
View file

@ -0,0 +1,13 @@
export class IndexUserNullDistinct1746813431756 {
name = 'Indexusernulldistinct1746813431756'
async up(queryRunner) {
await queryRunner.query(`DROP INDEX IF EXISTS "IDX_5deb01ae162d1d70b80d064c27"`);
await queryRunner.query(`CREATE UNIQUE INDEX "IDX_5deb01ae162d1d70b80d064c27" ON "user" ("usernameLower", "host") NULLS NOT DISTINCT`);
}
async down(queryRunner) {
await queryRunner.query(`DROP INDEX IF EXISTS "IDX_5deb01ae162d1d70b80d064c27"`);
await queryRunner.query(`CREATE UNIQUE INDEX "IDX_5deb01ae162d1d70b80d064c27" ON "user" ("usernameLower", "host") `);
}
}

18
packages/backend/migration/1747023091463-add_meta_translationTimeout.js Normal file
View file

@ -0,0 +1,18 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class AddMetaTranslationTimeout1747023091463 {
name = 'AddMetaTranslationTimeout1747023091463'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" ADD "translationTimeout" integer NOT NULL DEFAULT '5000'`);
await queryRunner.query(`COMMENT ON COLUMN "meta"."translationTimeout" IS 'Timeout in milliseconds for translation API requests'`);
}
async down(queryRunner) {
await queryRunner.query(`COMMENT ON COLUMN "meta"."translationTimeout" IS 'Timeout in milliseconds for translation API requests'`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "translationTimeout"`);
}
}

16
packages/backend/migration/1747934911491-rename_followingVisibility.js Normal file
View file

@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class RenameFollowingVisibility1747934911491 {
name = 'RenameFollowingVisibility1747934911491'
async up(queryRunner) {
await queryRunner.query(`ALTER TYPE "public"."user_profile_followingvisibility_enum" RENAME TO "user_profile_followingVisibility_enum"`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TYPE "public"."user_profile_followingVisibility_enum" RENAME TO "user_profile_followingvisibility_enum"`);
}
}

30
packages/backend/migration/1747935197708-add_entity_comments.js Normal file
View file

@ -0,0 +1,30 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class AddEntityComments1747935197708 {
name = 'AddEntityComments1747935197708'
async up(queryRunner) {
await queryRunner.query(`COMMENT ON COLUMN "user"."backgroundId" IS 'The ID of background DriveFile.'`);
await queryRunner.query(`COMMENT ON COLUMN "user"."isSilenced" IS 'Whether the User is silenced.'`);
await queryRunner.query(`COMMENT ON COLUMN "user"."noindex" IS 'Whether the User''s notes dont get indexed.'`);
await queryRunner.query(`COMMENT ON COLUMN "user"."speakAsCat" IS 'Whether the User speaks in nya.'`);
await queryRunner.query(`COMMENT ON COLUMN "user_profile"."listenbrainz" IS 'The ListenBrainz username of the User.'`);
await queryRunner.query(`COMMENT ON COLUMN "note"."updatedAt" IS 'The update time of the Note.'`);
await queryRunner.query(`COMMENT ON COLUMN "meta"."trustedLinkUrlPatterns" IS 'An array of URL strings or regex that can be used to omit warnings about redirects to external sites. Separate them with spaces to specify AND, and enclose them with slashes to specify regular expressions. Each item is regarded as an OR.'`);
await queryRunner.query(`COMMENT ON COLUMN "note_edit"."oldDate" IS 'The old date from before the edit'`);
}
async down(queryRunner) {
await queryRunner.query(`COMMENT ON COLUMN "note_edit"."oldDate" IS NULL`);
await queryRunner.query(`COMMENT ON COLUMN "meta"."trustedLinkUrlPatterns" IS NULL`);
await queryRunner.query(`COMMENT ON COLUMN "note"."updatedAt" IS 'The updated date of the Note.'`);
await queryRunner.query(`COMMENT ON COLUMN "user_profile"."listenbrainz" IS 'listenbrainz username to fetch currently playing.'`);
await queryRunner.query(`COMMENT ON COLUMN "user"."speakAsCat" IS 'Whether to speak as a cat if chosen.'`);
await queryRunner.query(`COMMENT ON COLUMN "user"."noindex" IS NULL`);
await queryRunner.query(`COMMENT ON COLUMN "user"."isSilenced" IS NULL`);
await queryRunner.query(`COMMENT ON COLUMN "user"."backgroundId" IS NULL`);
}
}

16
packages/backend/migration/1747937504140-fix-system_webhook-updatedAt-default.js Normal file
View file

@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class FixSystemWebhookUpdatedAtDefault1747937504140 {
name = 'FixSystemWebhookUpdatedAtDefault1747937504140'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "system_webhook" ALTER COLUMN "updatedAt" SET DEFAULT now()`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "system_webhook" ALTER COLUMN "updatedAt" SET DEFAULT CURRENT_TIMESTAMP`);
}
}

16
packages/backend/migration/1747937670341-fix-abuse_report_notification_recipient-updatedAt-default.js Normal file
View file

@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class FixAbuseReportNotificationRecipientUpdatedAtDefault1747937670341 {
name = 'FixAbuseReportNotificationRecipientUpdatedAtDefault1747937670341'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "abuse_report_notification_recipient" ALTER COLUMN "updatedAt" SET DEFAULT now()`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "abuse_report_notification_recipient" ALTER COLUMN "updatedAt" SET DEFAULT CURRENT_TIMESTAMP`);
}
}

16
packages/backend/migration/1747937796573-fix-flash-visibility-nullable.js Normal file
View file

@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class FixFlashVisibilityNullable1747937796573 {
name = 'FixFlashVisibilityNullable1747937796573'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "flash" ALTER COLUMN "visibility" SET NOT NULL`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "flash" ALTER COLUMN "visibility" DROP NOT NULL`);
}
}

18
packages/backend/migration/1747938136399-fix-abuse_report_notification_recipient-defaults.js Normal file
View file

@ -0,0 +1,18 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class FixAbuseReportNotificationRecipientDefaults1747938136399 {
name = 'FixAbuseReportNotificationRecipientDefaults1747938136399'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "abuse_report_notification_recipient" ALTER COLUMN "userId" DROP DEFAULT`);
await queryRunner.query(`ALTER TABLE "abuse_report_notification_recipient" ALTER COLUMN "systemWebhookId" DROP DEFAULT`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "abuse_report_notification_recipient" ALTER COLUMN "systemWebhookId" SET DEFAULT NULL`);
await queryRunner.query(`ALTER TABLE "abuse_report_notification_recipient" ALTER COLUMN "userId" SET DEFAULT NULL`);
}
}

16
packages/backend/migration/1747938263980-fix-meta-urlPreviewUserAgent-default.js Normal file
View file

@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class FixMetaUrlPreviewUserAgentDefault1747938263980 {
name = 'FixMetaUrlPreviewUserAgentDefault1747938263980'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" ALTER COLUMN "urlPreviewUserAgent" DROP DEFAULT`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" ALTER COLUMN "urlPreviewUserAgent" SET DEFAULT NULL`);
}
}

38
packages/backend/migration/1747938628395-add-missing-indexes.js Normal file
View file

@ -0,0 +1,38 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class AddMissingIndexes1747938628395 {
name = 'AddMissingIndexes1747938628395'
async up(queryRunner) {
// Some instances have duplicate list entries
await queryRunner.query(`
DELETE FROM "user_list_membership"
WHERE "id" NOT IN (
SELECT MIN("id")
FROM "user_list_membership"
GROUP BY "userId", "userListId"
)`);
// Some instances already have these indexes, for an unknown reason
await queryRunner.query(`DROP INDEX IF EXISTS "public"."IDX_e4f3094c43f2d665e6030b0337"`);
await queryRunner.query(`DROP INDEX IF EXISTS "public"."IDX_cddcaf418dc4d392ecfcca842a"`);
await queryRunner.query(`DROP INDEX IF EXISTS "public"."IDX_021015e6683570ae9f6b0c62be"`);
await queryRunner.query(`DROP INDEX IF EXISTS "public"."IDX_58699f75b9cf904f5f007909cb"`);
// Now the actual migration
await queryRunner.query(`CREATE INDEX "IDX_58699f75b9cf904f5f007909cb" ON "user_profile" ("birthday") `);
await queryRunner.query(`CREATE INDEX "IDX_021015e6683570ae9f6b0c62be" ON "user_list_membership" ("userId") `);
await queryRunner.query(`CREATE INDEX "IDX_cddcaf418dc4d392ecfcca842a" ON "user_list_membership" ("userListId") `);
await queryRunner.query(`CREATE UNIQUE INDEX "IDX_e4f3094c43f2d665e6030b0337" ON "user_list_membership" ("userId", "userListId") `);
}
async down(queryRunner) {
await queryRunner.query(`DROP INDEX "public"."IDX_e4f3094c43f2d665e6030b0337"`);
await queryRunner.query(`DROP INDEX "public"."IDX_cddcaf418dc4d392ecfcca842a"`);
await queryRunner.query(`DROP INDEX "public"."IDX_021015e6683570ae9f6b0c62be"`);
await queryRunner.query(`DROP INDEX "public"."IDX_58699f75b9cf904f5f007909cb"`);
}
}

16
packages/backend/migration/1747944466178-alter-meta-defaultLike-not-null.js Normal file
View file

@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class AlterMetaDefaultLikeNotNull1747944466178 {
name = 'AlterMetaDefaultLikeNotNull1747944466178'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" ALTER COLUMN "defaultLike" SET NOT NULL`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" ALTER COLUMN "defaultLike" DROP NOT NULL`);
}
}

16
packages/backend/migration/1748096357260-AddAttributionDomains.js Normal file
View file

@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: piuvas and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class AddAttributionDomains1748096357260 {
name = 'AddAttributionDomains1748096357260'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "user" ADD "attributionDomains" text array NOT NULL DEFAULT '{}'`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "user" DROP COLUMN "attributionDomains"`);
}
}

16
packages/backend/migration/1748104955717-index_IDX_instance_host_key.js Normal file
View file

@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class IndexIDXInstanceHostKey1748104955717 {
name = 'IndexIDXInstanceHostKey1748104955717'
async up(queryRunner) {
await queryRunner.query(`CREATE UNIQUE INDEX "IDX_instance_host_key" ON "instance" (((lower(reverse("host")) || '.')::text) text_pattern_ops)`);
}
async down(queryRunner) {
await queryRunner.query(`DROP INDEX "IDX_instance_host_key"`);
}
}

85
packages/backend/migration/1748105111513-add_instance_block_columns.js Normal file
View file

@ -0,0 +1,85 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
/**
* @typedef {import('typeorm').MigrationInterface} MigrationInterface
* @typedef {{ blockedHosts: string[], silencedHosts: string[], mediaSilencedHosts: string[], federationHosts: string[], bubbleInstances: string[] }} Meta
*/
/**
* @class
* @implements {MigrationInterface}
*/
export class AddInstanceBlockColumns1748105111513 {
name = 'AddInstanceBlockColumns1748105111513'
async up(queryRunner) {
// Schema migration
await queryRunner.query(`ALTER TABLE "instance" ADD "isBlocked" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`COMMENT ON COLUMN "instance"."isBlocked" IS 'True if this instance is blocked from federation.'`);
await queryRunner.query(`ALTER TABLE "instance" ADD "isAllowListed" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`COMMENT ON COLUMN "instance"."isAllowListed" IS 'True if this instance is allow-listed.'`);
await queryRunner.query(`ALTER TABLE "instance" ADD "isBubbled" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`COMMENT ON COLUMN "instance"."isBubbled" IS 'True if this instance is part of the local bubble.'`);
await queryRunner.query(`ALTER TABLE "instance" ADD "isSilenced" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`COMMENT ON COLUMN "instance"."isSilenced" IS 'True if this instance is silenced.'`);
await queryRunner.query(`ALTER TABLE "instance" ADD "isMediaSilenced" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`COMMENT ON COLUMN "instance"."isMediaSilenced" IS 'True if this instance is media-silenced.'`);
// Data migration
/** @type {Meta[]} */
const metas = await queryRunner.query(`SELECT "blockedHosts", "silencedHosts", "mediaSilencedHosts", "federationHosts", "bubbleInstances" FROM "meta"`);
if (metas.length > 0) {
/** @type {Meta} */
const meta = metas[0];
// Blocked hosts
if (meta.blockedHosts.length > 0) {
const patterns = buildPatterns(meta.blockedHosts);
await queryRunner.query(`UPDATE "instance" SET "isBlocked" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]);
}
// Silenced hosts
if (meta.silencedHosts.length > 0) {
const patterns = buildPatterns(meta.silencedHosts);
await queryRunner.query(`UPDATE "instance" SET "isSilenced" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]);
}
// Media silenced hosts
if (meta.mediaSilencedHosts.length > 0) {
const patterns = buildPatterns(meta.mediaSilencedHosts);
await queryRunner.query(`UPDATE "instance" SET "isMediaSilenced" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]);
}
// Allow-listed hosts
if (meta.federationHosts.length > 0) {
const patterns = buildPatterns(meta.federationHosts);
await queryRunner.query(`UPDATE "instance" SET "isAllowListed" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]);
}
// Bubbled hosts
if (meta.bubbleInstances.length > 0) {
const patterns = buildPatterns(meta.bubbleInstances);
await queryRunner.query(`UPDATE "instance" SET "isBubbled" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]);
}
}
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isMediaSilenced"`);
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isSilenced"`);
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isBubbled"`);
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isAllowListed"`);
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isBlocked"`);
}
}
/**
* @param {string[]} input
* @returns {string[]}
*/
function buildPatterns(input) {
return input.map(i => i.toLowerCase().split('').reverse().join('') + '.%');
}

44
packages/backend/migration/1748128176881-add_instance_foreign_keys.js Normal file
View file

@ -0,0 +1,44 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
/**
* @typedef {import('typeorm').MigrationInterface} MigrationInterface
*/
/**
* @class
* @implements {MigrationInterface}
*/
export class AddInstanceForeignKeys1748128176881 {
name = 'AddInstanceForeignKeys1748128176881'
async up(queryRunner) {
// Fix-up: Some older instances have users without a matching instance entry
await queryRunner.query(`
INSERT INTO "instance" ("id", "host", "firstRetrievedAt")
SELECT
MIN("id"),
"host",
COALESCE(MIN("lastFetchedAt"), CURRENT_TIMESTAMP)
FROM "user"
WHERE
"host" IS NOT NULL AND
NOT EXISTS (select 1 from "instance" where "instance"."host" = "user"."host")
GROUP BY "host"
`);
await queryRunner.query(`ALTER TABLE "user" ADD CONSTRAINT "FK_user_host" FOREIGN KEY ("host") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`);
await queryRunner.query(`ALTER TABLE "note" ADD CONSTRAINT "FK_note_userHost" FOREIGN KEY ("userHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`);
await queryRunner.query(`ALTER TABLE "note" ADD CONSTRAINT "FK_note_replyUserHost" FOREIGN KEY ("replyUserHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`);
await queryRunner.query(`ALTER TABLE "note" ADD CONSTRAINT "FK_note_renoteUserHost" FOREIGN KEY ("renoteUserHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "note" DROP CONSTRAINT "FK_note_renoteUserHost"`);
await queryRunner.query(`ALTER TABLE "note" DROP CONSTRAINT "FK_note_replyUserHost"`);
await queryRunner.query(`ALTER TABLE "note" DROP CONSTRAINT "FK_note_userHost"`);
await queryRunner.query(`ALTER TABLE "user" DROP CONSTRAINT "FK_user_host"`);
}
}

26
packages/backend/migration/1748137683887-add_instance_foreign_keys_to_following.js Normal file
View file

@ -0,0 +1,26 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
/**
* @typedef {import('typeorm').MigrationInterface} MigrationInterface
*/
/**
* @class
* @implements {MigrationInterface}
*/
export class AddInstanceForeignKeysToFollowing1748137683887 {
name = 'AddInstanceForeignKeysToFollowing1748137683887'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "following" ADD CONSTRAINT "FK_following_followerHost" FOREIGN KEY ("followerHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`);
await queryRunner.query(`ALTER TABLE "following" ADD CONSTRAINT "FK_following_followeeHost" FOREIGN KEY ("followeeHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "following" DROP CONSTRAINT "FK_following_followeeHost"`);
await queryRunner.query(`ALTER TABLE "following" DROP CONSTRAINT "FK_following_followerHost"`);
}
}

25
packages/backend/migration/1748191631151-analyze_instance-user-note-following.js Normal file
View file

@ -0,0 +1,25 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
/**
* @typedef {import('typeorm').MigrationInterface} MigrationInterface
*/
/**
* @class
* @implements {MigrationInterface}
*/
export class AnalyzeInstanceUserNoteFollowing1748191631151 {
name = 'AnalyzeInstanceUserNoteFollowing1748191631151'
async up(queryRunner) {
// Refresh statistics for tables impacted by new indexes.
// This helps the query planner to efficiently use them without waiting for the next full vacuum.
await queryRunner.query(`ANALYZE "instance", "user", "following", "note"`);
}
async down(queryRunner) {
}
}

22
packages/backend/migration/1748990452958-replace_note-userHost_index.js Normal file
View file

@ -0,0 +1,22 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class ReplaceNoteUserHostIndex1748990452958 {
name = 'ReplaceNoteUserHostIndex1748990452958'
async up(queryRunner) {
await queryRunner.query(`DROP INDEX "public"."IDX_7125a826ab192eb27e11d358a5"`);
await queryRunner.query(`
create index "IDX_note_userHost_id"
on "note" ("userHost", "id" desc)
nulls not distinct`);
await queryRunner.query(`comment on index "IDX_note_userHost_id" is 'User host with ID included'`);
}
async down(queryRunner) {
await queryRunner.query(`drop index if exists "IDX_note_userHost_id"`);
await queryRunner.query(`CREATE INDEX "IDX_7125a826ab192eb27e11d358a5" ON "note" ("userHost") `);
}
}

22
packages/backend/migration/1748990662839-fix-IDX_instance_host_key.js Normal file
View file

@ -0,0 +1,22 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class FixIDXInstanceHostKey1748990662839 {
async up(queryRunner) {
// must include host for index-only scans: https://www.postgresql.org/docs/current/indexes-index-only-scans.html
await queryRunner.query(`DROP INDEX "public"."IDX_instance_host_key"`);
await queryRunner.query(`
create index "IDX_instance_host_key"
on "instance" ((lower(reverse("host"::text)) || '.'::text) text_pattern_ops)
include ("host")
`);
await queryRunner.query(`comment on index "IDX_instance_host_key" is 'Expression index for finding instances by base domain'`);
}
async down(queryRunner) {
await queryRunner.query(`DROP INDEX "public"."IDX_instance_host_key"`);
await queryRunner.query(`CREATE UNIQUE INDEX "IDX_instance_host_key" ON "instance" (((lower(reverse("host")) || '.')::text) text_pattern_ops)`);
}
}

19
packages/backend/migration/1748991828473-create-IDX_note_for_timelines.js Normal file
View file

@ -0,0 +1,19 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class CreateIDXNoteForTimelines1748991828473 {
async up(queryRunner) {
await queryRunner.query(`
create index "IDX_note_for_timelines"
on "note" ("id" desc, "channelId", "visibility", "userHost")
include ("userId", "userHost", "replyId", "replyUserId", "replyUserHost", "renoteId", "renoteUserId", "renoteUserHost")
NULLS NOT DISTINCT`);
await queryRunner.query(`comment on index "IDX_note_for_timelines" is 'Covering index for timeline queries'`);
}
async down(queryRunner) {
await queryRunner.query(`DROP INDEX "IDX_note_for_timelines"`);
}
}

17
packages/backend/migration/1748992017688-create-IDX_instance_host_filters.js Normal file
View file

@ -0,0 +1,17 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class CreateIDXInstanceHostFilters1748992017688 {
async up(queryRunner) {
await queryRunner.query(`
create index "IDX_instance_host_filters"
on "instance" ("host", "isBlocked", "isSilenced", "isMediaSilenced", "isAllowListed", "isBubbled", "suspensionState")`);
await queryRunner.query(`comment on index "IDX_instance_host_filters" is 'Covering index for host filter queries'`);
}
async down(queryRunner) {
await queryRunner.query(`DROP INDEX "IDX_instance_host_filters"`);
}
}

28
packages/backend/migration/1748992128683-create-statistics.js Normal file
View file

@ -0,0 +1,28 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class CreateStatistics1748992128683 {
async up(queryRunner) {
await queryRunner.query(`CREATE STATISTICS "STTS_instance_isBlocked_isBubbled" (mcv) ON "isBlocked", "isBubbled" FROM "instance"`);
await queryRunner.query(`CREATE STATISTICS "STTS_instance_isBlocked_isSilenced" (mcv) ON "isBlocked", "isSilenced" FROM "instance"`);
await queryRunner.query(`CREATE STATISTICS "STTS_note_replyId_replyUserId_replyUserHost" (dependencies) ON "replyId", "replyUserId", "replyUserHost" FROM "note"`)
await queryRunner.query(`CREATE STATISTICS "STTS_note_renoteId_renoteUserId_renoteUserHost" (dependencies) ON "renoteId", "renoteUserId", "renoteUserHost" FROM "note"`);
await queryRunner.query(`CREATE STATISTICS "STTS_note_userId_userHost" (mcv) ON "userId", "userHost" FROM "note"`);
await queryRunner.query(`CREATE STATISTICS "STTS_note_replyUserId_replyUserHost" (mcv) ON "replyUserId", "replyUserHost" FROM "note"`);
await queryRunner.query(`CREATE STATISTICS "STTS_note_renoteUserId_renoteUserHost" (mcv) ON "renoteUserId", "renoteUserHost" FROM "note"`);
await queryRunner.query(`ANALYZE "note", "instance"`);
}
async down(queryRunner) {
await queryRunner.query(`DROP STATISTICS "STTS_instance_isBlocked_isBubbled"`);
await queryRunner.query(`DROP STATISTICS "STTS_instance_isBlocked_isSilenced"`);
await queryRunner.query(`DROP STATISTICS "STTS_note_replyId_replyUserId_replyUserHost"`);
await queryRunner.query(`DROP STATISTICS "STTS_note_renoteId_renoteUserId_renoteUserHost"`);
await queryRunner.query(`DROP STATISTICS "STTS_note_userId_userHost"`);
await queryRunner.query(`DROP STATISTICS "STTS_note_replyUserId_replyUserHost"`);
await queryRunner.query(`DROP STATISTICS "STTS_note_renoteUserId_renoteUserHost"`);
await queryRunner.query(`ANALYZE "note", "instance"`);
}
}

28
packages/backend/migration/1749097536193-fix-IDX_note_for_timeline.js Normal file
View file

@ -0,0 +1,28 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class FixIDXNoteForTimeline1749097536193 {
async up(queryRunner) {
await queryRunner.query('drop index "IDX_note_for_timelines"');
await queryRunner.query(`
create index "IDX_note_for_timelines"
on "note" ("id" desc, "channelId", "visibility", "userHost")
include ("userId", "replyId", "replyUserId", "replyUserHost", "renoteId", "renoteUserId", "renoteUserHost", "threadId")
NULLS NOT DISTINCT
`);
await queryRunner.query(`comment on index "IDX_note_for_timelines" is 'Covering index for timeline queries'`);
}
async down(queryRunner) {
await queryRunner.query('drop index "IDX_note_for_timelines"');
await queryRunner.query(`
create index "IDX_note_for_timelines"
on "note" ("id" desc, "channelId", "visibility", "userHost")
include ("userId", "userHost", "replyId", "replyUserId", "replyUserHost", "renoteId", "renoteUserId", "renoteUserHost")
NULLS NOT DISTINCT
`);
await queryRunner.query(`comment on index "IDX_note_for_timelines" is 'Covering index for timeline queries'`);
}
}

16
packages/backend/migration/1749229288946-create-IDX_note_url.js Normal file
View file

@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class CreateIDXNoteUrl1749229288946 {
name = 'CreateIDXNoteUrl1749229288946'
async up(queryRunner) {
await queryRunner.query(`CREATE INDEX IF NOT EXISTS "IDX_note_url" ON "note" ("url") `);
}
async down(queryRunner) {
await queryRunner.query(`DROP INDEX "public"."IDX_note_url"`);
}
}

17
packages/backend/migration/1749267016885-remove-IDX_instance_host_filters.js Normal file
View file

@ -0,0 +1,17 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class RemoveIDXInstanceHostFilters1749267016885 {
async up(queryRunner) {
await queryRunner.query(`DROP INDEX IF EXISTS "IDX_instance_host_filters"`);
}
async down(queryRunner) {
await queryRunner.query(`
create index "IDX_instance_host_filters"
on "instance" ("host", "isBlocked", "isSilenced", "isMediaSilenced", "isAllowListed", "isBubbled", "suspensionState")`);
await queryRunner.query(`comment on index "IDX_instance_host_filters" is 'Covering index for host filter queries'`);
}
}

22
packages/backend/migration/1749523586531-add-note_thread_muting-isPostMute.js Normal file
View file

@ -0,0 +1,22 @@
/**
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class AddNoteThreadMutingIsPostMute1749523586531 {
name = 'AddNoteThreadMutingIsPostMute1749523586531'
async up(queryRunner) {
await queryRunner.query(`DROP INDEX "public"."IDX_ae7aab18a2641d3e5f25e0c4ea"`);
await queryRunner.query(`ALTER TABLE "note_thread_muting" ADD "isPostMute" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`COMMENT ON COLUMN "note_thread_muting"."isPostMute" IS 'If true, then this mute applies only to the referenced note. If false (default), then it applies to all replies as well.'`);
await queryRunner.query(`CREATE UNIQUE INDEX "IDX_01f7ab05099400012e9a7fd42b" ON "note_thread_muting" ("userId", "threadId", "isPostMute") `);
}
async down(queryRunner) {
await queryRunner.query(`DROP INDEX "public"."IDX_01f7ab05099400012e9a7fd42b"`);
await queryRunner.query(`COMMENT ON COLUMN "note_thread_muting"."isPostMute" IS 'If true, then this mute applies only to the referenced note. If false (default), then it applies to all replies as well.'`);
await queryRunner.query(`ALTER TABLE "note_thread_muting" DROP COLUMN "isPostMute"`);
await queryRunner.query(`CREATE UNIQUE INDEX "IDX_ae7aab18a2641d3e5f25e0c4ea" ON "note_thread_muting" ("userId", "threadId") `);
}
}

14
packages/backend/migration/1750541176036-userDescriptionText.js Normal file
View file

@ -0,0 +1,14 @@
/*
* SPDX-FileCopyrightText: Lillychan and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class UserDescriptionText1750541176036 {
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "user_profile" ALTER COLUMN "description" TYPE TEXT`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "user_profile" ALTER COLUMN "description" TYPE character varying(2048)`);
}
}

19
packages/backend/migration/1750591589187-registry-unique-constraints.js Normal file
View file

@ -0,0 +1,19 @@
/*
* SPDX-FileCopyrightText: dakkar and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class RegistryUniqueConstraints1750591589187 {
async up(queryRunner) {
await queryRunner.query(`DELETE FROM "registry_item" WHERE "id" IN (
SELECT t."id" FROM (
SELECT *, ROW_NUMBER() OVER (PARTITION BY "userId","key","scope","domain" ORDER BY "updatedAt" DESC) rn
FROM "registry_item"
) t WHERE t.rn>1)`);
await queryRunner.query(`CREATE UNIQUE INDEX "IDX_d9c48d580287308f8c1f674946" ON "registry_item" ("userId", "key", "scope", "domain") NULLS NOT DISTINCT`);
}
async down(queryRunner) {
await queryRunner.query(`DROP INDEX "public"."IDX_d9c48d580287308f8c1f674946"`);
}
}

16
packages/backend/migration/1751912435779-announcement-confetti.js Normal file
View file

@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: bunnybeam and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class AnnouncementConfetti1751912435779 {
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "announcement" ADD "confetti" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`CREATE INDEX "IDX_94aabe9f742bc9808264a1c97c" ON "announcement" ("confetti") `);
}
async down(queryRunner) {
await queryRunner.query(`DROP INDEX "public"."IDX_94aabe9f742bc9808264a1c97c"`);
await queryRunner.query(`ALTER TABLE "announcement" DROP COLUMN "confetti"`);
}
}

14
packages/backend/migration/1752352800438-announcement-forRoles.js Normal file
View file

@ -0,0 +1,14 @@
/*
* SPDX-FileCopyrightText: наб and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class AnnouncementForRoles1752352800438 {
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "announcement" ADD "forRoles" text[] DEFAULT '{}'`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "announcement" DROP COLUMN "forRoles"`);
}
}

14
packages/backend/migration/1752377661219-UserPending-ip.js Normal file
View file

@ -0,0 +1,14 @@
/*
* SPDX-FileCopyrightText: наб and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class UserPendingIp1752377661219 {
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "user_pending" ADD "requestOriginIp" varchar(128)`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "user_pending" DROP COLUMN "requestOriginIp"`);
}
}

11
packages/backend/migration/1752607599852-split-descriptions.js Normal file
View file

@ -0,0 +1,11 @@
export class SplitDescriptions1752607599852 {
name = 'SplitDescriptions1752607599852'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" ADD COLUMN "about" TEXT`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "about"`);
}
}

16
packages/backend/migration/1753574755478-change-chat_message-text-type.js Normal file
View file

@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class ChangeChatMessageTextType1753574755478 {
name = 'ChangeChatMessageTextType1753574755478'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "chat_message" ALTER COLUMN "text" TYPE text`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "chat_message" ALTER COLUMN "text" TYPE varchar(4096)`);
}
}

8
packages/backend/migration/js/migration-config.js Normal file
View file

@ -0,0 +1,8 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
export function isConcurrentIndexMigrationEnabled() {
return process.env.MISSKEY_MIGRATION_CREATE_INDEX_CONCURRENTLY === '1';
}

4
packages/backend/ormconfig.js
View file

@ -1,6 +1,7 @@
import { DataSource } from 'typeorm';
import { loadConfig } from './built/config.js';
import { entities } from './built/postgres.js';
import { isConcurrentIndexMigrationEnabled } from "./migration/js/migration-config.js";
const config = loadConfig();
@ -14,8 +15,9 @@ export default new DataSource({
extra: {
...config.db.extra,
// migrations may be very slow, give them longer to run (that 10*1000 comes from postgres.ts)
statement_timeout: (config.db.extra?.statement_timeout ?? 1000 * 10) * 10,
statement_timeout: (config.db.extra?.statement_timeout ?? 1000 * 10) * 100,
},
entities: entities,
migrations: ['migration/*.js'],
migrationsTransactionMode: isConcurrentIndexMigrationEnabled() ? 'each' : 'all',
});

219
packages/backend/package.json
View file

@ -10,6 +10,9 @@
"start": "node ./built/boot/entry.js",
"start:test": "cross-env NODE_ENV=test node ./built/boot/entry.js",
"migrate": "pnpm typeorm migration:run -d ormconfig.js",
"migrate:revert": "pnpm typeorm migration:revert -d ormconfig.js",
"migrate:generate": "pnpm typeorm migration:generate -d ormconfig.js",
"migrate:create": "pnpm typeorm migration:create",
"revert": "pnpm typeorm migration:revert -d ormconfig.js",
"check:connect": "node ./scripts/check_connect.js",
"build": "swc src -d built -D --strip-leading-paths",
@ -22,12 +25,12 @@
"typecheck": "pnpm --filter megalodon build && tsc --noEmit && tsc -p test --noEmit && tsc -p test-federation --noEmit",
"eslint": "eslint --quiet \"{src,test-federation,test,js,@types}/**/*.{js,jsx,ts,tsx,vue}\" --cache",
"lint": "pnpm typecheck && pnpm eslint",
"jest": "cross-env NODE_ENV=test node --experimental-vm-modules --experimental-import-meta-resolve node_modules/jest/bin/jest.js --forceExit --config jest.config.unit.cjs",
"jest:e2e": "cross-env NODE_ENV=test node --experimental-vm-modules --experimental-import-meta-resolve node_modules/jest/bin/jest.js --forceExit --config jest.config.e2e.cjs",
"jest:fed": "node --experimental-vm-modules --experimental-import-meta-resolve node_modules/jest/bin/jest.js --forceExit --config jest.config.fed.cjs",
"jest-and-coverage": "cross-env NODE_ENV=test node --experimental-vm-modules --experimental-import-meta-resolve node_modules/jest/bin/jest.js --coverage --forceExit --config jest.config.unit.cjs",
"jest-and-coverage:e2e": "cross-env NODE_ENV=test node --experimental-vm-modules --experimental-import-meta-resolve node_modules/jest/bin/jest.js --coverage --forceExit --config jest.config.e2e.cjs",
"jest-clear": "cross-env NODE_ENV=test node --experimental-vm-modules --experimental-import-meta-resolve node_modules/jest/bin/jest.js --clearCache",
"jest": "cross-env NODE_ENV=test node ./jest.js --forceExit --config jest.config.unit.cjs",
"jest:e2e": "cross-env NODE_ENV=test node ./jest.js --forceExit --config jest.config.e2e.cjs",
"jest:fed": "node ./jest.js --forceExit --config jest.config.fed.cjs",
"jest-and-coverage": "cross-env NODE_ENV=test node ./jest.js --coverage --forceExit --config jest.config.unit.cjs",
"jest-and-coverage:e2e": "cross-env NODE_ENV=test node ./jest.js --coverage --forceExit --config jest.config.e2e.cjs",
"jest-clear": "cross-env NODE_ENV=test node ./jest.js --clearCache",
"test": "pnpm jest",
"test:e2e": "pnpm build && pnpm build:test && pnpm jest:e2e",
"test:fed": "pnpm jest:fed",
@ -37,18 +40,18 @@
},
"optionalDependencies": {
"@swc/core-android-arm64": "1.3.11",
"@swc/core-darwin-arm64": "1.3.56",
"@swc/core-darwin-x64": "1.3.56",
"@swc/core-darwin-arm64": "1.11.24",
"@swc/core-darwin-x64": "1.11.24",
"@swc/core-freebsd-x64": "1.3.11",
"@swc/core-linux-arm-gnueabihf": "1.3.56",
"@swc/core-linux-arm64-gnu": "1.3.56",
"@swc/core-linux-arm64-musl": "1.3.56",
"@swc/core-linux-x64-gnu": "1.3.56",
"@swc/core-linux-x64-musl": "1.3.56",
"@swc/core-win32-arm64-msvc": "1.3.56",
"@swc/core-win32-ia32-msvc": "1.3.56",
"@swc/core-win32-x64-msvc": "1.3.56",
"bufferutil": "4.0.7",
"@swc/core-linux-arm-gnueabihf": "1.11.24",
"@swc/core-linux-arm64-gnu": "1.11.24",
"@swc/core-linux-arm64-musl": "1.11.24",
"@swc/core-linux-x64-gnu": "1.11.24",
"@swc/core-linux-x64-musl": "1.11.24",
"@swc/core-win32-arm64-msvc": "1.11.24",
"@swc/core-win32-ia32-msvc": "1.11.24",
"@swc/core-win32-x64-msvc": "1.11.24",
"bufferutil": "4.0.9",
"slacc-android-arm-eabi": "0.0.10",
"slacc-android-arm64": "0.0.10",
"slacc-darwin-arm64": "0.0.10",
@ -62,191 +65,177 @@
"slacc-linux-x64-musl": "0.0.10",
"slacc-win32-arm64-msvc": "0.0.10",
"slacc-win32-x64-msvc": "0.0.10",
"utf-8-validate": "6.0.3"
"utf-8-validate": "6.0.5"
},
"dependencies": {
"@aws-sdk/client-s3": "3.620.0",
"@aws-sdk/lib-storage": "3.620.0",
"@bull-board/api": "6.5.0",
"@bull-board/fastify": "6.5.0",
"@bull-board/ui": "6.5.0",
"@aws-sdk/client-s3": "3.797.0",
"@aws-sdk/lib-storage": "3.797.0",
"@discordapp/twemoji": "15.1.0",
"@fastify/accepts": "5.0.1",
"@fastify/cookie": "11.0.1",
"@fastify/cors": "10.0.1",
"@fastify/express": "4.0.1",
"@fastify/http-proxy": "10.0.1",
"@fastify/multipart": "9.0.1",
"@fastify/static": "8.0.2",
"@fastify/view": "10.0.1",
"@misskey-dev/sharp-read-bmp": "1.2.0",
"@misskey-dev/summaly": "5.1.0",
"@nestjs/common": "10.4.7",
"@nestjs/core": "10.4.7",
"@nestjs/testing": "10.4.7",
"@fastify/accepts": "5.0.2",
"@fastify/cookie": "11.0.2",
"@fastify/cors": "10.1.0",
"@fastify/express": "4.0.2",
"@fastify/http-proxy": "10.0.2",
"@fastify/multipart": "9.0.3",
"@fastify/static": "8.1.1",
"@fastify/view": "10.0.2",
"@misskey-dev/sharp-read-bmp": "1.3.0",
"@misskey-dev/summaly": "npm:@transfem-org/summaly@5.2.2",
"@nestjs/common": "11.1.0",
"@nestjs/core": "11.1.0",
"@nestjs/testing": "11.1.0",
"@peertube/http-signature": "1.7.0",
"@sentry/node": "8.38.0",
"@sentry/profiling-node": "8.38.0",
"@simplewebauthn/server": "10.0.1",
"@sinonjs/fake-timers": "11.2.2",
"@sentry/node": "8.55.0",
"@sentry/profiling-node": "8.55.0",
"@simplewebauthn/server": "12.0.0",
"@sinonjs/fake-timers": "11.3.1",
"@smithy/node-http-handler": "2.5.0",
"@swc/cli": "0.3.12",
"@swc/core": "1.9.2",
"@transfem-org/sfm-js": "0.24.5",
"mfm-js": "npm:@transfem-org/sfm-js@0.24.8",
"@twemoji/parser": "15.1.1",
"@types/psl": "^1.1.3",
"accepts": "1.3.8",
"ajv": "8.17.1",
"archiver": "7.0.1",
"argon2": "^0.40.1",
"async-mutex": "0.5.0",
"argon2": "0.43.0",
"axios": "1.7.4",
"bcryptjs": "2.4.3",
"blurhash": "2.0.5",
"body-parser": "1.20.3",
"bullmq": "5.26.1",
"bullmq": "5.51.1",
"cacheable-lookup": "7.0.0",
"canvas": "^3.1.0",
"canvas": "3.1.0",
"cbor": "9.0.2",
"chalk": "5.3.0",
"chalk": "5.4.1",
"chalk-template": "1.1.0",
"chokidar": "3.6.0",
"cli-highlight": "2.1.11",
"cheerio": "1.0.0",
"cli-highlight": "npm:@transfem-org/cli-highlight@2.1.12",
"color-convert": "2.0.1",
"content-disposition": "0.5.4",
"date-fns": "2.30.0",
"deep-email-validator": "0.1.21",
"fast-xml-parser": "4.4.1",
"fastify": "5.0.0",
"fastify-multer": "^2.0.3",
"dom-serializer": "2.0.0",
"domhandler": "5.0.3",
"domutils": "3.2.2",
"fastify": "5.3.2",
"fastify-raw-body": "5.0.0",
"feed": "4.2.2",
"file-type": "19.6.0",
"fluent-ffmpeg": "2.1.3",
"form-data": "4.0.1",
"form-data": "4.0.2",
"glob": "11.0.0",
"got": "14.4.4",
"happy-dom": "15.11.4",
"got": "14.4.7",
"hpagent": "1.2.0",
"htmlescape": "1.1.1",
"http-link-header": "1.1.3",
"ioredis": "5.4.1",
"htmlparser2": "9.1.0",
"ioredis": "5.6.1",
"ip-cidr": "4.0.2",
"ipaddr.js": "2.2.0",
"is-svg": "5.1.0",
"js-yaml": "4.1.0",
"jsdom": "24.1.1",
"json5": "2.2.3",
"jsonld": "8.3.2",
"jsrsasign": "11.1.0",
"juice": "11.0.0",
"jsonld": "8.3.3",
"juice": "11.0.1",
"megalodon": "workspace:*",
"meilisearch": "0.45.0",
"microformats-parser": "2.0.2",
"meilisearch": "0.50.0",
"mime-types": "2.1.35",
"misskey-js": "workspace:*",
"misskey-reversi": "workspace:*",
"moment": "^2.30.1",
"moment": "2.30.1",
"ms": "3.0.0-canary.1",
"nanoid": "5.0.8",
"nanoid": "5.1.5",
"nested-property": "4.0.0",
"node-fetch": "3.3.2",
"nodemailer": "6.9.16",
"oauth": "0.10.0",
"oauth2orize": "1.12.0",
"oauth2orize-pkce": "0.1.2",
"nodemailer": "6.10.1",
"os-utils": "0.0.14",
"otpauth": "9.3.4",
"parse5": "7.2.1",
"pg": "8.13.1",
"otpauth": "9.4.0",
"pg": "8.15.6",
"pkce-challenge": "4.1.0",
"probe-image-size": "7.2.3",
"promise-limit": "2.7.0",
"proxy-addr": "^2.0.7",
"psl": "^1.13.0",
"proxy-addr": "2.0.7",
"psl": "1.15.0",
"pug": "3.0.3",
"qrcode": "1.5.4",
"random-seed": "0.3.0",
"ratelimiter": "3.4.1",
"re2": "1.21.4",
"redis-info": "3.1.0",
"redis-lock": "0.1.4",
"reflect-metadata": "0.2.2",
"rename": "1.0.4",
"rss-parser": "3.13.0",
"rxjs": "7.8.1",
"sanitize-html": "2.13.1",
"secure-json-parse": "2.7.0",
"sharp": "0.33.5",
"sanitize-html": "2.16.0",
"secure-json-parse": "3.0.2",
"sharp": "0.34.1",
"semver": "7.7.1",
"slacc": "0.0.10",
"strict-event-emitter-types": "2.0.0",
"stringz": "2.1.0",
"systeminformation": "5.23.5",
"systeminformation": "5.25.11",
"tinycolor2": "1.6.0",
"tmp": "0.2.3",
"tsc-alias": "1.8.10",
"tsc-alias": "1.8.15",
"tsconfig-paths": "4.2.0",
"typeorm": "0.3.20",
"typescript": "5.6.3",
"ulid": "2.3.0",
"uuid": "^9.0.1",
"typeorm": "0.3.22",
"typescript": "5.8.3",
"ulid": "2.4.0",
"uuid": "11.1.0",
"vary": "1.1.2",
"web-push": "3.6.7",
"ws": "8.18.0",
"ws": "8.18.1",
"xev": "3.0.2"
},
"devDependencies": {
"@jest/globals": "29.7.0",
"@nestjs/platform-express": "10.4.7",
"@simplewebauthn/types": "10.0.0",
"@swc/jest": "0.2.37",
"@nestjs/platform-express": "11.1.0",
"@sentry/vue": "9.14.0",
"@simplewebauthn/types": "12.0.0",
"@swc/cli": "0.7.3",
"@swc/core": "1.11.24",
"@swc/jest": "0.2.38",
"@types/accepts": "1.3.7",
"@types/archiver": "6.0.3",
"@types/bcryptjs": "2.4.6",
"@types/body-parser": "1.19.5",
"@types/color-convert": "2.0.4",
"@types/content-disposition": "0.5.8",
"@types/fluent-ffmpeg": "2.1.27",
"@types/htmlescape": "1.1.3",
"@types/http-link-header": "1.0.7",
"@types/jest": "29.5.14",
"@types/js-yaml": "4.0.9",
"@types/jsdom": "21.1.7",
"@types/jsonld": "1.5.15",
"@types/jsrsasign": "10.5.14",
"@types/jsrsasign": "10.5.15",
"@types/mime-types": "2.1.4",
"@types/ms": "0.7.34",
"@types/node": "22.9.0",
"@types/nodemailer": "6.4.16",
"@types/node": "22.15.2",
"@types/nodemailer": "6.4.17",
"@types/oauth": "0.9.6",
"@types/oauth2orize": "1.11.5",
"@types/oauth2orize-pkce": "0.1.2",
"@types/pg": "8.11.10",
"@types/proxy-addr": "^2.0.3",
"@types/pg": "8.11.14",
"@types/proxy-addr": "2.0.3",
"@types/psl": "1.1.3",
"@types/pug": "2.0.10",
"@types/qrcode": "1.5.5",
"@types/random-seed": "0.3.5",
"@types/ratelimiter": "3.4.6",
"@types/redis-info": "3.0.3",
"@types/rename": "1.0.7",
"@types/sanitize-html": "2.13.0",
"@types/semver": "7.5.8",
"@types/sanitize-html": "2.15.0",
"@types/semver": "7.7.0",
"@types/simple-oauth2": "5.0.7",
"@types/sinonjs__fake-timers": "8.1.5",
"@types/supertest": "6.0.3",
"@types/tinycolor2": "1.4.6",
"@types/tmp": "0.2.6",
"@types/uuid": "^9.0.4",
"@types/vary": "1.1.3",
"@types/web-push": "3.6.4",
"@types/ws": "8.5.13",
"@typescript-eslint/eslint-plugin": "7.17.0",
"@typescript-eslint/parser": "7.17.0",
"aws-sdk-client-mock": "4.0.1",
"@types/ws": "8.18.1",
"@typescript-eslint/eslint-plugin": "8.31.0",
"@typescript-eslint/parser": "8.31.0",
"aws-sdk-client-mock": "4.1.0",
"cross-env": "7.0.3",
"eslint-plugin-import": "2.30.0",
"execa": "8.0.1",
"eslint-plugin-import": "2.31.0",
"execa": "9.5.2",
"fkill": "9.0.0",
"jest": "29.7.0",
"jest-mock": "29.7.0",
"nodemon": "3.1.7",
"pid-port": "1.0.0",
"simple-oauth2": "5.1.0"
"nodemon": "3.1.10",
"pid-port": "1.0.2",
"simple-oauth2": "5.1.0",
"supertest": "7.1.0"
}
}

2
packages/backend/scripts/check_connect.js
View file

@ -9,7 +9,7 @@ import { createPostgresDataSource } from '../built/postgres.js';
const config = loadConfig();
// createPostgresDataSource handels primaries and replicas automatically.
// createPostgresDataSource handles primaries and replicas automatically.
// usually, it only opens connections first use, so we force it using
// .initialize()
async function connectToPostgres(){

33
packages/backend/src/GlobalModule.ts
View file

@ -14,6 +14,7 @@ import { createPostgresDataSource } from './postgres.js';
import { RepositoryModule } from './models/RepositoryModule.js';
import { allSettled } from './misc/promise-tracker.js';
import { GlobalEvents } from './core/GlobalEventService.js';
import Logger from './logger.js';
import type { Provider, OnApplicationShutdown } from '@nestjs/common';
const $config: Provider = {
@ -24,8 +25,13 @@ const $config: Provider = {
const $db: Provider = {
provide: DI.db,
useFactory: async (config) => {
const db = createPostgresDataSource(config);
return await db.initialize();
try {
const db = createPostgresDataSource(config);
return await db.initialize();
} catch (e) {
console.error('failed to initialize database connection', e);
throw e;
}
},
inject: [DI.config],
};
@ -141,7 +147,7 @@ const $meta: Provider = {
for (const key in body.after) {
(meta as any)[key] = (body.after as any)[key];
}
meta.proxyAccount = null; // joinなカラムは通常取ってこないので
meta.rootUser = null; // joinなカラムは通常取ってこないので
break;
}
default:
@ -164,6 +170,8 @@ const $meta: Provider = {
exports: [$config, $db, $meta, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, $redisForReactions, $redisForRateLimit, RepositoryModule],
})
export class GlobalModule implements OnApplicationShutdown {
private readonly logger = new Logger('global');
constructor(
@Inject(DI.db) private db: DataSource,
@Inject(DI.redis) private redisClient: Redis.Redis,
@ -176,17 +184,18 @@ export class GlobalModule implements OnApplicationShutdown {
public async dispose(): Promise<void> {
// Wait for all potential DB queries
this.logger.info('Finalizing active promises...');
await allSettled();
// And then disconnect from DB
await Promise.all([
this.db.destroy(),
this.redisClient.disconnect(),
this.redisForPub.disconnect(),
this.redisForSub.disconnect(),
this.redisForTimelines.disconnect(),
this.redisForReactions.disconnect(),
this.redisForRateLimit.disconnect(),
]);
this.logger.info('Disconnected from data sources...');
await this.db.destroy();
this.redisClient.disconnect();
this.redisForPub.disconnect();
this.redisForSub.disconnect();
this.redisForTimelines.disconnect();
this.redisForReactions.disconnect();
this.redisForRateLimit.disconnect();
this.logger.info('Global module disposed.');
}
async onApplicationShutdown(signal: string): Promise<void> {

2
packages/backend/src/boot/common.ts
View file

@ -19,6 +19,7 @@ export async function server() {
const app = await NestFactory.createApplicationContext(MainModule, {
logger: new NestLogger(),
});
app.enableShutdownHooks();
const serverService = app.get(ServerService);
await serverService.launch();
@ -39,6 +40,7 @@ export async function jobQueue() {
const jobQueue = await NestFactory.createApplicationContext(QueueProcessorModule, {
logger: new NestLogger(),
});
jobQueue.enableShutdownHooks();
jobQueue.get(QueueProcessorService).start();
jobQueue.get(ChartManagementService).start();

36
packages/backend/src/boot/entry.ts
View file

@ -9,6 +9,7 @@
import cluster from 'node:cluster';
import { EventEmitter } from 'node:events';
import { inspect } from 'node:util';
import chalk from 'chalk';
import Xev from 'xev';
import Logger from '@/logger.js';
@ -53,18 +54,41 @@ async function main() {
// Display detail of unhandled promise rejection
if (!envOption.quiet) {
process.on('unhandledRejection', console.dir);
process.on('unhandledRejection', e => {
logger.error('Unhandled rejection:', inspect(e));
});
}
process.on('uncaughtException', (err) => {
// Workaround for https://github.com/node-fetch/node-fetch/issues/954
if (String(err).match(/^TypeError: .+ is an? url with embedded credentials.$/)) {
logger.debug('Suppressed node-fetch issue#954, but the current job may fail.');
return;
}
// Workaround for https://github.com/node-fetch/node-fetch/issues/1845
if (String(err) === 'TypeError: Cannot read properties of undefined (reading \'body\')') {
logger.debug('Suppressed node-fetch issue#1845, but the current job may fail.');
return;
}
// Throw all other errors to avoid inconsistent state.
// (per NodeJS docs, it's unsafe to suppress arbitrary errors in an uncaughtException handler.)
throw err;
});
// Display detail of uncaught exception
process.on('uncaughtException', err => {
try {
logger.error(err);
console.trace(err);
} catch { }
process.on('uncaughtExceptionMonitor', (err, origin) => {
logger.error(`Uncaught exception (${origin}):`, err);
});
// Dying away...
process.on('disconnect', () => {
logger.warn('IPC channel disconnected! The process may soon die.');
});
process.on('beforeExit', code => {
logger.warn(`Event loop died! Process will exit with code ${code}.`);
});
process.on('exit', code => {
logger.info(`The process is going to exit with code ${code}`);
});

20
packages/backend/src/boot/master.ts
View file

@ -8,6 +8,7 @@ import { fileURLToPath } from 'node:url';
import { dirname } from 'node:path';
import * as os from 'node:os';
import cluster from 'node:cluster';
import * as net from 'node:net';
import chalk from 'chalk';
import chalkTemplate from 'chalk-template';
import * as Sentry from '@sentry/node';
@ -18,7 +19,6 @@ import type { Config } from '@/config.js';
import { showMachineInfo } from '@/misc/show-machine-info.js';
import { envOption } from '@/env.js';
import { jobQueue, server } from './common.js';
import * as net from 'node:net';
const _filename = fileURLToPath(import.meta.url);
const _dirname = dirname(_filename);
@ -51,7 +51,7 @@ function greet() {
}
bootLogger.info('Welcome to Sharkey!');
bootLogger.info(`Sharkey v${meta.version}`, null, true);
bootLogger.info(`Sharkey v${meta.gitVersion ?? meta.version}`, null, true);
}
/**
@ -74,7 +74,7 @@ export async function masterMain() {
process.exit(1);
}
bootLogger.succ('Sharkey initialized');
bootLogger.info('Sharkey initialized');
if (config.sentryForBackend) {
Sentry.init({
@ -91,7 +91,7 @@ export async function masterMain() {
maxBreadcrumbs: 0,
// Set release version
release: 'Sharkey@' + meta.version,
release: 'Sharkey@' + (meta.gitVersion ?? meta.version),
...config.sentryForBackend.options,
});
@ -140,10 +140,10 @@ export async function masterMain() {
}
if (envOption.onlyQueue) {
bootLogger.succ('Queue started', null, true);
bootLogger.info('Queue started', null, true);
} else {
const addressString = net.isIPv6(config.address) ? `[${config.address}]` : config.address;
bootLogger.succ(config.socket ? `Now listening on socket ${config.socket} on ${config.url}` : `Now listening on ${addressString}:${config.port} on ${config.url}`, null, true);
bootLogger.info(config.socket ? `Now listening on socket ${config.socket} on ${config.url}` : `Now listening on ${addressString}:${config.port} on ${config.url}`, null, true);
}
}
@ -172,7 +172,7 @@ function loadConfigBoot(): Config {
config = loadConfig();
} catch (exception) {
if (typeof exception === 'string') {
configLogger.error(exception);
configLogger.error('Exception loading config:', exception);
process.exit(1);
} else if ((exception as any).code === 'ENOENT') {
configLogger.error('Configuration file not found', null, true);
@ -181,7 +181,7 @@ function loadConfigBoot(): Config {
throw exception;
}
configLogger.succ('Loaded');
configLogger.info('Loaded');
return config;
}
@ -195,7 +195,7 @@ async function connectDb(): Promise<void> {
dbLogger.info('Connecting...');
await initDb();
const v = await db.query('SHOW server_version').then(x => x[0].server_version);
dbLogger.succ(`Connected: v${v}`);
dbLogger.info(`Connected: v${v}`);
} catch (err) {
dbLogger.error('Cannot connect', null, true);
dbLogger.error(err);
@ -211,7 +211,7 @@ async function spawnWorkers(limit = 1) {
bootLogger.info(`Starting ${workers} worker${workers === 1 ? '' : 's'}...`);
await Promise.all([...Array(workers)].map(spawnWorker));
bootLogger.succ('All workers started');
bootLogger.info('All workers started');
}
function spawnWorker(): Promise<void> {

2
packages/backend/src/boot/worker.ts
View file

@ -37,7 +37,7 @@ export async function workerMain() {
maxBreadcrumbs: 0,
// Set release version
release: "Sharkey@" + meta.version,
release: "Sharkey@" + (meta.gitVersion ?? meta.version),
...config.sentryForBackend.options,
});

167
packages/backend/src/config.ts
View file

@ -8,8 +8,12 @@ import { fileURLToPath } from 'node:url';
import { dirname, resolve } from 'node:path';
import * as yaml from 'js-yaml';
import { globSync } from 'glob';
import * as Sentry from '@sentry/node';
import ipaddr from 'ipaddr.js';
import Logger from './logger.js';
import type * as Sentry from '@sentry/node';
import type * as SentryVue from '@sentry/vue';
import type { RedisOptions } from 'ioredis';
import type { IPv4, IPv6 } from 'ipaddr.js';
type RedisOptionsSource = Partial<RedisOptions> & {
host?: string;
@ -37,6 +41,7 @@ type Source = {
db?: string;
user?: string;
pass?: string;
slowQueryThreshold?: number;
disableCache?: boolean;
extra?: { [x: string]: string };
};
@ -66,7 +71,12 @@ type Source = {
scope?: 'local' | 'global' | string[];
};
sentryForBackend?: { options: Partial<Sentry.NodeOptions>; enableNodeProfiling: boolean; };
sentryForFrontend?: { options: Partial<Sentry.NodeOptions> };
sentryForFrontend?: {
options: Partial<SentryVue.BrowserOptions> & { dsn: string };
vueIntegration?: SentryVue.VueIntegrationOptions | null;
browserTracingIntegration?: Parameters<typeof SentryVue.browserTracingIntegration>[0] | null;
replayIntegration?: Parameters<typeof SentryVue.replayIntegration>[0] | null;
};
publishTarballInsteadOfProvideRepositoryUrl?: boolean;
@ -76,7 +86,8 @@ type Source = {
proxySmtp?: string;
proxyBypassHosts?: string[];
allowedPrivateNetworks?: string[];
allowedPrivateNetworks?: PrivateNetworkSource[];
disallowExternalApRedirect?: boolean;
maxFileSize?: number;
maxNoteLength?: number;
@ -85,6 +96,8 @@ type Source = {
maxRemoteNoteLength?: number;
maxAltTextLength?: number;
maxRemoteAltTextLength?: number;
maxBioLength?: number;
maxRemoteBioLength?: number;
clusterLimit?: number;
@ -102,6 +115,7 @@ type Source = {
deliverJobMaxAttempts?: number;
inboxJobMaxAttempts?: number;
mediaDirectory?: string;
mediaProxy?: string;
proxyRemoteFiles?: boolean;
videoThumbnailGenerator?: string;
@ -126,9 +140,10 @@ type Source = {
logging?: {
sql?: {
disableQueryTruncation? : boolean,
enableQueryParamLogging? : boolean,
}
disableQueryTruncation?: boolean,
enableQueryParamLogging?: boolean,
};
verbose?: boolean;
}
activityLogging?: {
@ -136,8 +151,70 @@ type Source = {
preSave?: boolean;
maxAge?: number;
};
websocketCompression?: boolean;
customHtml?: {
head?: string;
}
};
const configLogger = new Logger('config');
export type PrivateNetworkSource = string | { network?: string, ports?: number[] };
export type PrivateNetwork = {
/**
* CIDR IP/netmask definition of the IP range to match.
*/
cidr: CIDR;
/**
* List of ports to match.
* If undefined, then all ports match.
* If empty, then NO ports match.
*/
ports?: number[];
};
export type CIDR = [ip: IPv4 | IPv6, prefixLength: number];
export function parsePrivateNetworks(patterns: PrivateNetworkSource[]): PrivateNetwork[];
export function parsePrivateNetworks(patterns: undefined): undefined;
export function parsePrivateNetworks(patterns: PrivateNetworkSource[] | undefined): PrivateNetwork[] | undefined;
export function parsePrivateNetworks(patterns: PrivateNetworkSource[] | undefined): PrivateNetwork[] | undefined {
if (!patterns) return undefined;
return patterns
.map(e => {
if (typeof(e) === 'string') {
const cidr = parseIpOrMask(e);
if (cidr) {
return { cidr } satisfies PrivateNetwork;
}
} else if (e.network) {
const cidr = parseIpOrMask(e.network);
if (cidr) {
return { cidr, ports: e.ports } satisfies PrivateNetwork;
}
}
configLogger.warn('Skipping invalid entry in allowedPrivateNetworks: ', e);
return null;
})
.filter(p => p != null);
}
function parseIpOrMask(ipOrMask: string): CIDR | null {
if (ipaddr.isValidCIDR(ipOrMask)) {
return ipaddr.parseCIDR(ipOrMask);
}
if (ipaddr.isValid(ipOrMask)) {
const ip = ipaddr.parse(ipOrMask);
return [ip, 32];
}
return null;
}
export type Config = {
url: string;
port: number;
@ -151,6 +228,7 @@ export type Config = {
db: string;
user: string;
pass: string;
slowQueryThreshold?: number;
disableCache?: boolean;
extra?: { [x: string]: string };
};
@ -176,7 +254,8 @@ export type Config = {
proxy: string | undefined;
proxySmtp: string | undefined;
proxyBypassHosts: string[] | undefined;
allowedPrivateNetworks: string[] | undefined;
allowedPrivateNetworks: PrivateNetwork[] | undefined;
disallowExternalApRedirect: boolean;
maxFileSize: number;
maxNoteLength: number;
maxRemoteNoteLength: number;
@ -184,6 +263,8 @@ export type Config = {
maxRemoteCwLength: number;
maxAltTextLength: number;
maxRemoteAltTextLength: number;
maxBioLength: number;
maxRemoteBioLength: number;
clusterLimit: number | undefined;
id: string;
outgoingAddress: string | undefined;
@ -200,12 +281,14 @@ export type Config = {
customMOTD: string[] | undefined;
signToActivityPubGet: boolean;
attachLdSignatureForRelays: boolean;
/** @deprecated Use MiMeta.allowUnsignedFetch instead */
checkActivityPubGetSignature: boolean | undefined;
logging?: {
sql?: {
disableQueryTruncation? : boolean,
enableQueryParamLogging? : boolean,
}
disableQueryTruncation?: boolean,
enableQueryParamLogging?: boolean,
};
verbose?: boolean;
}
version: string;
@ -224,6 +307,7 @@ export type Config = {
frontendManifestExists: boolean;
frontendEmbedEntry: string;
frontendEmbedManifestExists: boolean;
mediaDirectory: string;
mediaProxy: string;
externalMediaProxyEnabled: boolean;
videoThumbnailGenerator: string | null;
@ -234,7 +318,12 @@ export type Config = {
redisForReactions: RedisOptions & RedisOptionsSource;
redisForRateLimit: RedisOptions & RedisOptionsSource;
sentryForBackend: { options: Partial<Sentry.NodeOptions>; enableNodeProfiling: boolean; } | undefined;
sentryForFrontend: { options: Partial<Sentry.NodeOptions> } | undefined;
sentryForFrontend: {
options: Partial<SentryVue.BrowserOptions> & { dsn: string };
vueIntegration?: SentryVue.VueIntegrationOptions | null;
browserTracingIntegration?: Parameters<typeof SentryVue.browserTracingIntegration>[0] | null;
replayIntegration?: Parameters<typeof SentryVue.replayIntegration>[0] | null;
} | undefined;
perChannelMaxNoteCacheCount: number;
perUserNotificationsMaxCount: number;
deactivateAntennaThreshold: number;
@ -252,6 +341,12 @@ export type Config = {
preSave: boolean;
maxAge: number;
};
websocketCompression?: boolean;
customHtml: {
head: string;
}
};
export type FulltextSearchProvider = 'sqlLike' | 'sqlPgroonga' | 'meilisearch' | 'sqlTsvector';
@ -262,7 +357,7 @@ const _dirname = dirname(_filename);
/**
* Path of configuration directory
*/
const dir = `${_dirname}/../../../.config`;
const dir = process.env.MISSKEY_CONFIG_DIR ?? `${_dirname}/../../../.config`;
/**
* Path of configuration file
@ -289,11 +384,14 @@ export function loadConfig(): Config {
if (configFiles.length === 0
&& !process.env['MK_WARNED_ABOUT_CONFIG']) {
console.log('No config files loaded, check if this is intentional');
configLogger.warn('No config files loaded, check if this is intentional');
process.env['MK_WARNED_ABOUT_CONFIG'] = '1';
}
const config = configFiles.map(path => fs.readFileSync(path, 'utf-8'))
const config = configFiles.map(path => {
configLogger.info(`Reading configuration from ${path}`);
return fs.readFileSync(path, 'utf-8');
})
.map(contents => yaml.load(contents) as Source)
.reduce(
(acc: Source, cur: Source) => Object.assign(acc, cur),
@ -303,7 +401,7 @@ export function loadConfig(): Config {
applyEnvOverrides(config);
const url = tryCreateUrl(config.url ?? process.env.MISSKEY_URL ?? '');
const version = meta.version;
const version = meta.gitVersion ?? meta.version;
const host = url.host;
const hostname = url.hostname;
const scheme = url.protocol.replace(/:$/, '');
@ -319,6 +417,10 @@ export function loadConfig(): Config {
const internalMediaProxy = `${scheme}://${host}/proxy`;
const redis = convertRedisOptions(config.redis, host);
// nullish => 300 (default)
// 0 => undefined (disabled)
const slowQueryThreshold = (config.db.slowQueryThreshold ?? 300) || undefined;
return {
version,
publishTarballInsteadOfProvideRepositoryUrl: !!config.publishTarballInsteadOfProvideRepositoryUrl,
@ -337,7 +439,7 @@ export function loadConfig(): Config {
apiUrl: `${scheme}://${host}/api`,
authUrl: `${scheme}://${host}/auth`,
driveUrl: `${scheme}://${host}/files`,
db: { ...config.db, db: dbDb, user: dbUser, pass: dbPass },
db: { ...config.db, db: dbDb, user: dbUser, pass: dbPass, slowQueryThreshold },
dbReplications: config.dbReplications,
dbSlaves: config.dbSlaves,
fulltextSearch: config.fulltextSearch,
@ -354,7 +456,8 @@ export function loadConfig(): Config {
proxy: config.proxy,
proxySmtp: config.proxySmtp,
proxyBypassHosts: config.proxyBypassHosts,
allowedPrivateNetworks: config.allowedPrivateNetworks,
allowedPrivateNetworks: parsePrivateNetworks(config.allowedPrivateNetworks),
disallowExternalApRedirect: config.disallowExternalApRedirect ?? false,
maxFileSize: config.maxFileSize ?? 262144000,
maxNoteLength: config.maxNoteLength ?? 3000,
maxRemoteNoteLength: config.maxRemoteNoteLength ?? 100000,
@ -362,6 +465,8 @@ export function loadConfig(): Config {
maxRemoteCwLength: config.maxRemoteCwLength ?? 5000,
maxAltTextLength: config.maxAltTextLength ?? 20000,
maxRemoteAltTextLength: config.maxRemoteAltTextLength ?? 100000,
maxBioLength: config.maxBioLength ?? 1500,
maxRemoteBioLength: config.maxRemoteBioLength ?? 15000,
clusterLimit: config.clusterLimit,
outgoingAddress: config.outgoingAddress,
outgoingAddressFamily: config.outgoingAddressFamily,
@ -378,6 +483,7 @@ export function loadConfig(): Config {
signToActivityPubGet: config.signToActivityPubGet ?? true,
attachLdSignatureForRelays: config.attachLdSignatureForRelays ?? true,
checkActivityPubGetSignature: config.checkActivityPubGetSignature,
mediaDirectory: config.mediaDirectory ?? resolve(_dirname, '../../../files'),
mediaProxy: externalMediaProxy ?? internalMediaProxy,
externalMediaProxyEnabled: externalMediaProxy !== null && externalMediaProxy !== internalMediaProxy,
videoThumbnailGenerator: config.videoThumbnailGenerator ?
@ -400,10 +506,18 @@ export function loadConfig(): Config {
preSave: config.activityLogging?.preSave ?? false,
maxAge: config.activityLogging?.maxAge ?? (1000 * 60 * 60 * 24 * 30),
},
websocketCompression: config.websocketCompression ?? false,
customHtml: {
head: config.customHtml?.head ?? '',
},
};
}
function tryCreateUrl(url: string) {
if (!url) {
throw new Error('Failed to load: no "url" property found in config. Please check the value of "MISSKEY_CONFIG_DIR" and "MISSKEY_CONFIG_YML", and verify that all configuration files are correct.');
}
try {
return new URL(url);
} catch (e) {
@ -478,7 +592,7 @@ function applyEnvOverrides(config: Source) {
}
}
function _step2name(step: string|number): string {
function _step2name(step: string | number): string {
return step.toString().replaceAll(/[^a-z0-9]+/gi, '').toUpperCase();
}
@ -534,22 +648,27 @@ function applyEnvOverrides(config: Source) {
// these are all the settings that can be overridden
_apply_top([['url', 'port', 'address', 'socket', 'chmodSocket', 'disableHsts', 'id', 'dbReplications']]);
_apply_top(['db', ['host', 'port', 'db', 'user', 'pass', 'disableCache']]);
_apply_top([['url', 'port', 'address', 'socket', 'chmodSocket', 'disableHsts', 'id', 'dbReplications', 'websocketCompression']]);
_apply_top(['db', ['host', 'port', 'db', 'user', 'pass', 'slowQueryThreshold', 'disableCache']]);
_apply_top(['dbSlaves', Array.from((config.dbSlaves ?? []).keys()), ['host', 'port', 'db', 'user', 'pass']]);
_apply_top([
['redis', 'redisForPubsub', 'redisForJobQueue', 'redisForTimelines', 'redisForReactions', 'redisForRateLimit'],
['host', 'port', 'username', 'pass', 'db', 'prefix'],
]);
_apply_top(['fulltextSearch', 'provider']);
_apply_top(['meilisearch', ['host', 'port', 'apikey', 'ssl', 'index', 'scope']]);
_apply_top(['meilisearch', ['host', 'port', 'apiKey', 'ssl', 'index', 'scope']]);
_apply_top([['sentryForFrontend', 'sentryForBackend'], 'options', ['dsn', 'profileSampleRate', 'serverName', 'includeLocalVariables', 'proxy', 'keepAlive', 'caCerts']]);
_apply_top(['sentryForBackend', 'enableNodeProfiling']);
_apply_top(['sentryForFrontend', 'vueIntegration', ['attachProps', 'attachErrorHandler']]);
_apply_top(['sentryForFrontend', 'vueIntegration', 'tracingOptions', 'timeout']);
_apply_top(['sentryForFrontend', 'browserTracingIntegration', 'routeLabel']);
_apply_top([['clusterLimit', 'deliverJobConcurrency', 'inboxJobConcurrency', 'relashionshipJobConcurrency', 'deliverJobPerSec', 'inboxJobPerSec', 'relashionshipJobPerSec', 'deliverJobMaxAttempts', 'inboxJobMaxAttempts']]);
_apply_top([['outgoingAddress', 'outgoingAddressFamily', 'proxy', 'proxySmtp', 'mediaProxy', 'proxyRemoteFiles', 'videoThumbnailGenerator']]);
_apply_top([['maxFileSize', 'maxNoteLength', 'maxRemoteNoteLength', 'maxAltTextLength', 'maxRemoteAltTextLength', 'pidFile', 'filePermissionBits']]);
_apply_top([['outgoingAddress', 'outgoingAddressFamily', 'proxy', 'proxySmtp', 'mediaDirectory', 'mediaProxy', 'proxyRemoteFiles', 'videoThumbnailGenerator']]);
_apply_top([['maxFileSize', 'maxNoteLength', 'maxRemoteNoteLength', 'maxAltTextLength', 'maxRemoteAltTextLength', 'maxBioLength', 'maxRemoteBioLength', 'pidFile', 'filePermissionBits']]);
_apply_top(['import', ['downloadTimeout', 'maxFileSize']]);
_apply_top([['signToActivityPubGet', 'checkActivityPubGetSignature', 'setupPassword']]);
_apply_top([['signToActivityPubGet', 'checkActivityPubGetSignature', 'setupPassword', 'disallowExternalApRedirect']]);
_apply_top(['logging', 'sql', ['disableQueryTruncation', 'enableQueryParamLogging']]);
_apply_top(['logging', ['verbose']]);
_apply_top(['activityLogging', ['enabled', 'preSave', 'maxAge']]);
_apply_top(['customHtml', ['head']]);
}

6
packages/backend/src/const.ts
View file

@ -70,3 +70,9 @@ https://github.com/sindresorhus/file-type/blob/main/supported.js
https://github.com/sindresorhus/file-type/blob/main/core.js
https://developer.mozilla.org/en-US/docs/Web/Media/Formats/Containers
*/
export const instanceUnsignedFetchOptions = ['never', 'always', 'essential'] as const;
export type InstanceUnsignedFetchOption = (typeof instanceUnsignedFetchOptions)[number];
export const userUnsignedFetchOptions = ['never', 'always', 'essential', 'staff'] as const;
export type UserUnsignedFetchOption = (typeof userUnsignedFetchOptions)[number];

32
packages/backend/src/core/AbuseReportNotificationService.ts
View file

@ -82,6 +82,28 @@ export class AbuseReportNotificationService implements OnApplicationShutdown {
}
}
/**
* Collects all email addresses that a abuse report should be sent to.
*/
@bindThis
public async getRecipientEMailAddresses(): Promise<string[]> {
const recipientEMailAddresses = await this.fetchEMailRecipients().then(it => it
.filter(it => it.isActive && it.userProfile?.emailVerified)
.map(it => it.userProfile?.email)
.filter(x => x != null),
);
if (this.meta.email) {
recipientEMailAddresses.push(this.meta.email);
}
if (this.meta.maintainerEmail) {
recipientEMailAddresses.push(this.meta.maintainerEmail);
}
return recipientEMailAddresses;
}
/**
* Mailを用いて{@link abuseReports}の内容を管理者各位に通知する.
* メールアドレスの送信先は以下の通り.
@ -96,15 +118,7 @@ export class AbuseReportNotificationService implements OnApplicationShutdown {
return;
}
const recipientEMailAddresses = await this.fetchEMailRecipients().then(it => it
.filter(it => it.isActive && it.userProfile?.emailVerified)
.map(it => it.userProfile?.email)
.filter(x => x != null),
);
recipientEMailAddresses.push(
...(this.meta.email ? [this.meta.email] : []),
);
const recipientEMailAddresses = await this.getRecipientEMailAddresses();
if (recipientEMailAddresses.length <= 0) {
return;

16
packages/backend/src/core/AbuseReportService.ts
View file

@ -10,9 +10,11 @@ import { bindThis } from '@/decorators.js';
import type { AbuseUserReportsRepository, MiAbuseUserReport, MiUser, UsersRepository } from '@/models/_.js';
import { AbuseReportNotificationService } from '@/core/AbuseReportNotificationService.js';
import { QueueService } from '@/core/QueueService.js';
import { InstanceActorService } from '@/core/InstanceActorService.js';
import { ApRendererService } from '@/core/activitypub/ApRendererService.js';
import { ModerationLogService } from '@/core/ModerationLogService.js';
import { SystemAccountService } from '@/core/SystemAccountService.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import { trackPromise } from '@/misc/promise-tracker.js';
import { IdService } from './IdService.js';
@Injectable()
@ -27,7 +29,7 @@ export class AbuseReportService {
private idService: IdService,
private abuseReportNotificationService: AbuseReportNotificationService,
private queueService: QueueService,
private instanceActorService: InstanceActorService,
private systemAccountService: SystemAccountService,
private apRendererService: ApRendererService,
private moderationLogService: ModerationLogService,
) {
@ -67,11 +69,11 @@ export class AbuseReportService {
reports.push(report);
}
return Promise.all([
trackPromise(Promise.all([
this.abuseReportNotificationService.notifyAdminStream(reports),
this.abuseReportNotificationService.notifySystemWebhook(reports, 'abuseReport'),
this.abuseReportNotificationService.notifyMail(reports),
]);
]));
}
/**
@ -125,18 +127,18 @@ export class AbuseReportService {
const report = await this.abuseUserReportsRepository.findOneByOrFail({ id: reportId });
if (report.targetUserHost == null) {
throw new Error('The target user host is null.');
throw new IdentifiableError('0b1ce202-b2c1-4ee4-8af4-2742a51b383d', 'The target user host is null.');
}
if (report.forwarded) {
throw new Error('The report has already been forwarded.');
throw new IdentifiableError('5c008bdf-f0e8-4154-9f34-804e114516d7', 'The report has already been forwarded.');
}
await this.abuseUserReportsRepository.update(report.id, {
forwarded: true,
});
const actor = await this.instanceActorService.getInstanceActor();
const actor = await this.systemAccountService.fetch('actor');
const targetUser = await this.usersRepository.findOneByOrFail({ id: report.targetUserId });
const flag = this.apRendererService.renderFlag(actor, targetUser.uri!, report.comment);

71
packages/backend/src/core/AccountMoveService.ts
View file

@ -20,10 +20,13 @@ import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js';
import { ApDeliverManagerService } from '@/core/activitypub/ApDeliverManagerService.js';
import { ApRendererService } from '@/core/activitypub/ApRendererService.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { ProxyAccountService } from '@/core/ProxyAccountService.js';
import { FederatedInstanceService } from '@/core/FederatedInstanceService.js';
import InstanceChart from '@/core/chart/charts/instance.js';
import PerUserFollowingChart from '@/core/chart/charts/per-user-following.js';
import { SystemAccountService } from '@/core/SystemAccountService.js';
import { RoleService } from '@/core/RoleService.js';
import { AntennaService } from '@/core/AntennaService.js';
import { CacheService } from '@/core/CacheService.js';
@Injectable()
export class AccountMoveService {
@ -58,12 +61,15 @@ export class AccountMoveService {
private apRendererService: ApRendererService,
private apDeliverManagerService: ApDeliverManagerService,
private globalEventService: GlobalEventService,
private proxyAccountService: ProxyAccountService,
private perUserFollowingChart: PerUserFollowingChart,
private federatedInstanceService: FederatedInstanceService,
private instanceChart: InstanceChart,
private relayService: RelayService,
private queueService: QueueService,
private systemAccountService: SystemAccountService,
private roleService: RoleService,
private antennaService: AntennaService,
private readonly cacheService: CacheService,
) {
}
@ -91,23 +97,22 @@ export class AccountMoveService {
const srcPerson = await this.apRendererService.renderPerson(src);
const updateAct = this.apRendererService.addContext(this.apRendererService.renderUpdate(srcPerson, src));
await this.apDeliverManagerService.deliverToFollowers(src, updateAct);
this.relayService.deliverToRelays(src, updateAct);
await this.relayService.deliverToRelays(src, updateAct);
// Deliver Move activity to the followers of the old account
const moveAct = this.apRendererService.addContext(this.apRendererService.renderMove(src, dst));
await this.apDeliverManagerService.deliverToFollowers(src, moveAct);
await this.relayService.deliverToRelays(src, moveAct);
// Publish meUpdated event
const iObj = await this.userEntityService.pack(src.id, src, { schema: 'MeDetailed', includeSecrets: true });
this.globalEventService.publishMainStream(src.id, 'meUpdated', iObj);
// Unfollow after 24 hours
const followings = await this.followingsRepository.findBy({
followerId: src.id,
});
this.queueService.createDelayedUnfollowJob(followings.map(following => ({
const followings = await this.cacheService.userFollowingsCache.fetch(src.id);
this.queueService.createDelayedUnfollowJob(Array.from(followings.keys()).map(followeeId => ({
from: { id: src.id },
to: { id: following.followeeId },
to: { id: followeeId },
})), process.env.NODE_ENV === 'test' ? 10000 : 1000 * 60 * 60 * 24);
await this.postMoveProcess(src, dst);
@ -123,19 +128,19 @@ export class AccountMoveService {
this.copyBlocking(src, dst),
this.copyMutings(src, dst),
this.deleteScheduledNotes(src),
this.copyRoles(src, dst),
this.updateLists(src, dst),
this.antennaService.onMoveAccount(src, dst),
]);
} catch {
/* skip if any error happens */
}
// follow the new account
const proxy = await this.proxyAccountService.fetch();
const followings = await this.followingsRepository.findBy({
followeeId: src.id,
followerHost: IsNull(), // follower is local
followerId: proxy ? Not(proxy.id) : undefined,
});
const proxy = await this.systemAccountService.fetch('proxy');
const followings = await this.cacheService.userFollowersCache.fetch(src.id)
.then(fs => Array.from(fs.values())
.filter(f => f.followerHost == null && f.followerId !== proxy.id));
const followJobs = followings.map(following => ({
from: { id: following.followerId },
to: { id: dst.id },
@ -220,6 +225,32 @@ export class AccountMoveService {
});
}
@bindThis
public async copyRoles(src: ThinUser, dst: ThinUser): Promise<void> {
// Insert new roles with the same values except userId
// role service may have cache for roles so retrieve roles from service
const [oldRoleAssignments, roles] = await Promise.all([
this.roleService.getUserAssigns(src.id),
this.roleService.getRoles(),
]);
if (oldRoleAssignments.length === 0) return;
// No promise all since the only async operation is writing to the database
for (const oldRoleAssignment of oldRoleAssignments) {
const role = roles.find(x => x.id === oldRoleAssignment.roleId);
if (role == null) continue; // Very unlikely however removing role may cause this case
if (!role.preserveAssignmentOnMoveAccount) continue;
try {
await this.roleService.assign(dst.id, role.id, oldRoleAssignment.expiresAt);
} catch (e) {
if (e instanceof RoleService.AlreadyAssignedError) continue;
throw e;
}
}
}
/**
* Update lists while moving accounts.
* - No removal of the old account from the lists
@ -269,10 +300,8 @@ export class AccountMoveService {
// Have the proxy account follow the new account in the same way as UserListService.push
if (this.userEntityService.isRemoteUser(dst)) {
const proxy = await this.proxyAccountService.fetch();
if (proxy) {
this.queueService.createFollowJob([{ from: { id: proxy.id }, to: { id: dst.id } }]);
}
const proxy = await this.systemAccountService.fetch('proxy');
this.queueService.createFollowJob([{ from: { id: proxy.id }, to: { id: dst.id } }]);
}
}
@ -287,9 +316,9 @@ export class AccountMoveService {
await this.usersRepository.decrement({ id: In(localFollowerIds) }, 'followingCount', 1);
// Decrease follower counts of local followees by 1.
const oldFollowings = await this.followingsRepository.findBy({ followerId: oldAccount.id });
if (oldFollowings.length > 0) {
await this.usersRepository.decrement({ id: In(oldFollowings.map(following => following.followeeId)) }, 'followersCount', 1);
const oldFollowings = await this.cacheService.userFollowingsCache.fetch(oldAccount.id);
if (oldFollowings.size > 0) {
await this.usersRepository.decrement({ id: In(Array.from(oldFollowings.keys())) }, 'followersCount', 1);
}
// Update instance stats by decreasing remote followers count by the number of local followers who were following the old account.

9
packages/backend/src/core/AccountUpdateService.ts
View file

@ -27,15 +27,12 @@ export class AccountUpdateService {
}
@bindThis
public async publishToFollowers(userId: MiUser['id']) {
const user = await this.usersRepository.findOneBy({ id: userId });
if (user == null) throw new Error('user not found');
public async publishToFollowers(user: MiUser) {
// フォロワーがリモートユーザーかつ投稿者がローカルユーザーならUpdateを配信
if (this.userEntityService.isLocalUser(user)) {
const content = this.apRendererService.addContext(this.apRendererService.renderUpdate(await this.apRendererService.renderPerson(user), user));
this.apDeliverManagerService.deliverToFollowers(user, content);
this.relayService.deliverToRelays(user, content);
await this.apDeliverManagerService.deliverToFollowers(user, content);
await this.relayService.deliverToRelays(user, content);
}
}
}

82
packages/backend/src/core/AchievementService.ts
View file

@ -9,87 +9,7 @@ import type { MiUser } from '@/models/User.js';
import { DI } from '@/di-symbols.js';
import { bindThis } from '@/decorators.js';
import { NotificationService } from '@/core/NotificationService.js';
export const ACHIEVEMENT_TYPES = [
'notes1',
'notes10',
'notes100',
'notes500',
'notes1000',
'notes5000',
'notes10000',
'notes20000',
'notes30000',
'notes40000',
'notes50000',
'notes60000',
'notes70000',
'notes80000',
'notes90000',
'notes100000',
'login3',
'login7',
'login15',
'login30',
'login60',
'login100',
'login200',
'login300',
'login400',
'login500',
'login600',
'login700',
'login800',
'login900',
'login1000',
'passedSinceAccountCreated1',
'passedSinceAccountCreated2',
'passedSinceAccountCreated3',
'loggedInOnBirthday',
'loggedInOnNewYearsDay',
'noteClipped1',
'noteFavorited1',
'myNoteFavorited1',
'profileFilled',
'markedAsCat',
'following1',
'following10',
'following50',
'following100',
'following300',
'followers1',
'followers10',
'followers50',
'followers100',
'followers300',
'followers500',
'followers1000',
'collectAchievements30',
'viewAchievements3min',
'iLoveMisskey',
'foundTreasure',
'client30min',
'client60min',
'noteDeletedWithin1min',
'postedAtLateNight',
'postedAt0min0sec',
'selfQuote',
'htl20npm',
'viewInstanceChart',
'outputHelloWorldOnScratchpad',
'open3windows',
'driveFolderCircularReference',
'reactWithoutRead',
'clickedClickHere',
'justPlainLucky',
'setNameToSyuilo',
'cookieClicked',
'brainDiver',
'smashTestNotificationButton',
'tutorialCompleted',
'bubbleGameExplodingHead',
'bubbleGameDoubleExplodingHead',
] as const;
import { ACHIEVEMENT_TYPES } from '@/models/UserProfile.js';
@Injectable()
export class AchievementService {

11
packages/backend/src/core/AnnouncementService.ts
View file

@ -14,6 +14,7 @@ import { IdService } from '@/core/IdService.js';
import { AnnouncementEntityService } from '@/core/entities/AnnouncementEntityService.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
import { ModerationLogService } from '@/core/ModerationLogService.js';
import { RoleService } from '@/core/RoleService.js';
@Injectable()
export class AnnouncementService {
@ -31,6 +32,7 @@ export class AnnouncementService {
private globalEventService: GlobalEventService,
private moderationLogService: ModerationLogService,
private announcementEntityService: AnnouncementEntityService,
private roleService: RoleService,
) {
}
@ -46,6 +48,7 @@ export class AnnouncementService {
const readsQuery = this.announcementReadsRepository.createQueryBuilder('read')
.select('read.announcementId')
.where('read.userId = :userId', { userId: user.id });
const roles = await this.roleService.getUserRoles(user);
const q = this.announcementsRepository.createQueryBuilder('announcement')
.where('announcement.isActive = true')
@ -58,6 +61,10 @@ export class AnnouncementService {
qb.orWhere('announcement.forExistingUsers = false');
qb.orWhere('announcement.id > :userId', { userId: user.id });
}))
.andWhere(new Brackets(qb => {
qb.orWhere('announcement.forRoles && :roles', { roles: roles.map((r) => r.id) });
qb.orWhere('announcement.forRoles = \'{}\'');
}))
.andWhere(`announcement.id NOT IN (${ readsQuery.getQuery() })`);
q.setParameters(readsQuery.getParameters());
@ -76,8 +83,10 @@ export class AnnouncementService {
icon: values.icon,
display: values.display,
forExistingUsers: values.forExistingUsers,
forRoles: values.forRoles,
silence: values.silence,
needConfirmationToRead: values.needConfirmationToRead,
confetti: values.confetti,
userId: values.userId,
});
@ -128,8 +137,10 @@ export class AnnouncementService {
display: values.display,
icon: values.icon,
forExistingUsers: values.forExistingUsers,
forRoles: values.forRoles,
silence: values.silence,
needConfirmationToRead: values.needConfirmationToRead,
confetti: values.confetti,
isActive: values.isActive,
});

72
packages/backend/src/core/AntennaService.ts
View file

@ -5,18 +5,20 @@
import { Inject, Injectable } from '@nestjs/common';
import * as Redis from 'ioredis';
import { In } from 'typeorm';
import { FanoutTimelineService } from '@/core/FanoutTimelineService.js';
import type { GlobalEvents } from '@/core/GlobalEventService.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
import { UtilityService } from '@/core/UtilityService.js';
import { bindThis } from '@/decorators.js';
import { DI } from '@/di-symbols.js';
import * as Acct from '@/misc/acct.js';
import type { Packed } from '@/misc/json-schema.js';
import type { AntennasRepository, UserListMembershipsRepository } from '@/models/_.js';
import type { MiAntenna } from '@/models/Antenna.js';
import type { MiNote } from '@/models/Note.js';
import type { MiUser } from '@/models/User.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
import * as Acct from '@/misc/acct.js';
import type { Packed } from '@/misc/json-schema.js';
import { DI } from '@/di-symbols.js';
import type { AntennasRepository, UserListMembershipsRepository } from '@/models/_.js';
import { UtilityService } from '@/core/UtilityService.js';
import { bindThis } from '@/decorators.js';
import type { GlobalEvents } from '@/core/GlobalEventService.js';
import { FanoutTimelineService } from '@/core/FanoutTimelineService.js';
import { CacheService } from './CacheService.js';
import type { OnApplicationShutdown } from '@nestjs/common';
@Injectable()
@ -37,6 +39,7 @@ export class AntennaService implements OnApplicationShutdown {
@Inject(DI.userListMembershipsRepository)
private userListMembershipsRepository: UserListMembershipsRepository,
private cacheService: CacheService,
private utilityService: UtilityService,
private globalEventService: GlobalEventService,
private fanoutTimelineService: FanoutTimelineService,
@ -111,8 +114,7 @@ export class AntennaService implements OnApplicationShutdown {
@bindThis
public async checkHitAntenna(antenna: MiAntenna, note: (MiNote | Packed<'Note'>), noteUser: { id: MiUser['id']; username: string; host: string | null; isBot: boolean; }): Promise<boolean> {
if (note.visibility === 'specified') return false;
if (note.visibility === 'followers') return false;
if (antenna.excludeNotesInSensitiveChannel && note.channel?.isSensitive) return false;
if (antenna.excludeBots && noteUser.isBot) return false;
@ -120,6 +122,19 @@ export class AntennaService implements OnApplicationShutdown {
if (!antenna.withReplies && note.replyId != null) return false;
if (note.visibility === 'specified') {
if (note.userId !== antenna.userId) {
if (note.visibleUserIds == null) return false;
if (!note.visibleUserIds.includes(antenna.userId)) return false;
}
}
if (note.visibility === 'followers') {
const followings = await this.cacheService.userFollowingsCache.fetch(antenna.userId);
const isFollowing = followings.has(note.userId);
if (!isFollowing && antenna.userId !== note.userId) return false;
}
if (antenna.src === 'home') {
// TODO
} else if (antenna.src === 'list') {
@ -210,6 +225,41 @@ export class AntennaService implements OnApplicationShutdown {
return this.antennas;
}
@bindThis
public async onMoveAccount(src: MiUser, dst: MiUser): Promise<void> {
// There is a possibility for users to add the srcUser to their antennas, but it's low, so we don't check it.
// Get MiAntenna[] from cache and filter to select antennas with the src user is in the users list
const srcUserAcct = this.utilityService.getFullApAccount(src.username, src.host).toLowerCase();
const antennasToMigrate = (await this.getAntennas()).filter(antenna => {
return antenna.users.some(user => {
const { username, host } = Acct.parse(user);
return this.utilityService.getFullApAccount(username, host).toLowerCase() === srcUserAcct;
});
});
if (antennasToMigrate.length === 0) return;
const antennaIds = antennasToMigrate.map(x => x.id);
// Update the antennas by appending dst users acct to the users list
const dstUserAcct = '@' + Acct.toString({ username: dst.username, host: dst.host });
await this.antennasRepository.createQueryBuilder('antenna')
.update()
.set({
users: () => 'array_append(antenna.users, :dstUserAcct)',
})
.where('antenna.id IN (:...antennaIds)', { antennaIds })
.setParameters({ dstUserAcct })
.execute();
// announce update to event
for (const newAntenna of await this.antennasRepository.findBy({ id: In(antennaIds) })) {
this.globalEventService.publishInternalEvent('antennaUpdated', newAntenna);
}
}
@bindThis
public dispose(): void {
this.redisForSub.off('message', this.onRedisMessage);

18
packages/backend/src/core/ApLogService.ts
View file

@ -139,6 +139,24 @@ export class ApLogService {
}
}
/**
* Deletes all logged inbox activities from a user or users
* @param userIds IDs of the users to delete
*/
public async deleteInboxLogs(userIds: string | string[]): Promise<number> {
if (Array.isArray(userIds)) {
const logsDeleted = await this.apInboxLogsRepository.delete({
authUserId: In(userIds),
});
return logsDeleted.affected ?? 0;
} else {
const logsDeleted = await this.apInboxLogsRepository.delete({
authUserId: userIds,
});
return logsDeleted.affected ?? 0;
}
}
/**
* Deletes all expired AP logs and garbage-collects the AP context cache.
* Returns the total number of deleted rows.

101
packages/backend/src/core/BunnyService.ts Normal file
View file

@ -0,0 +1,101 @@
/*
* SPDX-FileCopyrightText: marie and sharkey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
import * as https from 'node:https';
import * as fs from 'node:fs';
import { Readable } from 'node:stream';
import { finished } from 'node:stream/promises';
import { Injectable } from '@nestjs/common';
import type { MiMeta } from '@/models/Meta.js';
import { HttpRequestService } from '@/core/HttpRequestService.js';
import { bindThis } from '@/decorators.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import Logger from '@/logger.js';
@Injectable()
export class BunnyService {
private bunnyCdnLogger: Logger;
constructor(
private httpRequestService: HttpRequestService,
) {
this.bunnyCdnLogger = new Logger('bunnycdn', 'blue');
}
@bindThis
public getBunnyInfo(meta: MiMeta) {
if (!meta.objectStorageEndpoint || !meta.objectStorageBucket || !meta.objectStorageSecretKey) {
throw new IdentifiableError('689ee33f-f97c-479a-ac49-1b9f8140bf90', 'Failed to use BunnyCDN, One of the required fields is missing.');
}
return {
endpoint: meta.objectStorageEndpoint,
/*
The way S3 works is that the Secret Key is essentially the password for the API but Bunny calls their password AccessKey so we call it accessKey here.
Bunny also doesn't specify a username/s3 access key when doing HTTP API requests so we end up not using our Access Key field from the form.
*/
accessKey: meta.objectStorageSecretKey,
zone: meta.objectStorageBucket,
fullUrl: `https://${meta.objectStorageEndpoint}/${meta.objectStorageBucket}`,
};
}
@bindThis
public usingBunnyCDN(meta: MiMeta) {
return meta.objectStorageEndpoint && meta.objectStorageEndpoint.endsWith('bunnycdn.com');
}
@bindThis
public async upload(meta: MiMeta, path: string, input: fs.ReadStream | Buffer) {
const client = this.getBunnyInfo(meta);
// Required to convert the buffer from webpublic and thumbnail to a ReadableStream for PUT
const data = Buffer.isBuffer(input) ? Readable.from(input) : input;
const agent = this.httpRequestService.getAgentByUrl(new URL(`${client.fullUrl}/${path}`), !meta.objectStorageUseProxy, true);
// Seperation of path and host/domain is required here
const options = {
method: 'PUT',
host: client.endpoint,
path: `/${client.zone}/${path}`,
headers: {
AccessKey: client.accessKey,
'Content-Type': 'application/octet-stream',
},
agent: agent,
};
const req = https.request(options);
// Log and return if BunnyCDN detects wrong data (return is used to prevent console spam as this event occurs multiple times)
req.on('response', (res) => {
if (res.statusCode === 401) {
this.bunnyCdnLogger.error('Invalid AccessKey or region hostname');
data.destroy();
return;
}
});
req.on('error', (error) => {
this.bunnyCdnLogger.error('Unhandled error', error);
data.destroy();
throw new IdentifiableError('689ee33f-f97c-479a-ac49-1b9f8140bf91', 'An error has occurred while connecting to BunnyCDN', true, error);
});
data.pipe(req).on('finish', () => {
data.destroy();
});
// wait till stream gets destroyed upon finish of piping to prevent the UI from showing the upload as success way too early
await finished(data);
}
@bindThis
public delete(meta: MiMeta, file: string) {
const client = this.getBunnyInfo(meta);
return this.httpRequestService.send(`${client.fullUrl}/${file}`, { method: 'DELETE', headers: { AccessKey: client.accessKey } });
}
}

520
packages/backend/src/core/CacheService.ts
View file

@ -5,27 +5,54 @@
import { Inject, Injectable } from '@nestjs/common';
import * as Redis from 'ioredis';
import type { BlockingsRepository, FollowingsRepository, MutingsRepository, RenoteMutingsRepository, MiUserProfile, UserProfilesRepository, UsersRepository, MiFollowing } from '@/models/_.js';
import { In, IsNull } from 'typeorm';
import type { BlockingsRepository, FollowingsRepository, MutingsRepository, RenoteMutingsRepository, MiUserProfile, UserProfilesRepository, UsersRepository, MiNote, MiFollowing, NoteThreadMutingsRepository } from '@/models/_.js';
import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js';
import type { MiLocalUser, MiUser } from '@/models/User.js';
import { QuantumKVCache } from '@/misc/QuantumKVCache.js';
import type { MiLocalUser, MiRemoteUser, MiUser } from '@/models/User.js';
import { DI } from '@/di-symbols.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { bindThis } from '@/decorators.js';
import type { GlobalEvents } from '@/core/GlobalEventService.js';
import type { InternalEventTypes } from '@/core/GlobalEventService.js';
import { InternalEventService } from '@/core/InternalEventService.js';
import type { OnApplicationShutdown } from '@nestjs/common';
export interface FollowStats {
localFollowing: number;
localFollowers: number;
remoteFollowing: number;
remoteFollowers: number;
}
export interface CachedTranslation {
sourceLang: string | undefined;
text: string | undefined;
}
export interface CachedTranslationEntity {
l?: string;
t?: string;
u?: number;
}
@Injectable()
export class CacheService implements OnApplicationShutdown {
public userByIdCache: MemoryKVCache<MiUser>;
public localUserByNativeTokenCache: MemoryKVCache<MiLocalUser | null>;
public localUserByIdCache: MemoryKVCache<MiLocalUser>;
public uriPersonCache: MemoryKVCache<MiUser | null>;
public userProfileCache: RedisKVCache<MiUserProfile>;
public userMutingsCache: RedisKVCache<Set<string>>;
public userBlockingCache: RedisKVCache<Set<string>>;
public userBlockedCache: RedisKVCache<Set<string>>; // NOTE: 「被」Blockキャッシュ
public renoteMutingsCache: RedisKVCache<Set<string>>;
public userFollowingsCache: RedisKVCache<Record<string, Pick<MiFollowing, 'withReplies'> | undefined>>;
public userProfileCache: QuantumKVCache<MiUserProfile>;
public userMutingsCache: QuantumKVCache<Set<string>>;
public userBlockingCache: QuantumKVCache<Set<string>>;
public userBlockedCache: QuantumKVCache<Set<string>>; // NOTE: 「被」Blockキャッシュ
public renoteMutingsCache: QuantumKVCache<Set<string>>;
public threadMutingsCache: QuantumKVCache<Set<string>>;
public noteMutingsCache: QuantumKVCache<Set<string>>;
public userFollowingsCache: QuantumKVCache<Map<string, Omit<MiFollowing, 'isFollowerHibernated'>>>;
public userFollowersCache: QuantumKVCache<Map<string, Omit<MiFollowing, 'isFollowerHibernated'>>>;
public hibernatedUserCache: QuantumKVCache<boolean>;
protected userFollowStatsCache = new MemoryKVCache<FollowStats>(1000 * 60 * 10); // 10 minutes
protected translationsCache: RedisKVCache<CachedTranslationEntity>;
constructor(
@Inject(DI.redis)
@ -52,7 +79,11 @@ export class CacheService implements OnApplicationShutdown {
@Inject(DI.followingsRepository)
private followingsRepository: FollowingsRepository,
@Inject(DI.noteThreadMutingsRepository)
private readonly noteThreadMutingsRepository: NoteThreadMutingsRepository,
private userEntityService: UserEntityService,
private readonly internalEventService: InternalEventService,
) {
//this.onMessage = this.onMessage.bind(this);
@ -61,76 +92,202 @@ export class CacheService implements OnApplicationShutdown {
this.localUserByIdCache = new MemoryKVCache<MiLocalUser>(1000 * 60 * 5); // 5m
this.uriPersonCache = new MemoryKVCache<MiUser | null>(1000 * 60 * 5); // 5m
this.userProfileCache = new RedisKVCache<MiUserProfile>(this.redisClient, 'userProfile', {
this.userProfileCache = new QuantumKVCache(this.internalEventService, 'userProfile', {
lifetime: 1000 * 60 * 30, // 30m
memoryCacheLifetime: 1000 * 60, // 1m
fetcher: (key) => this.userProfilesRepository.findOneByOrFail({ userId: key }),
toRedisConverter: (value) => JSON.stringify(value),
fromRedisConverter: (value) => JSON.parse(value), // TODO: date型の考慮
bulkFetcher: userIds => this.userProfilesRepository.findBy({ userId: In(userIds) }).then(ps => ps.map(p => [p.userId, p])),
});
this.userMutingsCache = new RedisKVCache<Set<string>>(this.redisClient, 'userMutings', {
this.userMutingsCache = new QuantumKVCache<Set<string>>(this.internalEventService, 'userMutings', {
lifetime: 1000 * 60 * 30, // 30m
memoryCacheLifetime: 1000 * 60, // 1m
fetcher: (key) => this.mutingsRepository.find({ where: { muterId: key }, select: ['muteeId'] }).then(xs => new Set(xs.map(x => x.muteeId))),
toRedisConverter: (value) => JSON.stringify(Array.from(value)),
fromRedisConverter: (value) => new Set(JSON.parse(value)),
bulkFetcher: muterIds => this.mutingsRepository
.createQueryBuilder('muting')
.select('"muting"."muterId"', 'muterId')
.addSelect('array_agg("muting"."muteeId")', 'muteeIds')
.where({ muterId: In(muterIds) })
.groupBy('muting.muterId')
.getRawMany<{ muterId: string, muteeIds: string[] }>()
.then(ms => ms.map(m => [m.muterId, new Set(m.muteeIds)])),
});
this.userBlockingCache = new RedisKVCache<Set<string>>(this.redisClient, 'userBlocking', {
this.userBlockingCache = new QuantumKVCache<Set<string>>(this.internalEventService, 'userBlocking', {
lifetime: 1000 * 60 * 30, // 30m
memoryCacheLifetime: 1000 * 60, // 1m
fetcher: (key) => this.blockingsRepository.find({ where: { blockerId: key }, select: ['blockeeId'] }).then(xs => new Set(xs.map(x => x.blockeeId))),
toRedisConverter: (value) => JSON.stringify(Array.from(value)),
fromRedisConverter: (value) => new Set(JSON.parse(value)),
bulkFetcher: blockerIds => this.blockingsRepository
.createQueryBuilder('blocking')
.select('"blocking"."blockerId"', 'blockerId')
.addSelect('array_agg("blocking"."blockeeId")', 'blockeeIds')
.where({ blockerId: In(blockerIds) })
.groupBy('blocking.blockerId')
.getRawMany<{ blockerId: string, blockeeIds: string[] }>()
.then(ms => ms.map(m => [m.blockerId, new Set(m.blockeeIds)])),
});
this.userBlockedCache = new RedisKVCache<Set<string>>(this.redisClient, 'userBlocked', {
this.userBlockedCache = new QuantumKVCache<Set<string>>(this.internalEventService, 'userBlocked', {
lifetime: 1000 * 60 * 30, // 30m
memoryCacheLifetime: 1000 * 60, // 1m
fetcher: (key) => this.blockingsRepository.find({ where: { blockeeId: key }, select: ['blockerId'] }).then(xs => new Set(xs.map(x => x.blockerId))),
toRedisConverter: (value) => JSON.stringify(Array.from(value)),
fromRedisConverter: (value) => new Set(JSON.parse(value)),
bulkFetcher: blockeeIds => this.blockingsRepository
.createQueryBuilder('blocking')
.select('"blocking"."blockeeId"', 'blockeeId')
.addSelect('array_agg("blocking"."blockeeId")', 'blockeeIds')
.where({ blockeeId: In(blockeeIds) })
.groupBy('blocking.blockeeId')
.getRawMany<{ blockeeId: string, blockerIds: string[] }>()
.then(ms => ms.map(m => [m.blockeeId, new Set(m.blockerIds)])),
});
this.renoteMutingsCache = new RedisKVCache<Set<string>>(this.redisClient, 'renoteMutings', {
this.renoteMutingsCache = new QuantumKVCache<Set<string>>(this.internalEventService, 'renoteMutings', {
lifetime: 1000 * 60 * 30, // 30m
memoryCacheLifetime: 1000 * 60, // 1m
fetcher: (key) => this.renoteMutingsRepository.find({ where: { muterId: key }, select: ['muteeId'] }).then(xs => new Set(xs.map(x => x.muteeId))),
toRedisConverter: (value) => JSON.stringify(Array.from(value)),
fromRedisConverter: (value) => new Set(JSON.parse(value)),
bulkFetcher: muterIds => this.renoteMutingsRepository
.createQueryBuilder('muting')
.select('"muting"."muterId"', 'muterId')
.addSelect('array_agg("muting"."muteeId")', 'muteeIds')
.where({ muterId: In(muterIds) })
.groupBy('muting.muterId')
.getRawMany<{ muterId: string, muteeIds: string[] }>()
.then(ms => ms.map(m => [m.muterId, new Set(m.muteeIds)])),
});
this.userFollowingsCache = new RedisKVCache<Record<string, Pick<MiFollowing, 'withReplies'> | undefined>>(this.redisClient, 'userFollowings', {
this.threadMutingsCache = new QuantumKVCache<Set<string>>(this.internalEventService, 'threadMutings', {
lifetime: 1000 * 60 * 30, // 30m
memoryCacheLifetime: 1000 * 60, // 1m
fetcher: (key) => this.followingsRepository.find({ where: { followerId: key }, select: ['followeeId', 'withReplies'] }).then(xs => {
const obj: Record<string, Pick<MiFollowing, 'withReplies'> | undefined> = {};
for (const x of xs) {
obj[x.followeeId] = { withReplies: x.withReplies };
fetcher: muterId => this.noteThreadMutingsRepository
.find({ where: { userId: muterId, isPostMute: false }, select: { threadId: true } })
.then(ms => new Set(ms.map(m => m.threadId))),
bulkFetcher: muterIds => this.noteThreadMutingsRepository
.createQueryBuilder('muting')
.select('"muting"."userId"', 'userId')
.addSelect('array_agg("muting"."threadId")', 'threadIds')
.groupBy('"muting"."userId"')
.where({ userId: In(muterIds), isPostMute: false })
.getRawMany<{ userId: string, threadIds: string[] }>()
.then(ms => ms.map(m => [m.userId, new Set(m.threadIds)])),
});
this.noteMutingsCache = new QuantumKVCache<Set<string>>(this.internalEventService, 'noteMutings', {
lifetime: 1000 * 60 * 30, // 30m
fetcher: muterId => this.noteThreadMutingsRepository
.find({ where: { userId: muterId, isPostMute: true }, select: { threadId: true } })
.then(ms => new Set(ms.map(m => m.threadId))),
bulkFetcher: muterIds => this.noteThreadMutingsRepository
.createQueryBuilder('muting')
.select('"muting"."userId"', 'userId')
.addSelect('array_agg("muting"."threadId")', 'threadIds')
.groupBy('"muting"."userId"')
.where({ userId: In(muterIds), isPostMute: true })
.getRawMany<{ userId: string, threadIds: string[] }>()
.then(ms => ms.map(m => [m.userId, new Set(m.threadIds)])),
});
this.userFollowingsCache = new QuantumKVCache<Map<string, Omit<MiFollowing, 'isFollowerHibernated'>>>(this.internalEventService, 'userFollowings', {
lifetime: 1000 * 60 * 30, // 30m
fetcher: (key) => this.followingsRepository.findBy({ followerId: key }).then(xs => new Map(xs.map(f => [f.followeeId, f]))),
bulkFetcher: followerIds => this.followingsRepository
.findBy({ followerId: In(followerIds) })
.then(fs => fs
.reduce((groups, f) => {
let group = groups.get(f.followerId);
if (!group) {
group = new Map();
groups.set(f.followerId, group);
}
group.set(f.followeeId, f);
return groups;
}, new Map<string, Map<string, Omit<MiFollowing, 'isFollowerHibernated'>>>)),
});
this.userFollowersCache = new QuantumKVCache<Map<string, Omit<MiFollowing, 'isFollowerHibernated'>>>(this.internalEventService, 'userFollowers', {
lifetime: 1000 * 60 * 30, // 30m
fetcher: followeeId => this.followingsRepository.findBy({ followeeId: followeeId }).then(xs => new Map(xs.map(x => [x.followerId, x]))),
bulkFetcher: followeeIds => this.followingsRepository
.findBy({ followeeId: In(followeeIds) })
.then(fs => fs
.reduce((groups, f) => {
let group = groups.get(f.followeeId);
if (!group) {
group = new Map();
groups.set(f.followeeId, group);
}
group.set(f.followerId, f);
return groups;
}, new Map<string, Map<string, Omit<MiFollowing, 'isFollowerHibernated'>>>)),
});
this.hibernatedUserCache = new QuantumKVCache<boolean>(this.internalEventService, 'hibernatedUsers', {
lifetime: 1000 * 60 * 30, // 30m
fetcher: async userId => {
const { isHibernated } = await this.usersRepository.findOneOrFail({
where: { id: userId },
select: { isHibernated: true },
});
return isHibernated;
},
bulkFetcher: async userIds => {
const results = await this.usersRepository.find({
where: { id: In(userIds) },
select: { id: true, isHibernated: true },
});
return results.map(({ id, isHibernated }) => [id, isHibernated]);
},
onChanged: async userIds => {
// We only update local copies since each process will get this event, but we can have user objects in multiple different caches.
// Before doing anything else we must "find" all the objects to update.
const userObjects = new Map<string, MiUser[]>();
const toUpdate: string[] = [];
for (const uid of userIds) {
const toAdd: MiUser[] = [];
const localUserById = this.localUserByIdCache.get(uid);
if (localUserById) toAdd.push(localUserById);
const userById = this.userByIdCache.get(uid);
if (userById) toAdd.push(userById);
if (toAdd.length > 0) {
toUpdate.push(uid);
userObjects.set(uid, toAdd);
}
}
return obj;
}),
toRedisConverter: (value) => JSON.stringify(value),
fromRedisConverter: (value) => JSON.parse(value),
// In many cases, we won't have to do anything.
// Skipping the DB fetch ensures that this remains a single-step synchronous process.
if (toUpdate.length > 0) {
const hibernations = await this.usersRepository.find({ where: { id: In(toUpdate) }, select: { id: true, isHibernated: true } });
for (const { id, isHibernated } of hibernations) {
const users = userObjects.get(id);
if (users) {
for (const u of users) {
u.isHibernated = isHibernated;
}
}
}
}
},
});
this.translationsCache = new RedisKVCache<CachedTranslationEntity>(this.redisClient, 'translations', {
lifetime: 1000 * 60 * 60 * 24 * 7, // 1 week,
memoryCacheLifetime: 1000 * 60, // 1 minute
});
// NOTE: チャンネルのフォロー状況キャッシュはChannelFollowingServiceで行っている
this.redisForSub.on('message', this.onMessage);
this.internalEventService.on('userChangeSuspendedState', this.onUserEvent);
this.internalEventService.on('userChangeDeletedState', this.onUserEvent);
this.internalEventService.on('remoteUserUpdated', this.onUserEvent);
this.internalEventService.on('localUserUpdated', this.onUserEvent);
this.internalEventService.on('userChangeSuspendedState', this.onUserEvent);
this.internalEventService.on('userTokenRegenerated', this.onTokenEvent);
this.internalEventService.on('follow', this.onFollowEvent);
this.internalEventService.on('unfollow', this.onFollowEvent);
}
@bindThis
private async onMessage(_: string, data: string): Promise<void> {
const obj = JSON.parse(data);
if (obj.channel === 'internal') {
const { type, body } = obj.message as GlobalEvents['internal']['payload'];
switch (type) {
case 'userChangeSuspendedState':
case 'userChangeDeletedState':
case 'remoteUserUpdated':
case 'localUserUpdated': {
private async onUserEvent<E extends 'userChangeSuspendedState' | 'userChangeDeletedState' | 'remoteUserUpdated' | 'localUserUpdated'>(body: InternalEventTypes[E], _: E, isLocal: boolean): Promise<void> {
{
{
{
const user = await this.usersRepository.findOneBy({ id: body.id });
if (user == null) {
this.userByIdCache.delete(body.id);
@ -140,6 +297,20 @@ export class CacheService implements OnApplicationShutdown {
this.uriPersonCache.delete(k);
}
}
if (isLocal) {
await Promise.all([
this.userProfileCache.delete(body.id),
this.userMutingsCache.delete(body.id),
this.userBlockingCache.delete(body.id),
this.userBlockedCache.delete(body.id),
this.renoteMutingsCache.delete(body.id),
this.userFollowingsCache.delete(body.id),
this.userFollowersCache.delete(body.id),
this.hibernatedUserCache.delete(body.id),
this.threadMutingsCache.delete(body.id),
this.noteMutingsCache.delete(body.id),
]);
}
} else {
this.userByIdCache.set(user.id, user);
for (const [k, v] of this.uriPersonCache.entries) {
@ -152,24 +323,54 @@ export class CacheService implements OnApplicationShutdown {
this.localUserByIdCache.set(user.id, user);
}
}
break;
}
case 'userTokenRegenerated': {
}
}
}
@bindThis
private async onTokenEvent<E extends 'userTokenRegenerated'>(body: InternalEventTypes[E]): Promise<void> {
{
{
{
const user = await this.usersRepository.findOneByOrFail({ id: body.id }) as MiLocalUser;
this.localUserByNativeTokenCache.delete(body.oldToken);
this.localUserByNativeTokenCache.set(body.newToken, user);
break;
}
}
}
}
@bindThis
private async onFollowEvent<E extends 'follow' | 'unfollow'>(body: InternalEventTypes[E], type: E): Promise<void> {
{
switch (type) {
case 'follow': {
const follower = this.userByIdCache.get(body.followerId);
if (follower) follower.followingCount++;
const followee = this.userByIdCache.get(body.followeeId);
if (followee) followee.followersCount++;
this.userFollowingsCache.delete(body.followerId);
await Promise.all([
this.userFollowingsCache.delete(body.followerId),
this.userFollowersCache.delete(body.followeeId),
]);
this.userFollowStatsCache.delete(body.followerId);
this.userFollowStatsCache.delete(body.followeeId);
break;
}
default:
case 'unfollow': {
const follower = this.userByIdCache.get(body.followerId);
if (follower) follower.followingCount--;
const followee = this.userByIdCache.get(body.followeeId);
if (followee) followee.followersCount--;
await Promise.all([
this.userFollowingsCache.delete(body.followerId),
this.userFollowersCache.delete(body.followeeId),
]);
this.userFollowStatsCache.delete(body.followerId);
this.userFollowStatsCache.delete(body.followeeId);
break;
}
}
}
}
@ -179,9 +380,212 @@ export class CacheService implements OnApplicationShutdown {
return this.userByIdCache.fetch(userId, () => this.usersRepository.findOneByOrFail({ id: userId }));
}
@bindThis
public async findLocalUserById(userId: MiUser['id']): Promise<MiLocalUser | null> {
return await this.localUserByIdCache.fetchMaybe(userId, async () => {
return await this.usersRepository.findOneBy({ id: userId, host: IsNull() }) as MiLocalUser | null ?? undefined;
}) ?? null;
}
@bindThis
public async findRemoteUserById(userId: MiUser['id']): Promise<MiRemoteUser | null> {
const user = await this.findUserById(userId);
if (user.host == null) {
return null;
}
return user as MiRemoteUser;
}
@bindThis
public findOptionalUserById(userId: MiUser['id']) {
return this.userByIdCache.fetchMaybe(userId, async () => await this.usersRepository.findOneBy({ id: userId }) ?? undefined);
}
@bindThis
public async getFollowStats(userId: MiUser['id']): Promise<FollowStats> {
return await this.userFollowStatsCache.fetch(userId, async () => {
const stats = {
localFollowing: 0,
localFollowers: 0,
remoteFollowing: 0,
remoteFollowers: 0,
};
const followings = await this.followingsRepository.findBy([
{ followerId: userId },
{ followeeId: userId },
]);
for (const following of followings) {
if (following.followerId === userId) {
// increment following; user is a follower of someone else
if (following.followeeHost == null) {
stats.localFollowing++;
} else {
stats.remoteFollowing++;
}
} else if (following.followeeId === userId) {
// increment followers; user is followed by someone else
if (following.followerHost == null) {
stats.localFollowers++;
} else {
stats.remoteFollowers++;
}
} else {
// Should never happen
}
}
// Infer remote-remote followers heuristically, since we don't track that info directly.
const user = await this.findUserById(userId);
if (user.host !== null) {
stats.remoteFollowing = Math.max(0, user.followingCount - stats.localFollowing);
stats.remoteFollowers = Math.max(0, user.followersCount - stats.localFollowers);
}
return stats;
});
}
@bindThis
public async getCachedTranslation(note: MiNote, targetLang: string): Promise<CachedTranslation | null> {
const cacheKey = `${note.id}@${targetLang}`;
// Use cached translation, if present and up-to-date
const cached = await this.translationsCache.get(cacheKey);
if (cached && cached.u === note.updatedAt?.valueOf()) {
return {
sourceLang: cached.l,
text: cached.t,
};
}
// No cache entry :(
return null;
}
@bindThis
public async setCachedTranslation(note: MiNote, targetLang: string, translation: CachedTranslation): Promise<void> {
const cacheKey = `${note.id}@${targetLang}`;
await this.translationsCache.set(cacheKey, {
l: translation.sourceLang,
t: translation.text,
u: note.updatedAt?.valueOf(),
});
}
@bindThis
public async getUsers(userIds: Iterable<string>): Promise<Map<string, MiUser>> {
const users = new Map<string, MiUser>;
const toFetch: string[] = [];
for (const userId of userIds) {
const fromCache = this.userByIdCache.get(userId);
if (fromCache) {
users.set(userId, fromCache);
} else {
toFetch.push(userId);
}
}
if (toFetch.length > 0) {
const fetched = await this.usersRepository.findBy({
id: In(toFetch),
});
for (const user of fetched) {
users.set(user.id, user);
this.userByIdCache.set(user.id, user);
}
}
return users;
}
@bindThis
public async isFollowing(follower: string | { id: string }, followee: string | { id: string }): Promise<boolean> {
const followerId = typeof(follower) === 'string' ? follower : follower.id;
const followeeId = typeof(followee) === 'string' ? followee : followee.id;
// This lets us use whichever one is in memory, falling back to DB fetch via userFollowingsCache.
return this.userFollowersCache.get(followeeId)?.has(followerId)
?? (await this.userFollowingsCache.fetch(followerId)).has(followeeId);
}
/**
* Returns all hibernated followers.
*/
@bindThis
public async getHibernatedFollowers(followeeId: string): Promise<MiFollowing[]> {
const followers = await this.getFollowersWithHibernation(followeeId);
return followers.filter(f => f.isFollowerHibernated);
}
/**
* Returns all non-hibernated followers.
*/
@bindThis
public async getNonHibernatedFollowers(followeeId: string): Promise<MiFollowing[]> {
const followers = await this.getFollowersWithHibernation(followeeId);
return followers.filter(f => !f.isFollowerHibernated);
}
/**
* Returns follower relations with populated isFollowerHibernated.
* If you don't need this field, then please use userFollowersCache directly for reduced overhead.
*/
@bindThis
public async getFollowersWithHibernation(followeeId: string): Promise<MiFollowing[]> {
const followers = await this.userFollowersCache.fetch(followeeId);
const hibernations = await this.hibernatedUserCache.fetchMany(followers.keys()).then(fs => fs.reduce((map, f) => {
map.set(f[0], f[1]);
return map;
}, new Map<string, boolean>));
return Array.from(followers.values()).map(following => ({
...following,
isFollowerHibernated: hibernations.get(following.followerId) ?? false,
}));
}
/**
* Refreshes follower and following relations for the given user.
*/
@bindThis
public async refreshFollowRelationsFor(userId: string): Promise<void> {
const followings = await this.userFollowingsCache.refresh(userId);
const followees = Array.from(followings.values()).map(f => f.followeeId);
await this.userFollowersCache.deleteMany(followees);
}
@bindThis
public clear(): void {
this.userByIdCache.clear();
this.localUserByNativeTokenCache.clear();
this.localUserByIdCache.clear();
this.uriPersonCache.clear();
this.userProfileCache.clear();
this.userMutingsCache.clear();
this.userBlockingCache.clear();
this.userBlockedCache.clear();
this.renoteMutingsCache.clear();
this.userFollowingsCache.clear();
this.userFollowStatsCache.clear();
this.translationsCache.clear();
}
@bindThis
public dispose(): void {
this.redisForSub.off('message', this.onMessage);
this.internalEventService.off('userChangeSuspendedState', this.onUserEvent);
this.internalEventService.off('userChangeDeletedState', this.onUserEvent);
this.internalEventService.off('remoteUserUpdated', this.onUserEvent);
this.internalEventService.off('localUserUpdated', this.onUserEvent);
this.internalEventService.off('userChangeSuspendedState', this.onUserEvent);
this.internalEventService.off('userTokenRegenerated', this.onTokenEvent);
this.internalEventService.off('follow', this.onFollowEvent);
this.internalEventService.off('unfollow', this.onFollowEvent);
this.userByIdCache.dispose();
this.localUserByNativeTokenCache.dispose();
this.localUserByIdCache.dispose();
@ -191,7 +595,11 @@ export class CacheService implements OnApplicationShutdown {
this.userBlockingCache.dispose();
this.userBlockedCache.dispose();
this.renoteMutingsCache.dispose();
this.threadMutingsCache.dispose();
this.noteMutingsCache.dispose();
this.userFollowingsCache.dispose();
this.userFollowersCache.dispose();
this.hibernatedUserCache.dispose();
}
@bindThis

33
packages/backend/src/core/CaptchaService.ts
View file

@ -9,7 +9,10 @@ import { bindThis } from '@/decorators.js';
import { MetaService } from '@/core/MetaService.js';
import { MiMeta } from '@/models/Meta.js';
import Logger from '@/logger.js';
import { LoggerService } from './LoggerService.js';
import { LoggerService } from '@/core/LoggerService.js';
import { CaptchaError } from '@/misc/captcha-error.js';
export { CaptchaError } from '@/misc/captcha-error.js';
export const supportedCaptchaProviders = ['none', 'hcaptcha', 'mcaptcha', 'recaptcha', 'turnstile', 'fc', 'testcaptcha'] as const;
export type CaptchaProvider = typeof supportedCaptchaProviders[number];
@ -47,27 +50,15 @@ export type CaptchaSetting = {
siteKey: string | null;
secretKey: string | null;
}
}
export class CaptchaError extends Error {
public readonly code: CaptchaErrorCode;
public readonly cause?: unknown;
constructor(code: CaptchaErrorCode, message: string, cause?: unknown) {
super(message);
this.code = code;
this.cause = cause;
this.name = 'CaptchaError';
}
}
};
export type CaptchaSaveSuccess = {
success: true;
}
};
export type CaptchaSaveFailure = {
success: false;
error: CaptchaError;
}
};
export type CaptchaSaveResult = CaptchaSaveSuccess | CaptchaSaveFailure;
type CaptchaResponse = {
@ -117,7 +108,7 @@ export class CaptchaService {
}
const result = await this.getCaptchaResponse('https://www.recaptcha.net/recaptcha/api/siteverify', secret, response).catch(err => {
throw new CaptchaError(captchaErrorCodes.requestFailed, `recaptcha-request-failed: ${err}`);
throw new CaptchaError(captchaErrorCodes.requestFailed, `recaptcha-request-failed: ${err}`, err);
});
if (result.success !== true) {
@ -133,7 +124,7 @@ export class CaptchaService {
}
const result = await this.getCaptchaResponse('https://hcaptcha.com/siteverify', secret, response).catch(err => {
throw new CaptchaError(captchaErrorCodes.requestFailed, `hcaptcha-request-failed: ${err}`);
throw new CaptchaError(captchaErrorCodes.requestFailed, `hcaptcha-request-failed: ${err}`, err);
});
if (result.success !== true) {
@ -209,7 +200,7 @@ export class CaptchaService {
}
const result = await this.getCaptchaResponse('https://challenges.cloudflare.com/turnstile/v0/siteverify', secret, response).catch(err => {
throw new CaptchaError(captchaErrorCodes.requestFailed, `turnstile-request-failed: ${err}`);
throw new CaptchaError(captchaErrorCodes.requestFailed, `turnstile-request-failed: ${err}`, err);
});
if (result.success !== true) {
@ -375,7 +366,7 @@ export class CaptchaService {
throw new CaptchaError(captchaErrorCodes.invalidParameters, 'frc-failed: secret and captureResult are required');
}
await this.verifyFriendlyCaptcha(params.captchaResult, params.captchaResult);
await this.verifyFriendlyCaptcha(params.secret, params.captchaResult);
await this.updateMeta(provider, params);
},
}[provider];
@ -386,7 +377,7 @@ export class CaptchaService {
this.logger.info(err);
const error = err instanceof CaptchaError
? err
: new CaptchaError(captchaErrorCodes.unknown, `unknown error: ${err}`);
: new CaptchaError(captchaErrorCodes.unknown, `unknown error: ${err}`, err);
return {
success: false,
error,

29
packages/backend/src/core/ChannelFollowingService.ts
View file

@ -9,14 +9,15 @@ import { DI } from '@/di-symbols.js';
import type { ChannelFollowingsRepository } from '@/models/_.js';
import { MiChannel } from '@/models/_.js';
import { IdService } from '@/core/IdService.js';
import { GlobalEvents, GlobalEventService } from '@/core/GlobalEventService.js';
import { GlobalEvents, GlobalEventService, InternalEventTypes } from '@/core/GlobalEventService.js';
import { bindThis } from '@/decorators.js';
import type { MiLocalUser } from '@/models/User.js';
import { RedisKVCache } from '@/misc/cache.js';
import { QuantumKVCache } from '@/misc/QuantumKVCache.js';
import { InternalEventService } from './InternalEventService.js';
@Injectable()
export class ChannelFollowingService implements OnModuleInit {
public userFollowingChannelsCache: RedisKVCache<Set<string>>;
public userFollowingChannelsCache: QuantumKVCache<Set<string>>;
constructor(
@Inject(DI.redis)
@ -27,19 +28,18 @@ export class ChannelFollowingService implements OnModuleInit {
private channelFollowingsRepository: ChannelFollowingsRepository,
private idService: IdService,
private globalEventService: GlobalEventService,
private readonly internalEventService: InternalEventService,
) {
this.userFollowingChannelsCache = new RedisKVCache<Set<string>>(this.redisClient, 'userFollowingChannels', {
this.userFollowingChannelsCache = new QuantumKVCache<Set<string>>(this.internalEventService, 'userFollowingChannels', {
lifetime: 1000 * 60 * 30, // 30m
memoryCacheLifetime: 1000 * 60, // 1m
fetcher: (key) => this.channelFollowingsRepository.find({
where: { followerId: key },
select: ['followeeId'],
}).then(xs => new Set(xs.map(x => x.followeeId))),
toRedisConverter: (value) => JSON.stringify(Array.from(value)),
fromRedisConverter: (value) => new Set(JSON.parse(value)),
});
this.redisForSub.on('message', this.onMessage);
this.internalEventService.on('followChannel', this.onMessage);
this.internalEventService.on('unfollowChannel', this.onMessage);
}
onModuleInit() {
@ -79,18 +79,15 @@ export class ChannelFollowingService implements OnModuleInit {
}
@bindThis
private async onMessage(_: string, data: string): Promise<void> {
const obj = JSON.parse(data);
if (obj.channel === 'internal') {
const { type, body } = obj.message as GlobalEvents['internal']['payload'];
private async onMessage<E extends 'followChannel' | 'unfollowChannel'>(body: InternalEventTypes[E], type: E): Promise<void> {
{
switch (type) {
case 'followChannel': {
this.userFollowingChannelsCache.refresh(body.userId);
await this.userFollowingChannelsCache.delete(body.userId);
break;
}
case 'unfollowChannel': {
this.userFollowingChannelsCache.delete(body.userId);
await this.userFollowingChannelsCache.delete(body.userId);
break;
}
}
@ -99,6 +96,8 @@ export class ChannelFollowingService implements OnModuleInit {
@bindThis
public dispose(): void {
this.internalEventService.off('followChannel', this.onMessage);
this.internalEventService.off('unfollowChannel', this.onMessage);
this.userFollowingChannelsCache.dispose();
}

925
packages/backend/src/core/ChatService.ts Normal file
View file

@ -0,0 +1,925 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Inject, Injectable } from '@nestjs/common';
import * as Redis from 'ioredis';
import { Brackets } from 'typeorm';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { QueueService } from '@/core/QueueService.js';
import { IdService } from '@/core/IdService.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { ChatEntityService } from '@/core/entities/ChatEntityService.js';
import { ApRendererService } from '@/core/activitypub/ApRendererService.js';
import { PushNotificationService } from '@/core/PushNotificationService.js';
import { bindThis } from '@/decorators.js';
import type { ChatApprovalsRepository, ChatMessagesRepository, ChatRoomInvitationsRepository, ChatRoomMembershipsRepository, ChatRoomsRepository, MiChatMessage, MiChatRoom, MiChatRoomMembership, MiDriveFile, MiUser, MutingsRepository, UsersRepository } from '@/models/_.js';
import { UserBlockingService } from '@/core/UserBlockingService.js';
import { QueryService } from '@/core/QueryService.js';
import { RoleService } from '@/core/RoleService.js';
import { UserFollowingService } from '@/core/UserFollowingService.js';
import { MiChatRoomInvitation } from '@/models/ChatRoomInvitation.js';
import { Packed } from '@/misc/json-schema.js';
import { sqlLikeEscape } from '@/misc/sql-like-escape.js';
import { CustomEmojiService } from '@/core/CustomEmojiService.js';
import { emojiRegex } from '@/misc/emoji-regex.js';
import { NotificationService } from '@/core/NotificationService.js';
import { ModerationLogService } from '@/core/ModerationLogService.js';
const MAX_ROOM_MEMBERS = 30;
const MAX_REACTIONS_PER_MESSAGE = 100;
const isCustomEmojiRegexp = /^:([\w+-]+)(?:@\.)?:$/;
// TODO: ReactionServiceのやつと共通化
function normalizeEmojiString(x: string) {
const match = emojiRegex.exec(x);
if (match) {
// 合字を含む1つの絵文字
const unicode = match[0];
// 異体字セレクタ除去
return unicode.match('\u200d') ? unicode : unicode.replace(/\ufe0f/g, '');
} else {
throw new Error('invalid emoji');
}
}
@Injectable()
export class ChatService {
constructor(
@Inject(DI.config)
private config: Config,
@Inject(DI.redis)
private redisClient: Redis.Redis,
@Inject(DI.usersRepository)
private usersRepository: UsersRepository,
@Inject(DI.chatMessagesRepository)
private chatMessagesRepository: ChatMessagesRepository,
@Inject(DI.chatApprovalsRepository)
private chatApprovalsRepository: ChatApprovalsRepository,
@Inject(DI.chatRoomsRepository)
private chatRoomsRepository: ChatRoomsRepository,
@Inject(DI.chatRoomInvitationsRepository)
private chatRoomInvitationsRepository: ChatRoomInvitationsRepository,
@Inject(DI.chatRoomMembershipsRepository)
private chatRoomMembershipsRepository: ChatRoomMembershipsRepository,
@Inject(DI.mutingsRepository)
private mutingsRepository: MutingsRepository,
private userEntityService: UserEntityService,
private chatEntityService: ChatEntityService,
private idService: IdService,
private globalEventService: GlobalEventService,
private apRendererService: ApRendererService,
private queueService: QueueService,
private pushNotificationService: PushNotificationService,
private notificationService: NotificationService,
private userBlockingService: UserBlockingService,
private queryService: QueryService,
private roleService: RoleService,
private userFollowingService: UserFollowingService,
private customEmojiService: CustomEmojiService,
private moderationLogService: ModerationLogService,
) {
}
@bindThis
public async getChatAvailability(userId: MiUser['id']): Promise<{ read: boolean; write: boolean; }> {
const policies = await this.roleService.getUserPolicies(userId);
switch (policies.chatAvailability) {
case 'available':
return {
read: true,
write: true,
};
case 'readonly':
return {
read: true,
write: false,
};
case 'unavailable':
return {
read: false,
write: false,
};
default:
throw new Error('invalid chat availability (unreachable)');
}
}
/** getChatAvailabilityの糖衣。主にAPI呼び出し時に走らせて、権限的に問題ない場合はそのまま続行する */
@bindThis
public async checkChatAvailability(userId: MiUser['id'], permission: 'read' | 'write') {
const policy = await this.getChatAvailability(userId);
if (policy[permission] === false) {
throw new Error('ROLE_PERMISSION_DENIED');
}
}
@bindThis
public async createMessageToUser(fromUser: { id: MiUser['id']; host: MiUser['host']; }, toUser: MiUser, params: {
text?: string | null;
file?: MiDriveFile | null;
uri?: string | null;
}): Promise<Packed<'ChatMessageLiteFor1on1'>> {
if (fromUser.id === toUser.id) {
throw new Error('yourself');
}
const approvals = await this.chatApprovalsRepository.createQueryBuilder('approval')
.where(new Brackets(qb => { // 自分が相手を許可しているか
qb.where('approval.userId = :fromUserId', { fromUserId: fromUser.id })
.andWhere('approval.otherId = :toUserId', { toUserId: toUser.id });
}))
.orWhere(new Brackets(qb => { // 相手が自分を許可しているか
qb.where('approval.userId = :toUserId', { toUserId: toUser.id })
.andWhere('approval.otherId = :fromUserId', { fromUserId: fromUser.id });
}))
.take(2)
.getMany();
const otherApprovedMe = approvals.some(approval => approval.userId === toUser.id);
const iApprovedOther = approvals.some(approval => approval.userId === fromUser.id);
if (!otherApprovedMe) {
if (toUser.chatScope === 'none') {
throw new Error('recipient is cannot chat (none)');
} else if (toUser.chatScope === 'followers') {
const isFollower = await this.userFollowingService.isFollowing(fromUser.id, toUser.id);
if (!isFollower) {
throw new Error('recipient is cannot chat (followers)');
}
} else if (toUser.chatScope === 'following') {
const isFollowing = await this.userFollowingService.isFollowing(toUser.id, fromUser.id);
if (!isFollowing) {
throw new Error('recipient is cannot chat (following)');
}
} else if (toUser.chatScope === 'mutual') {
const isMutual = await this.userFollowingService.isMutual(fromUser.id, toUser.id);
if (!isMutual) {
throw new Error('recipient is cannot chat (mutual)');
}
}
}
if (!(await this.getChatAvailability(toUser.id)).write) {
throw new Error('recipient is cannot chat (policy)');
}
const blocked = await this.userBlockingService.checkBlocked(toUser.id, fromUser.id);
if (blocked) {
throw new Error('blocked');
}
const message = {
id: this.idService.gen(),
fromUserId: fromUser.id,
toUserId: toUser.id,
text: params.text ? params.text.trim() : null,
fileId: params.file ? params.file.id : null,
reads: [],
uri: params.uri ?? null,
} satisfies Partial<MiChatMessage>;
const inserted = await this.chatMessagesRepository.insertOne(message);
// 相手を許可しておく
if (!iApprovedOther) {
this.chatApprovalsRepository.insertOne({
id: this.idService.gen(),
userId: fromUser.id,
otherId: toUser.id,
});
}
const packedMessage = await this.chatEntityService.packMessageLiteFor1on1(inserted);
if (this.userEntityService.isLocalUser(toUser)) {
const redisPipeline = this.redisClient.pipeline();
redisPipeline.set(`newUserChatMessageExists:${toUser.id}:${fromUser.id}`, message.id);
redisPipeline.sadd(`newChatMessagesExists:${toUser.id}`, `user:${fromUser.id}`);
redisPipeline.exec();
}
if (this.userEntityService.isLocalUser(fromUser)) {
// 自分のストリーム
this.globalEventService.publishChatUserStream(fromUser.id, toUser.id, 'message', packedMessage);
}
if (this.userEntityService.isLocalUser(toUser)) {
// 相手のストリーム
this.globalEventService.publishChatUserStream(toUser.id, fromUser.id, 'message', packedMessage);
}
// 3秒経っても既読にならなかったらイベント発行
if (this.userEntityService.isLocalUser(toUser)) {
setTimeout(async () => {
const marker = await this.redisClient.get(`newUserChatMessageExists:${toUser.id}:${fromUser.id}`);
if (marker == null) return; // 既読
const packedMessageForTo = await this.chatEntityService.packMessageDetailed(inserted, toUser);
this.globalEventService.publishMainStream(toUser.id, 'newChatMessage', packedMessageForTo);
this.pushNotificationService.pushNotification(toUser.id, 'newChatMessage', packedMessageForTo);
}, 3000);
}
return packedMessage;
}
@bindThis
public async createMessageToRoom(fromUser: { id: MiUser['id']; host: MiUser['host']; }, toRoom: MiChatRoom, params: {
text?: string | null;
file?: MiDriveFile | null;
uri?: string | null;
}): Promise<Packed<'ChatMessageLiteForRoom'>> {
const memberships = (await this.chatRoomMembershipsRepository.findBy({ roomId: toRoom.id })).map(m => ({
userId: m.userId,
isMuted: m.isMuted,
})).concat({ // ownerはmembershipレコードを作らないため
userId: toRoom.ownerId,
isMuted: false,
});
if (!memberships.some(member => member.userId === fromUser.id)) {
throw new Error('you are not a member of the room');
}
const membershipsOtherThanMe = memberships.filter(member => member.userId !== fromUser.id);
const message = {
id: this.idService.gen(),
fromUserId: fromUser.id,
toRoomId: toRoom.id,
text: params.text ? params.text.trim() : null,
fileId: params.file ? params.file.id : null,
reads: [],
uri: params.uri ?? null,
} satisfies Partial<MiChatMessage>;
const inserted = await this.chatMessagesRepository.insertOne(message);
const packedMessage = await this.chatEntityService.packMessageLiteForRoom(inserted);
this.globalEventService.publishChatRoomStream(toRoom.id, 'message', packedMessage);
const redisPipeline = this.redisClient.pipeline();
for (const membership of membershipsOtherThanMe) {
if (membership.isMuted) continue;
redisPipeline.set(`newRoomChatMessageExists:${membership.userId}:${toRoom.id}`, message.id);
redisPipeline.sadd(`newChatMessagesExists:${membership.userId}`, `room:${toRoom.id}`);
}
redisPipeline.exec();
// 3秒経っても既読にならなかったらイベント発行
setTimeout(async () => {
const redisPipeline = this.redisClient.pipeline();
for (const membership of membershipsOtherThanMe) {
redisPipeline.get(`newRoomChatMessageExists:${membership.userId}:${toRoom.id}`);
}
const markers = await redisPipeline.exec();
if (markers == null) throw new Error('redis error');
if (markers.every(marker => marker[1] == null)) return;
const packedMessageForTo = await this.chatEntityService.packMessageDetailed(inserted);
for (let i = 0; i < membershipsOtherThanMe.length; i++) {
const marker = markers[i][1];
if (marker == null) continue;
this.globalEventService.publishMainStream(membershipsOtherThanMe[i].userId, 'newChatMessage', packedMessageForTo);
this.pushNotificationService.pushNotification(membershipsOtherThanMe[i].userId, 'newChatMessage', packedMessageForTo);
}
}, 3000);
return packedMessage;
}
@bindThis
public async readUserChatMessage(
readerId: MiUser['id'],
senderId: MiUser['id'],
): Promise<void> {
const redisPipeline = this.redisClient.pipeline();
redisPipeline.del(`newUserChatMessageExists:${readerId}:${senderId}`);
redisPipeline.srem(`newChatMessagesExists:${readerId}`, `user:${senderId}`);
await redisPipeline.exec();
}
@bindThis
public async readRoomChatMessage(
readerId: MiUser['id'],
roomId: MiChatRoom['id'],
): Promise<void> {
const redisPipeline = this.redisClient.pipeline();
redisPipeline.del(`newRoomChatMessageExists:${readerId}:${roomId}`);
redisPipeline.srem(`newChatMessagesExists:${readerId}`, `room:${roomId}`);
await redisPipeline.exec();
}
@bindThis
public findMessageById(messageId: MiChatMessage['id']) {
return this.chatMessagesRepository.findOneBy({ id: messageId });
}
@bindThis
public findMyMessageById(userId: MiUser['id'], messageId: MiChatMessage['id']) {
return this.chatMessagesRepository.findOneBy({ id: messageId, fromUserId: userId });
}
@bindThis
public async hasPermissionToViewRoomTimeline(meId: MiUser['id'], room: MiChatRoom) {
if (await this.isRoomMember(room, meId)) {
return true;
} else {
const iAmModerator = await this.roleService.isModerator({ id: meId });
if (iAmModerator) {
return true;
}
return false;
}
}
@bindThis
public async deleteMessage(message: MiChatMessage) {
await this.chatMessagesRepository.delete(message.id);
if (message.toUserId) {
const [fromUser, toUser] = await Promise.all([
this.usersRepository.findOneByOrFail({ id: message.fromUserId }),
this.usersRepository.findOneByOrFail({ id: message.toUserId }),
]);
if (this.userEntityService.isLocalUser(fromUser)) this.globalEventService.publishChatUserStream(message.fromUserId, message.toUserId, 'deleted', message.id);
if (this.userEntityService.isLocalUser(toUser)) this.globalEventService.publishChatUserStream(message.toUserId, message.fromUserId, 'deleted', message.id);
if (this.userEntityService.isLocalUser(fromUser) && this.userEntityService.isRemoteUser(toUser)) {
//const activity = this.apRendererService.addContext(this.apRendererService.renderDelete(this.apRendererService.renderTombstone(`${this.config.url}/notes/${message.id}`), fromUser));
//this.queueService.deliver(fromUser, activity, toUser.inbox);
}
} else if (message.toRoomId) {
this.globalEventService.publishChatRoomStream(message.toRoomId, 'deleted', message.id);
}
}
@bindThis
public async userTimeline(meId: MiUser['id'], otherId: MiUser['id'], limit: number, sinceId?: MiChatMessage['id'] | null, untilId?: MiChatMessage['id'] | null) {
const query = this.queryService.makePaginationQuery(this.chatMessagesRepository.createQueryBuilder('message'), sinceId, untilId)
.andWhere(new Brackets(qb => {
qb
.where(new Brackets(qb => {
qb
.where('message.fromUserId = :meId')
.andWhere('message.toUserId = :otherId');
}))
.orWhere(new Brackets(qb => {
qb
.where('message.fromUserId = :otherId')
.andWhere('message.toUserId = :meId');
}));
}))
.setParameter('meId', meId)
.setParameter('otherId', otherId);
const messages = await query.take(limit).getMany();
return messages;
}
@bindThis
public async roomTimeline(roomId: MiChatRoom['id'], limit: number, sinceId?: MiChatMessage['id'] | null, untilId?: MiChatMessage['id'] | null) {
const query = this.queryService.makePaginationQuery(this.chatMessagesRepository.createQueryBuilder('message'), sinceId, untilId)
.andWhere('message.toRoomId = :roomId', { roomId })
.leftJoinAndSelect('message.file', 'file')
.leftJoinAndSelect('message.fromUser', 'fromUser');
const messages = await query.take(limit).getMany();
return messages;
}
@bindThis
public async userHistory(meId: MiUser['id'], limit: number): Promise<MiChatMessage[]> {
const history: MiChatMessage[] = [];
const mutingQuery = this.mutingsRepository.createQueryBuilder('muting')
.select('muting.muteeId')
.where('muting.muterId = :muterId', { muterId: meId });
for (let i = 0; i < limit; i++) {
const found = history.map(m => (m.fromUserId === meId) ? m.toUserId! : m.fromUserId!);
const query = this.chatMessagesRepository.createQueryBuilder('message')
.orderBy('message.id', 'DESC')
.where(new Brackets(qb => {
qb
.where('message.fromUserId = :meId', { meId: meId })
.orWhere('message.toUserId = :meId', { meId: meId });
}))
.andWhere('message.toRoomId IS NULL')
.andWhere(`message.fromUserId NOT IN (${ mutingQuery.getQuery() })`)
.andWhere(`message.toUserId NOT IN (${ mutingQuery.getQuery() })`);
if (found.length > 0) {
query.andWhere('message.fromUserId NOT IN (:...found)', { found: found });
query.andWhere('message.toUserId NOT IN (:...found)', { found: found });
}
query.setParameters(mutingQuery.getParameters());
const message = await query.getOne();
if (message) {
history.push(message);
} else {
break;
}
}
return history;
}
@bindThis
public async roomHistory(meId: MiUser['id'], limit: number): Promise<MiChatMessage[]> {
// TODO: 一回のクエリにまとめられるかも
const [memberRoomIds, ownedRoomIds] = await Promise.all([
this.chatRoomMembershipsRepository.findBy({
userId: meId,
}).then(xs => xs.map(x => x.roomId)),
this.chatRoomsRepository.findBy({
ownerId: meId,
}).then(xs => xs.map(x => x.id)),
]);
const roomIds = memberRoomIds.concat(ownedRoomIds);
if (memberRoomIds.length === 0 && ownedRoomIds.length === 0) {
return [];
}
const history: MiChatMessage[] = [];
for (let i = 0; i < limit; i++) {
const found = history.map(m => m.toRoomId!);
const query = this.chatMessagesRepository.createQueryBuilder('message')
.orderBy('message.id', 'DESC')
.where('message.toRoomId IN (:...roomIds)', { roomIds });
if (found.length > 0) {
query.andWhere('message.toRoomId NOT IN (:...found)', { found: found });
}
const message = await query.getOne();
if (message) {
history.push(message);
} else {
break;
}
}
return history;
}
@bindThis
public async getUserReadStateMap(userId: MiUser['id'], otherIds: MiUser['id'][]) {
const readStateMap: Record<MiUser['id'], boolean> = {};
const redisPipeline = this.redisClient.pipeline();
for (const otherId of otherIds) {
redisPipeline.get(`newUserChatMessageExists:${userId}:${otherId}`);
}
const markers = await redisPipeline.exec();
if (markers == null) throw new Error('redis error');
for (let i = 0; i < otherIds.length; i++) {
const marker = markers[i][1];
readStateMap[otherIds[i]] = marker == null;
}
return readStateMap;
}
@bindThis
public async getRoomReadStateMap(userId: MiUser['id'], roomIds: MiChatRoom['id'][]) {
const readStateMap: Record<MiChatRoom['id'], boolean> = {};
const redisPipeline = this.redisClient.pipeline();
for (const roomId of roomIds) {
redisPipeline.get(`newRoomChatMessageExists:${userId}:${roomId}`);
}
const markers = await redisPipeline.exec();
if (markers == null) throw new Error('redis error');
for (let i = 0; i < roomIds.length; i++) {
const marker = markers[i][1];
readStateMap[roomIds[i]] = marker == null;
}
return readStateMap;
}
@bindThis
public async hasUnreadMessages(userId: MiUser['id']) {
const card = await this.redisClient.scard(`newChatMessagesExists:${userId}`);
return card > 0;
}
@bindThis
public async createRoom(owner: MiUser, params: Partial<{
name: string;
description: string;
}>) {
const room = {
id: this.idService.gen(),
name: params.name,
description: params.description,
ownerId: owner.id,
} satisfies Partial<MiChatRoom>;
const created = await this.chatRoomsRepository.insertOne(room);
return created;
}
@bindThis
public async hasPermissionToDeleteRoom(meId: MiUser['id'], room: MiChatRoom) {
if (room.ownerId === meId) {
return true;
}
const iAmModerator = await this.roleService.isModerator({ id: meId });
if (iAmModerator) {
return true;
}
return false;
}
@bindThis
public async deleteRoom(room: MiChatRoom, deleter?: MiUser) {
await this.chatRoomsRepository.delete(room.id);
if (deleter) {
const deleterIsModerator = await this.roleService.isModerator(deleter);
if (deleterIsModerator) {
this.moderationLogService.log(deleter, 'deleteChatRoom', {
roomId: room.id,
room: room,
});
}
}
}
@bindThis
public async findMyRoomById(ownerId: MiUser['id'], roomId: MiChatRoom['id']) {
return this.chatRoomsRepository.findOneBy({ id: roomId, ownerId: ownerId });
}
@bindThis
public async findRoomById(roomId: MiChatRoom['id']) {
return this.chatRoomsRepository.findOne({ where: { id: roomId }, relations: ['owner'] });
}
@bindThis
public async isRoomMember(room: MiChatRoom, userId: MiUser['id']) {
if (room.ownerId === userId) return true;
const membership = await this.chatRoomMembershipsRepository.findOneBy({ roomId: room.id, userId });
return membership != null;
}
@bindThis
public async createRoomInvitation(inviterId: MiUser['id'], roomId: MiChatRoom['id'], inviteeId: MiUser['id']) {
if (inviterId === inviteeId) {
throw new Error('yourself');
}
const room = await this.chatRoomsRepository.findOneByOrFail({ id: roomId, ownerId: inviterId });
if (await this.isRoomMember(room, inviteeId)) {
throw new Error('already member');
}
const existingInvitation = await this.chatRoomInvitationsRepository.findOneBy({ roomId, userId: inviteeId });
if (existingInvitation) {
throw new Error('already invited');
}
const membershipsCount = await this.chatRoomMembershipsRepository.countBy({ roomId });
if (membershipsCount >= MAX_ROOM_MEMBERS) {
throw new Error('room is full');
}
// TODO: cehck block
const invitation = {
id: this.idService.gen(),
roomId: room.id,
userId: inviteeId,
} satisfies Partial<MiChatRoomInvitation>;
const created = await this.chatRoomInvitationsRepository.insertOne(invitation);
this.notificationService.createNotification(inviteeId, 'chatRoomInvitationReceived', {
invitationId: invitation.id,
}, inviterId);
return created;
}
@bindThis
public async getSentRoomInvitationsWithPagination(roomId: MiChatRoom['id'], limit: number, sinceId?: MiChatRoomInvitation['id'] | null, untilId?: MiChatRoomInvitation['id'] | null) {
const query = this.queryService.makePaginationQuery(this.chatRoomInvitationsRepository.createQueryBuilder('invitation'), sinceId, untilId)
.andWhere('invitation.roomId = :roomId', { roomId });
const invitations = await query.take(limit).getMany();
return invitations;
}
@bindThis
public async getOwnedRoomsWithPagination(ownerId: MiUser['id'], limit: number, sinceId?: MiChatRoom['id'] | null, untilId?: MiChatRoom['id'] | null) {
const query = this.queryService.makePaginationQuery(this.chatRoomsRepository.createQueryBuilder('room'), sinceId, untilId)
.andWhere('room.ownerId = :ownerId', { ownerId });
const rooms = await query.take(limit).getMany();
return rooms;
}
@bindThis
public async getReceivedRoomInvitationsWithPagination(userId: MiUser['id'], limit: number, sinceId?: MiChatRoomInvitation['id'] | null, untilId?: MiChatRoomInvitation['id'] | null) {
const query = this.queryService.makePaginationQuery(this.chatRoomInvitationsRepository.createQueryBuilder('invitation'), sinceId, untilId)
.andWhere('invitation.userId = :userId', { userId })
.andWhere('invitation.ignored = FALSE');
const invitations = await query.take(limit).getMany();
return invitations;
}
@bindThis
public async joinToRoom(userId: MiUser['id'], roomId: MiChatRoom['id']) {
const invitation = await this.chatRoomInvitationsRepository.findOneByOrFail({ roomId, userId });
const membershipsCount = await this.chatRoomMembershipsRepository.countBy({ roomId });
if (membershipsCount >= MAX_ROOM_MEMBERS) {
throw new Error('room is full');
}
const membership = {
id: this.idService.gen(),
roomId: roomId,
userId: userId,
} satisfies Partial<MiChatRoomMembership>;
// TODO: transaction
await this.chatRoomMembershipsRepository.insertOne(membership);
await this.chatRoomInvitationsRepository.delete(invitation.id);
}
@bindThis
public async ignoreRoomInvitation(userId: MiUser['id'], roomId: MiChatRoom['id']) {
const invitation = await this.chatRoomInvitationsRepository.findOneByOrFail({ roomId, userId });
await this.chatRoomInvitationsRepository.update(invitation.id, { ignored: true });
}
@bindThis
public async leaveRoom(userId: MiUser['id'], roomId: MiChatRoom['id']) {
const membership = await this.chatRoomMembershipsRepository.findOneByOrFail({ roomId, userId });
await this.chatRoomMembershipsRepository.delete(membership.id);
}
@bindThis
public async muteRoom(userId: MiUser['id'], roomId: MiChatRoom['id'], mute: boolean) {
const membership = await this.chatRoomMembershipsRepository.findOneByOrFail({ roomId, userId });
await this.chatRoomMembershipsRepository.update(membership.id, { isMuted: mute });
}
@bindThis
public async updateRoom(room: MiChatRoom, params: {
name?: string;
description?: string;
}): Promise<MiChatRoom> {
return this.chatRoomsRepository.createQueryBuilder().update()
.set(params)
.where('id = :id', { id: room.id })
.returning('*')
.execute()
.then((response) => {
return response.raw[0];
});
}
@bindThis
public async getRoomMembershipsWithPagination(roomId: MiChatRoom['id'], limit: number, sinceId?: MiChatRoomMembership['id'] | null, untilId?: MiChatRoomMembership['id'] | null) {
const query = this.queryService.makePaginationQuery(this.chatRoomMembershipsRepository.createQueryBuilder('membership'), sinceId, untilId)
.andWhere('membership.roomId = :roomId', { roomId });
const memberships = await query.take(limit).getMany();
return memberships;
}
@bindThis
public async searchMessages(meId: MiUser['id'], query: string, limit: number, params: {
userId?: MiUser['id'] | null;
roomId?: MiChatRoom['id'] | null;
}) {
const q = this.chatMessagesRepository.createQueryBuilder('message');
if (params.userId) {
q.andWhere(new Brackets(qb => {
qb
.where(new Brackets(qb => {
qb
.where('message.fromUserId = :meId')
.andWhere('message.toUserId = :otherId');
}))
.orWhere(new Brackets(qb => {
qb
.where('message.fromUserId = :otherId')
.andWhere('message.toUserId = :meId');
}));
}))
.setParameter('meId', meId)
.setParameter('otherId', params.userId);
} else if (params.roomId) {
q.where('message.toRoomId = :roomId', { roomId: params.roomId });
} else {
const membershipsQuery = this.chatRoomMembershipsRepository.createQueryBuilder('membership')
.select('membership.roomId')
.where('membership.userId = :meId', { meId: meId });
const ownedRoomsQuery = this.chatRoomsRepository.createQueryBuilder('room')
.select('room.id')
.where('room.ownerId = :meId', { meId });
q.andWhere(new Brackets(qb => {
qb
.where('message.fromUserId = :meId')
.orWhere('message.toUserId = :meId')
.orWhere(`message.toRoomId IN (${membershipsQuery.getQuery()})`)
.orWhere(`message.toRoomId IN (${ownedRoomsQuery.getQuery()})`);
}));
q.setParameters(membershipsQuery.getParameters());
q.setParameters(ownedRoomsQuery.getParameters());
}
q.andWhere('LOWER(message.text) LIKE :q', { q: `%${ sqlLikeEscape(query.toLowerCase()) }%` });
q.leftJoinAndSelect('message.file', 'file');
q.leftJoinAndSelect('message.fromUser', 'fromUser');
q.leftJoinAndSelect('message.toUser', 'toUser');
q.leftJoinAndSelect('message.toRoom', 'toRoom');
q.leftJoinAndSelect('toRoom.owner', 'toRoomOwner');
const messages = await q.orderBy('message.id', 'DESC').take(limit).getMany();
return messages;
}
@bindThis
public async react(messageId: MiChatMessage['id'], userId: MiUser['id'], reaction_: string) {
let reaction;
const custom = reaction_.match(isCustomEmojiRegexp);
if (custom == null) {
reaction = normalizeEmojiString(reaction_);
} else {
const name = custom[1];
const emoji = (await this.customEmojiService.localEmojisCache.fetch()).get(name);
if (emoji == null) {
throw new Error('no such emoji');
} else {
reaction = `:${name}:`;
}
}
const message = await this.chatMessagesRepository.findOneByOrFail({ id: messageId });
if (message.fromUserId === userId) {
throw new Error('cannot react to own message');
}
if (message.toRoomId === null && message.toUserId !== userId) {
throw new Error('cannot react to others message');
}
if (message.reactions.length >= MAX_REACTIONS_PER_MESSAGE) {
throw new Error('too many reactions');
}
const room = message.toRoomId ? await this.chatRoomsRepository.findOneByOrFail({ id: message.toRoomId }) : null;
if (room) {
if (!await this.isRoomMember(room, userId)) {
throw new Error('cannot react to others message');
}
}
await this.chatMessagesRepository.createQueryBuilder().update()
.set({
reactions: () => `array_append("reactions", '${userId}/${reaction}')`,
})
.where('id = :id', { id: message.id })
.execute();
if (room) {
this.globalEventService.publishChatRoomStream(room.id, 'react', {
messageId: message.id,
user: await this.userEntityService.pack(userId),
reaction,
});
} else {
this.globalEventService.publishChatUserStream(message.fromUserId, message.toUserId!, 'react', {
messageId: message.id,
reaction,
});
this.globalEventService.publishChatUserStream(message.toUserId!, message.fromUserId, 'react', {
messageId: message.id,
reaction,
});
}
}
@bindThis
public async unreact(messageId: MiChatMessage['id'], userId: MiUser['id'], reaction_: string) {
let reaction;
const custom = reaction_.match(isCustomEmojiRegexp);
if (custom == null) {
reaction = normalizeEmojiString(reaction_);
} else { // 削除されたカスタム絵文字のリアクションを削除したいかもしれないので絵文字の存在チェックはする必要なし
const name = custom[1];
reaction = `:${name}:`;
}
// NOTE: 自分のリアクションを(あれば)削除するだけなので諸々の権限チェックは必要なし
const message = await this.chatMessagesRepository.findOneByOrFail({ id: messageId });
const room = message.toRoomId ? await this.chatRoomsRepository.findOneByOrFail({ id: message.toRoomId }) : null;
await this.chatMessagesRepository.createQueryBuilder().update()
.set({
reactions: () => `array_remove("reactions", '${userId}/${reaction}')`,
})
.where('id = :id', { id: message.id })
.execute();
// TODO: 実際に削除が行われたときのみイベントを発行する
if (room) {
this.globalEventService.publishChatRoomStream(room.id, 'unreact', {
messageId: message.id,
user: await this.userEntityService.pack(userId),
reaction,
});
} else {
this.globalEventService.publishChatUserStream(message.fromUserId, message.toUserId!, 'unreact', {
messageId: message.id,
reaction,
});
this.globalEventService.publishChatUserStream(message.toUserId!, message.fromUserId, 'unreact', {
messageId: message.id,
reaction,
});
}
}
@bindThis
public async getMyMemberships(userId: MiUser['id'], limit: number, sinceId?: MiChatRoomMembership['id'] | null, untilId?: MiChatRoomMembership['id'] | null) {
const query = this.queryService.makePaginationQuery(this.chatRoomMembershipsRepository.createQueryBuilder('membership'), sinceId, untilId)
.andWhere('membership.userId = :userId', { userId });
const memberships = await query.take(limit).getMany();
return memberships;
}
}

60
packages/backend/src/core/CoreModule.ts
View file

@ -19,6 +19,7 @@ import { TimeService } from '@/core/TimeService.js';
import { EnvService } from '@/core/EnvService.js';
import { ApUtilityService } from '@/core/activitypub/ApUtilityService.js';
import { ApLogService } from '@/core/ApLogService.js';
import { UpdateInstanceQueue } from '@/core/UpdateInstanceQueue.js';
import { AccountMoveService } from './AccountMoveService.js';
import { AccountUpdateService } from './AccountUpdateService.js';
import { AnnouncementService } from './AnnouncementService.js';
@ -27,7 +28,6 @@ import { AppLockService } from './AppLockService.js';
import { AchievementService } from './AchievementService.js';
import { AvatarDecorationService } from './AvatarDecorationService.js';
import { CaptchaService } from './CaptchaService.js';
import { CreateSystemUserService } from './CreateSystemUserService.js';
import { CustomEmojiService } from './CustomEmojiService.js';
import { DeleteAccountService } from './DeleteAccountService.js';
import { DownloadService } from './DownloadService.js';
@ -40,7 +40,8 @@ import { HashtagService } from './HashtagService.js';
import { HttpRequestService } from './HttpRequestService.js';
import { IdService } from './IdService.js';
import { ImageProcessingService } from './ImageProcessingService.js';
import { InstanceActorService } from './InstanceActorService.js';
import { SystemAccountService } from './SystemAccountService.js';
import { InternalEventService } from './InternalEventService.js';
import { InternalStorageService } from './InternalStorageService.js';
import { MetaService } from './MetaService.js';
import { MfmService } from './MfmService.js';
@ -50,7 +51,6 @@ import { NoteEditService } from './NoteEditService.js';
import { NoteDeleteService } from './NoteDeleteService.js';
import { LatestNoteService } from './LatestNoteService.js';
import { NotePiningService } from './NotePiningService.js';
import { NoteReadService } from './NoteReadService.js';
import { NotificationService } from './NotificationService.js';
import { PollService } from './PollService.js';
import { PushNotificationService } from './PushNotificationService.js';
@ -60,6 +60,7 @@ import { ReactionsBufferingService } from './ReactionsBufferingService.js';
import { RelayService } from './RelayService.js';
import { RoleService } from './RoleService.js';
import { S3Service } from './S3Service.js';
import { BunnyService } from './BunnyService.js';
import { SignupService } from './SignupService.js';
import { WebAuthnService } from './WebAuthnService.js';
import { UserBlockingService } from './UserBlockingService.js';
@ -74,7 +75,6 @@ import { UserSuspendService } from './UserSuspendService.js';
import { UserAuthService } from './UserAuthService.js';
import { VideoProcessingService } from './VideoProcessingService.js';
import { UserWebhookService } from './UserWebhookService.js';
import { ProxyAccountService } from './ProxyAccountService.js';
import { UtilityService } from './UtilityService.js';
import { FileInfoService } from './FileInfoService.js';
import { SearchService } from './SearchService.js';
@ -82,6 +82,7 @@ import { ClipService } from './ClipService.js';
import { FeaturedService } from './FeaturedService.js';
import { FanoutTimelineService } from './FanoutTimelineService.js';
import { ChannelFollowingService } from './ChannelFollowingService.js';
import { ChatService } from './ChatService.js';
import { RegistryApiService } from './RegistryApiService.js';
import { ReversiService } from './ReversiService.js';
@ -107,6 +108,7 @@ import { AppEntityService } from './entities/AppEntityService.js';
import { AuthSessionEntityService } from './entities/AuthSessionEntityService.js';
import { BlockingEntityService } from './entities/BlockingEntityService.js';
import { ChannelEntityService } from './entities/ChannelEntityService.js';
import { ChatEntityService } from './entities/ChatEntityService.js';
import { ClipEntityService } from './entities/ClipEntityService.js';
import { DriveFileEntityService } from './entities/DriveFileEntityService.js';
import { DriveFolderEntityService } from './entities/DriveFolderEntityService.js';
@ -173,7 +175,6 @@ const $AppLockService: Provider = { provide: 'AppLockService', useExisting: AppL
const $AchievementService: Provider = { provide: 'AchievementService', useExisting: AchievementService };
const $AvatarDecorationService: Provider = { provide: 'AvatarDecorationService', useExisting: AvatarDecorationService };
const $CaptchaService: Provider = { provide: 'CaptchaService', useExisting: CaptchaService };
const $CreateSystemUserService: Provider = { provide: 'CreateSystemUserService', useExisting: CreateSystemUserService };
const $CustomEmojiService: Provider = { provide: 'CustomEmojiService', useExisting: CustomEmojiService };
const $DeleteAccountService: Provider = { provide: 'DeleteAccountService', useExisting: DeleteAccountService };
const $DownloadService: Provider = { provide: 'DownloadService', useExisting: DownloadService };
@ -186,7 +187,7 @@ const $HashtagService: Provider = { provide: 'HashtagService', useExisting: Hash
const $HttpRequestService: Provider = { provide: 'HttpRequestService', useExisting: HttpRequestService };
const $IdService: Provider = { provide: 'IdService', useExisting: IdService };
const $ImageProcessingService: Provider = { provide: 'ImageProcessingService', useExisting: ImageProcessingService };
const $InstanceActorService: Provider = { provide: 'InstanceActorService', useExisting: InstanceActorService };
const $InternalEventService: Provider = { provide: 'InternalEventService', useExisting: InternalEventService };
const $InternalStorageService: Provider = { provide: 'InternalStorageService', useExisting: InternalStorageService };
const $MetaService: Provider = { provide: 'MetaService', useExisting: MetaService };
const $MfmService: Provider = { provide: 'MfmService', useExisting: MfmService };
@ -196,10 +197,9 @@ const $NoteEditService: Provider = { provide: 'NoteEditService', useExisting: No
const $NoteDeleteService: Provider = { provide: 'NoteDeleteService', useExisting: NoteDeleteService };
const $LatestNoteService: Provider = { provide: 'LatestNoteService', useExisting: LatestNoteService };
const $NotePiningService: Provider = { provide: 'NotePiningService', useExisting: NotePiningService };
const $NoteReadService: Provider = { provide: 'NoteReadService', useExisting: NoteReadService };
const $NotificationService: Provider = { provide: 'NotificationService', useExisting: NotificationService };
const $PollService: Provider = { provide: 'PollService', useExisting: PollService };
const $ProxyAccountService: Provider = { provide: 'ProxyAccountService', useExisting: ProxyAccountService };
const $SystemAccountService: Provider = { provide: 'SystemAccountService', useExisting: SystemAccountService };
const $PushNotificationService: Provider = { provide: 'PushNotificationService', useExisting: PushNotificationService };
const $QueryService: Provider = { provide: 'QueryService', useExisting: QueryService };
const $ReactionService: Provider = { provide: 'ReactionService', useExisting: ReactionService };
@ -207,6 +207,7 @@ const $ReactionsBufferingService: Provider = { provide: 'ReactionsBufferingServi
const $RelayService: Provider = { provide: 'RelayService', useExisting: RelayService };
const $RoleService: Provider = { provide: 'RoleService', useExisting: RoleService };
const $S3Service: Provider = { provide: 'S3Service', useExisting: S3Service };
const $BunnyService: Provider = { provide: 'BunnyService', useExisting: BunnyService };
const $SignupService: Provider = { provide: 'SignupService', useExisting: SignupService };
const $WebAuthnService: Provider = { provide: 'WebAuthnService', useExisting: WebAuthnService };
const $UserBlockingService: Provider = { provide: 'UserBlockingService', useExisting: UserBlockingService };
@ -220,6 +221,7 @@ const $UserRenoteMutingService: Provider = { provide: 'UserRenoteMutingService',
const $UserSearchService: Provider = { provide: 'UserSearchService', useExisting: UserSearchService };
const $UserSuspendService: Provider = { provide: 'UserSuspendService', useExisting: UserSuspendService };
const $UserAuthService: Provider = { provide: 'UserAuthService', useExisting: UserAuthService };
const $UpdateInstanceQueue: Provider = { provide: 'UpdateInstanceQueue', useExisting: UpdateInstanceQueue };
const $VideoProcessingService: Provider = { provide: 'VideoProcessingService', useExisting: VideoProcessingService };
const $UserWebhookService: Provider = { provide: 'UserWebhookService', useExisting: UserWebhookService };
const $SystemWebhookService: Provider = { provide: 'SystemWebhookService', useExisting: SystemWebhookService };
@ -233,6 +235,7 @@ const $FeaturedService: Provider = { provide: 'FeaturedService', useExisting: Fe
const $FanoutTimelineService: Provider = { provide: 'FanoutTimelineService', useExisting: FanoutTimelineService };
const $FanoutTimelineEndpointService: Provider = { provide: 'FanoutTimelineEndpointService', useExisting: FanoutTimelineEndpointService };
const $ChannelFollowingService: Provider = { provide: 'ChannelFollowingService', useExisting: ChannelFollowingService };
const $ChatService: Provider = { provide: 'ChatService', useExisting: ChatService };
const $RegistryApiService: Provider = { provide: 'RegistryApiService', useExisting: RegistryApiService };
const $ReversiService: Provider = { provide: 'ReversiService', useExisting: ReversiService };
const $TimeService: Provider = { provide: 'TimeService', useExisting: TimeService };
@ -261,6 +264,7 @@ const $AppEntityService: Provider = { provide: 'AppEntityService', useExisting:
const $AuthSessionEntityService: Provider = { provide: 'AuthSessionEntityService', useExisting: AuthSessionEntityService };
const $BlockingEntityService: Provider = { provide: 'BlockingEntityService', useExisting: BlockingEntityService };
const $ChannelEntityService: Provider = { provide: 'ChannelEntityService', useExisting: ChannelEntityService };
const $ChatEntityService: Provider = { provide: 'ChatEntityService', useExisting: ChatEntityService };
const $ClipEntityService: Provider = { provide: 'ClipEntityService', useExisting: ClipEntityService };
const $DriveFileEntityService: Provider = { provide: 'DriveFileEntityService', useExisting: DriveFileEntityService };
const $DriveFolderEntityService: Provider = { provide: 'DriveFolderEntityService', useExisting: DriveFolderEntityService };
@ -331,7 +335,6 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
AchievementService,
AvatarDecorationService,
CaptchaService,
CreateSystemUserService,
CustomEmojiService,
DeleteAccountService,
DownloadService,
@ -344,7 +347,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
HttpRequestService,
IdService,
ImageProcessingService,
InstanceActorService,
InternalEventService,
InternalStorageService,
MetaService,
MfmService,
@ -354,10 +357,9 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
NoteDeleteService,
LatestNoteService,
NotePiningService,
NoteReadService,
NotificationService,
PollService,
ProxyAccountService,
SystemAccountService,
PushNotificationService,
QueryService,
ReactionService,
@ -365,6 +367,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
RelayService,
RoleService,
S3Service,
BunnyService,
SignupService,
WebAuthnService,
UserBlockingService,
@ -378,6 +381,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
UserSearchService,
UserSuspendService,
UserAuthService,
UpdateInstanceQueue,
VideoProcessingService,
UserWebhookService,
SystemWebhookService,
@ -391,6 +395,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
FanoutTimelineService,
FanoutTimelineEndpointService,
ChannelFollowingService,
ChatService,
RegistryApiService,
ReversiService,
TimeService,
@ -419,6 +424,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
AuthSessionEntityService,
BlockingEntityService,
ChannelEntityService,
ChatEntityService,
ClipEntityService,
DriveFileEntityService,
DriveFolderEntityService,
@ -485,7 +491,6 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
$AchievementService,
$AvatarDecorationService,
$CaptchaService,
$CreateSystemUserService,
$CustomEmojiService,
$DeleteAccountService,
$DownloadService,
@ -498,7 +503,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
$HttpRequestService,
$IdService,
$ImageProcessingService,
$InstanceActorService,
$InternalEventService,
$InternalStorageService,
$MetaService,
$MfmService,
@ -508,10 +513,9 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
$NoteDeleteService,
$LatestNoteService,
$NotePiningService,
$NoteReadService,
$NotificationService,
$PollService,
$ProxyAccountService,
$SystemAccountService,
$PushNotificationService,
$QueryService,
$ReactionService,
@ -519,6 +523,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
$RelayService,
$RoleService,
$S3Service,
$BunnyService,
$SignupService,
$WebAuthnService,
$UserBlockingService,
@ -532,6 +537,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
$UserSearchService,
$UserSuspendService,
$UserAuthService,
$UpdateInstanceQueue,
$VideoProcessingService,
$UserWebhookService,
$SystemWebhookService,
@ -545,6 +551,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
$FanoutTimelineService,
$FanoutTimelineEndpointService,
$ChannelFollowingService,
$ChatService,
$RegistryApiService,
$ReversiService,
$TimeService,
@ -573,6 +580,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
$AuthSessionEntityService,
$BlockingEntityService,
$ChannelEntityService,
$ChatEntityService,
$ClipEntityService,
$DriveFileEntityService,
$DriveFolderEntityService,
@ -640,7 +648,6 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
AchievementService,
AvatarDecorationService,
CaptchaService,
CreateSystemUserService,
CustomEmojiService,
DeleteAccountService,
DownloadService,
@ -653,7 +660,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
HttpRequestService,
IdService,
ImageProcessingService,
InstanceActorService,
InternalEventService,
InternalStorageService,
MetaService,
MfmService,
@ -663,10 +670,9 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
NoteDeleteService,
LatestNoteService,
NotePiningService,
NoteReadService,
NotificationService,
PollService,
ProxyAccountService,
SystemAccountService,
PushNotificationService,
QueryService,
ReactionService,
@ -674,6 +680,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
RelayService,
RoleService,
S3Service,
BunnyService,
SignupService,
WebAuthnService,
UserBlockingService,
@ -687,6 +694,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
UserSearchService,
UserSuspendService,
UserAuthService,
UpdateInstanceQueue,
VideoProcessingService,
UserWebhookService,
SystemWebhookService,
@ -700,6 +708,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
FanoutTimelineService,
FanoutTimelineEndpointService,
ChannelFollowingService,
ChatService,
RegistryApiService,
ReversiService,
TimeService,
@ -727,6 +736,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
AuthSessionEntityService,
BlockingEntityService,
ChannelEntityService,
ChatEntityService,
ClipEntityService,
DriveFileEntityService,
DriveFolderEntityService,
@ -793,7 +803,6 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
$AchievementService,
$AvatarDecorationService,
$CaptchaService,
$CreateSystemUserService,
$CustomEmojiService,
$DeleteAccountService,
$DownloadService,
@ -806,7 +815,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
$HttpRequestService,
$IdService,
$ImageProcessingService,
$InstanceActorService,
$InternalEventService,
$InternalStorageService,
$MetaService,
$MfmService,
@ -816,10 +825,9 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
$NoteDeleteService,
$LatestNoteService,
$NotePiningService,
$NoteReadService,
$NotificationService,
$PollService,
$ProxyAccountService,
$SystemAccountService,
$PushNotificationService,
$QueryService,
$ReactionService,
@ -827,6 +835,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
$RelayService,
$RoleService,
$S3Service,
$BunnyService,
$SignupService,
$WebAuthnService,
$UserBlockingService,
@ -840,6 +849,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
$UserSearchService,
$UserSuspendService,
$UserAuthService,
$UpdateInstanceQueue,
$VideoProcessingService,
$UserWebhookService,
$SystemWebhookService,
@ -852,6 +862,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
$FanoutTimelineService,
$FanoutTimelineEndpointService,
$ChannelFollowingService,
$ChatService,
$RegistryApiService,
$ReversiService,
$TimeService,
@ -879,6 +890,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
$AuthSessionEntityService,
$BlockingEntityService,
$ChannelEntityService,
$ChatEntityService,
$ClipEntityService,
$DriveFileEntityService,
$DriveFolderEntityService,

88
packages/backend/src/core/CreateSystemUserService.ts
View file

@ -1,88 +0,0 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { randomUUID } from 'node:crypto';
import { Inject, Injectable } from '@nestjs/common';
import * as argon2 from 'argon2';
//import bcrypt from 'bcryptjs';
import { IsNull, DataSource } from 'typeorm';
import { genRsaKeyPair } from '@/misc/gen-key-pair.js';
import { MiUser } from '@/models/User.js';
import { MiUserProfile } from '@/models/UserProfile.js';
import { IdService } from '@/core/IdService.js';
import { MiUserKeypair } from '@/models/UserKeypair.js';
import { MiUsedUsername } from '@/models/UsedUsername.js';
import { DI } from '@/di-symbols.js';
import generateNativeUserToken from '@/misc/generate-native-user-token.js';
import { bindThis } from '@/decorators.js';
@Injectable()
export class CreateSystemUserService {
constructor(
@Inject(DI.db)
private db: DataSource,
private idService: IdService,
) {
}
@bindThis
public async createSystemUser(username: string): Promise<MiUser> {
const password = randomUUID();
// Generate hash of password
//const salt = await bcrypt.genSalt(8);
const hash = await argon2.hash(password);
// Generate secret
const secret = generateNativeUserToken();
const keyPair = await genRsaKeyPair();
let account!: MiUser;
// Start transaction
await this.db.transaction(async transactionalEntityManager => {
const exist = await transactionalEntityManager.findOneBy(MiUser, {
usernameLower: username.toLowerCase(),
host: IsNull(),
});
if (exist) throw new Error('the user is already exists');
account = await transactionalEntityManager.insert(MiUser, {
id: this.idService.gen(),
username: username,
usernameLower: username.toLowerCase(),
host: null,
token: secret,
isRoot: false,
isLocked: true,
isExplorable: false,
approved: true,
isBot: true,
}).then(x => transactionalEntityManager.findOneByOrFail(MiUser, x.identifiers[0]));
await transactionalEntityManager.insert(MiUserKeypair, {
publicKey: keyPair.publicKey,
privateKey: keyPair.privateKey,
userId: account.id,
});
await transactionalEntityManager.insert(MiUserProfile, {
userId: account.id,
autoAcceptFollowed: false,
password: hash,
});
await transactionalEntityManager.insert(MiUsedUsername, {
createdAt: new Date(),
username: username.toLowerCase(),
});
});
return account;
}
}

32
packages/backend/src/core/DeleteAccountService.ts
View file

@ -5,7 +5,7 @@
import { Inject, Injectable } from '@nestjs/common';
import { Not, IsNull } from 'typeorm';
import type { FollowingsRepository, MiUser, UsersRepository } from '@/models/_.js';
import type { FollowingsRepository, MiMeta, MiUser, UsersRepository } from '@/models/_.js';
import { QueueService } from '@/core/QueueService.js';
import { DI } from '@/di-symbols.js';
import { bindThis } from '@/decorators.js';
@ -13,11 +13,15 @@ import { GlobalEventService } from '@/core/GlobalEventService.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { ApRendererService } from '@/core/activitypub/ApRendererService.js';
import { ModerationLogService } from '@/core/ModerationLogService.js';
import { SystemAccountService } from '@/core/SystemAccountService.js';
import { isSystemAccount } from '@/misc/is-system-account.js';
@Injectable()
export class DeleteAccountService {
constructor(
@Inject(DI.meta)
private meta: MiMeta,
@Inject(DI.usersRepository)
private usersRepository: UsersRepository,
@ -29,6 +33,7 @@ export class DeleteAccountService {
private queueService: QueueService,
private globalEventService: GlobalEventService,
private moderationLogService: ModerationLogService,
private systemAccountService: SystemAccountService,
) {
}
@ -37,9 +42,13 @@ export class DeleteAccountService {
id: string;
host: string | null;
}, moderator?: MiUser): Promise<void> {
if (this.meta.rootUserId === user.id) throw new Error('cannot delete a root account');
const _user = await this.usersRepository.findOneByOrFail({ id: user.id });
if (_user.isRoot) throw new Error('cannot delete a root account');
if (isSystemAccount(_user)) throw new Error('cannot delete a system account');
if (isSystemAccount(_user)) {
throw new Error('cannot delete a system account');
}
if (moderator != null) {
this.moderationLogService.log(moderator, 'deleteAccount', {
@ -54,8 +63,6 @@ export class DeleteAccountService {
// 知り得る全SharedInboxにDelete配信
const content = this.apRendererService.addContext(this.apRendererService.renderDelete(this.userEntityService.genLocalUserUri(user.id), user));
const queue: string[] = [];
const followings = await this.followingsRepository.find({
where: [
{ followerSharedInbox: Not(IsNull()) },
@ -64,22 +71,17 @@ export class DeleteAccountService {
select: ['followerSharedInbox', 'followeeSharedInbox'],
});
const inboxes = followings.map(x => x.followerSharedInbox ?? x.followeeSharedInbox);
const inboxes = followings.map(x => [x.followerSharedInbox ?? x.followeeSharedInbox as string, true] as const);
const queue = new Map<string, true>(inboxes);
for (const inbox of inboxes) {
if (inbox != null && !queue.includes(inbox)) queue.push(inbox);
}
await this.queueService.deliverMany(user, content, queue);
for (const inbox of queue) {
this.queueService.deliver(user, content, inbox, true);
}
this.queueService.createDeleteAccountJob(user, {
await this.queueService.createDeleteAccountJob(user, {
soft: false,
});
} else {
// リモートユーザーの削除は、完全にDBから物理削除してしまうと再度連合してきてアカウントが復活する可能性があるため、soft指定する
this.queueService.createDeleteAccountJob(user, {
await this.queueService.createDeleteAccountJob(user, {
soft: true,
});
}

27
packages/backend/src/core/DownloadService.ts
View file

@ -18,6 +18,8 @@ import { LoggerService } from '@/core/LoggerService.js';
import type Logger from '@/logger.js';
import { bindThis } from '@/decorators.js';
import { renderInlineError } from '@/misc/render-inline-error.js';
import { UtilityService } from '@/core/UtilityService.js';
@Injectable()
export class DownloadService {
@ -29,15 +31,18 @@ export class DownloadService {
private httpRequestService: HttpRequestService,
private loggerService: LoggerService,
private readonly utilityService: UtilityService,
) {
this.logger = this.loggerService.getLogger('download');
}
@bindThis
public async downloadUrl(url: string, path: string, options: { timeout?: number, operationTimeout?: number, maxSize?: number} = {} ): Promise<{
public async downloadUrl(url: string, path: string, options: { timeout?: number, operationTimeout?: number, maxSize?: number } = {} ): Promise<{
filename: string;
}> {
this.logger.info(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`);
this.utilityService.assertUrl(url);
this.logger.debug(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`);
const timeout = options.timeout ?? 30 * 1000;
const operationTimeout = options.operationTimeout ?? 60 * 1000;
@ -60,8 +65,8 @@ export class DownloadService {
request: operationTimeout, // whole operation timeout
},
agent: {
http: this.httpRequestService.httpAgent,
https: this.httpRequestService.httpsAgent,
http: this.httpRequestService.getAgentForHttp(urlObj, true),
https: this.httpRequestService.getAgentForHttps(urlObj, true),
},
http2: false, // default
retry: {
@ -86,7 +91,7 @@ export class DownloadService {
filename = parsed.parameters.filename;
}
} catch (e) {
this.logger.warn(`Failed to parse content-disposition: ${contentDisposition}`, { stack: e });
this.logger.warn(`Failed to parse content-disposition ${contentDisposition}: ${renderInlineError(e)}`);
}
}
}).on('downloadProgress', (progress: Got.Progress) => {
@ -100,13 +105,17 @@ export class DownloadService {
await stream.pipeline(req, fs.createWriteStream(path));
} catch (e) {
if (e instanceof Got.HTTPError) {
throw new StatusError(`${e.response.statusCode} ${e.response.statusMessage}`, e.response.statusCode, e.response.statusMessage);
} else {
throw new StatusError(`download error from ${url}`, e.response.statusCode, e.response.statusMessage, e);
} else if (e instanceof Got.RequestError || e instanceof Got.AbortError) {
throw new Error(String(e), { cause: e });
} else if (e instanceof Error) {
throw e;
} else {
throw new Error(String(e), { cause: e });
}
}
this.logger.succ(`Download finished: ${chalk.cyan(url)}`);
this.logger.info(`Download finished: ${chalk.cyan(url)}`);
return {
filename,
@ -118,7 +127,7 @@ export class DownloadService {
// Create temp file
const [path, cleanup] = await createTemp();
this.logger.info(`text file: Temp file is ${path}`);
this.logger.debug(`text file: Temp file is ${path}`);
try {
// write content at URL to temp file

155
packages/backend/src/core/DriveService.ts
View file

@ -44,6 +44,9 @@ import { correctFilename } from '@/misc/correct-filename.js';
import { isMimeImage } from '@/misc/is-mime-image.js';
import { ModerationLogService } from '@/core/ModerationLogService.js';
import { UtilityService } from '@/core/UtilityService.js';
import { BunnyService } from '@/core/BunnyService.js';
import { renderInlineError } from '@/misc/render-inline-error.js';
import { LoggerService } from './LoggerService.js';
type AddFileArgs = {
/** User who wish to add file */
@ -121,6 +124,7 @@ export class DriveService {
private downloadService: DownloadService,
private internalStorageService: InternalStorageService,
private s3Service: S3Service,
private bunnyService: BunnyService,
private imageProcessingService: ImageProcessingService,
private videoProcessingService: VideoProcessingService,
private globalEventService: GlobalEventService,
@ -131,8 +135,10 @@ export class DriveService {
private perUserDriveChart: PerUserDriveChart,
private instanceChart: InstanceChart,
private utilityService: UtilityService,
loggerService: LoggerService,
) {
const logger = new Logger('drive', 'blue');
const logger = loggerService.getLogger('drive', 'blue');
this.registerLogger = logger.createSubLogger('register', 'yellow');
this.downloaderLogger = logger.createSubLogger('downloader');
this.deleteLogger = logger.createSubLogger('delete');
@ -148,12 +154,23 @@ export class DriveService {
@bindThis
private async save(file: MiDriveFile, path: string, name: string, info: FileInfo): Promise<MiDriveFile> {
const type = info.type.mime;
const hash = info.md5;
const size = info.size;
let hash = info.md5;
let size = info.size;
// thunbnail, webpublic を必要なら生成
const alts = await this.generateAlts(path, type, !file.uri);
if (type && type.startsWith('video/')) {
try {
await this.videoProcessingService.webOptimizeVideo(path, type);
const newInfo = await this.fileInfoService.getFileInfo(path);
hash = newInfo.md5;
size = newInfo.size;
} catch (err) {
this.registerLogger.warn(`Video optimization failed: ${renderInlineError(err)}`);
}
}
if (this.meta.useObjectStorage) {
//#region ObjectStorage params
let [ext] = (name.match(/\.([a-zA-Z0-9_-]+)$/) ?? ['']);
@ -177,7 +194,8 @@ export class DriveService {
?? `${ this.meta.objectStorageUseSSL ? 'https' : 'http' }://${ this.meta.objectStorageEndpoint }${ this.meta.objectStoragePort ? `:${this.meta.objectStoragePort}` : '' }/${ this.meta.objectStorageBucket }`;
// for original
const key = `${this.meta.objectStoragePrefix}/${randomUUID()}${ext}`;
const prefix = this.meta.objectStoragePrefix ? `${this.meta.objectStoragePrefix}/` : '';
const key = `${prefix}${randomUUID()}${ext}`;
const url = `${ baseUrl }/${ key }`;
// for alts
@ -188,24 +206,24 @@ export class DriveService {
//#endregion
//#region Uploads
this.registerLogger.info(`uploading original: ${key}`);
this.registerLogger.debug(`uploading original: ${key}`);
const uploads = [
this.upload(key, fs.createReadStream(path), type, null, name),
];
if (alts.webpublic) {
webpublicKey = `${this.meta.objectStoragePrefix}/webpublic-${randomUUID()}.${alts.webpublic.ext}`;
webpublicKey = `${prefix}webpublic-${randomUUID()}.${alts.webpublic.ext}`;
webpublicUrl = `${ baseUrl }/${ webpublicKey }`;
this.registerLogger.info(`uploading webpublic: ${webpublicKey}`);
this.registerLogger.debug(`uploading webpublic: ${webpublicKey}`);
uploads.push(this.upload(webpublicKey, alts.webpublic.data, alts.webpublic.type, alts.webpublic.ext, name));
}
if (alts.thumbnail) {
thumbnailKey = `${this.meta.objectStoragePrefix}/thumbnail-${randomUUID()}.${alts.thumbnail.ext}`;
thumbnailKey = `${prefix}thumbnail-${randomUUID()}.${alts.thumbnail.ext}`;
thumbnailUrl = `${ baseUrl }/${ thumbnailKey }`;
this.registerLogger.info(`uploading thumbnail: ${thumbnailKey}`);
this.registerLogger.debug(`uploading thumbnail: ${thumbnailKey}`);
uploads.push(this.upload(thumbnailKey, alts.thumbnail.data, alts.thumbnail.type, alts.thumbnail.ext, `${name}.thumbnail`));
}
@ -249,11 +267,11 @@ export class DriveService {
const [url, thumbnailUrl, webpublicUrl] = await Promise.all(promises);
if (thumbnailUrl) {
this.registerLogger.info(`thumbnail stored: ${thumbnailAccessKey}`);
this.registerLogger.debug(`thumbnail stored: ${thumbnailAccessKey}`);
}
if (webpublicUrl) {
this.registerLogger.info(`web stored: ${webpublicAccessKey}`);
this.registerLogger.debug(`web stored: ${webpublicAccessKey}`);
}
file.storedInternal = true;
@ -297,7 +315,7 @@ export class DriveService {
thumbnail,
};
} catch (err) {
this.registerLogger.warn(`GenerateVideoThumbnail failed: ${err}`);
this.registerLogger.warn(`GenerateVideoThumbnail failed: ${renderInlineError(err)}`);
return {
webpublic: null,
thumbnail: null,
@ -330,7 +348,7 @@ export class DriveService {
metadata.height && metadata.height <= 2048
);
} catch (err) {
this.registerLogger.warn(`sharp failed: ${err}`);
this.registerLogger.warn(`sharp failed: ${renderInlineError(err)}`);
return {
webpublic: null,
thumbnail: null,
@ -341,7 +359,7 @@ export class DriveService {
let webpublic: IImage | null = null;
if (generateWeb && !satisfyWebpublic && !isAnimated) {
this.registerLogger.info('creating web image');
this.registerLogger.debug('creating web image');
try {
if (['image/jpeg', 'image/webp', 'image/avif'].includes(type)) {
@ -352,12 +370,12 @@ export class DriveService {
this.registerLogger.debug('web image not created (not an required image)');
}
} catch (err) {
this.registerLogger.warn('web image not created (an error occurred)', err as Error);
this.registerLogger.warn(`web image not created: ${renderInlineError(err)}`);
}
} else {
if (satisfyWebpublic) this.registerLogger.info('web image not created (original satisfies webpublic)');
else if (isAnimated) this.registerLogger.info('web image not created (animated image)');
else this.registerLogger.info('web image not created (from remote)');
if (satisfyWebpublic) this.registerLogger.debug('web image not created (original satisfies webpublic)');
else if (isAnimated) this.registerLogger.debug('web image not created (animated image)');
else this.registerLogger.debug('web image not created (from remote)');
}
// #endregion webpublic
@ -371,7 +389,7 @@ export class DriveService {
thumbnail = await this.imageProcessingService.convertSharpToWebp(img, 498, 422);
}
} catch (err) {
this.registerLogger.warn('thumbnail not created (an error occurred)', err as Error);
this.registerLogger.warn(`Error creating thumbnail: ${renderInlineError(err)}`);
}
// #endregion thumbnail
@ -405,20 +423,22 @@ export class DriveService {
);
if (this.meta.objectStorageSetPublicRead) params.ACL = 'public-read';
await this.s3Service.upload(this.meta, params)
.then(
result => {
if ('Bucket' in result) { // CompleteMultipartUploadCommandOutput
this.registerLogger.debug(`Uploaded: ${result.Bucket}/${result.Key} => ${result.Location}`);
} else { // AbortMultipartUploadCommandOutput
this.registerLogger.error(`Upload Result Aborted: key = ${key}, filename = ${filename}`);
}
})
.catch(
err => {
this.registerLogger.error(`Upload Failed: key = ${key}, filename = ${filename}`, err);
},
);
try {
if (this.bunnyService.usingBunnyCDN(this.meta)) {
await this.bunnyService.upload(this.meta, key, stream);
} else {
const result = await this.s3Service.upload(this.meta, params);
if ('Bucket' in result) { // CompleteMultipartUploadCommandOutput
this.registerLogger.debug(`Uploaded: ${result.Bucket}/${result.Key} => ${result.Location}`);
} else { // AbortMultipartUploadCommandOutput
this.registerLogger.error(`Upload Result Aborted: key = ${key}, filename = ${filename}`);
throw new Error('S3 upload aborted');
}
}
} catch (err) {
this.registerLogger.error(`Upload Failed: key = ${key}, filename = ${filename}: ${renderInlineError(err)}`);
throw err;
}
}
// Expire oldest file (without avatar or banner) of remote user
@ -476,7 +496,6 @@ export class DriveService {
}: AddFileArgs): Promise<MiDriveFile> {
const userRoleNSFW = user && (await this.roleService.getUserPolicies(user.id)).alwaysMarkNsfw;
const info = await this.fileInfoService.getFileInfo(path);
this.registerLogger.info(`${JSON.stringify(info)}`);
// detect name
const detectedName = correctFilename(
@ -486,6 +505,8 @@ export class DriveService {
ext ?? info.type.ext,
);
this.registerLogger.debug(`Detected file info: ${JSON.stringify(info)}`);
if (user && !force) {
// Check if there is a file with the same hash
const matched = await this.driveFilesRepository.findOneBy({
@ -494,7 +515,7 @@ export class DriveService {
});
if (matched) {
this.registerLogger.info(`file with same hash is found: ${matched.id}`);
this.registerLogger.debug(`file with same hash is found: ${matched.id}`);
if (sensitive && !matched.isSensitive) {
// The file is federated as sensitive for this time, but was federated as non-sensitive before.
// Therefore, update the file to sensitive.
@ -514,11 +535,23 @@ export class DriveService {
const policies = await this.roleService.getUserPolicies(user.id);
const driveCapacity = 1024 * 1024 * policies.driveCapacityMb;
const maxFileSize = 1024 * 1024 * policies.maxFileSizeMb;
this.registerLogger.debug('drive capacity override applied');
this.registerLogger.debug(`overrideCap: ${driveCapacity}bytes, usage: ${usage}bytes, u+s: ${usage + info.size}bytes`);
if (maxFileSize < info.size) {
if (isLocalUser) {
throw new IdentifiableError('f9e4e5f3-4df4-40b5-b400-f236945f7073', 'Max file size exceeded.');
} else {
// For remote users, throwing an exception will break Activity processing.
// Instead, force "link" mode which does not cache the file locally.
isLink = true;
}
}
// If usage limit exceeded
if (driveCapacity < usage + info.size) {
// Repeat the "!isLink" check because it could be set to true by the previous block.
if (driveCapacity < usage + info.size && !isLink) {
if (isLocalUser) {
throw new IdentifiableError('c6244ed2-a39a-4e1c-bf93-f0fbd7764fa6', 'No free space.', true);
}
@ -610,14 +643,14 @@ export class DriveService {
} catch (err) {
// duplicate key error (when already registered)
if (isDuplicateKeyValueError(err)) {
this.registerLogger.info(`already registered ${file.uri}`);
this.registerLogger.debug(`already registered ${file.uri}`);
file = await this.driveFilesRepository.findOneBy({
uri: file.uri!,
userId: user ? user.id : IsNull(),
}) as MiDriveFile;
} else {
this.registerLogger.error(err as Error);
this.registerLogger.error('Error in drive register', err as Error);
throw err;
}
}
@ -625,7 +658,7 @@ export class DriveService {
file = await (this.save(file, path, detectedName, info));
}
this.registerLogger.succ(`drive file has been created ${file.id}`);
this.registerLogger.info(`Created file ${file.id} (${detectedName}) of type ${info.type.mime} for user ${user?.id ?? '<none>'}`);
if (user) {
this.driveFileEntityService.pack(file, { self: true }).then(packedFile => {
@ -708,14 +741,14 @@ export class DriveService {
@bindThis
public async deleteFile(file: MiDriveFile, isExpired = false, deleter?: MiUser) {
if (file.storedInternal) {
this.internalStorageService.del(file.accessKey!);
this.deleteLocalFile(file.accessKey!);
if (file.thumbnailUrl) {
this.internalStorageService.del(file.thumbnailAccessKey!);
this.deleteLocalFile(file.thumbnailAccessKey!);
}
if (file.webpublicUrl) {
this.internalStorageService.del(file.webpublicAccessKey!);
this.deleteLocalFile(file.webpublicAccessKey!);
}
} else if (!file.isLink) {
this.queueService.createDeleteObjectStorageFileJob(file.accessKey!);
@ -737,14 +770,14 @@ export class DriveService {
const promises = [];
if (file.storedInternal) {
promises.push(this.internalStorageService.del(file.accessKey!));
promises.push(this.deleteLocalFile(file.accessKey!));
if (file.thumbnailUrl) {
promises.push(this.internalStorageService.del(file.thumbnailAccessKey!));
promises.push(this.deleteLocalFile(file.thumbnailAccessKey!));
}
if (file.webpublicUrl) {
promises.push(this.internalStorageService.del(file.webpublicAccessKey!));
promises.push(this.deleteLocalFile(file.webpublicAccessKey!));
}
} else if (!file.isLink) {
promises.push(this.deleteObjectStorageFile(file.accessKey!));
@ -814,11 +847,14 @@ export class DriveService {
Bucket: this.meta.objectStorageBucket,
Key: key,
} as DeleteObjectCommandInput;
await this.s3Service.delete(this.meta, param);
if (this.bunnyService.usingBunnyCDN(this.meta)) {
await this.bunnyService.delete(this.meta, key);
} else {
await this.s3Service.delete(this.meta, param);
}
} catch (err: any) {
if (err.name === 'NoSuchKey') {
this.deleteLogger.warn(`The object storage had no such key to delete: ${key}. Skipping this.`, err as Error);
this.deleteLogger.warn(`The object storage had no such key to delete: ${key}. Skipping this.`);
return;
} else {
throw new Error(`Failed to delete the file from the object storage with the given key: ${key}`, {
@ -828,6 +864,22 @@ export class DriveService {
}
}
@bindThis
public async deleteLocalFile(key: string) {
try {
await this.internalStorageService.del(key);
} catch (err: any) {
if (err.code === 'ENOENT') {
this.deleteLogger.warn(`The file to delete did not exist: ${key}. Skipping this.`);
return;
} else {
throw new Error(`Failed to delete the file: ${key}`, {
cause: err,
});
}
}
}
@bindThis
public async uploadFromUrl({
url,
@ -855,13 +907,10 @@ export class DriveService {
}
const driveFile = await this.addFile({ user, path, name, comment, folderId, force, isLink, url, uri, sensitive, requestIp, requestHeaders });
this.downloaderLogger.succ(`Got: ${driveFile.id}`);
this.downloaderLogger.debug(`Upload succeeded: created file ${driveFile.id}`);
return driveFile!;
} catch (err) {
this.downloaderLogger.error(`Failed to create drive file: ${err}`, {
url: url,
e: err,
});
this.downloaderLogger.error(`Failed to create drive file from ${url}: ${renderInlineError(err)}`);
throw err;
} finally {
cleanup();

7
packages/backend/src/core/EmailService.ts
View file

@ -164,6 +164,13 @@ export class EmailService {
available: boolean;
reason: null | 'used' | 'format' | 'disposable' | 'mx' | 'smtp' | 'banned' | 'network' | 'blacklist';
}> {
if (!this.utilityService.validateEmailFormat(emailAddress)) {
return {
available: false,
reason: 'format',
};
}
const exist = await this.userProfilesRepository.countBy({
emailVerified: true,
email: emailAddress,

45
packages/backend/src/core/FanoutTimelineEndpointService.ts
View file

@ -8,10 +8,12 @@ import { DI } from '@/di-symbols.js';
import { bindThis } from '@/decorators.js';
import type { MiUser } from '@/models/User.js';
import type { MiNote } from '@/models/Note.js';
import type { MiMeta } from '@/models/Meta.js';
import { Packed } from '@/misc/json-schema.js';
import type { NotesRepository } from '@/models/_.js';
import { NoteEntityService } from '@/core/entities/NoteEntityService.js';
import { FanoutTimelineName, FanoutTimelineService } from '@/core/FanoutTimelineService.js';
import { UtilityService } from '@/core/UtilityService.js';
import { isUserRelated } from '@/misc/is-user-related.js';
import { isQuote, isRenote } from '@/misc/is-renote.js';
import { CacheService } from '@/core/CacheService.js';
@ -30,10 +32,12 @@ type TimelineOptions = {
alwaysIncludeMyNotes?: boolean;
ignoreAuthorFromBlock?: boolean;
ignoreAuthorFromMute?: boolean;
ignoreAuthorFromInstanceBlock?: boolean;
excludeNoFiles?: boolean;
excludeReplies?: boolean;
excludeBots?: boolean;
excludePureRenotes: boolean;
ignoreAuthorFromUserSuspension?: boolean;
dbFallback: (untilId: string | null, sinceId: string | null, limit: number) => Promise<MiNote[]>,
};
@ -43,9 +47,13 @@ export class FanoutTimelineEndpointService {
@Inject(DI.notesRepository)
private notesRepository: NotesRepository,
@Inject(DI.meta)
private meta: MiMeta,
private noteEntityService: NoteEntityService,
private cacheService: CacheService,
private fanoutTimelineService: FanoutTimelineService,
private utilityService: UtilityService,
) {
}
@ -55,7 +63,7 @@ export class FanoutTimelineEndpointService {
}
@bindThis
private async getMiNotes(ps: TimelineOptions): Promise<MiNote[]> {
async getMiNotes(ps: TimelineOptions): Promise<MiNote[]> {
// 呼び出し元と以下の処理をシンプルにするためにdbFallbackを置き換える
if (!ps.useDbFallback) ps.dbFallback = () => Promise.resolve([]);
@ -125,6 +133,36 @@ export class FanoutTimelineEndpointService {
};
}
{
const parentFilter = filter;
filter = (note) => {
if (!ps.ignoreAuthorFromInstanceBlock) {
if (note.userInstance?.isBlocked) return false;
}
if (note.userId !== note.renoteUserId && note.renoteUserInstance?.isBlocked) return false;
if (note.userId !== note.replyUserId && note.replyUserInstance?.isBlocked) return false;
return parentFilter(note);
};
}
{
const parentFilter = filter;
filter = (note) => {
const noteJoined = note as MiNote & {
renoteUser: MiUser | null;
replyUser: MiUser | null;
};
if (!ps.ignoreAuthorFromUserSuspension) {
if (note.user!.isSuspended) return false;
}
if (note.userId !== note.renoteUserId && noteJoined.renoteUser?.isSuspended) return false;
if (note.userId !== note.replyUserId && noteJoined.replyUser?.isSuspended) return false;
return parentFilter(note);
};
}
const redisTimeline: MiNote[] = [];
let readFromRedis = 0;
let lastSuccessfulRate = 1; // rateをキャッシュする？
@ -174,7 +212,10 @@ export class FanoutTimelineEndpointService {
.leftJoinAndSelect('note.renote', 'renote')
.leftJoinAndSelect('reply.user', 'replyUser')
.leftJoinAndSelect('renote.user', 'renoteUser')
.leftJoinAndSelect('note.channel', 'channel');
.leftJoinAndSelect('note.channel', 'channel')
.leftJoinAndSelect('note.userInstance', 'userInstance')
.leftJoinAndSelect('note.replyUserInstance', 'replyUserInstance')
.leftJoinAndSelect('note.renoteUserInstance', 'renoteUserInstance');
const notes = (await query.getMany()).filter(noteFilter);

3
packages/backend/src/core/FanoutTimelineService.ts
View file

@ -9,7 +9,7 @@ import { DI } from '@/di-symbols.js';
import { bindThis } from '@/decorators.js';
import { IdService } from '@/core/IdService.js';
export type FanoutTimelineName =
export type FanoutTimelineName = (
// home timeline
| `homeTimeline:${string}`
| `homeTimelineWithFiles:${string}` // only notes with files are included
@ -37,6 +37,7 @@ export type FanoutTimelineName =
// role timelines
| `roleTimeline:${string}` // any notes are included
);
@Injectable()
export class FanoutTimelineService {

30
packages/backend/src/core/FeaturedService.ts
View file

@ -8,6 +8,7 @@ import * as Redis from 'ioredis';
import type { MiGalleryPost, MiNote, MiUser } from '@/models/_.js';
import { DI } from '@/di-symbols.js';
import { bindThis } from '@/decorators.js';
import { RoleService } from '@/core/RoleService.js';
const GLOBAL_NOTES_RANKING_WINDOW = 1000 * 60 * 60 * 24 * 3; // 3日ごと
export const GALLERY_POSTS_RANKING_WINDOW = 1000 * 60 * 60 * 24 * 3; // 3日ごと
@ -21,6 +22,8 @@ export class FeaturedService {
constructor(
@Inject(DI.redis)
private redisClient: Redis.Redis, // TODO: 専用のRedisサーバーを設定できるようにする
private readonly roleService: RoleService,
) {
}
@ -31,7 +34,14 @@ export class FeaturedService {
}
@bindThis
private async updateRankingOf(name: string, windowRange: number, element: string, score = 1): Promise<void> {
private async updateRankingOf(name: string, windowRange: number, element: string, score: number, userId: string | null): Promise<void> {
if (userId) {
const policies = await this.roleService.getUserPolicies(userId);
if (!policies.canTrend) {
return;
}
}
const currentWindow = this.getCurrentWindow(windowRange);
const redisTransaction = this.redisClient.multi();
redisTransaction.zincrby(
@ -89,28 +99,28 @@ export class FeaturedService {
}
@bindThis
public updateGlobalNotesRanking(noteId: MiNote['id'], score = 1): Promise<void> {
return this.updateRankingOf('featuredGlobalNotesRanking', GLOBAL_NOTES_RANKING_WINDOW, noteId, score);
public updateGlobalNotesRanking(note: Pick<MiNote, 'id' | 'userId'>, score = 1): Promise<void> {
return this.updateRankingOf('featuredGlobalNotesRanking', GLOBAL_NOTES_RANKING_WINDOW, note.id, score, note.userId);
}
@bindThis
public updateGalleryPostsRanking(galleryPostId: MiGalleryPost['id'], score = 1): Promise<void> {
return this.updateRankingOf('featuredGalleryPostsRanking', GALLERY_POSTS_RANKING_WINDOW, galleryPostId, score);
public updateGalleryPostsRanking(galleryPost: Pick<MiGalleryPost, 'id' | 'userId'>, score = 1): Promise<void> {
return this.updateRankingOf('featuredGalleryPostsRanking', GALLERY_POSTS_RANKING_WINDOW, galleryPost.id, score, galleryPost.userId);
}
@bindThis
public updateInChannelNotesRanking(channelId: MiNote['channelId'], noteId: MiNote['id'], score = 1): Promise<void> {
return this.updateRankingOf(`featuredInChannelNotesRanking:${channelId}`, GLOBAL_NOTES_RANKING_WINDOW, noteId, score);
public updateInChannelNotesRanking(channelId: MiNote['channelId'], note: Pick<MiNote, 'id' | 'userId'>, score = 1): Promise<void> {
return this.updateRankingOf(`featuredInChannelNotesRanking:${channelId}`, GLOBAL_NOTES_RANKING_WINDOW, note.id, score, note.userId);
}
@bindThis
public updatePerUserNotesRanking(userId: MiUser['id'], noteId: MiNote['id'], score = 1): Promise<void> {
return this.updateRankingOf(`featuredPerUserNotesRanking:${userId}`, PER_USER_NOTES_RANKING_WINDOW, noteId, score);
public updatePerUserNotesRanking(userId: MiUser['id'], note: Pick<MiNote, 'id'>, score = 1): Promise<void> {
return this.updateRankingOf(`featuredPerUserNotesRanking:${userId}`, PER_USER_NOTES_RANKING_WINDOW, note.id, score, userId);
}
@bindThis
public updateHashtagsRanking(hashtag: string, score = 1): Promise<void> {
return this.updateRankingOf('featuredHashtagsRanking', HASHTAG_RANKING_WINDOW, hashtag, score);
return this.updateRankingOf('featuredHashtagsRanking', HASHTAG_RANKING_WINDOW, hashtag, score, null);
}
@bindThis

103
packages/backend/src/core/FederatedInstanceService.ts
View file

@ -5,23 +5,24 @@
import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
import * as Redis from 'ioredis';
import { QueryFailedError } from 'typeorm';
import type { InstancesRepository } from '@/models/_.js';
import type { InstancesRepository, MiMeta } from '@/models/_.js';
import type { MiInstance } from '@/models/Instance.js';
import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js';
import { MemoryKVCache } from '@/misc/cache.js';
import { IdService } from '@/core/IdService.js';
import { DI } from '@/di-symbols.js';
import { UtilityService } from '@/core/UtilityService.js';
import { bindThis } from '@/decorators.js';
import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js';
import type { GlobalEvents } from '@/core/GlobalEventService.js';
import { Serialized } from '@/types.js';
import { diffArrays, diffArraysSimple } from '@/misc/diff-arrays.js';
@Injectable()
export class FederatedInstanceService implements OnApplicationShutdown {
public federatedInstanceCache: RedisKVCache<MiInstance | null>;
private readonly federatedInstanceCache: MemoryKVCache<MiInstance | null>;
constructor(
@Inject(DI.redis)
private redisClient: Redis.Redis,
@Inject(DI.redisForSub)
private redisForSub: Redis.Redis,
@Inject(DI.instancesRepository)
private instancesRepository: InstancesRepository,
@ -29,67 +30,46 @@ export class FederatedInstanceService implements OnApplicationShutdown {
private utilityService: UtilityService,
private idService: IdService,
) {
this.federatedInstanceCache = new RedisKVCache<MiInstance | null>(this.redisClient, 'federatedInstance', {
lifetime: 1000 * 60 * 30, // 30m
memoryCacheLifetime: 1000 * 60 * 3, // 3m
fetcher: (key) => this.instancesRepository.findOneBy({ host: key }),
toRedisConverter: (value) => JSON.stringify(value),
fromRedisConverter: (value) => {
const parsed = JSON.parse(value);
if (parsed == null) return null;
return {
...parsed,
firstRetrievedAt: new Date(parsed.firstRetrievedAt),
latestRequestReceivedAt: parsed.latestRequestReceivedAt ? new Date(parsed.latestRequestReceivedAt) : null,
infoUpdatedAt: parsed.infoUpdatedAt ? new Date(parsed.infoUpdatedAt) : null,
notRespondingSince: parsed.notRespondingSince ? new Date(parsed.notRespondingSince) : null,
};
},
});
this.federatedInstanceCache = new MemoryKVCache(1000 * 60 * 3); // 3m
this.redisForSub.on('message', this.onMessage);
}
@bindThis
public async fetchOrRegister(host: string): Promise<MiInstance> {
host = this.utilityService.toPuny(host);
const cached = await this.federatedInstanceCache.get(host);
const cached = this.federatedInstanceCache.get(host);
if (cached) return cached;
const index = await this.instancesRepository.findOneBy({ host });
let index = await this.instancesRepository.findOneBy({ host });
if (index == null) {
let i;
try {
i = await this.instancesRepository.insertOne({
await this.instancesRepository.createQueryBuilder('instance')
.insert()
.values({
id: this.idService.gen(),
host,
firstRetrievedAt: new Date(),
});
} catch (e: unknown) {
if (e instanceof QueryFailedError) {
if (isDuplicateKeyValueError(e)) {
i = await this.instancesRepository.findOneBy({ host });
}
}
isBlocked: this.utilityService.isBlockedHost(host),
isSilenced: this.utilityService.isSilencedHost(host),
isMediaSilenced: this.utilityService.isMediaSilencedHost(host),
isAllowListed: this.utilityService.isAllowListedHost(host),
isBubbled: this.utilityService.isBubbledHost(host),
})
.orIgnore()
.execute();
if (i == null) {
throw e;
}
}
this.federatedInstanceCache.set(host, i);
return i;
} else {
this.federatedInstanceCache.set(host, index);
return index;
index = await this.instancesRepository.findOneByOrFail({ host });
}
this.federatedInstanceCache.set(host, index);
return index;
}
@bindThis
public async fetch(host: string): Promise<MiInstance | null> {
host = this.utilityService.toPuny(host);
const cached = await this.federatedInstanceCache.get(host);
const cached = this.federatedInstanceCache.get(host);
if (cached !== undefined) return cached;
const index = await this.instancesRepository.findOneBy({ host });
@ -117,8 +97,35 @@ export class FederatedInstanceService implements OnApplicationShutdown {
this.federatedInstanceCache.set(result.host, result);
}
private syncCache(before: Serialized<MiMeta | undefined>, after: Serialized<MiMeta>): void {
const changed =
diffArraysSimple(before?.blockedHosts, after.blockedHosts) ||
diffArraysSimple(before?.silencedHosts, after.silencedHosts) ||
diffArraysSimple(before?.mediaSilencedHosts, after.mediaSilencedHosts) ||
diffArraysSimple(before?.federationHosts, after.federationHosts) ||
diffArraysSimple(before?.bubbleInstances, after.bubbleInstances);
if (changed) {
// We have to clear the whole thing, otherwise subdomains won't be synced.
this.federatedInstanceCache.clear();
}
}
@bindThis
private async onMessage(_: string, data: string): Promise<void> {
const obj = JSON.parse(data);
if (obj.channel === 'internal') {
const { type, body } = obj.message as GlobalEvents['internal']['payload'];
if (type === 'metaUpdated') {
this.syncCache(body.before, body.after);
}
}
}
@bindThis
public dispose(): void {
this.redisForSub.off('message', this.onMessage);
this.federatedInstanceCache.dispose();
}

62
packages/backend/src/core/FetchInstanceMetadataService.ts
View file

@ -5,9 +5,9 @@
import { URL } from 'node:url';
import { Inject, Injectable } from '@nestjs/common';
import { JSDOM } from 'jsdom';
import tinycolor from 'tinycolor2';
import * as Redis from 'ioredis';
import { load as cheerio } from 'cheerio/slim';
import type { MiInstance } from '@/models/Instance.js';
import type Logger from '@/logger.js';
import { DI } from '@/di-symbols.js';
@ -15,7 +15,8 @@ import { LoggerService } from '@/core/LoggerService.js';
import { HttpRequestService } from '@/core/HttpRequestService.js';
import { bindThis } from '@/decorators.js';
import { FederatedInstanceService } from '@/core/FederatedInstanceService.js';
import type { DOMWindow } from 'jsdom';
import { renderInlineError } from '@/misc/render-inline-error.js';
import type { CheerioAPI } from 'cheerio/slim';
type NodeInfo = {
openRegistrations?: unknown;
@ -90,7 +91,7 @@ export class FetchInstanceMetadataService {
}
}
this.logger.info(`Fetching metadata of ${instance.host} ...`);
this.logger.debug(`Fetching metadata of ${instance.host} ...`);
const [info, dom, manifest] = await Promise.all([
this.fetchNodeinfo(instance).catch(() => null),
@ -106,7 +107,7 @@ export class FetchInstanceMetadataService {
this.getDescription(info, dom, manifest).catch(() => null),
]);
this.logger.succ(`Successfuly fetched metadata of ${instance.host}`);
this.logger.debug(`Successfuly fetched metadata of ${instance.host}`);
const updates = {
infoUpdatedAt: new Date(),
@ -128,9 +129,9 @@ export class FetchInstanceMetadataService {
await this.federatedInstanceService.update(instance.id, updates);
this.logger.succ(`Successfuly updated metadata of ${instance.host}`);
this.logger.info(`Successfully updated metadata of ${instance.host}`);
} catch (e) {
this.logger.error(`Failed to update metadata of ${instance.host}: ${e}`);
this.logger.error(`Failed to update metadata of ${instance.host}: ${renderInlineError(e)}`);
} finally {
await this.unlock(host);
}
@ -138,7 +139,7 @@ export class FetchInstanceMetadataService {
@bindThis
private async fetchNodeinfo(instance: MiInstance): Promise<NodeInfo> {
this.logger.info(`Fetching nodeinfo of ${instance.host} ...`);
this.logger.debug(`Fetching nodeinfo of ${instance.host} ...`);
try {
const wellknown = await this.httpRequestService.getJson('https://' + instance.host + '/.well-known/nodeinfo')
@ -170,28 +171,25 @@ export class FetchInstanceMetadataService {
throw err.statusCode ?? err.message;
});
this.logger.succ(`Successfuly fetched nodeinfo of ${instance.host}`);
this.logger.debug(`Successfuly fetched nodeinfo of ${instance.host}`);
return info as NodeInfo;
} catch (err) {
this.logger.error(`Failed to fetch nodeinfo of ${instance.host}: ${err}`);
this.logger.warn(`Failed to fetch nodeinfo of ${instance.host}: ${renderInlineError(err)}`);
throw err;
}
}
@bindThis
private async fetchDom(instance: MiInstance): Promise<Document> {
this.logger.info(`Fetching HTML of ${instance.host} ...`);
private async fetchDom(instance: MiInstance): Promise<CheerioAPI> {
this.logger.debug(`Fetching HTML of ${instance.host} ...`);
const url = 'https://' + instance.host;
const html = await this.httpRequestService.getHtml(url);
const { window } = new JSDOM(html);
const doc = window.document;
return doc;
return cheerio(html);
}
@bindThis
@ -206,12 +204,15 @@ export class FetchInstanceMetadataService {
}
@bindThis
private async fetchFaviconUrl(instance: MiInstance, doc: Document | null): Promise<string | null> {
private async fetchFaviconUrl(instance: MiInstance, doc: CheerioAPI | null): Promise<string | null> {
const url = 'https://' + instance.host;
if (doc) {
// https://github.com/misskey-dev/misskey/pull/8220#issuecomment-1025104043
const href = Array.from(doc.getElementsByTagName('link')).reverse().find(link => link.relList.contains('icon'))?.href;
const href = doc('link[rel][href]')
.filter((_, link) => link.attribs.rel.split(' ').includes('icon'))
.last()
.attr('href');
if (href) {
return (new URL(href, url)).href;
@ -232,7 +233,7 @@ export class FetchInstanceMetadataService {
}
@bindThis
private async fetchIconUrl(instance: MiInstance, doc: Document | null, manifest: Record<string, any> | null): Promise<string | null> {
private async fetchIconUrl(instance: MiInstance, doc: CheerioAPI | null, manifest: Record<string, any> | null): Promise<string | null> {
if (manifest && manifest.icons && manifest.icons.length > 0 && manifest.icons[0].src) {
const url = 'https://' + instance.host;
return (new URL(manifest.icons[0].src, url)).href;
@ -242,13 +243,16 @@ export class FetchInstanceMetadataService {
const url = 'https://' + instance.host;
// https://github.com/misskey-dev/misskey/pull/8220#issuecomment-1025104043
const links = Array.from(doc.getElementsByTagName('link')).reverse();
const links = Array.from(doc('link[rel][href]')).reverse().map(link => ({
rel: link.attribs.rel.split(' '),
href: link.attribs.href,
}));
// https://github.com/misskey-dev/misskey/pull/8220/files/0ec4eba22a914e31b86874f12448f88b3e58dd5a#r796487559
const href =
[
links.find(link => link.relList.contains('apple-touch-icon-precomposed'))?.href,
links.find(link => link.relList.contains('apple-touch-icon'))?.href,
links.find(link => link.relList.contains('icon'))?.href,
links.find(link => link.rel.includes('apple-touch-icon-precomposed'))?.href,
links.find(link => link.rel.includes('apple-touch-icon'))?.href,
links.find(link => link.rel.includes('icon'))?.href,
]
.find(href => href);
@ -261,8 +265,8 @@ export class FetchInstanceMetadataService {
}
@bindThis
private async getThemeColor(info: NodeInfo | null, doc: Document | null, manifest: Record<string, any> | null): Promise<string | null> {
const themeColor = info?.metadata?.themeColor ?? doc?.querySelector('meta[name="theme-color"]')?.getAttribute('content') ?? manifest?.theme_color;
private async getThemeColor(info: NodeInfo | null, doc: CheerioAPI | null, manifest: Record<string, any> | null): Promise<string | null> {
const themeColor = info?.metadata?.themeColor ?? doc?.('meta[name="theme-color"][content]').attr('content') ?? manifest?.theme_color;
if (themeColor) {
const color = new tinycolor(themeColor);
@ -273,7 +277,7 @@ export class FetchInstanceMetadataService {
}
@bindThis
private async getSiteName(info: NodeInfo | null, doc: Document | null, manifest: Record<string, any> | null): Promise<string | null> {
private async getSiteName(info: NodeInfo | null, doc: CheerioAPI | null, manifest: Record<string, any> | null): Promise<string | null> {
if (info && info.metadata) {
if (typeof info.metadata.nodeName === 'string') {
return info.metadata.nodeName;
@ -283,7 +287,7 @@ export class FetchInstanceMetadataService {
}
if (doc) {
const og = doc.querySelector('meta[property="og:title"]')?.getAttribute('content');
const og = doc('meta[property="og:title"][content]').attr('content');
if (og) {
return og;
@ -298,7 +302,7 @@ export class FetchInstanceMetadataService {
}
@bindThis
private async getDescription(info: NodeInfo | null, doc: Document | null, manifest: Record<string, any> | null): Promise<string | null> {
private async getDescription(info: NodeInfo | null, doc: CheerioAPI | null, manifest: Record<string, any> | null): Promise<string | null> {
if (info && info.metadata) {
if (typeof info.metadata.nodeDescription === 'string') {
return info.metadata.nodeDescription;
@ -308,12 +312,12 @@ export class FetchInstanceMetadataService {
}
if (doc) {
const meta = doc.querySelector('meta[name="description"]')?.getAttribute('content');
const meta = doc('meta[name="description"][content]').attr('content');
if (meta) {
return meta;
}
const og = doc.querySelector('meta[property="og:description"]')?.getAttribute('content');
const og = doc('meta[property="og:description"][content]').attr('content');
if (og) {
return og;
}

9
packages/backend/src/core/FileInfoService.ts
View file

@ -46,11 +46,13 @@ const TYPE_SVG = {
@Injectable()
export class FileInfoService {
private logger: Logger;
private ffprobeLogger: Logger;
constructor(
private loggerService: LoggerService,
) {
this.logger = this.loggerService.getLogger('file-info');
this.ffprobeLogger = this.logger.createSubLogger('ffprobe');
}
/**
@ -162,20 +164,19 @@ export class FileInfoService {
*/
@bindThis
private hasVideoTrackOnVideoFile(path: string): Promise<boolean> {
const sublogger = this.logger.createSubLogger('ffprobe');
sublogger.info(`Checking the video file. File path: ${path}`);
this.ffprobeLogger.debug(`Checking the video file. File path: ${path}`);
return new Promise((resolve) => {
try {
FFmpeg.ffprobe(path, (err, metadata) => {
if (err) {
sublogger.warn(`Could not check the video file. Returns true. File path: ${path}`, err);
this.ffprobeLogger.warn(`Could not check the video file. Returns true. File path: ${path}`, err);
resolve(true);
return;
}
resolve(metadata.streams.some((stream) => stream.codec_type === 'video'));
});
} catch (err) {
sublogger.warn(`Could not check the video file. Returns true. File path: ${path}`, err as Error);
this.ffprobeLogger.warn(`Could not check the video file. Returns true. File path: ${path}`, err as Error);
resolve(true);
}
});

55
packages/backend/src/core/GlobalEventService.ts
View file

@ -20,7 +20,7 @@ import type { MiPage } from '@/models/Page.js';
import type { MiWebhook } from '@/models/Webhook.js';
import type { MiSystemWebhook } from '@/models/SystemWebhook.js';
import type { MiMeta } from '@/models/Meta.js';
import { MiAvatarDecoration, MiReversiGame, MiRole, MiRoleAssignment } from '@/models/_.js';
import { MiAvatarDecoration, MiChatMessage, MiChatRoom, MiReversiGame, MiRole, MiRoleAssignment } from '@/models/_.js';
import type { Packed } from '@/misc/json-schema.js';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
@ -72,12 +72,8 @@ export interface MainEventTypes {
readAllNotifications: undefined;
notificationFlushed: undefined;
unreadNotification: Packed<'Notification'>;
unreadMention: MiNote['id'];
readAllUnreadMentions: undefined;
unreadSpecifiedNote: MiNote['id'];
readAllUnreadSpecifiedNotes: undefined;
readAllAntennas: undefined;
unreadAntenna: MiAntenna;
newChatMessage: Packed<'ChatMessage'>;
readAllAnnouncements: undefined;
myTokenRegenerated: undefined;
signin: {
@ -168,6 +164,21 @@ export interface AdminEventTypes {
};
}
export interface ChatEventTypes {
message: Packed<'ChatMessageLite'>;
deleted: Packed<'ChatMessageLite'>['id'];
react: {
reaction: string;
user?: Packed<'UserLite'>;
messageId: MiChatMessage['id'];
};
unreact: {
reaction: string;
user?: Packed<'UserLite'>;
messageId: MiChatMessage['id'];
};
}
export interface ReversiEventTypes {
matched: {
game: Packed<'ReversiGameDetailed'>;
@ -207,7 +218,7 @@ export interface ReversiGameEventTypes {
type Events<T extends object> = { [K in keyof T]: { type: K; body: T[K]; } };
type EventUnionFromDictionary<
T extends object,
U = Events<T>
U = Events<T>,
> = U[keyof U];
type SerializedAll<T> = {
@ -216,7 +227,7 @@ type SerializedAll<T> = {
type UndefinedAsNullAll<T> = {
[K in keyof T]: T[K] extends undefined ? null : T[K];
}
};
export interface InternalEventTypes {
userChangeSuspendedState: { id: MiUser['id']; isSuspended: MiUser['isSuspended']; };
@ -254,6 +265,7 @@ export interface InternalEventTypes {
unmute: { muterId: MiUser['id']; muteeId: MiUser['id']; };
userListMemberAdded: { userListId: MiUserList['id']; memberId: MiUser['id']; };
userListMemberRemoved: { userListId: MiUserList['id']; memberId: MiUser['id']; };
quantumCacheUpdated: { name: string, keys: string[] };
}
type EventTypesToEventPayload<T> = EventUnionFromDictionary<UndefinedAsNullAll<SerializedAll<T>>>;
@ -300,6 +312,14 @@ export type GlobalEvents = {
name: 'notesStream';
payload: Serialized<Packed<'Note'>>;
};
chatUser: {
name: `chatUserStream:${MiUser['id']}-${MiUser['id']}`;
payload: EventTypesToEventPayload<ChatEventTypes>;
};
chatRoom: {
name: `chatRoomStream:${MiChatRoom['id']}`;
payload: EventTypesToEventPayload<ChatEventTypes>;
};
reversi: {
name: `reversiStream:${MiUser['id']}`;
payload: EventTypesToEventPayload<ReversiEventTypes>;
@ -334,12 +354,12 @@ export class GlobalEventService {
}
@bindThis
private publish(channel: StreamChannels, type: string | null, value?: any): void {
private async publish(channel: StreamChannels, type: string | null, value?: any): Promise<void> {
const message = type == null ? value : value == null ?
{ type: type, body: null } :
{ type: type, body: value };
this.redisForPub.publish(this.config.host, JSON.stringify({
await this.redisForPub.publish(this.config.host, JSON.stringify({
channel: channel,
message: message,
}));
@ -350,6 +370,11 @@ export class GlobalEventService {
this.publish('internal', type, typeof value === 'undefined' ? null : value);
}
@bindThis
public async publishInternalEventAsync<K extends keyof InternalEventTypes>(type: K, value?: InternalEventTypes[K]): Promise<void> {
await this.publish('internal', type, typeof value === 'undefined' ? null : value);
}
@bindThis
public publishBroadcastStream<K extends keyof BroadcastTypes>(type: K, value?: BroadcastTypes[K]): void {
this.publish('broadcast', type, typeof value === 'undefined' ? null : value);
@ -398,6 +423,16 @@ export class GlobalEventService {
this.publish(`adminStream:${userId}`, type, typeof value === 'undefined' ? null : value);
}
@bindThis
public publishChatUserStream<K extends keyof ChatEventTypes>(fromUserId: MiUser['id'], toUserId: MiUser['id'], type: K, value?: ChatEventTypes[K]): void {
this.publish(`chatUserStream:${fromUserId}-${toUserId}`, type, typeof value === 'undefined' ? null : value);
}
@bindThis
public publishChatRoomStream<K extends keyof ChatEventTypes>(toRoomId: MiChatRoom['id'], type: K, value?: ChatEventTypes[K]): void {
this.publish(`chatRoomStream:${toRoomId}`, type, typeof value === 'undefined' ? null : value);
}
@bindThis
public publishReversiStream<K extends keyof ReversiEventTypes>(userId: MiUser['id'], type: K, value?: ReversiEventTypes[K]): void {
this.publish(`reversiStream:${userId}`, type, typeof value === 'undefined' ? null : value);

167
packages/backend/src/core/HttpRequestService.ts
View file

@ -12,20 +12,64 @@ import fetch from 'node-fetch';
import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent';
import { Inject, Injectable } from '@nestjs/common';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import type { Config, PrivateNetwork } from '@/config.js';
import { StatusError } from '@/misc/status-error.js';
import { bindThis } from '@/decorators.js';
import { validateContentTypeSetAsActivityPub } from '@/core/activitypub/misc/validator.js';
import { IObject } from '@/core/activitypub/type.js';
import { ApUtilityService } from './activitypub/ApUtilityService.js';
import type { IObject, IObjectWithId } from '@/core/activitypub/type.js';
import { UtilityService } from '@/core/UtilityService.js';
import { ApUtilityService } from '@/core/activitypub/ApUtilityService.js';
import type { Response } from 'node-fetch';
import type { URL } from 'node:url';
import type { Socket } from 'node:net';
export type HttpRequestSendOptions = {
throwErrorWhenResponseNotOk: boolean;
validators?: ((res: Response) => void)[];
};
export async function isPrivateUrl(url: URL, lookup: net.LookupFunction): Promise<boolean> {
const ip = await resolveIp(url, lookup);
return ip.range() !== 'unicast';
}
export async function resolveIp(url: URL, lookup: net.LookupFunction) {
if (ipaddr.isValid(url.hostname)) {
return ipaddr.parse(url.hostname);
}
const resolvedIp = await new Promise<string>((resolve, reject) => {
lookup(url.hostname, {}, (err, address) => {
if (err) reject(err);
else resolve(address as string);
});
});
return ipaddr.parse(resolvedIp);
}
export function isAllowedPrivateIp(allowedPrivateNetworks: PrivateNetwork[] | undefined, ip: string, port?: number): boolean {
const parsedIp = ipaddr.parse(ip);
for (const { cidr, ports } of allowedPrivateNetworks ?? []) {
if (cidr[0].kind() === parsedIp.kind() && parsedIp.match(cidr)) {
if (ports == null || (port != null && ports.includes(port))) {
return false;
}
}
}
return parsedIp.range() !== 'unicast';
}
export function validateSocketConnect(allowedPrivateNetworks: PrivateNetwork[] | undefined, socket: Socket): void {
const address = socket.remoteAddress;
if (address && ipaddr.isValid(address)) {
if (isAllowedPrivateIp(allowedPrivateNetworks, address, socket.remotePort)) {
socket.destroy(new Error(`Blocked address: ${address}`));
}
}
}
declare module 'node:http' {
interface Agent {
createConnection(options: net.NetConnectOpts, callback?: (err: unknown, stream: net.Socket) => void): net.Socket;
@ -44,31 +88,12 @@ class HttpRequestServiceAgent extends http.Agent {
public createConnection(options: net.NetConnectOpts, callback?: (err: unknown, stream: net.Socket) => void): net.Socket {
const socket = super.createConnection(options, callback)
.on('connect', () => {
const address = socket.remoteAddress;
if (process.env.NODE_ENV === 'production') {
if (address && ipaddr.isValid(address)) {
if (this.isPrivateIp(address)) {
socket.destroy(new Error(`Blocked address: ${address}`));
}
}
validateSocketConnect(this.config.allowedPrivateNetworks, socket);
}
});
return socket;
}
@bindThis
private isPrivateIp(ip: string): boolean {
const parsedIp = ipaddr.parse(ip);
for (const net of this.config.allowedPrivateNetworks ?? []) {
const cidr = ipaddr.parseCIDR(net);
if (cidr[0].kind() === parsedIp.kind() && parsedIp.match(ipaddr.parseCIDR(net))) {
return false;
}
}
return parsedIp.range() !== 'unicast';
}
}
class HttpsRequestServiceAgent extends https.Agent {
@ -83,31 +108,12 @@ class HttpsRequestServiceAgent extends https.Agent {
public createConnection(options: net.NetConnectOpts, callback?: (err: unknown, stream: net.Socket) => void): net.Socket {
const socket = super.createConnection(options, callback)
.on('connect', () => {
const address = socket.remoteAddress;
if (process.env.NODE_ENV === 'production') {
if (address && ipaddr.isValid(address)) {
if (this.isPrivateIp(address)) {
socket.destroy(new Error(`Blocked address: ${address}`));
}
}
validateSocketConnect(this.config.allowedPrivateNetworks, socket);
}
});
return socket;
}
@bindThis
private isPrivateIp(ip: string): boolean {
const parsedIp = ipaddr.parse(ip);
for (const net of this.config.allowedPrivateNetworks ?? []) {
const cidr = ipaddr.parseCIDR(net);
if (cidr[0].kind() === parsedIp.kind() && parsedIp.match(ipaddr.parseCIDR(net))) {
return false;
}
}
return parsedIp.range() !== 'unicast';
}
}
@Injectable()
@ -115,37 +121,43 @@ export class HttpRequestService {
/**
* Get http non-proxy agent (without local address filtering)
*/
private httpNative: http.Agent;
private readonly httpNative: http.Agent;
/**
* Get https non-proxy agent (without local address filtering)
*/
private httpsNative: https.Agent;
private readonly httpsNative: https.Agent;
/**
* Get http non-proxy agent
*/
private http: http.Agent;
private readonly http: http.Agent;
/**
* Get https non-proxy agent
*/
private https: https.Agent;
private readonly https: https.Agent;
/**
* Get http proxy or non-proxy agent
*/
public httpAgent: http.Agent;
public readonly httpAgent: http.Agent;
/**
* Get https proxy or non-proxy agent
*/
public httpsAgent: https.Agent;
public readonly httpsAgent: https.Agent;
/**
* Get shared DNS resolver
*/
public readonly lookup: net.LookupFunction;
constructor(
@Inject(DI.config)
private config: Config,
private readonly apUtilityService: ApUtilityService,
private readonly utilityService: UtilityService,
) {
const cache = new CacheableLookup({
maxTtl: 3600, // 1hours
@ -153,6 +165,8 @@ export class HttpRequestService {
lookup: false, // nativeのdns.lookupにfallbackしない
});
this.lookup = cache.lookup as unknown as net.LookupFunction;
const agentOption = {
keepAlive: true,
keepAliveMsecs: 30 * 1000,
@ -198,7 +212,7 @@ export class HttpRequestService {
/**
* Get agent by URL
* @param url URL
* @param bypassProxy Allways bypass proxy
* @param bypassProxy Always bypass proxy
* @param isLocalAddressAllowed
*/
@bindThis
@ -216,8 +230,40 @@ export class HttpRequestService {
}
}
/**
* Get agent for http by URL
* @param url URL
* @param isLocalAddressAllowed
*/
@bindThis
public async getActivityJson(url: string, isLocalAddressAllowed = false): Promise<IObject> {
public getAgentForHttp(url: URL, isLocalAddressAllowed = false): http.Agent {
if ((this.config.proxyBypassHosts ?? []).includes(url.hostname)) {
return isLocalAddressAllowed
? this.httpNative
: this.http;
} else {
return this.httpAgent;
}
}
/**
* Get agent for https by URL
* @param url URL
* @param isLocalAddressAllowed
*/
@bindThis
public getAgentForHttps(url: URL, isLocalAddressAllowed = false): https.Agent {
if ((this.config.proxyBypassHosts ?? []).includes(url.hostname)) {
return isLocalAddressAllowed
? this.httpsNative
: this.https;
} else {
return this.httpsAgent;
}
}
@bindThis
public async getActivityJson(url: string, isLocalAddressAllowed = false, allowAnonymous = false): Promise<IObjectWithId> {
const res = await this.send(url, {
method: 'GET',
headers: {
@ -235,9 +281,13 @@ export class HttpRequestService {
// Make sure the object ID matches the final URL (which is where it actually exists).
// The caller (ApResolverService) will verify the ID against the original / entry URL, which ensures that all three match.
this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url);
if (allowAnonymous && activity.id == null) {
activity.id = res.url;
} else {
this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url);
}
return activity;
return activity as IObjectWithId;
}
@bindThis
@ -279,6 +329,7 @@ export class HttpRequestService {
timeout?: number,
size?: number,
isLocalAddressAllowed?: boolean,
allowHttp?: boolean,
} = {},
extra: HttpRequestSendOptions = {
throwErrorWhenResponseNotOk: true,
@ -287,6 +338,10 @@ export class HttpRequestService {
): Promise<Response> {
const timeout = args.timeout ?? 5000;
const parsedUrl = new URL(url);
const allowHttp = args.allowHttp || await isPrivateUrl(parsedUrl, this.lookup);
this.utilityService.assertUrl(parsedUrl, allowHttp);
const controller = new AbortController();
setTimeout(() => {
controller.abort();
@ -294,7 +349,7 @@ export class HttpRequestService {
const isLocalAddressAllowed = args.isLocalAddressAllowed ?? false;
const res = await fetch(url, {
const res = await fetch(parsedUrl, {
method: args.method ?? 'GET',
headers: {
'User-Agent': this.config.userAgent,
@ -307,7 +362,7 @@ export class HttpRequestService {
});
if (!res.ok && extra.throwErrorWhenResponseNotOk) {
throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText);
throw new StatusError(`request error from ${url}`, res.status, res.statusText);
}
if (res.ok) {

26
packages/backend/src/core/IdService.ts
View file

@ -7,13 +7,13 @@ import { Inject, Injectable } from '@nestjs/common';
import { ulid } from 'ulid';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { genAid, isSafeAidT, parseAid } from '@/misc/id/aid.js';
import { genAidx, isSafeAidxT, parseAidx } from '@/misc/id/aidx.js';
import { genMeid, isSafeMeidT, parseMeid } from '@/misc/id/meid.js';
import { genMeidg, isSafeMeidgT, parseMeidg } from '@/misc/id/meidg.js';
import { genObjectId, isSafeObjectIdT, parseObjectId } from '@/misc/id/object-id.js';
import { genAid, isSafeAidT, parseAid, parseAidFull } from '@/misc/id/aid.js';
import { genAidx, isSafeAidxT, parseAidx, parseAidxFull } from '@/misc/id/aidx.js';
import { genMeid, isSafeMeidT, parseMeid, parseMeidFull } from '@/misc/id/meid.js';
import { genMeidg, isSafeMeidgT, parseMeidg, parseMeidgFull } from '@/misc/id/meidg.js';
import { genObjectId, isSafeObjectIdT, parseObjectId, parseObjectIdFull } from '@/misc/id/object-id.js';
import { bindThis } from '@/decorators.js';
import { parseUlid } from '@/misc/id/ulid.js';
import { parseUlid, parseUlidFull } from '@/misc/id/ulid.js';
@Injectable()
export class IdService {
@ -70,4 +70,18 @@ export class IdService {
default: throw new Error('unrecognized id generation method');
}
}
// Note: additional is at most 64 bits
@bindThis
public parseFull(id: string): { date: number; additional: bigint; } {
switch (this.method) {
case 'aid': return parseAidFull(id);
case 'aidx': return parseAidxFull(id);
case 'objectid': return parseObjectIdFull(id);
case 'meid': return parseMeidFull(id);
case 'meidg': return parseMeidgFull(id);
case 'ulid': return parseUlidFull(id);
default: throw new Error('unrecognized id generation method');
}
}
}

1
packages/backend/src/core/ImageProcessingService.ts
View file

@ -34,6 +34,7 @@ export const webpDefault: sharp.WebpOptions = {
smartSubsample: true,
mixed: true,
effort: 2,
loop: 0,
};
export const avifDefault: sharp.AvifOptions = {

57
packages/backend/src/core/InstanceActorService.ts
View file

@ -1,57 +0,0 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Inject, Injectable } from '@nestjs/common';
import { IsNull, Not } from 'typeorm';
import type { MiLocalUser } from '@/models/User.js';
import type { UsersRepository } from '@/models/_.js';
import { MemorySingleCache } from '@/misc/cache.js';
import { DI } from '@/di-symbols.js';
import { CreateSystemUserService } from '@/core/CreateSystemUserService.js';
import { bindThis } from '@/decorators.js';
const ACTOR_USERNAME = 'instance.actor' as const;
@Injectable()
export class InstanceActorService {
private cache: MemorySingleCache<MiLocalUser>;
constructor(
@Inject(DI.usersRepository)
private usersRepository: UsersRepository,
private createSystemUserService: CreateSystemUserService,
) {
this.cache = new MemorySingleCache<MiLocalUser>(Infinity);
}
@bindThis
public async realLocalUsersPresent(): Promise<boolean> {
return await this.usersRepository.existsBy({
host: IsNull(),
username: Not(ACTOR_USERNAME),
});
}
@bindThis
public async getInstanceActor(): Promise<MiLocalUser> {
const cached = this.cache.get();
if (cached) return cached;
const user = await this.usersRepository.findOneBy({
host: IsNull(),
username: ACTOR_USERNAME,
}) as MiLocalUser | undefined;
if (user) {
this.cache.set(user);
return user;
} else {
const created = await this.createSystemUserService.createSystemUser(ACTOR_USERNAME) as MiLocalUser;
this.cache.set(created);
return created;
}
}
}

103
packages/backend/src/core/InternalEventService.ts Normal file
View file

@ -0,0 +1,103 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
import Redis from 'ioredis';
import { DI } from '@/di-symbols.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
import type { GlobalEvents, InternalEventTypes } from '@/core/GlobalEventService.js';
import { bindThis } from '@/decorators.js';
export type Listener<K extends keyof InternalEventTypes> = (value: InternalEventTypes[K], key: K, isLocal: boolean) => void | Promise<void>;
export interface ListenerProps {
ignoreLocal?: boolean,
ignoreRemote?: boolean,
}
@Injectable()
export class InternalEventService implements OnApplicationShutdown {
private readonly listeners = new Map<keyof InternalEventTypes, Map<Listener<keyof InternalEventTypes>, ListenerProps>>();
constructor(
@Inject(DI.redisForSub)
private readonly redisForSub: Redis.Redis,
private readonly globalEventService: GlobalEventService,
) {
this.redisForSub.on('message', this.onMessage);
}
@bindThis
public on<K extends keyof InternalEventTypes>(type: K, listener: Listener<K>, props?: ListenerProps): void {
let set = this.listeners.get(type);
if (!set) {
set = new Map();
this.listeners.set(type, set);
}
// Functionally, this is just a set with metadata on the values.
set.set(listener as Listener<keyof InternalEventTypes>, props ?? {});
}
@bindThis
public off<K extends keyof InternalEventTypes>(type: K, listener: Listener<K>): void {
this.listeners.get(type)?.delete(listener as Listener<keyof InternalEventTypes>);
}
@bindThis
public async emit<K extends keyof InternalEventTypes>(type: K, value: InternalEventTypes[K]): Promise<void> {
await this.emitInternal(type, value, true);
await this.globalEventService.publishInternalEventAsync(type, { ...value, _pid: process.pid });
}
@bindThis
private async emitInternal<K extends keyof InternalEventTypes>(type: K, value: InternalEventTypes[K], isLocal: boolean): Promise<void> {
const listeners = this.listeners.get(type);
if (!listeners) {
return;
}
const promises: Promise<void>[] = [];
for (const [listener, props] of listeners) {
if ((isLocal && !props.ignoreLocal) || (!isLocal && !props.ignoreRemote)) {
const promise = Promise.resolve(listener(value, type, isLocal));
promises.push(promise);
}
}
await Promise.all(promises);
}
@bindThis
private async onMessage(_: string, data: string): Promise<void> {
const obj = JSON.parse(data);
if (obj.channel === 'internal') {
const { type, body } = obj.message as GlobalEvents['internal']['payload'];
if (!isLocalInternalEvent(body) || body._pid !== process.pid) {
await this.emitInternal(type, body as InternalEventTypes[keyof InternalEventTypes], false);
}
}
}
@bindThis
public dispose(): void {
this.redisForSub.off('message', this.onMessage);
this.listeners.clear();
}
@bindThis
public onApplicationShutdown(): void {
this.dispose();
}
}
interface LocalInternalEvent {
_pid: number;
}
function isLocalInternalEvent(body: object): body is LocalInternalEvent {
return '_pid' in body && typeof(body._pid) === 'number';
}

11
packages/backend/src/core/InternalStorageService.ts
View file

@ -6,18 +6,11 @@
import * as fs from 'node:fs';
import { copyFile, unlink, writeFile, chmod } from 'node:fs/promises';
import * as Path from 'node:path';
import { fileURLToPath } from 'node:url';
import { dirname } from 'node:path';
import { Inject, Injectable } from '@nestjs/common';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { bindThis } from '@/decorators.js';
const _filename = fileURLToPath(import.meta.url);
const _dirname = dirname(_filename);
const path = Path.resolve(_dirname, '../../../../files');
@Injectable()
export class InternalStorageService {
constructor(
@ -25,12 +18,12 @@ export class InternalStorageService {
private config: Config,
) {
// No one should erase the working directory *while the server is running*.
fs.mkdirSync(path, { recursive: true });
fs.mkdirSync(this.config.mediaDirectory, { recursive: true });
}
@bindThis
public resolvePath(key: string) {
return Path.resolve(path, key);
return Path.resolve(this.config.mediaDirectory, key);
}
@bindThis

25
packages/backend/src/core/LatestNoteService.ts
View file

@ -7,6 +7,7 @@ import { DI } from '@/di-symbols.js';
import type { LatestNotesRepository, NotesRepository } from '@/models/_.js';
import { LoggerService } from '@/core/LoggerService.js';
import Logger from '@/logger.js';
import { QueryService } from './QueryService.js';
@Injectable()
export class LatestNoteService {
@ -14,11 +15,12 @@ export class LatestNoteService {
constructor(
@Inject(DI.notesRepository)
private notesRepository: NotesRepository,
private readonly notesRepository: NotesRepository,
@Inject(DI.latestNotesRepository)
private latestNotesRepository: LatestNotesRepository,
private readonly latestNotesRepository: LatestNotesRepository,
private readonly queryService: QueryService,
loggerService: LoggerService,
) {
this.logger = loggerService.getLogger('LatestNoteService');
@ -91,7 +93,7 @@ export class LatestNoteService {
// Find the newest remaining note for the user.
// We exclude DMs and pure renotes.
const nextLatest = await this.notesRepository
const query = this.notesRepository
.createQueryBuilder('note')
.select()
.where({
@ -106,18 +108,11 @@ export class LatestNoteService {
? Not(null)
: null,
})
.andWhere(`
(
note."renoteId" IS NULL
OR note.text IS NOT NULL
OR note.cw IS NOT NULL
OR note."replyId" IS NOT NULL
OR note."hasPoll"
OR note."fileIds" != '{}'
)
`)
.orderBy({ id: 'DESC' })
.getOne();
.orderBy({ id: 'DESC' });
this.queryService.andIsNotRenote(query, 'note');
const nextLatest = await query.getOne();
if (!nextLatest) return;
// Record it as the latest

Some files were not shown because too many files have changed in this diff Show more