misty.cafe/mistykey
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Merge branch 'develop' into upstream/2025.5.0

Browse source
This commit is contained in:
dakkar 2025-06-10 14:02:32 +01:00
commit 3ebf9c4a71
317 changed files with 6144 additions and 2603 deletions
Download patch file Download diff file Expand all files Collapse all files

10
.config/ci.yml
View file

@ -115,8 +115,14 @@ db:
user: postgres
pass: ci
# Whether disable Caching queries
#disableCache: true
## Log a warning to the server console if any query takes longer than this to complete.
## Measured in milliseconds; set to 0 to disable. (default: 300)
#slowQueryThreshold: 300
# If false, then query results will be cached in redis.
# If true (default), then queries will not be cached.
# This will reduce database load at the cost of increased Redis traffic and risk of bugs and unpredictable behavior.
#disableCache: false
# Extra Connection options
#extra:

10
.config/cypress-devcontainer.yml
View file

@ -57,8 +57,14 @@ db:
user: postgres
pass: postgres
# Whether disable Caching queries
#disableCache: true
## Log a warning to the server console if any query takes longer than this to complete.
## Measured in milliseconds; set to 0 to disable. (default: 300)
#slowQueryThreshold: 300
# If false, then query results will be cached in redis.
# If true (default), then queries will not be cached.
# This will reduce database load at the cost of increased Redis traffic and risk of bugs and unpredictable behavior.
#disableCache: false
# Extra Connection options
#extra:

10
.config/docker_example.yml
View file

@ -118,8 +118,14 @@ db:
user: example-misskey-user
pass: example-misskey-pass
# Whether disable Caching queries
#disableCache: true
## Log a warning to the server console if any query takes longer than this to complete.
## Measured in milliseconds; set to 0 to disable. (default: 300)
#slowQueryThreshold: 300
# If false, then query results will be cached in redis.
# If true (default), then queries will not be cached.
# This will reduce database load at the cost of increased Redis traffic and risk of bugs and unpredictable behavior.
#disableCache: false
# Extra Connection options
#extra:

10
.config/example.yml
View file

@ -121,8 +121,14 @@ db:
user: sharkey
pass: example-misskey-pass
# Whether disable Caching queries
#disableCache: true
## Log a warning to the server console if any query takes longer than this to complete.
## Measured in milliseconds; set to 0 to disable. (default: 300)
#slowQueryThreshold: 300
# If false, then query results will be cached in redis.
# If true (default), then queries will not be cached.
# This will reduce database load at the cost of increased Redis traffic and risk of bugs and unpredictable behavior.
#disableCache: false
# Extra Connection options
#extra:

4
.gitignore vendored
View file

@ -87,3 +87,7 @@ vite.config.local-dev.ts.timestamp-*
# Sharkey
/packages/megalodon/lib
# TypeScript
.tsbuildinfo
*.tsbuildinfo

2
CONTRIBUTING.md
View file

@ -690,7 +690,7 @@ seems to do a decent job)
* re-generate locales (`pnpm run build-assets`) and commit
* build the frontend: `rm -rf built/; NODE_ENV=development pnpm --filter=frontend --filter=frontend-embed --filter=frontend-shared build` (the `development` tells it to keep some of the original filenames in the built files)
* make sure there aren't any new `ti-*` classes (Tabler Icons), and replace them with appropriate `ph-*` ones (Phosphor Icons) in [`vite.replaceicons.ts`](packages/frontend/vite.replaceIcons.ts).
* This command should show you want to change: `grep -ohrP '(?<=["'\'']ti )(ti-(?!fw)[\w\-]+)' --exclude \*.map -- built/ | sort -u`.
* This command should show you want to change: `grep -ohrP '(?<=["'\''](ti )?)(ti-(?!fw)[\w\-]+)' --exclude \*.map -- built/ | sort -u`.
* NOTE: `ti-fw` is a special class that's defined by Misskey, leave it alone.
* After every change, re-build the frontend and check again, until there are no more `ti-*` classes in the built files.
* Commit!

3
cypress/tsconfig.json
View file

@ -2,7 +2,8 @@
"compilerOptions": {
"lib": ["dom", "es5"],
"target": "es5",
"types": ["cypress", "node"]
"types": ["cypress", "node"],
"incremental": true
},
"include": ["./**/*.ts"]
}

128
locales/index.d.ts vendored
View file

@ -12524,6 +12524,14 @@ export interface Locale extends ILocale {
* Displays content centered.
*/
"centerDescription": string;
/**
* Unix Time
*/
"unixtime": string;
/**
* Displays a timestamp in the viewer's current timezone.
*/
"unixtimeDescription": string;
/**
* Code (Inline)
*/
@ -13101,6 +13109,26 @@ export interface Locale extends ILocale {
* Users popular on {name}
*/
"popularUsersLocal": ParameterizedString<"name">;
/**
* Polls trending on {name}
*/
"pollsOnLocal": ParameterizedString<"name">;
/**
* Polls trending on the global network
*/
"pollsOnRemote": string;
/**
* Polls that have ended recently
*/
"pollsExpired": string;
/**
* Trending polls are disabled on this instance.
*/
"trendingPollsDisabled": string;
/**
* Please log in to view trending polls.
*/
"trendingPollsDisabledLogIn": string;
/**
* Silenced
*/
@ -13161,6 +13189,106 @@ export interface Locale extends ILocale {
* Timeout in milliseconds for translation API requests.
*/
"translationTimeoutCaption": string;
/**
* Staff notes
*/
"staffNotes": string;
/**
* Icon of {name}
*/
"instanceIconAlt": ParameterizedString<"name">;
/**
* Attribution Domains
*/
"attributionDomains": string;
/**
* A list of domains whose content can be attributed to you on link previews, separated by new-line. Any subdomain will also be valid. The following needs to be on the webpage:
*/
"attributionDomainsDescription": string;
/**
* Written by {user}
*/
"writtenBy": ParameterizedString<"user">;
/**
* Following (Pub)
*/
"followingPub": string;
/**
* Followers (Sub)
*/
"followersSub": string;
/**
* Well-known resources
*/
"wellKnownResources": string;
/**
* Last posted: {at}
*/
"lastPosted": ParameterizedString<"at">;
/**
* NSFW
*/
"nsfw": string;
/**
* Raw
*/
"raw": string;
/**
* CW
*/
"cw": string;
/**
* Media Silenced
*/
"mediaSilenced": string;
/**
* Bubble
*/
"bubble": string;
/**
* Verified
*/
"verified": string;
/**
* Not Verified
*/
"notVerified": string;
/**
* Hibernated
*/
"hibernated": string;
/**
* When replying to a post with a Content Warning, automatically use the same CW for the reply.
*/
"keepCwDescription": string;
/**
* Disabled (do not copy CWs)
*/
"keepCwDisabled": string;
/**
* Enabled (copy CWs verbatim)
*/
"keepCwEnabled": string;
/**
* Enabled (copy CW and prepend "RE:")
*/
"keepCwPrependRe": string;
/**
* Note controls
*/
"noteFooterLabel": string;
/**
* Packed user data in its raw form. Most of these fields are public and visible to all users.
*/
"rawUserDescription": string;
/**
* Extended user data in its raw form. These fields are private and can only be accessed by moderators.
*/
"rawInfoDescription": string;
/**
* ActivityPub user data in its raw form. These fields are public and accessible to other instances.
*/
"rawApDescription": string;
}
declare const locales: {
[lang: string]: Locale;

42
packages/backend/migration/1747938628395-add-missing-indexes.js
View file

@ -4,19 +4,35 @@
*/
export class AddMissingIndexes1747938628395 {
name = 'AddMissingIndexes1747938628395'
name = 'AddMissingIndexes1747938628395'
async up(queryRunner) {
await queryRunner.query(`CREATE INDEX "IDX_58699f75b9cf904f5f007909cb" ON "user_profile" ("birthday") `);
await queryRunner.query(`CREATE INDEX "IDX_021015e6683570ae9f6b0c62be" ON "user_list_membership" ("userId") `);
await queryRunner.query(`CREATE INDEX "IDX_cddcaf418dc4d392ecfcca842a" ON "user_list_membership" ("userListId") `);
await queryRunner.query(`CREATE UNIQUE INDEX "IDX_e4f3094c43f2d665e6030b0337" ON "user_list_membership" ("userId", "userListId") `);
}
async up(queryRunner) {
// Some instances have duplicate list entries
await queryRunner.query(`
DELETE FROM "user_list_membership"
WHERE "id" NOT IN (
SELECT MIN("id")
FROM "user_list_membership"
GROUP BY "userId", "userListId"
)`);
async down(queryRunner) {
await queryRunner.query(`DROP INDEX "public"."IDX_e4f3094c43f2d665e6030b0337"`);
await queryRunner.query(`DROP INDEX "public"."IDX_cddcaf418dc4d392ecfcca842a"`);
await queryRunner.query(`DROP INDEX "public"."IDX_021015e6683570ae9f6b0c62be"`);
await queryRunner.query(`DROP INDEX "public"."IDX_58699f75b9cf904f5f007909cb"`);
}
// Some instances already have these indexes, for an unknown reason
await queryRunner.query(`DROP INDEX IF EXISTS "public"."IDX_e4f3094c43f2d665e6030b0337"`);
await queryRunner.query(`DROP INDEX IF EXISTS "public"."IDX_cddcaf418dc4d392ecfcca842a"`);
await queryRunner.query(`DROP INDEX IF EXISTS "public"."IDX_021015e6683570ae9f6b0c62be"`);
await queryRunner.query(`DROP INDEX IF EXISTS "public"."IDX_58699f75b9cf904f5f007909cb"`);
// Now the actual migration
await queryRunner.query(`CREATE INDEX "IDX_58699f75b9cf904f5f007909cb" ON "user_profile" ("birthday") `);
await queryRunner.query(`CREATE INDEX "IDX_021015e6683570ae9f6b0c62be" ON "user_list_membership" ("userId") `);
await queryRunner.query(`CREATE INDEX "IDX_cddcaf418dc4d392ecfcca842a" ON "user_list_membership" ("userListId") `);
await queryRunner.query(`CREATE UNIQUE INDEX "IDX_e4f3094c43f2d665e6030b0337" ON "user_list_membership" ("userId", "userListId") `);
}
async down(queryRunner) {
await queryRunner.query(`DROP INDEX "public"."IDX_e4f3094c43f2d665e6030b0337"`);
await queryRunner.query(`DROP INDEX "public"."IDX_cddcaf418dc4d392ecfcca842a"`);
await queryRunner.query(`DROP INDEX "public"."IDX_021015e6683570ae9f6b0c62be"`);
await queryRunner.query(`DROP INDEX "public"."IDX_58699f75b9cf904f5f007909cb"`);
}
}

16
packages/backend/migration/1748096357260-AddAttributionDomains.js Normal file
View file

@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: piuvas and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class AddAttributionDomains1748096357260 {
name = 'AddAttributionDomains1748096357260'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "user" ADD "attributionDomains" text array NOT NULL DEFAULT '{}'`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "user" DROP COLUMN "attributionDomains"`);
}
}

16
packages/backend/migration/1748104955717-index_IDX_instance_host_key.js Normal file
View file

@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class IndexIDXInstanceHostKey1748104955717 {
name = 'IndexIDXInstanceHostKey1748104955717'
async up(queryRunner) {
await queryRunner.query(`CREATE UNIQUE INDEX "IDX_instance_host_key" ON "instance" (((lower(reverse("host")) || '.')::text) text_pattern_ops)`);
}
async down(queryRunner) {
await queryRunner.query(`DROP INDEX "IDX_instance_host_key"`);
}
}

85
packages/backend/migration/1748105111513-add_instance_block_columns.js Normal file
View file

@ -0,0 +1,85 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
/**
* @typedef {import('typeorm').MigrationInterface} MigrationInterface
* @typedef {{ blockedHosts: string[], silencedHosts: string[], mediaSilencedHosts: string[], federationHosts: string[], bubbleInstances: string[] }} Meta
*/
/**
* @class
* @implements {MigrationInterface}
*/
export class AddInstanceBlockColumns1748105111513 {
name = 'AddInstanceBlockColumns1748105111513'
async up(queryRunner) {
// Schema migration
await queryRunner.query(`ALTER TABLE "instance" ADD "isBlocked" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`COMMENT ON COLUMN "instance"."isBlocked" IS 'True if this instance is blocked from federation.'`);
await queryRunner.query(`ALTER TABLE "instance" ADD "isAllowListed" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`COMMENT ON COLUMN "instance"."isAllowListed" IS 'True if this instance is allow-listed.'`);
await queryRunner.query(`ALTER TABLE "instance" ADD "isBubbled" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`COMMENT ON COLUMN "instance"."isBubbled" IS 'True if this instance is part of the local bubble.'`);
await queryRunner.query(`ALTER TABLE "instance" ADD "isSilenced" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`COMMENT ON COLUMN "instance"."isSilenced" IS 'True if this instance is silenced.'`);
await queryRunner.query(`ALTER TABLE "instance" ADD "isMediaSilenced" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`COMMENT ON COLUMN "instance"."isMediaSilenced" IS 'True if this instance is media-silenced.'`);
// Data migration
/** @type {Meta[]} */
const metas = await queryRunner.query(`SELECT "blockedHosts", "silencedHosts", "mediaSilencedHosts", "federationHosts", "bubbleInstances" FROM "meta"`);
if (metas.length > 0) {
/** @type {Meta} */
const meta = metas[0];
// Blocked hosts
if (meta.blockedHosts.length > 0) {
const patterns = buildPatterns(meta.blockedHosts);
await queryRunner.query(`UPDATE "instance" SET "isBlocked" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]);
}
// Silenced hosts
if (meta.silencedHosts.length > 0) {
const patterns = buildPatterns(meta.silencedHosts);
await queryRunner.query(`UPDATE "instance" SET "isSilenced" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]);
}
// Media silenced hosts
if (meta.mediaSilencedHosts.length > 0) {
const patterns = buildPatterns(meta.mediaSilencedHosts);
await queryRunner.query(`UPDATE "instance" SET "isMediaSilenced" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]);
}
// Allow-listed hosts
if (meta.federationHosts.length > 0) {
const patterns = buildPatterns(meta.federationHosts);
await queryRunner.query(`UPDATE "instance" SET "isAllowListed" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]);
}
// Bubbled hosts
if (meta.bubbleInstances.length > 0) {
const patterns = buildPatterns(meta.bubbleInstances);
await queryRunner.query(`UPDATE "instance" SET "isBubbled" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]);
}
}
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isMediaSilenced"`);
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isSilenced"`);
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isBubbled"`);
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isAllowListed"`);
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isBlocked"`);
}
}
/**
* @param {string[]} input
* @returns {string[]}
*/
function buildPatterns(input) {
return input.map(i => i.toLowerCase().split('').reverse().join('') + '.%');
}

44
packages/backend/migration/1748128176881-add_instance_foreign_keys.js Normal file
View file

@ -0,0 +1,44 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
/**
* @typedef {import('typeorm').MigrationInterface} MigrationInterface
*/
/**
* @class
* @implements {MigrationInterface}
*/
export class AddInstanceForeignKeys1748128176881 {
name = 'AddInstanceForeignKeys1748128176881'
async up(queryRunner) {
// Fix-up: Some older instances have users without a matching instance entry
await queryRunner.query(`
INSERT INTO "instance" ("id", "host", "firstRetrievedAt")
SELECT
MIN("id"),
"host",
COALESCE(MIN("lastFetchedAt"), CURRENT_TIMESTAMP)
FROM "user"
WHERE
"host" IS NOT NULL AND
NOT EXISTS (select 1 from "instance" where "instance"."host" = "user"."host")
GROUP BY "host"
`);
await queryRunner.query(`ALTER TABLE "user" ADD CONSTRAINT "FK_user_host" FOREIGN KEY ("host") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`);
await queryRunner.query(`ALTER TABLE "note" ADD CONSTRAINT "FK_note_userHost" FOREIGN KEY ("userHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`);
await queryRunner.query(`ALTER TABLE "note" ADD CONSTRAINT "FK_note_replyUserHost" FOREIGN KEY ("replyUserHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`);
await queryRunner.query(`ALTER TABLE "note" ADD CONSTRAINT "FK_note_renoteUserHost" FOREIGN KEY ("renoteUserHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "note" DROP CONSTRAINT "FK_note_renoteUserHost"`);
await queryRunner.query(`ALTER TABLE "note" DROP CONSTRAINT "FK_note_replyUserHost"`);
await queryRunner.query(`ALTER TABLE "note" DROP CONSTRAINT "FK_note_userHost"`);
await queryRunner.query(`ALTER TABLE "user" DROP CONSTRAINT "FK_user_host"`);
}
}

26
packages/backend/migration/1748137683887-add_instance_foreign_keys_to_following.js Normal file
View file

@ -0,0 +1,26 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
/**
* @typedef {import('typeorm').MigrationInterface} MigrationInterface
*/
/**
* @class
* @implements {MigrationInterface}
*/
export class AddInstanceForeignKeysToFollowing1748137683887 {
name = 'AddInstanceForeignKeysToFollowing1748137683887'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "following" ADD CONSTRAINT "FK_following_followerHost" FOREIGN KEY ("followerHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`);
await queryRunner.query(`ALTER TABLE "following" ADD CONSTRAINT "FK_following_followeeHost" FOREIGN KEY ("followeeHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "following" DROP CONSTRAINT "FK_following_followeeHost"`);
await queryRunner.query(`ALTER TABLE "following" DROP CONSTRAINT "FK_following_followerHost"`);
}
}

25
packages/backend/migration/1748191631151-analyze_instance-user-note-following.js Normal file
View file

@ -0,0 +1,25 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
/**
* @typedef {import('typeorm').MigrationInterface} MigrationInterface
*/
/**
* @class
* @implements {MigrationInterface}
*/
export class AnalyzeInstanceUserNoteFollowing1748191631151 {
name = 'AnalyzeInstanceUserNoteFollowing1748191631151'
async up(queryRunner) {
// Refresh statistics for tables impacted by new indexes.
// This helps the query planner to efficiently use them without waiting for the next full vacuum.
await queryRunner.query(`ANALYZE "instance", "user", "following", "note"`);
}
async down(queryRunner) {
}
}

22
packages/backend/migration/1748990452958-replace_note-userHost_index.js Normal file
View file

@ -0,0 +1,22 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class ReplaceNoteUserHostIndex1748990452958 {
name = 'ReplaceNoteUserHostIndex1748990452958'
async up(queryRunner) {
await queryRunner.query(`DROP INDEX "public"."IDX_7125a826ab192eb27e11d358a5"`);
await queryRunner.query(`
create index "IDX_note_userHost_id"
on "note" ("userHost", "id" desc)
nulls not distinct`);
await queryRunner.query(`comment on index "IDX_note_userHost_id" is 'User host with ID included'`);
}
async down(queryRunner) {
await queryRunner.query(`drop index if exists "IDX_note_userHost_id"`);
await queryRunner.query(`CREATE INDEX "IDX_7125a826ab192eb27e11d358a5" ON "note" ("userHost") `);
}
}

22
packages/backend/migration/1748990662839-fix-IDX_instance_host_key.js Normal file
View file

@ -0,0 +1,22 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class FixIDXInstanceHostKey1748990662839 {
async up(queryRunner) {
// must include host for index-only scans: https://www.postgresql.org/docs/current/indexes-index-only-scans.html
await queryRunner.query(`DROP INDEX "public"."IDX_instance_host_key"`);
await queryRunner.query(`
create index "IDX_instance_host_key"
on "instance" ((lower(reverse("host"::text)) || '.'::text) text_pattern_ops)
include ("host")
`);
await queryRunner.query(`comment on index "IDX_instance_host_key" is 'Expression index for finding instances by base domain'`);
}
async down(queryRunner) {
await queryRunner.query(`DROP INDEX "public"."IDX_instance_host_key"`);
await queryRunner.query(`CREATE UNIQUE INDEX "IDX_instance_host_key" ON "instance" (((lower(reverse("host")) || '.')::text) text_pattern_ops)`);
}
}

19
packages/backend/migration/1748991828473-create-IDX_note_for_timelines.js Normal file
View file

@ -0,0 +1,19 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class CreateIDXNoteForTimelines1748991828473 {
async up(queryRunner) {
await queryRunner.query(`
create index "IDX_note_for_timelines"
on "note" ("id" desc, "channelId", "visibility", "userHost")
include ("userId", "userHost", "replyId", "replyUserId", "replyUserHost", "renoteId", "renoteUserId", "renoteUserHost")
NULLS NOT DISTINCT`);
await queryRunner.query(`comment on index "IDX_note_for_timelines" is 'Covering index for timeline queries'`);
}
async down(queryRunner) {
await queryRunner.query(`DROP INDEX "IDX_note_for_timelines"`);
}
}

17
packages/backend/migration/1748992017688-create-IDX_instance_host_filters.js Normal file
View file

@ -0,0 +1,17 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class CreateIDXInstanceHostFilters1748992017688 {
async up(queryRunner) {
await queryRunner.query(`
create index "IDX_instance_host_filters"
on "instance" ("host", "isBlocked", "isSilenced", "isMediaSilenced", "isAllowListed", "isBubbled", "suspensionState")`);
await queryRunner.query(`comment on index "IDX_instance_host_filters" is 'Covering index for host filter queries'`);
}
async down(queryRunner) {
await queryRunner.query(`DROP INDEX "IDX_instance_host_filters"`);
}
}

28
packages/backend/migration/1748992128683-create-statistics.js Normal file
View file

@ -0,0 +1,28 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class CreateStatistics1748992128683 {
async up(queryRunner) {
await queryRunner.query(`CREATE STATISTICS "STTS_instance_isBlocked_isBubbled" (mcv) ON "isBlocked", "isBubbled" FROM "instance"`);
await queryRunner.query(`CREATE STATISTICS "STTS_instance_isBlocked_isSilenced" (mcv) ON "isBlocked", "isSilenced" FROM "instance"`);
await queryRunner.query(`CREATE STATISTICS "STTS_note_replyId_replyUserId_replyUserHost" (dependencies) ON "replyId", "replyUserId", "replyUserHost" FROM "note"`)
await queryRunner.query(`CREATE STATISTICS "STTS_note_renoteId_renoteUserId_renoteUserHost" (dependencies) ON "renoteId", "renoteUserId", "renoteUserHost" FROM "note"`);
await queryRunner.query(`CREATE STATISTICS "STTS_note_userId_userHost" (mcv) ON "userId", "userHost" FROM "note"`);
await queryRunner.query(`CREATE STATISTICS "STTS_note_replyUserId_replyUserHost" (mcv) ON "replyUserId", "replyUserHost" FROM "note"`);
await queryRunner.query(`CREATE STATISTICS "STTS_note_renoteUserId_renoteUserHost" (mcv) ON "renoteUserId", "renoteUserHost" FROM "note"`);
await queryRunner.query(`ANALYZE "note", "instance"`);
}
async down(queryRunner) {
await queryRunner.query(`DROP STATISTICS "STTS_instance_isBlocked_isBubbled"`);
await queryRunner.query(`DROP STATISTICS "STTS_instance_isBlocked_isSilenced"`);
await queryRunner.query(`DROP STATISTICS "STTS_note_replyId_replyUserId_replyUserHost"`);
await queryRunner.query(`DROP STATISTICS "STTS_note_renoteId_renoteUserId_renoteUserHost"`);
await queryRunner.query(`DROP STATISTICS "STTS_note_userId_userHost"`);
await queryRunner.query(`DROP STATISTICS "STTS_note_replyUserId_replyUserHost"`);
await queryRunner.query(`DROP STATISTICS "STTS_note_renoteUserId_renoteUserHost"`);
await queryRunner.query(`ANALYZE "note", "instance"`);
}
}

28
packages/backend/migration/1749097536193-fix-IDX_note_for_timeline.js Normal file
View file

@ -0,0 +1,28 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class FixIDXNoteForTimeline1749097536193 {
async up(queryRunner) {
await queryRunner.query('drop index "IDX_note_for_timelines"');
await queryRunner.query(`
create index "IDX_note_for_timelines"
on "note" ("id" desc, "channelId", "visibility", "userHost")
include ("userId", "replyId", "replyUserId", "replyUserHost", "renoteId", "renoteUserId", "renoteUserHost", "threadId")
NULLS NOT DISTINCT
`);
await queryRunner.query(`comment on index "IDX_note_for_timelines" is 'Covering index for timeline queries'`);
}
async down(queryRunner) {
await queryRunner.query('drop index "IDX_note_for_timelines"');
await queryRunner.query(`
create index "IDX_note_for_timelines"
on "note" ("id" desc, "channelId", "visibility", "userHost")
include ("userId", "userHost", "replyId", "replyUserId", "replyUserHost", "renoteId", "renoteUserId", "renoteUserHost")
NULLS NOT DISTINCT
`);
await queryRunner.query(`comment on index "IDX_note_for_timelines" is 'Covering index for timeline queries'`);
}
}

16
packages/backend/migration/1749229288946-create-IDX_note_url.js Normal file
View file

@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class CreateIDXNoteUrl1749229288946 {
name = 'CreateIDXNoteUrl1749229288946'
async up(queryRunner) {
await queryRunner.query(`CREATE INDEX IF NOT EXISTS "IDX_note_url" ON "note" ("url") `);
}
async down(queryRunner) {
await queryRunner.query(`DROP INDEX "public"."IDX_note_url"`);
}
}

17
packages/backend/migration/1749267016885-remove-IDX_instance_host_filters.js Normal file
View file

@ -0,0 +1,17 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class RemoveIDXInstanceHostFilters1749267016885 {
async up(queryRunner) {
await queryRunner.query(`DROP INDEX IF EXISTS "IDX_instance_host_filters"`);
}
async down(queryRunner) {
await queryRunner.query(`
create index "IDX_instance_host_filters"
on "instance" ("host", "isBlocked", "isSilenced", "isMediaSilenced", "isAllowListed", "isBubbled", "suspensionState")`);
await queryRunner.query(`comment on index "IDX_instance_host_filters" is 'Covering index for host filter queries'`);
}
}

3
packages/backend/package.json
View file

@ -10,6 +10,9 @@
"start": "node ./built/boot/entry.js",
"start:test": "cross-env NODE_ENV=test node ./built/boot/entry.js",
"migrate": "pnpm typeorm migration:run -d ormconfig.js",
"migrate:revert": "pnpm typeorm migration:revert -d ormconfig.js",
"migrate:generate": "pnpm typeorm migration:generate -d ormconfig.js",
"migrate:create": "pnpm typeorm migration:create",
"revert": "pnpm typeorm migration:revert -d ormconfig.js",
"check:connect": "node ./scripts/check_connect.js",
"build": "swc src -d built -D --strip-leading-paths",

16
packages/backend/src/boot/entry.ts
View file

@ -9,6 +9,7 @@
import cluster from 'node:cluster';
import { EventEmitter } from 'node:events';
import { inspect } from 'node:util';
import chalk from 'chalk';
import Xev from 'xev';
import Logger from '@/logger.js';
@ -53,15 +54,22 @@ async function main() {
// Display detail of unhandled promise rejection
if (!envOption.quiet) {
process.on('unhandledRejection', console.dir);
process.on('unhandledRejection', e => {
try {
logger.error('Unhandled rejection:', inspect(e));
} catch {
console.error('Unhandled rejection:', inspect(e));
}
});
}
// Display detail of uncaught exception
process.on('uncaughtException', err => {
try {
logger.error(err);
console.trace(err);
} catch { }
logger.error('Uncaught exception:', err);
} catch {
console.error('Uncaught exception:', err);
}
});
// Dying away...

14
packages/backend/src/boot/master.ts
View file

@ -74,7 +74,7 @@ export async function masterMain() {
process.exit(1);
}
bootLogger.succ('Sharkey initialized');
bootLogger.info('Sharkey initialized');
if (config.sentryForBackend) {
Sentry.init({
@ -140,10 +140,10 @@ export async function masterMain() {
}
if (envOption.onlyQueue) {
bootLogger.succ('Queue started', null, true);
bootLogger.info('Queue started', null, true);
} else {
const addressString = net.isIPv6(config.address) ? `[${config.address}]` : config.address;
bootLogger.succ(config.socket ? `Now listening on socket ${config.socket} on ${config.url}` : `Now listening on ${addressString}:${config.port} on ${config.url}`, null, true);
bootLogger.info(config.socket ? `Now listening on socket ${config.socket} on ${config.url}` : `Now listening on ${addressString}:${config.port} on ${config.url}`, null, true);
}
}
@ -172,7 +172,7 @@ function loadConfigBoot(): Config {
config = loadConfig();
} catch (exception) {
if (typeof exception === 'string') {
configLogger.error(exception);
configLogger.error('Exception loading config:', exception);
process.exit(1);
} else if ((exception as any).code === 'ENOENT') {
configLogger.error('Configuration file not found', null, true);
@ -181,7 +181,7 @@ function loadConfigBoot(): Config {
throw exception;
}
configLogger.succ('Loaded');
configLogger.info('Loaded');
return config;
}
@ -195,7 +195,7 @@ async function connectDb(): Promise<void> {
dbLogger.info('Connecting...');
await initDb();
const v = await db.query('SHOW server_version').then(x => x[0].server_version);
dbLogger.succ(`Connected: v${v}`);
dbLogger.info(`Connected: v${v}`);
} catch (err) {
dbLogger.error('Cannot connect', null, true);
dbLogger.error(err);
@ -211,7 +211,7 @@ async function spawnWorkers(limit = 1) {
bootLogger.info(`Starting ${workers} worker${workers === 1 ? '' : 's'}...`);
await Promise.all([...Array(workers)].map(spawnWorker));
bootLogger.succ('All workers started');
bootLogger.info('All workers started');
}
function spawnWorker(): Promise<void> {

26
packages/backend/src/config.ts
View file

@ -9,6 +9,7 @@ import { dirname, resolve } from 'node:path';
import * as yaml from 'js-yaml';
import { globSync } from 'glob';
import ipaddr from 'ipaddr.js';
import Logger from './logger.js';
import type * as Sentry from '@sentry/node';
import type * as SentryVue from '@sentry/vue';
import type { RedisOptions } from 'ioredis';
@ -40,6 +41,7 @@ type Source = {
db?: string;
user?: string;
pass?: string;
slowQueryThreshold?: number;
disableCache?: boolean;
extra?: { [x: string]: string };
};
@ -155,6 +157,8 @@ type Source = {
}
};
const configLogger = new Logger('config');
export type PrivateNetworkSource = string | { network?: string, ports?: number[] };
export type PrivateNetwork = {
@ -192,7 +196,7 @@ export function parsePrivateNetworks(patterns: PrivateNetworkSource[] | undefine
}
}
console.warn('[config] Skipping invalid entry in allowedPrivateNetworks: ', e);
configLogger.warn('Skipping invalid entry in allowedPrivateNetworks: ', e);
return null;
})
.filter(p => p != null);
@ -222,6 +226,7 @@ export type Config = {
db: string;
user: string;
pass: string;
slowQueryThreshold?: number;
disableCache?: boolean;
extra?: { [x: string]: string };
};
@ -375,11 +380,14 @@ export function loadConfig(): Config {
if (configFiles.length === 0
&& !process.env['MK_WARNED_ABOUT_CONFIG']) {
console.log('No config files loaded, check if this is intentional');
configLogger.warn('No config files loaded, check if this is intentional');
process.env['MK_WARNED_ABOUT_CONFIG'] = '1';
}
const config = configFiles.map(path => fs.readFileSync(path, 'utf-8'))
const config = configFiles.map(path => {
configLogger.info(`Reading configuration from ${path}`);
return fs.readFileSync(path, 'utf-8');
})
.map(contents => yaml.load(contents) as Source)
.reduce(
(acc: Source, cur: Source) => Object.assign(acc, cur),
@ -405,6 +413,10 @@ export function loadConfig(): Config {
const internalMediaProxy = `${scheme}://${host}/proxy`;
const redis = convertRedisOptions(config.redis, host);
// nullish => 300 (default)
// 0 => undefined (disabled)
const slowQueryThreshold = (config.db.slowQueryThreshold ?? 300) || undefined;
return {
version,
publishTarballInsteadOfProvideRepositoryUrl: !!config.publishTarballInsteadOfProvideRepositoryUrl,
@ -423,7 +435,7 @@ export function loadConfig(): Config {
apiUrl: `${scheme}://${host}/api`,
authUrl: `${scheme}://${host}/auth`,
driveUrl: `${scheme}://${host}/files`,
db: { ...config.db, db: dbDb, user: dbUser, pass: dbPass },
db: { ...config.db, db: dbDb, user: dbUser, pass: dbPass, slowQueryThreshold },
dbReplications: config.dbReplications,
dbSlaves: config.dbSlaves,
fulltextSearch: config.fulltextSearch,
@ -496,6 +508,10 @@ export function loadConfig(): Config {
}
function tryCreateUrl(url: string) {
if (!url) {
throw new Error('Failed to load: no "url" property found in config. Please check the value of "MISSKEY_CONFIG_DIR" and "MISSKEY_CONFIG_YML", and verify that all configuration files are correct.');
}
try {
return new URL(url);
} catch (e) {
@ -627,7 +643,7 @@ function applyEnvOverrides(config: Source) {
// these are all the settings that can be overridden
_apply_top([['url', 'port', 'address', 'socket', 'chmodSocket', 'disableHsts', 'id', 'dbReplications', 'websocketCompression']]);
_apply_top(['db', ['host', 'port', 'db', 'user', 'pass', 'disableCache']]);
_apply_top(['db', ['host', 'port', 'db', 'user', 'pass', 'slowQueryThreshold', 'disableCache']]);
_apply_top(['dbSlaves', Array.from((config.dbSlaves ?? []).keys()), ['host', 'port', 'db', 'user', 'pass']]);
_apply_top([
['redis', 'redisForPubsub', 'redisForJobQueue', 'redisForTimelines', 'redisForReactions', 'redisForRateLimit'],

5
packages/backend/src/core/AbuseReportService.ts
View file

@ -13,6 +13,7 @@ import { QueueService } from '@/core/QueueService.js';
import { ApRendererService } from '@/core/activitypub/ApRendererService.js';
import { ModerationLogService } from '@/core/ModerationLogService.js';
import { SystemAccountService } from '@/core/SystemAccountService.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import { IdService } from './IdService.js';
@Injectable()
@ -125,11 +126,11 @@ export class AbuseReportService {
const report = await this.abuseUserReportsRepository.findOneByOrFail({ id: reportId });
if (report.targetUserHost == null) {
throw new Error('The target user host is null.');
throw new IdentifiableError('0b1ce202-b2c1-4ee4-8af4-2742a51b383d', 'The target user host is null.');
}
if (report.forwarded) {
throw new Error('The report has already been forwarded.');
throw new IdentifiableError('5c008bdf-f0e8-4154-9f34-804e114516d7', 'The report has already been forwarded.');
}
await this.abuseUserReportsRepository.update(report.id, {

6
packages/backend/src/core/BunnyService.ts
View file

@ -80,15 +80,15 @@ export class BunnyService {
});
req.on('error', (error) => {
this.bunnyCdnLogger.error(error);
this.bunnyCdnLogger.error('Unhandled error', error);
data.destroy();
throw new IdentifiableError('689ee33f-f97c-479a-ac49-1b9f8140bf91', 'An error has occured during the connectiong to BunnyCDN');
throw new IdentifiableError('689ee33f-f97c-479a-ac49-1b9f8140bf91', 'An error has occurred while connecting to BunnyCDN', true, error);
});
data.pipe(req).on('finish', () => {
data.destroy();
});
// wait till stream gets destroyed upon finish of piping to prevent the UI from showing the upload as success way too early
await finished(data);
}

10
packages/backend/src/core/CaptchaService.ts
View file

@ -54,7 +54,7 @@ export class CaptchaError extends Error {
public readonly cause?: unknown;
constructor(code: CaptchaErrorCode, message: string, cause?: unknown) {
super(message);
super(message, cause ? { cause } : undefined);
this.code = code;
this.cause = cause;
this.name = 'CaptchaError';
@ -117,7 +117,7 @@ export class CaptchaService {
}
const result = await this.getCaptchaResponse('https://www.recaptcha.net/recaptcha/api/siteverify', secret, response).catch(err => {
throw new CaptchaError(captchaErrorCodes.requestFailed, `recaptcha-request-failed: ${err}`);
throw new CaptchaError(captchaErrorCodes.requestFailed, `recaptcha-request-failed: ${err}`, err);
});
if (result.success !== true) {
@ -133,7 +133,7 @@ export class CaptchaService {
}
const result = await this.getCaptchaResponse('https://hcaptcha.com/siteverify', secret, response).catch(err => {
throw new CaptchaError(captchaErrorCodes.requestFailed, `hcaptcha-request-failed: ${err}`);
throw new CaptchaError(captchaErrorCodes.requestFailed, `hcaptcha-request-failed: ${err}`, err);
});
if (result.success !== true) {
@ -209,7 +209,7 @@ export class CaptchaService {
}
const result = await this.getCaptchaResponse('https://challenges.cloudflare.com/turnstile/v0/siteverify', secret, response).catch(err => {
throw new CaptchaError(captchaErrorCodes.requestFailed, `turnstile-request-failed: ${err}`);
throw new CaptchaError(captchaErrorCodes.requestFailed, `turnstile-request-failed: ${err}`, err);
});
if (result.success !== true) {
@ -386,7 +386,7 @@ export class CaptchaService {
this.logger.info(err);
const error = err instanceof CaptchaError
? err
: new CaptchaError(captchaErrorCodes.unknown, `unknown error: ${err}`);
: new CaptchaError(captchaErrorCodes.unknown, `unknown error: ${err}`, err);
return {
success: false,
error,

17
packages/backend/src/core/DownloadService.ts
View file

@ -18,6 +18,7 @@ import { LoggerService } from '@/core/LoggerService.js';
import type Logger from '@/logger.js';
import { bindThis } from '@/decorators.js';
import { renderInlineError } from '@/misc/render-inline-error.js';
@Injectable()
export class DownloadService {
@ -37,7 +38,7 @@ export class DownloadService {
public async downloadUrl(url: string, path: string, options: { timeout?: number, operationTimeout?: number, maxSize?: number } = {} ): Promise<{
filename: string;
}> {
this.logger.info(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`);
this.logger.debug(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`);
const timeout = options.timeout ?? 30 * 1000;
const operationTimeout = options.operationTimeout ?? 60 * 1000;
@ -86,7 +87,7 @@ export class DownloadService {
filename = parsed.parameters.filename;
}
} catch (e) {
this.logger.warn(`Failed to parse content-disposition: ${contentDisposition}`, { stack: e });
this.logger.warn(`Failed to parse content-disposition ${contentDisposition}: ${renderInlineError(e)}`);
}
}
}).on('downloadProgress', (progress: Got.Progress) => {
@ -100,13 +101,17 @@ export class DownloadService {
await stream.pipeline(req, fs.createWriteStream(path));
} catch (e) {
if (e instanceof Got.HTTPError) {
throw new StatusError(`${e.response.statusCode} ${e.response.statusMessage}`, e.response.statusCode, e.response.statusMessage);
} else {
throw new StatusError(`download error from ${url}`, e.response.statusCode, e.response.statusMessage, e);
} else if (e instanceof Got.RequestError || e instanceof Got.AbortError) {
throw new Error(String(e), { cause: e });
} else if (e instanceof Error) {
throw e;
} else {
throw new Error(String(e), { cause: e });
}
}
this.logger.succ(`Download finished: ${chalk.cyan(url)}`);
this.logger.info(`Download finished: ${chalk.cyan(url)}`);
return {
filename,
@ -118,7 +123,7 @@ export class DownloadService {
// Create temp file
const [path, cleanup] = await createTemp();
this.logger.info(`text file: Temp file is ${path}`);
this.logger.debug(`text file: Temp file is ${path}`);
try {
// write content at URL to temp file

49
packages/backend/src/core/DriveService.ts
View file

@ -45,6 +45,7 @@ import { isMimeImage } from '@/misc/is-mime-image.js';
import { ModerationLogService } from '@/core/ModerationLogService.js';
import { UtilityService } from '@/core/UtilityService.js';
import { BunnyService } from '@/core/BunnyService.js';
import { renderInlineError } from '@/misc/render-inline-error.js';
import { LoggerService } from './LoggerService.js';
type AddFileArgs = {
@ -159,6 +160,14 @@ export class DriveService {
// thunbnail, webpublic を必要なら生成
const alts = await this.generateAlts(path, type, !file.uri);
if (type && type.startsWith('video/')) {
try {
await this.videoProcessingService.webOptimizeVideo(path, type);
} catch (err) {
this.registerLogger.warn(`Video optimization failed: ${err instanceof Error ? err.message : String(err)}`, { error: err });
}
}
if (this.meta.useObjectStorage) {
//#region ObjectStorage params
let [ext] = (name.match(/\.([a-zA-Z0-9_-]+)$/) ?? ['']);
@ -194,7 +203,7 @@ export class DriveService {
//#endregion
//#region Uploads
this.registerLogger.info(`uploading original: ${key}`);
this.registerLogger.debug(`uploading original: ${key}`);
const uploads = [
this.upload(key, fs.createReadStream(path), type, null, name),
];
@ -203,7 +212,7 @@ export class DriveService {
webpublicKey = `${prefix}webpublic-${randomUUID()}.${alts.webpublic.ext}`;
webpublicUrl = `${ baseUrl }/${ webpublicKey }`;
this.registerLogger.info(`uploading webpublic: ${webpublicKey}`);
this.registerLogger.debug(`uploading webpublic: ${webpublicKey}`);
uploads.push(this.upload(webpublicKey, alts.webpublic.data, alts.webpublic.type, alts.webpublic.ext, name));
}
@ -211,7 +220,7 @@ export class DriveService {
thumbnailKey = `${prefix}thumbnail-${randomUUID()}.${alts.thumbnail.ext}`;
thumbnailUrl = `${ baseUrl }/${ thumbnailKey }`;
this.registerLogger.info(`uploading thumbnail: ${thumbnailKey}`);
this.registerLogger.debug(`uploading thumbnail: ${thumbnailKey}`);
uploads.push(this.upload(thumbnailKey, alts.thumbnail.data, alts.thumbnail.type, alts.thumbnail.ext, `${name}.thumbnail`));
}
@ -255,11 +264,11 @@ export class DriveService {
const [url, thumbnailUrl, webpublicUrl] = await Promise.all(promises);
if (thumbnailUrl) {
this.registerLogger.info(`thumbnail stored: ${thumbnailAccessKey}`);
this.registerLogger.debug(`thumbnail stored: ${thumbnailAccessKey}`);
}
if (webpublicUrl) {
this.registerLogger.info(`web stored: ${webpublicAccessKey}`);
this.registerLogger.debug(`web stored: ${webpublicAccessKey}`);
}
file.storedInternal = true;
@ -303,7 +312,7 @@ export class DriveService {
thumbnail,
};
} catch (err) {
this.registerLogger.warn(`GenerateVideoThumbnail failed: ${err}`);
this.registerLogger.warn(`GenerateVideoThumbnail failed: ${renderInlineError(err)}`);
return {
webpublic: null,
thumbnail: null,
@ -336,7 +345,7 @@ export class DriveService {
metadata.height && metadata.height <= 2048
);
} catch (err) {
this.registerLogger.warn(`sharp failed: ${err}`);
this.registerLogger.warn(`sharp failed: ${renderInlineError(err)}`);
return {
webpublic: null,
thumbnail: null,
@ -347,7 +356,7 @@ export class DriveService {
let webpublic: IImage | null = null;
if (generateWeb && !satisfyWebpublic && !isAnimated) {
this.registerLogger.info('creating web image');
this.registerLogger.debug('creating web image');
try {
if (['image/jpeg', 'image/webp', 'image/avif'].includes(type)) {
@ -361,9 +370,9 @@ export class DriveService {
this.registerLogger.warn('web image not created (an error occurred)', err as Error);
}
} else {
if (satisfyWebpublic) this.registerLogger.info('web image not created (original satisfies webpublic)');
else if (isAnimated) this.registerLogger.info('web image not created (animated image)');
else this.registerLogger.info('web image not created (from remote)');
if (satisfyWebpublic) this.registerLogger.debug('web image not created (original satisfies webpublic)');
else if (isAnimated) this.registerLogger.debug('web image not created (animated image)');
else this.registerLogger.debug('web image not created (from remote)');
}
// #endregion webpublic
@ -490,7 +499,6 @@ export class DriveService {
}: AddFileArgs): Promise<MiDriveFile> {
const userRoleNSFW = user && (await this.roleService.getUserPolicies(user.id)).alwaysMarkNsfw;
const info = await this.fileInfoService.getFileInfo(path);
this.registerLogger.info(`${JSON.stringify(info)}`);
// detect name
const detectedName = correctFilename(
@ -500,6 +508,8 @@ export class DriveService {
ext ?? info.type.ext,
);
this.registerLogger.debug(`Detected file info: ${JSON.stringify(info)}`);
if (user && !force) {
// Check if there is a file with the same hash
const matched = await this.driveFilesRepository.findOneBy({
@ -508,7 +518,7 @@ export class DriveService {
});
if (matched) {
this.registerLogger.info(`file with same hash is found: ${matched.id}`);
this.registerLogger.debug(`file with same hash is found: ${matched.id}`);
if (sensitive && !matched.isSensitive) {
// The file is federated as sensitive for this time, but was federated as non-sensitive before.
// Therefore, update the file to sensitive.
@ -636,14 +646,14 @@ export class DriveService {
} catch (err) {
// duplicate key error (when already registered)
if (isDuplicateKeyValueError(err)) {
this.registerLogger.info(`already registered ${file.uri}`);
this.registerLogger.debug(`already registered ${file.uri}`);
file = await this.driveFilesRepository.findOneBy({
uri: file.uri!,
userId: user ? user.id : IsNull(),
}) as MiDriveFile;
} else {
this.registerLogger.error(err as Error);
this.registerLogger.error('Error in drive register', err as Error);
throw err;
}
}
@ -651,7 +661,7 @@ export class DriveService {
file = await (this.save(file, path, detectedName, info));
}
this.registerLogger.succ(`drive file has been created ${file.id}`);
this.registerLogger.info(`Created file ${file.id} (${detectedName}) of type ${info.type.mime} for user ${user?.id ?? '<none>'}`);
if (user) {
this.driveFileEntityService.pack(file, { self: true }).then(packedFile => {
@ -884,13 +894,10 @@ export class DriveService {
}
const driveFile = await this.addFile({ user, path, name, comment, folderId, force, isLink, url, uri, sensitive, requestIp, requestHeaders });
this.downloaderLogger.succ(`Got: ${driveFile.id}`);
this.downloaderLogger.debug(`Upload succeeded: created file ${driveFile.id}`);
return driveFile!;
} catch (err) {
this.downloaderLogger.error(`Failed to create drive file: ${err}`, {
url: url,
e: err,
});
this.downloaderLogger.error(`Failed to create drive file from ${url}: ${renderInlineError(err)}`);
throw err;
} finally {
cleanup();

11
packages/backend/src/core/FanoutTimelineEndpointService.ts
View file

@ -137,10 +137,10 @@ export class FanoutTimelineEndpointService {
const parentFilter = filter;
filter = (note) => {
if (!ps.ignoreAuthorFromInstanceBlock) {
if (this.utilityService.isBlockedHost(this.meta.blockedHosts, note.userHost)) return false;
if (note.userInstance?.isBlocked) return false;
}
if (note.userId !== note.renoteUserId && this.utilityService.isBlockedHost(this.meta.blockedHosts, note.renoteUserHost)) return false;
if (note.userId !== note.replyUserId && this.utilityService.isBlockedHost(this.meta.blockedHosts, note.replyUserHost)) return false;
if (note.userId !== note.renoteUserId && note.renoteUserInstance?.isBlocked) return false;
if (note.userId !== note.replyUserId && note.replyUserInstance?.isBlocked) return false;
return parentFilter(note);
};
@ -212,7 +212,10 @@ export class FanoutTimelineEndpointService {
.leftJoinAndSelect('note.renote', 'renote')
.leftJoinAndSelect('reply.user', 'replyUser')
.leftJoinAndSelect('renote.user', 'renoteUser')
.leftJoinAndSelect('note.channel', 'channel');
.leftJoinAndSelect('note.channel', 'channel')
.leftJoinAndSelect('note.userInstance', 'userInstance')
.leftJoinAndSelect('note.replyUserInstance', 'replyUserInstance')
.leftJoinAndSelect('note.renoteUserInstance', 'renoteUserInstance');
const notes = (await query.getMany()).filter(noteFilter);

103
packages/backend/src/core/FederatedInstanceService.ts
View file

@ -5,23 +5,24 @@
import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
import * as Redis from 'ioredis';
import { QueryFailedError } from 'typeorm';
import type { InstancesRepository } from '@/models/_.js';
import type { InstancesRepository, MiMeta } from '@/models/_.js';
import type { MiInstance } from '@/models/Instance.js';
import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js';
import { MemoryKVCache } from '@/misc/cache.js';
import { IdService } from '@/core/IdService.js';
import { DI } from '@/di-symbols.js';
import { UtilityService } from '@/core/UtilityService.js';
import { bindThis } from '@/decorators.js';
import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js';
import type { GlobalEvents } from '@/core/GlobalEventService.js';
import { Serialized } from '@/types.js';
import { diffArrays, diffArraysSimple } from '@/misc/diff-arrays.js';
@Injectable()
export class FederatedInstanceService implements OnApplicationShutdown {
public federatedInstanceCache: RedisKVCache<MiInstance | null>;
private readonly federatedInstanceCache: MemoryKVCache<MiInstance | null>;
constructor(
@Inject(DI.redis)
private redisClient: Redis.Redis,
@Inject(DI.redisForSub)
private redisForSub: Redis.Redis,
@Inject(DI.instancesRepository)
private instancesRepository: InstancesRepository,
@ -29,67 +30,46 @@ export class FederatedInstanceService implements OnApplicationShutdown {
private utilityService: UtilityService,
private idService: IdService,
) {
this.federatedInstanceCache = new RedisKVCache<MiInstance | null>(this.redisClient, 'federatedInstance', {
lifetime: 1000 * 60 * 30, // 30m
memoryCacheLifetime: 1000 * 60 * 3, // 3m
fetcher: (key) => this.instancesRepository.findOneBy({ host: key }),
toRedisConverter: (value) => JSON.stringify(value),
fromRedisConverter: (value) => {
const parsed = JSON.parse(value);
if (parsed == null) return null;
return {
...parsed,
firstRetrievedAt: new Date(parsed.firstRetrievedAt),
latestRequestReceivedAt: parsed.latestRequestReceivedAt ? new Date(parsed.latestRequestReceivedAt) : null,
infoUpdatedAt: parsed.infoUpdatedAt ? new Date(parsed.infoUpdatedAt) : null,
notRespondingSince: parsed.notRespondingSince ? new Date(parsed.notRespondingSince) : null,
};
},
});
this.federatedInstanceCache = new MemoryKVCache(1000 * 60 * 3); // 3m
this.redisForSub.on('message', this.onMessage);
}
@bindThis
public async fetchOrRegister(host: string): Promise<MiInstance> {
host = this.utilityService.toPuny(host);
const cached = await this.federatedInstanceCache.get(host);
const cached = this.federatedInstanceCache.get(host);
if (cached) return cached;
const index = await this.instancesRepository.findOneBy({ host });
let index = await this.instancesRepository.findOneBy({ host });
if (index == null) {
let i;
try {
i = await this.instancesRepository.insertOne({
await this.instancesRepository.createQueryBuilder('instance')
.insert()
.values({
id: this.idService.gen(),
host,
firstRetrievedAt: new Date(),
});
} catch (e: unknown) {
if (e instanceof QueryFailedError) {
if (isDuplicateKeyValueError(e)) {
i = await this.instancesRepository.findOneBy({ host });
}
}
isBlocked: this.utilityService.isBlockedHost(host),
isSilenced: this.utilityService.isSilencedHost(host),
isMediaSilenced: this.utilityService.isMediaSilencedHost(host),
isAllowListed: this.utilityService.isAllowListedHost(host),
isBubbled: this.utilityService.isBubbledHost(host),
})
.orIgnore()
.execute();
if (i == null) {
throw e;
}
}
this.federatedInstanceCache.set(host, i);
return i;
} else {
this.federatedInstanceCache.set(host, index);
return index;
index = await this.instancesRepository.findOneByOrFail({ host });
}
this.federatedInstanceCache.set(host, index);
return index;
}
@bindThis
public async fetch(host: string): Promise<MiInstance | null> {
host = this.utilityService.toPuny(host);
const cached = await this.federatedInstanceCache.get(host);
const cached = this.federatedInstanceCache.get(host);
if (cached !== undefined) return cached;
const index = await this.instancesRepository.findOneBy({ host });
@ -117,8 +97,35 @@ export class FederatedInstanceService implements OnApplicationShutdown {
this.federatedInstanceCache.set(result.host, result);
}
private syncCache(before: Serialized<MiMeta | undefined>, after: Serialized<MiMeta>): void {
const changed =
diffArraysSimple(before?.blockedHosts, after.blockedHosts) ||
diffArraysSimple(before?.silencedHosts, after.silencedHosts) ||
diffArraysSimple(before?.mediaSilencedHosts, after.mediaSilencedHosts) ||
diffArraysSimple(before?.federationHosts, after.federationHosts) ||
diffArraysSimple(before?.bubbleInstances, after.bubbleInstances);
if (changed) {
// We have to clear the whole thing, otherwise subdomains won't be synced.
this.federatedInstanceCache.clear();
}
}
@bindThis
private async onMessage(_: string, data: string): Promise<void> {
const obj = JSON.parse(data);
if (obj.channel === 'internal') {
const { type, body } = obj.message as GlobalEvents['internal']['payload'];
if (type === 'metaUpdated') {
this.syncCache(body.before, body.after);
}
}
}
@bindThis
public dispose(): void {
this.redisForSub.off('message', this.onMessage);
this.federatedInstanceCache.dispose();
}

17
packages/backend/src/core/FetchInstanceMetadataService.ts
View file

@ -15,6 +15,7 @@ import { LoggerService } from '@/core/LoggerService.js';
import { HttpRequestService } from '@/core/HttpRequestService.js';
import { bindThis } from '@/decorators.js';
import { FederatedInstanceService } from '@/core/FederatedInstanceService.js';
import { renderInlineError } from '@/misc/render-inline-error.js';
import type { CheerioAPI } from 'cheerio';
type NodeInfo = {
@ -90,7 +91,7 @@ export class FetchInstanceMetadataService {
}
}
this.logger.info(`Fetching metadata of ${instance.host} ...`);
this.logger.debug(`Fetching metadata of ${instance.host} ...`);
const [info, dom, manifest] = await Promise.all([
this.fetchNodeinfo(instance).catch(() => null),
@ -106,7 +107,7 @@ export class FetchInstanceMetadataService {
this.getDescription(info, dom, manifest).catch(() => null),
]);
this.logger.succ(`Successfuly fetched metadata of ${instance.host}`);
this.logger.debug(`Successfuly fetched metadata of ${instance.host}`);
const updates = {
infoUpdatedAt: new Date(),
@ -128,9 +129,9 @@ export class FetchInstanceMetadataService {
await this.federatedInstanceService.update(instance.id, updates);
this.logger.succ(`Successfuly updated metadata of ${instance.host}`);
this.logger.info(`Successfully updated metadata of ${instance.host}`);
} catch (e) {
this.logger.error(`Failed to update metadata of ${instance.host}: ${e}`);
this.logger.error(`Failed to update metadata of ${instance.host}: ${renderInlineError(e)}`);
} finally {
await this.unlock(host);
}
@ -138,7 +139,7 @@ export class FetchInstanceMetadataService {
@bindThis
private async fetchNodeinfo(instance: MiInstance): Promise<NodeInfo> {
this.logger.info(`Fetching nodeinfo of ${instance.host} ...`);
this.logger.debug(`Fetching nodeinfo of ${instance.host} ...`);
try {
const wellknown = await this.httpRequestService.getJson('https://' + instance.host + '/.well-known/nodeinfo')
@ -170,11 +171,11 @@ export class FetchInstanceMetadataService {
throw err.statusCode ?? err.message;
});
this.logger.succ(`Successfuly fetched nodeinfo of ${instance.host}`);
this.logger.debug(`Successfuly fetched nodeinfo of ${instance.host}`);
return info as NodeInfo;
} catch (err) {
this.logger.error(`Failed to fetch nodeinfo of ${instance.host}: ${err}`);
this.logger.warn(`Failed to fetch nodeinfo of ${instance.host}: ${renderInlineError(err)}`);
throw err;
}
@ -182,7 +183,7 @@ export class FetchInstanceMetadataService {
@bindThis
private async fetchDom(instance: MiInstance): Promise<CheerioAPI> {
this.logger.info(`Fetching HTML of ${instance.host} ...`);
this.logger.debug(`Fetching HTML of ${instance.host} ...`);
const url = 'https://' + instance.host;

9
packages/backend/src/core/FileInfoService.ts
View file

@ -46,11 +46,13 @@ const TYPE_SVG = {
@Injectable()
export class FileInfoService {
private logger: Logger;
private ffprobeLogger: Logger;
constructor(
private loggerService: LoggerService,
) {
this.logger = this.loggerService.getLogger('file-info');
this.ffprobeLogger = this.logger.createSubLogger('ffprobe');
}
/**
@ -162,20 +164,19 @@ export class FileInfoService {
*/
@bindThis
private hasVideoTrackOnVideoFile(path: string): Promise<boolean> {
const sublogger = this.logger.createSubLogger('ffprobe');
sublogger.info(`Checking the video file. File path: ${path}`);
this.ffprobeLogger.debug(`Checking the video file. File path: ${path}`);
return new Promise((resolve) => {
try {
FFmpeg.ffprobe(path, (err, metadata) => {
if (err) {
sublogger.warn(`Could not check the video file. Returns true. File path: ${path}`, err);
this.ffprobeLogger.warn(`Could not check the video file. Returns true. File path: ${path}`, err);
resolve(true);
return;
}
resolve(metadata.streams.some((stream) => stream.codec_type === 'video'));
});
} catch (err) {
sublogger.warn(`Could not check the video file. Returns true. File path: ${path}`, err as Error);
this.ffprobeLogger.warn(`Could not check the video file. Returns true. File path: ${path}`, err as Error);
resolve(true);
}
});

10
packages/backend/src/core/HttpRequestService.ts
View file

@ -235,7 +235,7 @@ export class HttpRequestService {
}
@bindThis
public async getActivityJson(url: string, isLocalAddressAllowed = false): Promise<IObjectWithId> {
public async getActivityJson(url: string, isLocalAddressAllowed = false, allowAnonymous = false): Promise<IObjectWithId> {
this.apUtilityService.assertApUrl(url);
const res = await this.send(url, {
@ -255,7 +255,11 @@ export class HttpRequestService {
// Make sure the object ID matches the final URL (which is where it actually exists).
// The caller (ApResolverService) will verify the ID against the original / entry URL, which ensures that all three match.
this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url);
if (allowAnonymous && activity.id == null) {
activity.id = res.url;
} else {
this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url);
}
return activity as IObjectWithId;
}
@ -327,7 +331,7 @@ export class HttpRequestService {
});
if (!res.ok && extra.throwErrorWhenResponseNotOk) {
throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText);
throw new StatusError(`request error from ${url}`, res.status, res.statusText);
}
if (res.ok) {

25
packages/backend/src/core/LatestNoteService.ts
View file

@ -7,6 +7,7 @@ import { DI } from '@/di-symbols.js';
import type { LatestNotesRepository, NotesRepository } from '@/models/_.js';
import { LoggerService } from '@/core/LoggerService.js';
import Logger from '@/logger.js';
import { QueryService } from './QueryService.js';
@Injectable()
export class LatestNoteService {
@ -14,11 +15,12 @@ export class LatestNoteService {
constructor(
@Inject(DI.notesRepository)
private notesRepository: NotesRepository,
private readonly notesRepository: NotesRepository,
@Inject(DI.latestNotesRepository)
private latestNotesRepository: LatestNotesRepository,
private readonly latestNotesRepository: LatestNotesRepository,
private readonly queryService: QueryService,
loggerService: LoggerService,
) {
this.logger = loggerService.getLogger('LatestNoteService');
@ -91,7 +93,7 @@ export class LatestNoteService {
// Find the newest remaining note for the user.
// We exclude DMs and pure renotes.
const nextLatest = await this.notesRepository
const query = this.notesRepository
.createQueryBuilder('note')
.select()
.where({
@ -106,18 +108,11 @@ export class LatestNoteService {
? Not(null)
: null,
})
.andWhere(`
(
note."renoteId" IS NULL
OR note.text IS NOT NULL
OR note.cw IS NOT NULL
OR note."replyId" IS NOT NULL
OR note."hasPoll"
OR note."fileIds" != '{}'
)
`)
.orderBy({ id: 'DESC' })
.getOne();
.orderBy({ id: 'DESC' });
this.queryService.andIsNotRenote(query, 'note');
const nextLatest = await query.getOne();
if (!nextLatest) return;
// Record it as the latest

113
packages/backend/src/core/MetaService.ts
View file

@ -4,7 +4,7 @@
*/
import { Inject, Injectable } from '@nestjs/common';
import { DataSource } from 'typeorm';
import { DataSource, EntityManager } from 'typeorm';
import * as Redis from 'ioredis';
import { DI } from '@/di-symbols.js';
import { MiMeta } from '@/models/Meta.js';
@ -12,6 +12,9 @@ import { GlobalEventService } from '@/core/GlobalEventService.js';
import { bindThis } from '@/decorators.js';
import type { GlobalEvents } from '@/core/GlobalEventService.js';
import { FeaturedService } from '@/core/FeaturedService.js';
import { MiInstance } from '@/models/Instance.js';
import { diffArrays } from '@/misc/diff-arrays.js';
import type { MetasRepository } from '@/models/_.js';
import type { OnApplicationShutdown } from '@nestjs/common';
@Injectable()
@ -26,6 +29,9 @@ export class MetaService implements OnApplicationShutdown {
@Inject(DI.db)
private db: DataSource,
@Inject(DI.metasRepository)
private readonly metasRepository: MetasRepository,
private featuredService: FeaturedService,
private globalEventService: GlobalEventService,
) {
@ -67,35 +73,35 @@ export class MetaService implements OnApplicationShutdown {
public async fetch(noCache = false): Promise<MiMeta> {
if (!noCache && this.cache) return this.cache;
return await this.db.transaction(async transactionalEntityManager => {
// 過去のバグでレコードが複数出来てしまっている可能性があるので新しいIDを優先する
const metas = await transactionalEntityManager.find(MiMeta, {
order: {
// 過去のバグでレコードが複数出来てしまっている可能性があるので新しいIDを優先する
let meta = await this.metasRepository.createQueryBuilder('meta')
.select()
.orderBy({
id: 'DESC',
})
.limit(1)
.getOne();
if (!meta) {
await this.metasRepository.createQueryBuilder('meta')
.insert()
.values({
id: 'x',
})
.orIgnore()
.execute();
meta = await this.metasRepository.createQueryBuilder('meta')
.select()
.orderBy({
id: 'DESC',
},
});
})
.limit(1)
.getOneOrFail();
}
const meta = metas[0];
if (meta) {
this.cache = meta;
return meta;
} else {
// metaが空のときfetchMetaが同時に呼ばれるとここが同時に呼ばれてしまうことがあるのでフェイルセーフなupsertを使う
const saved = await transactionalEntityManager
.upsert(
MiMeta,
{
id: 'x',
},
['id'],
)
.then((x) => transactionalEntityManager.findOneByOrFail(MiMeta, x.identifiers[0]));
this.cache = saved;
return saved;
}
});
this.cache = meta;
return meta;
}
@bindThis
@ -103,7 +109,7 @@ export class MetaService implements OnApplicationShutdown {
let before: MiMeta | undefined;
const updated = await this.db.transaction(async transactionalEntityManager => {
const metas = await transactionalEntityManager.find(MiMeta, {
const metas: (MiMeta | undefined)[] = await transactionalEntityManager.find(MiMeta, {
order: {
id: 'DESC',
},
@ -126,6 +132,10 @@ export class MetaService implements OnApplicationShutdown {
},
});
// Propagate changes to blockedHosts, silencedHosts, mediaSilencedHosts, federationInstances, and bubbleInstances to the relevant instance rows
// Do this inside the transaction to avoid potential race condition (when an instance gets registered while we're updating).
await this.persistBlocks(transactionalEntityManager, before ?? {}, afters[0]);
return afters[0];
});
@ -159,4 +169,49 @@ export class MetaService implements OnApplicationShutdown {
public onApplicationShutdown(signal?: string | undefined): void {
this.dispose();
}
private async persistBlocks(tem: EntityManager, before: Partial<MiMeta>, after: Partial<MiMeta>): Promise<void> {
await this.persistBlock(tem, before.blockedHosts, after.blockedHosts, 'isBlocked');
await this.persistBlock(tem, before.silencedHosts, after.silencedHosts, 'isSilenced');
await this.persistBlock(tem, before.mediaSilencedHosts, after.mediaSilencedHosts, 'isMediaSilenced');
await this.persistBlock(tem, before.federationHosts, after.federationHosts, 'isAllowListed');
await this.persistBlock(tem, before.bubbleInstances, after.bubbleInstances, 'isBubbled');
}
private async persistBlock(tem: EntityManager, before: string[] | undefined, after: string[] | undefined, field: keyof MiInstance): Promise<void> {
const { added, removed } = diffArrays(before, after);
if (removed.length > 0) {
await this.updateInstancesByHost(tem, field, false, removed);
}
if (added.length > 0) {
await this.updateInstancesByHost(tem, field, true, added);
}
}
private async updateInstancesByHost(tem: EntityManager, field: keyof MiInstance, value: boolean, hosts: string[]): Promise<void> {
// Use non-array queries when possible, as they are indexed and can be much faster.
if (hosts.length === 1) {
const pattern = genHostPattern(hosts[0]);
await tem
.createQueryBuilder(MiInstance, 'instance')
.update()
.set({ [field]: value })
.where('(lower(reverse("host")) || \'.\') LIKE :pattern', { pattern })
.execute();
} else if (hosts.length > 1) {
const patterns = hosts.map(host => genHostPattern(host));
await tem
.createQueryBuilder(MiInstance, 'instance')
.update()
.set({ [field]: value })
.where('(lower(reverse("host")) || \'.\') LIKE ANY (:patterns)', { patterns })
.execute();
}
}
}
function genHostPattern(host: string): string {
return host.toLowerCase().split('').reverse().join('') + '.%';
}

12
packages/backend/src/core/NoteCreateService.ts
View file

@ -296,7 +296,7 @@ export class NoteCreateService implements OnApplicationShutdown {
case 'followers':
// 他人のfollowers noteはreject
if (data.renote.userId !== user.id) {
throw new Error('Renote target is not public or home');
throw new IdentifiableError('b6352a84-e5cd-4b05-a26c-63437a6b98ba', 'Renote target is not public or home');
}
// Renote対象がfollowersならfollowersにする
@ -304,7 +304,7 @@ export class NoteCreateService implements OnApplicationShutdown {
break;
case 'specified':
// specified / direct noteはreject
throw new Error('Renote target is not public or home');
throw new IdentifiableError('b6352a84-e5cd-4b05-a26c-63437a6b98ba', 'Renote target is not public or home');
}
}
@ -317,7 +317,7 @@ export class NoteCreateService implements OnApplicationShutdown {
if (data.renote.userId !== user.id) {
const blocked = await this.userBlockingService.checkBlocked(data.renote.userId, user.id);
if (blocked) {
throw new Error('blocked');
throw new IdentifiableError('b6352a84-e5cd-4b05-a26c-63437a6b98ba', 'Renote target is blocked');
}
}
}
@ -489,10 +489,10 @@ export class NoteCreateService implements OnApplicationShutdown {
// should really not happen, but better safe than sorry
if (data.reply?.id === insert.id) {
throw new Error('A note can\'t reply to itself');
throw new IdentifiableError('ea93b7c2-3d6c-4e10-946b-00d50b1a75cb', 'A note can\'t reply to itself');
}
if (data.renote?.id === insert.id) {
throw new Error('A note can\'t renote itself');
throw new IdentifiableError('ea93b7c2-3d6c-4e10-946b-00d50b1a75cb', 'A note can\'t renote itself');
}
if (data.uri != null) insert.uri = data.uri;
@ -549,8 +549,6 @@ export class NoteCreateService implements OnApplicationShutdown {
throw err;
}
console.error(e);
throw e;
}
}

6
packages/backend/src/core/NoteEditService.ts
View file

@ -309,7 +309,7 @@ export class NoteEditService implements OnApplicationShutdown {
if (this.isRenote(data)) {
if (data.renote.id === oldnote.id) {
throw new UnrecoverableError(`edit failed for ${oldnote.id}: cannot renote itself`);
throw new IdentifiableError('ea93b7c2-3d6c-4e10-946b-00d50b1a75cb', `edit failed for ${oldnote.id}: cannot renote itself`);
}
switch (data.renote.visibility) {
@ -325,7 +325,7 @@ export class NoteEditService implements OnApplicationShutdown {
case 'followers':
// 他人のfollowers noteはreject
if (data.renote.userId !== user.id) {
throw new Error('Renote target is not public or home');
throw new IdentifiableError('b6352a84-e5cd-4b05-a26c-63437a6b98ba', 'Renote target is not public or home');
}
// Renote対象がfollowersならfollowersにする
@ -333,7 +333,7 @@ export class NoteEditService implements OnApplicationShutdown {
break;
case 'specified':
// specified / direct noteはreject
throw new Error('Renote target is not public or home');
throw new IdentifiableError('b6352a84-e5cd-4b05-a26c-63437a6b98ba', 'Renote target is not public or home');
}
}

4
packages/backend/src/core/NotePiningService.ts
View file

@ -61,7 +61,7 @@ export class NotePiningService {
});
if (note == null) {
throw new IdentifiableError('70c4e51f-5bea-449c-a030-53bee3cce202', 'No such note.');
throw new IdentifiableError('70c4e51f-5bea-449c-a030-53bee3cce202', `Note ${noteId} does not exist`);
}
await this.db.transaction(async tem => {
@ -102,7 +102,7 @@ export class NotePiningService {
});
if (note == null) {
throw new IdentifiableError('b302d4cf-c050-400a-bbb3-be208681f40c', 'No such note.');
throw new IdentifiableError('b302d4cf-c050-400a-bbb3-be208681f40c', `Note ${noteId} does not exist`);
}
this.userNotePiningsRepository.delete({

619
packages/backend/src/core/QueryService.ts
View file

@ -4,13 +4,14 @@
*/
import { Inject, Injectable } from '@nestjs/common';
import { Brackets, ObjectLiteral } from 'typeorm';
import { Brackets, Not, WhereExpressionBuilder } from 'typeorm';
import { DI } from '@/di-symbols.js';
import type { MiUser } from '@/models/User.js';
import type { UserProfilesRepository, FollowingsRepository, ChannelFollowingsRepository, BlockingsRepository, NoteThreadMutingsRepository, MutingsRepository, RenoteMutingsRepository, MiMeta } from '@/models/_.js';
import { MiInstance } from '@/models/Instance.js';
import type { UserProfilesRepository, FollowingsRepository, ChannelFollowingsRepository, BlockingsRepository, NoteThreadMutingsRepository, MutingsRepository, RenoteMutingsRepository, MiMeta, InstancesRepository } from '@/models/_.js';
import { bindThis } from '@/decorators.js';
import { IdService } from '@/core/IdService.js';
import type { SelectQueryBuilder } from 'typeorm';
import type { SelectQueryBuilder, ObjectLiteral } from 'typeorm';
@Injectable()
export class QueryService {
@ -36,6 +37,9 @@ export class QueryService {
@Inject(DI.renoteMutingsRepository)
private renoteMutingsRepository: RenoteMutingsRepository,
@Inject(DI.instancesRepository)
private readonly instancesRepository: InstancesRepository,
@Inject(DI.meta)
private meta: MiMeta,
@ -79,219 +83,484 @@ export class QueryService {
// ここでいうBlockedは被Blockedの意
@bindThis
public generateBlockedUserQueryForNotes(q: SelectQueryBuilder<any>, me: { id: MiUser['id'] }): void {
const blockingQuery = this.blockingsRepository.createQueryBuilder('blocking')
.select('blocking.blockerId')
.where('blocking.blockeeId = :blockeeId', { blockeeId: me.id });
public generateBlockedUserQueryForNotes<E extends ObjectLiteral>(q: SelectQueryBuilder<E>, me: { id: MiUser['id'] }): SelectQueryBuilder<E> {
// 投稿の作者にブロックされていない かつ
// 投稿の返信先の作者にブロックされていない かつ
// 投稿の引用元の作者にブロックされていない
q
.andWhere(`note.userId NOT IN (${ blockingQuery.getQuery() })`)
.andWhere(new Brackets(qb => {
qb
.where('note.replyUserId IS NULL')
.orWhere(`note.replyUserId NOT IN (${ blockingQuery.getQuery() })`);
}))
.andWhere(new Brackets(qb => {
qb
.where('note.renoteUserId IS NULL')
.orWhere(`note.renoteUserId NOT IN (${ blockingQuery.getQuery() })`);
}));
q.setParameters(blockingQuery.getParameters());
return this
.andNotBlockingUser(q, 'note.userId', ':meId')
.andWhere(new Brackets(qb => this
.orNotBlockingUser(qb, 'note.replyUserId', ':meId')
.orWhere('note.replyUserId IS NULL')))
.andWhere(new Brackets(qb => this
.orNotBlockingUser(qb, 'note.renoteUserId', ':meId')
.orWhere('note.renoteUserId IS NULL')))
.setParameters({ meId: me.id });
}
@bindThis
public generateBlockQueryForUsers(q: SelectQueryBuilder<any>, me: { id: MiUser['id'] }): void {
const blockingQuery = this.blockingsRepository.createQueryBuilder('blocking')
.select('blocking.blockeeId')
.where('blocking.blockerId = :blockerId', { blockerId: me.id });
const blockedQuery = this.blockingsRepository.createQueryBuilder('blocking')
.select('blocking.blockerId')
.where('blocking.blockeeId = :blockeeId', { blockeeId: me.id });
q.andWhere(`user.id NOT IN (${ blockingQuery.getQuery() })`);
q.setParameters(blockingQuery.getParameters());
q.andWhere(`user.id NOT IN (${ blockedQuery.getQuery() })`);
q.setParameters(blockedQuery.getParameters());
public generateBlockQueryForUsers<E extends ObjectLiteral>(q: SelectQueryBuilder<E>, me: { id: MiUser['id'] }): SelectQueryBuilder<E> {
this.andNotBlockingUser(q, ':meId', 'user.id');
this.andNotBlockingUser(q, 'user.id', ':me.id');
return q.setParameters({ meId: me.id });
}
@bindThis
public generateMutedNoteThreadQuery(q: SelectQueryBuilder<any>, me: { id: MiUser['id'] }): void {
const mutedQuery = this.noteThreadMutingsRepository.createQueryBuilder('threadMuted')
.select('threadMuted.threadId')
.where('threadMuted.userId = :userId', { userId: me.id });
q.andWhere(`note.id NOT IN (${ mutedQuery.getQuery() })`);
q.andWhere(new Brackets(qb => {
qb
.where('note.threadId IS NULL')
.orWhere(`note.threadId NOT IN (${ mutedQuery.getQuery() })`);
}));
q.setParameters(mutedQuery.getParameters());
public generateMutedNoteThreadQuery<E extends ObjectLiteral>(q: SelectQueryBuilder<E>, me: { id: MiUser['id'] }): SelectQueryBuilder<E> {
return this
.andNotMutingThread(q, ':meId', 'note.id')
.andWhere(new Brackets(qb => this
.orNotMutingThread(qb, ':meId', 'note.threadId')
.orWhere('note.threadId IS NULL')))
.setParameters({ meId: me.id });
}
@bindThis
public generateMutedUserQueryForNotes(q: SelectQueryBuilder<any>, me: { id: MiUser['id'] }, exclude?: { id: MiUser['id'] }): void {
const mutingQuery = this.mutingsRepository.createQueryBuilder('muting')
.select('muting.muteeId')
.where('muting.muterId = :muterId', { muterId: me.id });
if (exclude) {
mutingQuery.andWhere('muting.muteeId != :excludeId', { excludeId: exclude.id });
}
const mutingInstanceQuery = this.userProfilesRepository.createQueryBuilder('user_profile')
.select('user_profile.mutedInstances')
.where('user_profile.userId = :muterId', { muterId: me.id });
public generateMutedUserQueryForNotes<E extends ObjectLiteral>(q: SelectQueryBuilder<E>, me: { id: MiUser['id'] }, exclude?: { id: MiUser['id'] }): SelectQueryBuilder<E> {
// 投稿の作者をミュートしていない かつ
// 投稿の返信先の作者をミュートしていない かつ
// 投稿の引用元の作者をミュートしていない
q
.andWhere(`note.userId NOT IN (${ mutingQuery.getQuery() })`)
.andWhere(new Brackets(qb => {
qb
.where('note.replyUserId IS NULL')
.orWhere(`note.replyUserId NOT IN (${ mutingQuery.getQuery() })`);
}))
.andWhere(new Brackets(qb => {
qb
.where('note.renoteUserId IS NULL')
.orWhere(`note.renoteUserId NOT IN (${ mutingQuery.getQuery() })`);
}))
return this
.andNotMutingUser(q, ':meId', 'note.userId', exclude)
.andWhere(new Brackets(qb => this
.orNotMutingUser(qb, ':meId', 'note.replyUserId', exclude)
.orWhere('note.replyUserId IS NULL')))
.andWhere(new Brackets(qb => this
.orNotMutingUser(qb, ':meId', 'note.renoteUserId', exclude)
.orWhere('note.renoteUserId IS NULL')))
// TODO exclude should also pass a host to skip these instances
// mute instances
.andWhere(new Brackets(qb => {
qb
.andWhere('note.userHost IS NULL')
.orWhere(`NOT ((${ mutingInstanceQuery.getQuery() })::jsonb ? note.userHost)`);
}))
.andWhere(new Brackets(qb => {
qb
.where('note.replyUserHost IS NULL')
.orWhere(`NOT ((${ mutingInstanceQuery.getQuery() })::jsonb ? note.replyUserHost)`);
}))
.andWhere(new Brackets(qb => {
qb
.where('note.renoteUserHost IS NULL')
.orWhere(`NOT ((${ mutingInstanceQuery.getQuery() })::jsonb ? note.renoteUserHost)`);
}));
q.setParameters(mutingQuery.getParameters());
q.setParameters(mutingInstanceQuery.getParameters());
.andWhere(new Brackets(qb => this
.andNotMutingInstance(qb, ':meId', 'note.userHost')
.orWhere('note.userHost IS NULL')))
.andWhere(new Brackets(qb => this
.orNotMutingInstance(qb, ':meId', 'note.replyUserHost')
.orWhere('note.replyUserHost IS NULL')))
.andWhere(new Brackets(qb => this
.orNotMutingInstance(qb, ':meId', 'note.renoteUserHost')
.orWhere('note.renoteUserHost IS NULL')))
.setParameters({ meId: me.id });
}
@bindThis
public generateMutedUserQueryForUsers(q: SelectQueryBuilder<any>, me: { id: MiUser['id'] }): void {
const mutingQuery = this.mutingsRepository.createQueryBuilder('muting')
.select('muting.muteeId')
.where('muting.muterId = :muterId', { muterId: me.id });
q.andWhere(`user.id NOT IN (${ mutingQuery.getQuery() })`);
q.setParameters(mutingQuery.getParameters());
public generateMutedUserQueryForUsers<E extends ObjectLiteral>(q: SelectQueryBuilder<E>, me: { id: MiUser['id'] }): SelectQueryBuilder<E> {
return this
.andNotMutingUser(q, ':meId', 'user.id')
.setParameters({ meId: me.id });
}
// This intentionally skips isSuspended, isDeleted, makeNotesFollowersOnlyBefore, makeNotesHiddenBefore, and requireSigninToViewContents.
// NoteEntityService checks these automatically and calls hideNote() to hide them without breaking threads.
// For moderation purposes, you can set isSilenced to forcibly hide existing posts by a user.
@bindThis
public generateVisibilityQuery(q: SelectQueryBuilder<any>, me?: { id: MiUser['id'] } | null): void {
public generateVisibilityQuery<E extends ObjectLiteral>(q: SelectQueryBuilder<E>, me?: { id: MiUser['id'] } | null): SelectQueryBuilder<E> {
// This code must always be synchronized with the checks in Notes.isVisibleForMe.
if (me == null) {
q.andWhere(new Brackets(qb => {
qb
.where('note.visibility = \'public\'')
.orWhere('note.visibility = \'home\'');
}));
} else {
const followingQuery = this.followingsRepository.createQueryBuilder('following')
.select('following.followeeId')
.where('following.followerId = :meId');
return q.andWhere(new Brackets(qb => {
// Public post
qb.orWhere('note.visibility = \'public\'')
.orWhere('note.visibility = \'home\'');
q.andWhere(new Brackets(qb => {
if (me != null) {
qb
// 公開投稿である
.where(new Brackets(qb => {
qb
.where('note.visibility = \'public\'')
.orWhere('note.visibility = \'home\'');
}))
// または 自分自身
.orWhere('note.userId = :meId')
// または 自分宛て
// My post
.orWhere(':meId = note.userId')
// Reply to me
.orWhere(':meId = note.replyUserId')
// DM to me
.orWhere(':meIdAsList <@ note.visibleUserIds')
.orWhere(new Brackets(qb => {
qb
// または フォロワー宛ての投稿であり、
.where('note.visibility = \'followers\'')
.andWhere(new Brackets(qb => {
qb
// 自分がフォロワーである
.where(`note.userId IN (${ followingQuery.getQuery() })`)
// または 自分の投稿へのリプライ
.orWhere('note.replyUserId = :meId')
.orWhere(':meIdAsList <@ note.mentions');
}));
}));
// Mentions me
.orWhere(':meIdAsList <@ note.mentions')
// Followers-only post
.orWhere(new Brackets(qb => this
.andFollowingUser(qb, ':meId', 'note.userId')
.andWhere('note.visibility = \'followers\'')));
q.setParameters({ meId: me.id, meIdAsList: [me.id] });
}
}));
}
@bindThis
public generateMutedUserRenotesQueryForNotes<E extends ObjectLiteral>(q: SelectQueryBuilder<E>, me: { id: MiUser['id'] }): SelectQueryBuilder<E> {
return q
.andWhere(new Brackets(qb => this
.orNotMutingRenote(qb, ':meId', 'note.userId')
.orWhere('note.renoteId IS NULL')
.orWhere('note.text IS NOT NULL')
.orWhere('note.cw IS NOT NULL')
.orWhere('note.replyId IS NOT NULL')
.orWhere('note.hasPoll = true')
.orWhere('note.fileIds != \'{}\'')))
.setParameters({ meId: me.id });
}
@bindThis
public generateExcludedRenotesQueryForNotes<Q extends WhereExpressionBuilder>(q: Q): Q {
return this.andIsNotRenote(q, 'note');
}
@bindThis
public generateBlockedHostQueryForNote<E extends ObjectLiteral>(q: SelectQueryBuilder<E>, excludeAuthor?: boolean): SelectQueryBuilder<E> {
const checkFor = (key: 'user' | 'replyUser' | 'renoteUser') => this
.leftJoinInstance(q, `note.${key}Instance`, `${key}Instance`)
.andWhere(new Brackets(qb => {
qb
.orWhere(`"${key}Instance" IS NULL`) // local
.orWhere(`"${key}Instance"."isBlocked" = false`); // not blocked
if (excludeAuthor) {
qb.orWhere(`note.userId = note.${key}Id`); // author
}
}));
q.setParameters({ meId: me.id, meIdAsList: [me.id] });
if (!excludeAuthor) {
checkFor('user');
}
checkFor('replyUser');
checkFor('renoteUser');
return q;
}
@bindThis
public generateMutedUserRenotesQueryForNotes(q: SelectQueryBuilder<any>, me: { id: MiUser['id'] }): void {
public generateSilencedUserQueryForNotes<E extends ObjectLiteral>(q: SelectQueryBuilder<E>, me?: { id: MiUser['id'] } | null): SelectQueryBuilder<E> {
if (!me) {
return q.andWhere('user.isSilenced = false');
}
return this
.leftJoinInstance(q, 'note.userInstance', 'userInstance')
.andWhere(new Brackets(qb => this
// case 1: we are following the user
.orFollowingUser(qb, ':meId', 'note.userId')
// case 2: user not silenced AND instance not silenced
.orWhere(new Brackets(qbb => qbb
.andWhere(new Brackets(qbbb => qbbb
.orWhere('"userInstance"."isSilenced" = false')
.orWhere('"userInstance" IS NULL')))
.andWhere('user.isSilenced = false')))))
.setParameters({ meId: me.id });
}
/**
* Left-joins an instance in to the query with a given alias and optional condition.
* These calls are de-duplicated - multiple uses of the same alias are skipped.
*/
@bindThis
public leftJoinInstance<E extends ObjectLiteral>(q: SelectQueryBuilder<E>, relation: string | typeof MiInstance, alias: string, condition?: string): SelectQueryBuilder<E> {
// Skip if it's already joined, otherwise we'll get an error
if (!q.expressionMap.joinAttributes.some(j => j.alias.name === alias)) {
q.leftJoin(relation, alias, condition);
}
return q;
}
/**
* Adds OR condition that noteProp (note ID) refers to a quote.
* The prop should be an expression, not a raw value.
*/
@bindThis
public orIsQuote<Q extends WhereExpressionBuilder>(q: Q, noteProp: string): Q {
return this.addIsQuote(q, noteProp, 'orWhere');
}
/**
* Adds AND condition that noteProp (note ID) refers to a quote.
* The prop should be an expression, not a raw value.
*/
@bindThis
public andIsQuote<Q extends WhereExpressionBuilder>(q: Q, noteProp: string): Q {
return this.addIsQuote(q, noteProp, 'andWhere');
}
private addIsQuote<Q extends WhereExpressionBuilder>(q: Q, noteProp: string, join: 'andWhere' | 'orWhere'): Q {
return q[join](new Brackets(qb => qb
.andWhere(`${noteProp}.renoteId IS NOT NULL`)
.andWhere(new Brackets(qbb => qbb
.orWhere(`${noteProp}.text IS NOT NULL`)
.orWhere(`${noteProp}.cw IS NOT NULL`)
.orWhere(`${noteProp}.replyId IS NOT NULL`)
.orWhere(`${noteProp}.hasPoll = true`)
.orWhere(`${noteProp}.fileIds != '{}'`)))));
}
/**
* Adds OR condition that noteProp (note ID) does not refer to a quote.
* The prop should be an expression, not a raw value.
*/
@bindThis
public orIsNotQuote<Q extends WhereExpressionBuilder>(q: Q, noteProp: string): Q {
return this.addIsNotQuote(q, noteProp, 'orWhere');
}
/**
* Adds AND condition that noteProp (note ID) does not refer to a quote.
* The prop should be an expression, not a raw value.
*/
@bindThis
public andIsNotQuote<Q extends WhereExpressionBuilder>(q: Q, noteProp: string): Q {
return this.addIsNotQuote(q, noteProp, 'andWhere');
}
private addIsNotQuote<Q extends WhereExpressionBuilder>(q: Q, noteProp: string, join: 'andWhere' | 'orWhere'): Q {
return q[join](new Brackets(qb => qb
.orWhere(`${noteProp}.renoteId IS NULL`)
.orWhere(new Brackets(qb => qb
.andWhere(`${noteProp}.text IS NULL`)
.andWhere(`${noteProp}.cw IS NULL`)
.andWhere(`${noteProp}.replyId IS NULL`)
.andWhere(`${noteProp}.hasPoll = false`)
.andWhere(`${noteProp}.fileIds = '{}'`)))));
}
/**
* Adds OR condition that noteProp (note ID) refers to a renote.
* The prop should be an expression, not a raw value.
*/
@bindThis
public orIsRenote<Q extends WhereExpressionBuilder>(q: Q, noteProp: string): Q {
return this.addIsRenote(q, noteProp, 'orWhere');
}
/**
* Adds AND condition that noteProp (note ID) refers to a renote.
* The prop should be an expression, not a raw value.
*/
@bindThis
public andIsRenote<Q extends WhereExpressionBuilder>(q: Q, noteProp: string): Q {
return this.addIsRenote(q, noteProp, 'andWhere');
}
private addIsRenote<Q extends WhereExpressionBuilder>(q: Q, noteProp: string, join: 'andWhere' | 'orWhere'): Q {
return q[join](new Brackets(qb => qb
.andWhere(`${noteProp}.renoteId IS NOT NULL`)
.andWhere(`${noteProp}.text IS NULL`)
.andWhere(`${noteProp}.cw IS NULL`)
.andWhere(`${noteProp}.replyId IS NULL`)
.andWhere(`${noteProp}.hasPoll = false`)
.andWhere(`${noteProp}.fileIds = '{}'`)));
}
/**
* Adds OR condition that noteProp (note ID) does not refer to a renote.
* The prop should be an expression, not a raw value.
*/
@bindThis
public orIsNotRenote<Q extends WhereExpressionBuilder>(q: Q, noteProp: string): Q {
return this.addIsNotRenote(q, noteProp, 'orWhere');
}
/**
* Adds AND condition that noteProp (note ID) does not refer to a renote.
* The prop should be an expression, not a raw value.
*/
@bindThis
public andIsNotRenote<Q extends WhereExpressionBuilder>(q: Q, noteProp: string): Q {
return this.addIsNotRenote(q, noteProp, 'andWhere');
}
private addIsNotRenote<Q extends WhereExpressionBuilder>(q: Q, noteProp: string, join: 'andWhere' | 'orWhere'): Q {
return q[join](new Brackets(qb => qb
.orWhere(`${noteProp}.renoteId IS NULL`)
.orWhere(`${noteProp}.text IS NOT NULL`)
.orWhere(`${noteProp}.cw IS NOT NULL`)
.orWhere(`${noteProp}.replyId IS NOT NULL`)
.orWhere(`${noteProp}.hasPoll = true`)
.orWhere(`${noteProp}.fileIds != '{}'`)));
}
/**
* Adds OR condition that followerProp (user ID) is following followeeProp (user ID).
* Both props should be expressions, not raw values.
*/
@bindThis
public orFollowingUser<Q extends WhereExpressionBuilder>(q: Q, followerProp: string, followeeProp: string): Q {
return this.addFollowingUser(q, followerProp, followeeProp, 'orWhere');
}
/**
* Adds AND condition that followerProp (user ID) is following followeeProp (user ID).
* Both props should be expressions, not raw values.
*/
@bindThis
public andFollowingUser<Q extends WhereExpressionBuilder>(q: Q, followerProp: string, followeeProp: string): Q {
return this.addFollowingUser(q, followerProp, followeeProp, 'andWhere');
}
private addFollowingUser<Q extends WhereExpressionBuilder>(q: Q, followerProp: string, followeeProp: string, join: 'andWhere' | 'orWhere'): Q {
const followingQuery = this.followingsRepository.createQueryBuilder('following')
.select('1')
.andWhere(`following.followerId = ${followerProp}`)
.andWhere(`following.followeeId = ${followeeProp}`);
return q[join](`EXISTS (${followingQuery.getQuery()})`, followingQuery.getParameters());
};
/**
* Adds OR condition that followerProp (user ID) is following followeeProp (channel ID).
* Both props should be expressions, not raw values.
*/
@bindThis
public orFollowingChannel<Q extends WhereExpressionBuilder>(q: Q, followerProp: string, followeeProp: string): Q {
return this.addFollowingChannel(q, followerProp, followeeProp, 'orWhere');
}
/**
* Adds AND condition that followerProp (user ID) is following followeeProp (channel ID).
* Both props should be expressions, not raw values.
*/
@bindThis
public andFollowingChannel<Q extends WhereExpressionBuilder>(q: Q, followerProp: string, followeeProp: string): Q {
return this.addFollowingChannel(q, followerProp, followeeProp, 'andWhere');
}
private addFollowingChannel<Q extends WhereExpressionBuilder>(q: Q, followerProp: string, followeeProp: string, join: 'andWhere' | 'orWhere'): Q {
const followingQuery = this.channelFollowingsRepository.createQueryBuilder('following')
.select('1')
.andWhere(`following.followerId = ${followerProp}`)
.andWhere(`following.followeeId = ${followeeProp}`);
return q[join](`EXISTS (${followingQuery.getQuery()})`, followingQuery.getParameters());
}
/**
* Adds OR condition that blockerProp (user ID) is not blocking blockeeProp (user ID).
* Both props should be expressions, not raw values.
*/
@bindThis
public orNotBlockingUser<Q extends WhereExpressionBuilder>(q: Q, blockerProp: string, blockeeProp: string): Q {
return this.excludeBlockingUser(q, blockerProp, blockeeProp, 'orWhere');
}
/**
* Adds AND condition that blockerProp (user ID) is not blocking blockeeProp (user ID).
* Both props should be expressions, not raw values.
*/
@bindThis
public andNotBlockingUser<Q extends WhereExpressionBuilder>(q: Q, blockerProp: string, blockeeProp: string): Q {
return this.excludeBlockingUser(q, blockerProp, blockeeProp, 'andWhere');
}
private excludeBlockingUser<Q extends WhereExpressionBuilder>(q: Q, blockerProp: string, blockeeProp: string, join: 'andWhere' | 'orWhere'): Q {
const blockingQuery = this.blockingsRepository.createQueryBuilder('blocking')
.select('1')
.andWhere(`blocking.blockerId = ${blockerProp}`)
.andWhere(`blocking.blockeeId = ${blockeeProp}`);
return q[join](`NOT EXISTS (${blockingQuery.getQuery()})`, blockingQuery.getParameters());
};
/**
* Adds OR condition that muterProp (user ID) is not muting muteeProp (user ID).
* Both props should be expressions, not raw values.
*/
@bindThis
public orNotMutingUser<Q extends WhereExpressionBuilder>(q: Q, muterProp: string, muteeProp: string, exclude?: { id: MiUser['id'] }): Q {
return this.excludeMutingUser(q, muterProp, muteeProp, 'orWhere', exclude);
}
/**
* Adds AND condition that muterProp (user ID) is not muting muteeProp (user ID).
* Both props should be expressions, not raw values.
*/
@bindThis
public andNotMutingUser<Q extends WhereExpressionBuilder>(q: Q, muterProp: string, muteeProp: string, exclude?: { id: MiUser['id'] }): Q {
return this.excludeMutingUser(q, muterProp, muteeProp, 'andWhere', exclude);
}
private excludeMutingUser<Q extends WhereExpressionBuilder>(q: Q, muterProp: string, muteeProp: string, join: 'andWhere' | 'orWhere', exclude?: { id: MiUser['id'] }): Q {
const mutingQuery = this.mutingsRepository.createQueryBuilder('muting')
.select('1')
.andWhere(`muting.muterId = ${muterProp}`)
.andWhere(`muting.muteeId = ${muteeProp}`);
if (exclude) {
mutingQuery.andWhere({ muteeId: Not(exclude.id) });
}
return q[join](`NOT EXISTS (${mutingQuery.getQuery()})`, mutingQuery.getParameters());
}
/**
* Adds OR condition that muterProp (user ID) is not muting renotes by muteeProp (user ID).
* Both props should be expressions, not raw values.
*/
@bindThis
public orNotMutingRenote<Q extends WhereExpressionBuilder>(q: Q, muterProp: string, muteeProp: string): Q {
return this.excludeMutingRenote(q, muterProp, muteeProp, 'orWhere');
}
/**
* Adds AND condition that muterProp (user ID) is not muting renotes by muteeProp (user ID).
* Both props should be expressions, not raw values.
*/
@bindThis
public andNotMutingRenote<Q extends WhereExpressionBuilder>(q: Q, muterProp: string, muteeProp: string): Q {
return this.excludeMutingRenote(q, muterProp, muteeProp, 'andWhere');
}
private excludeMutingRenote<Q extends WhereExpressionBuilder>(q: Q, muterProp: string, muteeProp: string, join: 'andWhere' | 'orWhere'): Q {
const mutingQuery = this.renoteMutingsRepository.createQueryBuilder('renote_muting')
.select('renote_muting.muteeId')
.where('renote_muting.muterId = :muterId', { muterId: me.id });
.select('1')
.andWhere(`renote_muting.muterId = ${muterProp}`)
.andWhere(`renote_muting.muteeId = ${muteeProp}`);
q.andWhere(new Brackets(qb => {
qb
.where(new Brackets(qb => {
qb.where('note.renoteId IS NOT NULL');
qb.andWhere('note.text IS NULL');
qb.andWhere(`note.userId NOT IN (${ mutingQuery.getQuery() })`);
}))
.orWhere('note.renoteId IS NULL')
.orWhere('note.text IS NOT NULL');
}));
return q[join](`NOT EXISTS (${mutingQuery.getQuery()})`, mutingQuery.getParameters());
};
q.setParameters(mutingQuery.getParameters());
/**
* Adds OR condition that muterProp (user ID) is not muting muteeProp (instance host).
* Both props should be expressions, not raw values.
*/
@bindThis
public orNotMutingInstance<Q extends WhereExpressionBuilder>(q: Q, muterProp: string, muteeProp: string): Q {
return this.excludeMutingInstance(q, muterProp, muteeProp, 'orWhere');
}
/**
* Adds AND condition that muterProp (user ID) is not muting muteeProp (instance host).
* Both props should be expressions, not raw values.
*/
@bindThis
public generateBlockedHostQueryForNote(q: SelectQueryBuilder<any>, excludeAuthor?: boolean): void {
let nonBlockedHostQuery: (part: string) => string;
if (this.meta.blockedHosts.length === 0) {
nonBlockedHostQuery = () => '1=1';
} else {
nonBlockedHostQuery = (match: string) => `('.' || ${match}) NOT ILIKE ALL(select '%.' || x from (select unnest("blockedHosts") as x from "meta") t)`;
}
public andNotMutingInstance<Q extends WhereExpressionBuilder>(q: Q, muterProp: string, muteeProp: string): Q {
return this.excludeMutingInstance(q, muterProp, muteeProp, 'andWhere');
}
if (excludeAuthor) {
const instanceSuspension = (user: string) => new Brackets(qb => qb
.where(`note.${user}Id IS NULL`) // no corresponding user
.orWhere(`note.userId = note.${user}Id`)
.orWhere(`note.${user}Host IS NULL`) // local
.orWhere(nonBlockedHostQuery(`note.${user}Host`)));
private excludeMutingInstance<Q extends WhereExpressionBuilder>(q: Q, muterProp: string, muteeProp: string, join: 'andWhere' | 'orWhere'): Q {
const mutingInstanceQuery = this.userProfilesRepository.createQueryBuilder('user_profile')
.select('1')
.andWhere(`user_profile.userId = ${muterProp}`)
.andWhere(`"user_profile"."mutedInstances"::jsonb ? ${muteeProp}`);
q
.andWhere(instanceSuspension('replyUser'))
.andWhere(instanceSuspension('renoteUser'));
} else {
const instanceSuspension = (user: string) => new Brackets(qb => qb
.where(`note.${user}Id IS NULL`) // no corresponding user
.orWhere(`note.${user}Host IS NULL`) // local
.orWhere(nonBlockedHostQuery(`note.${user}Host`)));
return q[join](`NOT EXISTS (${mutingInstanceQuery.getQuery()})`, mutingInstanceQuery.getParameters());
}
q
.andWhere(instanceSuspension('user'))
.andWhere(instanceSuspension('replyUser'))
.andWhere(instanceSuspension('renoteUser'));
}
/**
* Adds OR condition that muterProp (user ID) is not muting muteeProp (note ID).
* Both props should be expressions, not raw values.
*/
@bindThis
public orNotMutingThread<Q extends WhereExpressionBuilder>(q: Q, muterProp: string, muteeProp: string): Q {
return this.excludeMutingThread(q, muterProp, muteeProp, 'orWhere');
}
/**
* Adds AND condition that muterProp (user ID) is not muting muteeProp (note ID).
* Both props should be expressions, not raw values.
*/
@bindThis
public andNotMutingThread<Q extends WhereExpressionBuilder>(q: Q, muterProp: string, muteeProp: string): Q {
return this.excludeMutingThread(q, muterProp, muteeProp, 'andWhere');
}
private excludeMutingThread<Q extends WhereExpressionBuilder>(q: Q, muterProp: string, muteeProp: string, join: 'andWhere' | 'orWhere'): Q {
const threadMutedQuery = this.noteThreadMutingsRepository.createQueryBuilder('threadMuted')
.select('1')
.andWhere(`threadMuted.userId = ${muterProp}`)
.andWhere(`threadMuted.threadId = ${muteeProp}`);
return q[join](`NOT EXISTS (${threadMutedQuery.getQuery()})`, threadMutedQuery.getParameters());
}
// Requirements: user replyUser renoteUser must be joined

50
packages/backend/src/core/ReactionService.ts
View file

@ -10,7 +10,7 @@ import { IdentifiableError } from '@/misc/identifiable-error.js';
import type { MiRemoteUser, MiUser } from '@/models/User.js';
import type { MiNote } from '@/models/Note.js';
import { IdService } from '@/core/IdService.js';
import type { MiNoteReaction } from '@/models/NoteReaction.js';
import { MiNoteReaction } from '@/models/NoteReaction.js';
import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
import { NotificationService } from '@/core/NotificationService.js';
@ -31,6 +31,7 @@ import { isQuote, isRenote } from '@/misc/is-renote.js';
import { ReactionsBufferingService } from '@/core/ReactionsBufferingService.js';
import { PER_NOTE_REACTION_USER_PAIR_CACHE_MAX } from '@/const.js';
import { CacheService } from '@/core/CacheService.js';
import type { DataSource } from 'typeorm';
const FALLBACK = '\u2764';
@ -89,6 +90,9 @@ export class ReactionService {
@Inject(DI.emojisRepository)
private emojisRepository: EmojisRepository,
@Inject(DI.db)
private readonly db: DataSource,
private utilityService: UtilityService,
private customEmojiService: CustomEmojiService,
private roleService: RoleService,
@ -113,7 +117,7 @@ export class ReactionService {
if (note.userId !== user.id) {
const blocked = await this.userBlockingService.checkBlocked(note.userId, user.id);
if (blocked) {
throw new IdentifiableError('e70412a4-7197-4726-8e74-f3e0deb92aa7');
throw new IdentifiableError('e70412a4-7197-4726-8e74-f3e0deb92aa7', 'Note not accessible for you.');
}
}
@ -176,26 +180,28 @@ export class ReactionService {
reaction,
};
try {
await this.noteReactionsRepository.insert(record);
} catch (e) {
if (isDuplicateKeyValueError(e)) {
const exists = await this.noteReactionsRepository.findOneByOrFail({
noteId: note.id,
userId: user.id,
});
const result = await this.db.transaction(async tem => {
await tem.createQueryBuilder(MiNoteReaction, 'noteReaction')
.insert()
.values(record)
.orIgnore()
.execute();
if (exists.reaction !== reaction) {
// 別のリアクションがすでにされていたら置き換える
await this.delete(user, note);
await this.noteReactionsRepository.insert(record);
} else {
// 同じリアクションがすでにされていたらエラー
throw new IdentifiableError('51c42bb4-931a-456b-bff7-e5a8a70dd298');
}
} else {
throw e;
return await tem.createQueryBuilder(MiNoteReaction, 'noteReaction')
.select()
.where({ noteId: note.id, userId: user.id })
.getOneOrFail();
});
if (result.id !== record.id) {
// Conflict with the same ID => nothing to do.
if (result.reaction === record.reaction) {
return;
}
// 別のリアクションがすでにされていたら置き換える
await this.delete(user, note);
await this.noteReactionsRepository.insert(record);
}
// Increment reactions count
@ -316,14 +322,14 @@ export class ReactionService {
});
if (exist == null) {
throw new IdentifiableError('60527ec9-b4cb-4a88-a6bd-32d3ad26817d', 'not reacted');
throw new IdentifiableError('60527ec9-b4cb-4a88-a6bd-32d3ad26817d', 'reaction does not exist');
}
// Delete reaction
const result = await this.noteReactionsRepository.delete(exist.id);
if (result.affected !== 1) {
throw new IdentifiableError('60527ec9-b4cb-4a88-a6bd-32d3ad26817d', 'not reacted');
throw new IdentifiableError('60527ec9-b4cb-4a88-a6bd-32d3ad26817d', 'reaction does not exist');
}
// Decrement reactions count

55
packages/backend/src/core/RemoteUserResolveService.ts
View file

@ -3,7 +3,6 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { URL } from 'node:url';
import { Inject, Injectable } from '@nestjs/common';
import chalk from 'chalk';
import { IsNull } from 'typeorm';
@ -18,6 +17,7 @@ import { RemoteLoggerService } from '@/core/RemoteLoggerService.js';
import { ApDbResolverService } from '@/core/activitypub/ApDbResolverService.js';
import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js';
import { bindThis } from '@/decorators.js';
import { renderInlineError } from '@/misc/render-inline-error.js';
@Injectable()
export class RemoteUserResolveService {
@ -44,27 +44,13 @@ export class RemoteUserResolveService {
const usernameLower = username.toLowerCase();
if (host == null) {
this.logger.info(`return local user: ${usernameLower}`);
return await this.usersRepository.findOneBy({ usernameLower, host: IsNull() }).then(u => {
if (u == null) {
throw new Error('user not found');
} else {
return u;
}
}) as MiLocalUser;
return await this.usersRepository.findOneByOrFail({ usernameLower, host: IsNull() }) as MiLocalUser;
}
host = this.utilityService.toPuny(host);
if (host === this.utilityService.toPuny(this.config.host)) {
this.logger.info(`return local user: ${usernameLower}`);
return await this.usersRepository.findOneBy({ usernameLower, host: IsNull() }).then(u => {
if (u == null) {
throw new Error('user not found');
} else {
return u;
}
}) as MiLocalUser;
return await this.usersRepository.findOneByOrFail({ usernameLower, host: IsNull() }) as MiLocalUser;
}
const user = await this.usersRepository.findOneBy({ usernameLower, host }) as MiRemoteUser | null;
@ -82,7 +68,7 @@ export class RemoteUserResolveService {
.getUserFromApId(self.href)
.then((u) => {
if (u == null) {
throw new Error('local user not found');
throw new Error(`local user not found: ${self.href}`);
} else {
return u;
}
@ -90,7 +76,7 @@ export class RemoteUserResolveService {
}
}
this.logger.succ(`return new remote user: ${chalk.magenta(acctLower)}`);
this.logger.info(`Fetching new remote user ${chalk.magenta(acctLower)} from ${self.href}`);
return await this.apPersonService.createPerson(self.href);
}
@ -101,18 +87,16 @@ export class RemoteUserResolveService {
lastFetchedAt: new Date(),
});
this.logger.info(`try resync: ${acctLower}`);
const self = await this.resolveSelf(acctLower);
if (user.uri !== self.href) {
// if uri mismatch, Fix (user@host <=> AP's Person id(RemoteUser.uri)) mapping.
this.logger.info(`uri missmatch: ${acctLower}`);
this.logger.info(`recovery missmatch uri for (username=${username}, host=${host}) from ${user.uri} to ${self.href}`);
this.logger.warn(`Detected URI mismatch for ${acctLower}`);
// validate uri
const uri = new URL(self.href);
if (uri.hostname !== host) {
throw new Error('Invalid uri');
const uriHost = this.utilityService.extractDbHost(self.href);
if (uriHost !== host) {
throw new Error(`Failed to correct URI for ${acctLower}: new URI ${self.href} has different host from previous URI ${user.uri}`);
}
await this.usersRepository.update({
@ -121,37 +105,28 @@ export class RemoteUserResolveService {
}, {
uri: self.href,
});
} else {
this.logger.info(`uri is fine: ${acctLower}`);
}
this.logger.info(`Corrected URI for ${acctLower} from ${user.uri} to ${self.href}`);
await this.apPersonService.updatePerson(self.href);
this.logger.info(`return resynced remote user: ${acctLower}`);
return await this.usersRepository.findOneBy({ uri: self.href }).then(u => {
if (u == null) {
throw new Error('user not found');
} else {
return u as MiLocalUser | MiRemoteUser;
}
});
return await this.usersRepository.findOneByOrFail({ uri: self.href }) as MiLocalUser | MiRemoteUser;
}
this.logger.info(`return existing remote user: ${acctLower}`);
return user;
}
@bindThis
private async resolveSelf(acctLower: string): Promise<ILink> {
this.logger.info(`WebFinger for ${chalk.yellow(acctLower)}`);
const finger = await this.webfingerService.webfinger(acctLower).catch(err => {
this.logger.error(`Failed to WebFinger for ${chalk.yellow(acctLower)}: ${ err.statusCode ?? err.message }`);
throw new Error(`Failed to WebFinger for ${acctLower}: ${ err.statusCode ?? err.message }`);
this.logger.error(`Failed to WebFinger for ${chalk.yellow(acctLower)}: ${renderInlineError(err)}`);
throw new Error(`Failed to WebFinger for ${acctLower}: error thrown`, { cause: err });
});
const self = finger.links.find(link => link.rel != null && link.rel.toLowerCase() === 'self');
if (!self) {
this.logger.error(`Failed to WebFinger for ${chalk.yellow(acctLower)}: self link not found`);
throw new Error('self link not found');
throw new Error(`Failed to WebFinger for ${acctLower}: self link not found`);
}
return self;
}

4
packages/backend/src/core/ReversiService.ts
View file

@ -587,6 +587,8 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit {
lastActiveDate: parsed.user1.lastActiveDate != null ? new Date(parsed.user1.lastActiveDate) : null,
lastFetchedAt: parsed.user1.lastFetchedAt != null ? new Date(parsed.user1.lastFetchedAt) : null,
movedAt: parsed.user1.movedAt != null ? new Date(parsed.user1.movedAt) : null,
instance: null,
userProfile: null,
} : null,
user2: parsed.user2 != null ? {
...parsed.user2,
@ -597,6 +599,8 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit {
lastActiveDate: parsed.user2.lastActiveDate != null ? new Date(parsed.user2.lastActiveDate) : null,
lastFetchedAt: parsed.user2.lastFetchedAt != null ? new Date(parsed.user2.lastFetchedAt) : null,
movedAt: parsed.user2.movedAt != null ? new Date(parsed.user2.movedAt) : null,
instance: null,
userProfile: null,
} : null,
};
} else {

4
packages/backend/src/core/RoleService.ts
View file

@ -86,10 +86,10 @@ export const DEFAULT_POLICIES: RolePolicies = {
canManageCustomEmojis: false,
canManageAvatarDecorations: false,
canSearchNotes: false,
canUseTranslator: true,
canUseTranslator: false,
canHideAds: false,
driveCapacityMb: 100,
maxFileSizeMb: 10,
maxFileSizeMb: 25,
alwaysMarkNsfw: false,
canUpdateBioMedia: true,
pinLimit: 5,

39
packages/backend/src/core/UtilityService.ts
View file

@ -51,22 +51,49 @@ export class UtilityService {
return regexp.test(email);
}
public isBlockedHost(host: string | null): boolean;
public isBlockedHost(blockedHosts: string[], host: string | null): boolean;
@bindThis
public isBlockedHost(blockedHosts: string[], host: string | null): boolean {
public isBlockedHost(blockedHostsOrHost: string[] | string | null, host?: string | null): boolean {
const blockedHosts = Array.isArray(blockedHostsOrHost) ? blockedHostsOrHost : this.meta.blockedHosts;
host = Array.isArray(blockedHostsOrHost) ? host : blockedHostsOrHost;
if (host == null) return false;
return blockedHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`));
}
public isSilencedHost(host: string | null): boolean;
public isSilencedHost(silencedHosts: string[], host: string | null): boolean;
@bindThis
public isSilencedHost(silencedHosts: string[] | undefined, host: string | null): boolean {
if (!silencedHosts || host == null) return false;
public isSilencedHost(silencedHostsOrHost: string[] | string | null, host?: string | null): boolean {
const silencedHosts = Array.isArray(silencedHostsOrHost) ? silencedHostsOrHost : this.meta.silencedHosts;
host = Array.isArray(silencedHostsOrHost) ? host : silencedHostsOrHost;
if (host == null) return false;
return silencedHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`));
}
public isMediaSilencedHost(host: string | null): boolean;
public isMediaSilencedHost(silencedHosts: string[], host: string | null): boolean;
@bindThis
public isMediaSilencedHost(silencedHosts: string[] | undefined, host: string | null): boolean {
if (!silencedHosts || host == null) return false;
return silencedHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`));
public isMediaSilencedHost(mediaSilencedHostsOrHost: string[] | string | null, host?: string | null): boolean {
const mediaSilencedHosts = Array.isArray(mediaSilencedHostsOrHost) ? mediaSilencedHostsOrHost : this.meta.mediaSilencedHosts;
host = Array.isArray(mediaSilencedHostsOrHost) ? host : mediaSilencedHostsOrHost;
if (host == null) return false;
return mediaSilencedHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`));
}
@bindThis
public isAllowListedHost(host: string | null): boolean {
if (host == null) return false;
return this.meta.federationHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`));
}
@bindThis
public isBubbledHost(host: string | null): boolean {
if (host == null) return false;
return this.meta.bubbleInstances.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`));
}
@bindThis

64
packages/backend/src/core/VideoProcessingService.ts
View file

@ -3,24 +3,41 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import fs from 'node:fs/promises';
import { Inject, Injectable } from '@nestjs/common';
import FFmpeg from 'fluent-ffmpeg';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { ImageProcessingService } from '@/core/ImageProcessingService.js';
import type { IImage } from '@/core/ImageProcessingService.js';
import { createTempDir } from '@/misc/create-temp.js';
import { createTemp, createTempDir } from '@/misc/create-temp.js';
import { bindThis } from '@/decorators.js';
import { appendQuery, query } from '@/misc/prelude/url.js';
import { LoggerService } from '@/core/LoggerService.js';
import type Logger from '@/logger.js';
// faststart is only supported for MP4, M4A, M4W and MOV files (the MOV family).
// WebM (and Matroska) files always support faststart-like behavior.
const supportedMimeTypes = new Map([
['video/mp4', 'mp4'],
['video/m4a', 'mp4'],
['video/m4v', 'mp4'],
['video/quicktime', 'mov'],
]);
@Injectable()
export class VideoProcessingService {
private readonly logger: Logger;
constructor(
@Inject(DI.config)
private config: Config,
private imageProcessingService: ImageProcessingService,
private loggerService: LoggerService,
) {
this.logger = this.loggerService.getLogger('video-processing');
}
@bindThis
@ -60,5 +77,50 @@ export class VideoProcessingService {
}),
);
}
/**
* Optimize video for web playback by adding faststart flag.
* This allows the video to start playing before it is fully downloaded.
* The original file is modified in-place.
* @param source Path to the video file
* @param mimeType The MIME type of the video
* @returns Promise that resolves when optimization is complete
*/
@bindThis
public async webOptimizeVideo(source: string, mimeType: string): Promise<void> {
const outputFormat = supportedMimeTypes.get(mimeType);
if (!outputFormat) {
this.logger.debug(`Skipping web optimization for unsupported MIME type: ${mimeType}`);
return;
}
const [tempPath, cleanup] = await createTemp();
try {
await new Promise<void>((resolve, reject) => {
FFmpeg(source)
.format(outputFormat) // Specify output format
.addOutputOptions('-c copy') // Copy streams without re-encoding
.addOutputOptions('-movflags +faststart')
.on('error', reject)
.on('end', async () => {
try {
// Replace original file with optimized version
await fs.copyFile(tempPath, source);
this.logger.info(`Web-optimized video: ${source}`);
resolve();
} catch (copyError) {
reject(copyError);
}
})
.save(tempPath);
});
} catch (error) {
this.logger.warn(`Failed to web-optimize video: ${source}`, { error });
throw error;
} finally {
cleanup();
}
}
}

16
packages/backend/src/core/WebAuthnService.ts
View file

@ -17,6 +17,8 @@ import type { Config } from '@/config.js';
import { bindThis } from '@/decorators.js';
import { MiUser } from '@/models/_.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import { LoggerService } from '@/core/LoggerService.js';
import Logger from '@/logger.js';
import type {
AuthenticationResponseJSON,
AuthenticatorTransportFuture,
@ -28,6 +30,8 @@ import type {
@Injectable()
export class WebAuthnService {
private readonly logger: Logger;
constructor(
@Inject(DI.config)
private config: Config,
@ -40,7 +44,9 @@ export class WebAuthnService {
@Inject(DI.userSecurityKeysRepository)
private userSecurityKeysRepository: UserSecurityKeysRepository,
loggerService: LoggerService,
) {
this.logger = loggerService.getLogger('web-authn');
}
@bindThis
@ -114,8 +120,8 @@ export class WebAuthnService {
requireUserVerification: true,
});
} catch (error) {
console.error(error);
throw new IdentifiableError('5c1446f8-8ca7-4d31-9f39-656afe9c5d87', 'verification failed');
this.logger.error(error as Error, 'Error authenticating webauthn');
throw new IdentifiableError('5c1446f8-8ca7-4d31-9f39-656afe9c5d87', 'verification failed', true, error);
}
const { verified } = verification;
@ -221,7 +227,7 @@ export class WebAuthnService {
requireUserVerification: true,
});
} catch (error) {
throw new IdentifiableError('b18c89a7-5b5e-4cec-bb5b-0419f332d430', `verification failed: ${error}`);
throw new IdentifiableError('b18c89a7-5b5e-4cec-bb5b-0419f332d430', `verification failed`, true, error);
}
const { verified, authenticationInfo } = verification;
@ -301,8 +307,8 @@ export class WebAuthnService {
requireUserVerification: true,
});
} catch (error) {
console.error(error);
throw new IdentifiableError('b18c89a7-5b5e-4cec-bb5b-0419f332d430', 'verification failed');
this.logger.error(error as Error, 'Error authenticating webauthn');
throw new IdentifiableError('b18c89a7-5b5e-4cec-bb5b-0419f332d430', 'verification failed', true, error);
}
const { verified, authenticationInfo } = verification;

3
packages/backend/src/core/WebfingerService.ts
View file

@ -9,6 +9,7 @@ import { XMLParser } from 'fast-xml-parser';
import { HttpRequestService } from '@/core/HttpRequestService.js';
import { bindThis } from '@/decorators.js';
import type Logger from '@/logger.js';
import { renderInlineError } from '@/misc/render-inline-error.js';
import { RemoteLoggerService } from './RemoteLoggerService.js';
export type ILink = {
@ -109,7 +110,7 @@ export class WebfingerService {
const template = (hostMeta['XRD']['Link'] as Array<any>).filter(p => p['@_rel'] === 'lrdd')[0]['@_template'];
return template.indexOf('{uri}') < 0 ? null : template;
} catch (err) {
this.logger.error(`error while request host-meta for ${url}: ${err}`);
this.logger.error(`error while request host-meta for ${url}: ${renderInlineError(err)}`);
return null;
}
}

9
packages/backend/src/core/WebhookTestService.ts
View file

@ -63,6 +63,7 @@ function generateDummyUser(override?: Partial<MiUser>): MiUser {
emojis: [],
score: 0,
host: null,
instance: null,
inbox: null,
sharedInbox: null,
featured: null,
@ -76,6 +77,8 @@ function generateDummyUser(override?: Partial<MiUser>): MiUser {
mandatoryCW: null,
rejectQuotes: false,
allowUnsignedFetch: 'staff',
userProfile: null,
attributionDomains: [],
...override,
};
}
@ -114,10 +117,13 @@ function generateDummyNote(override?: Partial<MiNote>): MiNote {
channelId: null,
channel: null,
userHost: null,
userInstance: null,
replyUserId: null,
replyUserHost: null,
replyUserInstance: null,
renoteUserId: null,
renoteUserHost: null,
renoteUserInstance: null,
updatedAt: null,
processErrors: [],
...override,
@ -358,8 +364,10 @@ export class WebhookTestService {
id: 'dummy-abuse-report1',
targetUserId: 'dummy-target-user',
targetUser: null,
targetUserInstance: null,
reporterId: 'dummy-reporter-user',
reporter: null,
reporterInstance: null,
assigneeId: null,
assignee: null,
resolved: false,
@ -451,6 +459,7 @@ export class WebhookTestService {
isAdmin: false,
isModerator: false,
isSystem: false,
instance: undefined,
...override,
};
}

19
packages/backend/src/core/activitypub/ApDbResolverService.ts
View file

@ -165,18 +165,23 @@ export class ApDbResolverService implements OnApplicationShutdown {
*/
@bindThis
public async refetchPublicKeyForApId(user: MiRemoteUser): Promise<MiUserPublickey | null> {
this.apLoggerService.logger.debug('Re-fetching public key for user', { userId: user.id, uri: user.uri });
this.apLoggerService.logger.debug(`Updating public key for user ${user.id} (${user.uri})`);
const oldKey = await this.apPersonService.findPublicKeyByUserId(user.id);
await this.apPersonService.updatePerson(user.uri);
const newKey = await this.apPersonService.findPublicKeyByUserId(user.id);
const key = await this.apPersonService.findPublicKeyByUserId(user.id);
if (key) {
this.apLoggerService.logger.info('Re-fetched public key for user', { userId: user.id, uri: user.uri });
if (newKey) {
if (oldKey && newKey.keyPem === oldKey.keyPem) {
this.apLoggerService.logger.debug(`Public key is up-to-date for user ${user.id} (${user.uri})`);
} else {
this.apLoggerService.logger.info(`Updated public key for user ${user.id} (${user.uri})`);
}
} else {
this.apLoggerService.logger.warn('Failed to re-fetch key for user', { userId: user.id, uri: user.uri });
this.apLoggerService.logger.warn(`Failed to update public key for user ${user.id} (${user.uri})`);
}
return key;
return newKey ?? oldKey;
}
@bindThis

5
packages/backend/src/core/activitypub/ApDeliverManagerService.ts
View file

@ -57,7 +57,7 @@ class DeliverManager {
) {
// 型で弾いてはいるが一応ローカルユーザーかチェック
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
if (actor.host != null) throw new Error('actor.host must be null');
if (actor.host != null) throw new Error(`deliver failed for ${actor.id}: host is not null`);
// パフォーマンス向上のためキューに突っ込むのはidのみに絞る
this.actor = {
@ -124,12 +124,13 @@ class DeliverManager {
select: {
followerSharedInbox: true,
followerInbox: true,
followerId: true,
},
});
for (const following of followers) {
const inbox = following.followerSharedInbox ?? following.followerInbox;
if (inbox === null) throw new UnrecoverableError(`inbox is null: following ${following.id}`);
if (inbox === null) throw new UnrecoverableError(`deliver failed for ${this.actor.id}: follower ${following.followerId} inbox is null`);
inboxes.set(inbox, following.followerSharedInbox != null);
}
}

88
packages/backend/src/core/activitypub/ApInboxService.ts
View file

@ -32,11 +32,12 @@ import { AbuseReportService } from '@/core/AbuseReportService.js';
import { FederatedInstanceService } from '@/core/FederatedInstanceService.js';
import { fromTuple } from '@/misc/from-tuple.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import { renderInlineError } from '@/misc/render-inline-error.js';
import InstanceChart from '@/core/chart/charts/instance.js';
import FederationChart from '@/core/chart/charts/federation.js';
import { FetchInstanceMetadataService } from '@/core/FetchInstanceMetadataService.js';
import { UpdateInstanceQueue } from '@/core/UpdateInstanceQueue.js';
import { getApHrefNullable, getApId, getApIds, getApType, getNullableApId, isAccept, isActor, isAdd, isAnnounce, isApObject, isBlock, isCollection, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isDislike, isMove, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost, isActivity, IObjectWithId } from './type.js';
import { getApHrefNullable, getApId, getApIds, getApType, getNullableApId, isAccept, isActor, isAdd, isAnnounce, isApObject, isBlock, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isDislike, isMove, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost, isActivity, IObjectWithId } from './type.js';
import { ApNoteService } from './models/ApNoteService.js';
import { ApLoggerService } from './ApLoggerService.js';
import { ApDbResolverService } from './ApDbResolverService.js';
@ -106,25 +107,29 @@ export class ApInboxService {
let result = undefined as string | void;
if (isCollectionOrOrderedCollection(activity)) {
const results = [] as [string, string | void][];
// eslint-disable-next-line no-param-reassign
resolver ??= this.apResolverService.createResolver();
const items = toArray(isCollection(activity) ? activity.items : activity.orderedItems);
if (items.length >= resolver.getRecursionLimit()) {
throw new Error(`skipping activity: collection would surpass recursion limit: ${this.utilityService.extractDbHost(actor.uri)}`);
}
for (const item of items) {
const act = await resolver.resolve(item);
if (act.id == null || this.utilityService.extractDbHost(act.id) !== this.utilityService.extractDbHost(actor.uri)) {
this.logger.debug('skipping activity: activity id is null or mismatching');
continue;
const items = await resolver.resolveCollectionItems(activity);
for (let i = 0; i < items.length; i++) {
const act = items[i];
if (act.id != null) {
if (this.utilityService.extractDbHost(act.id) !== this.utilityService.extractDbHost(actor.uri)) {
this.logger.warn('skipping activity: activity id mismatch');
continue;
}
} else {
// Activity ID should only be string or undefined.
act.id = undefined;
}
const id = getNullableApId(act) ?? `${getNullableApId(activity)}#${i}`;
try {
results.push([getApId(item), await this.performOneActivity(actor, act, resolver)]);
const result = await this.performOneActivity(actor, act, resolver);
results.push([id, result]);
} catch (err) {
if (err instanceof Error || typeof err === 'string') {
this.logger.error(err);
this.logger.error(`Unhandled error in activity ${id}:`, err);
} else {
throw err;
}
@ -144,7 +149,8 @@ export class ApInboxService {
if (actor.lastFetchedAt == null || Date.now() - actor.lastFetchedAt.getTime() > 1000 * 60 * 60 * 24) {
setImmediate(() => {
// 同一ユーザーの情報を再度処理するので、使用済みのresolverを再利用してはいけない
this.apPersonService.updatePerson(actor.uri);
this.apPersonService.updatePerson(actor.uri)
.catch(err => this.logger.error(`Failed to update person: ${renderInlineError(err)}`));
});
}
}
@ -217,6 +223,10 @@ export class ApInboxService {
const note = await this.apNoteService.resolveNote(object, { resolver });
if (!note) return `skip: target note not found ${targetUri}`;
if (note.userHost == null && note.localOnly) {
throw new IdentifiableError('12e23cec-edd9-442b-aa48-9c21f0c3b215', 'Cannot react to local-only note');
}
await this.apNoteService.extractEmojis(activity.tag ?? [], actor.host).catch(() => null);
try {
@ -246,7 +256,7 @@ export class ApInboxService {
resolver ??= this.apResolverService.createResolver();
const object = await resolver.resolve(activity.object).catch(err => {
this.logger.error(`Resolution failed: ${err}`);
this.logger.error(`Resolution failed: ${renderInlineError(err)}`);
throw err;
});
@ -319,7 +329,7 @@ export class ApInboxService {
if (targetUri.startsWith('bear:')) return 'skip: bearcaps url not supported.';
const target = await resolver.secureResolve(activityObject, uri).catch(e => {
this.logger.error(`Resolution failed: ${e}`);
this.logger.error(`Resolution failed: ${renderInlineError(e)}`);
throw e;
});
@ -350,27 +360,19 @@ export class ApInboxService {
}
// Announce対象をresolve
let renote;
try {
// The target ID is verified by secureResolve, so we know it shares host authority with the actor who sent it.
// This means we can pass that ID to resolveNote and avoid an extra fetch, which will fail if the note is private.
renote = await this.apNoteService.resolveNote(target, { resolver, sentFrom: getApId(target) });
if (renote == null) return 'announce target is null';
} catch (err) {
// 対象が4xxならスキップ
if (err instanceof StatusError) {
if (!err.isRetryable) {
return `skip: ignored announce target ${target.id} - ${err.statusCode}`;
}
return `Error in announce target ${target.id} - ${err.statusCode}`;
}
throw err;
}
// The target ID is verified by secureResolve, so we know it shares host authority with the actor who sent it.
// This means we can pass that ID to resolveNote and avoid an extra fetch, which will fail if the note is private.
const renote = await this.apNoteService.resolveNote(target, { resolver, sentFrom: getApId(target) });
if (renote == null) return 'announce target is null';
if (!await this.noteEntityService.isVisibleForMe(renote, actor.id)) {
return 'skip: invalid actor for this activity';
}
if (renote.userHost == null && renote.localOnly) {
throw new IdentifiableError('12e23cec-edd9-442b-aa48-9c21f0c3b215', 'Cannot renote a local-only note');
}
this.logger.info(`Creating the (Re)Note: ${uri}`);
const activityAudience = await this.apAudienceService.parseAudience(actor, activity.to, activity.cc, resolver);
@ -443,9 +445,11 @@ export class ApInboxService {
setImmediate(() => {
// Don't re-use the resolver, or it may throw recursion errors.
// Instead, create a new resolver with an appropriately-reduced recursion limit.
this.apPersonService.updatePerson(actor.uri, this.apResolverService.createResolver({
const subResolver = this.apResolverService.createResolver({
recursionLimit: resolver.getRecursionLimit() - resolver.getHistory().length,
}));
});
this.apPersonService.updatePerson(actor.uri, subResolver)
.catch(err => this.logger.error(`Failed to update person: ${renderInlineError(err)}`));
});
}
});
@ -500,7 +504,7 @@ export class ApInboxService {
resolver ??= this.apResolverService.createResolver();
const object = await resolver.resolve(activityObject).catch(e => {
this.logger.error(`Resolution failed: ${e}`);
this.logger.error(`Resolution failed: ${renderInlineError(e)}`);
throw e;
});
@ -537,12 +541,6 @@ export class ApInboxService {
await this.apNoteService.createNote(note, actor, resolver, silent);
return 'ok';
} catch (err) {
if (err instanceof StatusError && !err.isRetryable) {
return `skip: ${err.statusCode}`;
} else {
throw err;
}
} finally {
unlock();
}
@ -675,7 +673,7 @@ export class ApInboxService {
resolver ??= this.apResolverService.createResolver();
const object = await resolver.resolve(activity.object).catch(e => {
this.logger.error(`Resolution failed: ${e}`);
this.logger.error(`Resolution failed: ${renderInlineError(e)}`);
throw e;
});
@ -747,7 +745,7 @@ export class ApInboxService {
resolver ??= this.apResolverService.createResolver();
const object = await resolver.resolve(activity.object).catch(e => {
this.logger.error(`Resolution failed: ${e}`);
this.logger.error(`Resolution failed: ${renderInlineError(e)}`);
throw e;
});
@ -879,7 +877,7 @@ export class ApInboxService {
resolver ??= this.apResolverService.createResolver();
const object = await resolver.resolve(activity.object).catch(e => {
this.logger.error(`Resolution failed: ${e}`);
this.logger.error(`Resolution failed: ${renderInlineError(e)}`);
throw e;
});

1
packages/backend/src/core/activitypub/ApRendererService.ts
View file

@ -613,6 +613,7 @@ export class ApRendererService {
enableRss: user.enableRss,
speakAsCat: user.speakAsCat,
attachment: attachment.length ? attachment : undefined,
attributionDomains: user.attributionDomains,
};
if (user.movedToUri) {

13
packages/backend/src/core/activitypub/ApRequestService.ts
View file

@ -184,10 +184,11 @@ export class ApRequestService {
* Get AP object with http-signature
* @param user http-signature user
* @param url URL to fetch
* @param followAlternate
* @param allowAnonymous If a fetched object lacks an ID, then it will be auto-generated from the final URL. (default: false)
* @param followAlternate Whether to resolve HTML responses to their referenced canonical AP endpoint. (default: true)
*/
@bindThis
public async signedGet(url: string, user: { id: MiUser['id'] }, followAlternate?: boolean): Promise<IObjectWithId> {
public async signedGet(url: string, user: { id: MiUser['id'] }, allowAnonymous = false, followAlternate?: boolean): Promise<IObjectWithId> {
this.apUtilityService.assertApUrl(url);
const _followAlternate = followAlternate ?? true;
@ -258,7 +259,7 @@ export class ApRequestService {
if (alternate) {
const href = alternate.getAttribute('href');
if (href && this.apUtilityService.haveSameAuthority(url, href)) {
return await this.signedGet(href, user, false);
return await this.signedGet(href, user, allowAnonymous, false);
}
}
} catch {
@ -275,7 +276,11 @@ export class ApRequestService {
// Make sure the object ID matches the final URL (which is where it actually exists).
// The caller (ApResolverService) will verify the ID against the original / entry URL, which ensures that all three match.
this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url);
if (allowAnonymous && activity.id == null) {
activity.id = res.url;
} else {
this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url);
}
return activity as IObjectWithId;
}

204
packages/backend/src/core/activitypub/ApResolverService.ts
View file

@ -5,6 +5,7 @@
import { Inject, Injectable } from '@nestjs/common';
import { IsNull, Not } from 'typeorm';
import promiseLimit from 'promise-limit';
import type { MiLocalUser, MiRemoteUser } from '@/models/User.js';
import type { NotesRepository, PollsRepository, NoteReactionsRepository, UsersRepository, FollowRequestsRepository, MiMeta, SkApFetchLog } from '@/models/_.js';
import type { Config } from '@/config.js';
@ -19,11 +20,12 @@ import { ApLogService, calculateDurationSince, extractObjectContext } from '@/co
import { ApUtilityService } from '@/core/activitypub/ApUtilityService.js';
import { SystemAccountService } from '@/core/SystemAccountService.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import { getApId, getNullableApId, IObjectWithId, isCollectionOrOrderedCollection } from './type.js';
import { toArray } from '@/misc/prelude/array.js';
import { AnyCollection, getApId, getNullableApId, IObjectWithId, isCollection, isCollectionOrOrderedCollection, isCollectionPage, isOrderedCollection, isOrderedCollectionPage } from './type.js';
import { ApDbResolverService } from './ApDbResolverService.js';
import { ApRendererService } from './ApRendererService.js';
import { ApRequestService } from './ApRequestService.js';
import type { IObject, ICollection, IOrderedCollection, ApObject } from './type.js';
import type { IObject, ApObject, IAnonymousObject } from './type.js';
export class Resolver {
private history: Set<string>;
@ -63,34 +65,129 @@ export class Resolver {
return this.recursionLimit;
}
public async resolveCollection(value: string | IObjectWithId, allowAnonymous?: boolean, sentFromUri?: string): Promise<AnyCollection & IObjectWithId>;
public async resolveCollection(value: string | IObject, allowAnonymous: boolean | undefined, sentFromUri: string): Promise<AnyCollection & IObjectWithId>;
public async resolveCollection(value: string | IObject, allowAnonymous?: boolean, sentFromUri?: string): Promise<AnyCollection>;
@bindThis
public async resolveCollection(value: string | IObject): Promise<ICollection | IOrderedCollection> {
public async resolveCollection(value: string | IObject, allowAnonymous?: boolean, sentFromUri?: string): Promise<AnyCollection> {
const collection = typeof value === 'string'
? await this.resolve(value)
: value;
? sentFromUri
? await this.secureResolve(value, sentFromUri, allowAnonymous)
: await this.resolve(value, allowAnonymous)
: value; // TODO try and remove this eventually, as it's a major security foot-gun
if (isCollectionOrOrderedCollection(collection)) {
return collection;
} else {
throw new IdentifiableError('f100eccf-f347-43fb-9b45-96a0831fb635', `unrecognized collection type: ${collection.type}`);
throw new IdentifiableError('f100eccf-f347-43fb-9b45-96a0831fb635', `collection ${getApId(value)} has unsupported type: ${collection.type}`);
}
}
public async resolveCollectionItems(collection: IAnonymousObject, limit?: number | null, allowAnonymousItems?: true, concurrency?: number): Promise<IAnonymousObject[]>;
public async resolveCollectionItems(collection: string | IObjectWithId, limit?: number | null, allowAnonymousItems?: boolean, concurrency?: number): Promise<IObjectWithId[]>;
public async resolveCollectionItems(collection: string | IObject, limit?: number | null, allowAnonymousItems?: boolean, concurrency?: number): Promise<IObject[]>;
/**
* Recursively resolves items from a collection.
* Stops when reaching the resolution limit or an optional item limit - whichever is lower.
* This method supports Collection, OrderedCollection, and individual pages of either type.
* Malformed collections (mixing Ordered and un-Ordered types) are also supported.
* @param collection Collection to resolve from - can be a URL or object of any supported collection type.
* @param limit Maximum number of items to resolve. If null or undefined (default), then items will be resolved until reaching the recursion limit.
* @param allowAnonymousItems If true, collection items can be anonymous (lack an ID). If false (default), then an error is thrown when reaching an item without ID.
* @param concurrency Maximum number of items to resolve at once. (default: 4)
*/
@bindThis
public async resolveCollectionItems(collection: string | IObject, limit?: number | null, allowAnonymousItems?: boolean, concurrency = 4): Promise<IObject[]> {
const resolvedItems: IObject[] = [];
// This is pulled up to avoid code duplication below
const iterate = async(items: ApObject, current: AnyCollection) => {
const sentFrom = current.id;
const itemArr = toArray(items);
const itemLimit = limit ?? Number.MAX_SAFE_INTEGER;
const allowAnonymous = allowAnonymousItems ?? false;
await this.resolveItemArray(itemArr, sentFrom, itemLimit, concurrency, allowAnonymous, resolvedItems);
};
let current: AnyCollection | null = await this.resolveCollection(collection);
do {
// Iterate all items in the current page
if (current.items) {
await iterate(current.items, current);
}
if (current.orderedItems) {
await iterate(current.orderedItems, current);
}
if (this.history.size >= this.recursionLimit) {
// Stop when we reach the fetch limit
current = null;
} else if (limit != null && resolvedItems.length >= limit) {
// Stop when we reach the item limit
current = null;
} else if (isCollection(current) || isOrderedCollection(current)) {
// Continue to first page
current = current.first ? await this.resolveCollection(current.first, true, current.id) : null;
} else if (isCollectionPage(current) || isOrderedCollectionPage(current)) {
// Continue to next page
current = current.next ? await this.resolveCollection(current.next, true, current.id) : null;
} else {
// Stop in all other conditions
current = null;
}
} while (current != null);
return resolvedItems;
}
private async resolveItemArray(source: (string | IObject)[], sentFrom: undefined, itemLimit: number, concurrency: number, allowAnonymousItems: true, destination: IAnonymousObject[]): Promise<void>;
private async resolveItemArray(source: (string | IObject)[], sentFrom: string, itemLimit: number, concurrency: number, allowAnonymousItems: boolean, destination: IObjectWithId[]): Promise<void>;
private async resolveItemArray(source: (string | IObject)[], sentFrom: string | undefined, itemLimit: number, concurrency: number, allowAnonymousItems: boolean, destination: IObject[]): Promise<void>;
private async resolveItemArray(source: (string | IObject)[], sentFrom: string | undefined, itemLimit: number, concurrency: number, allowAnonymousItems: boolean, destination: IObject[]): Promise<void> {
const recursionLimit = this.recursionLimit - this.history.size;
const batchLimit = Math.min(source.length, recursionLimit, itemLimit);
const limiter = promiseLimit<IObject>(concurrency);
const batch = await Promise.all(source
.slice(0, batchLimit)
.map(item => limiter(async () => {
if (sentFrom) {
// Use secureResolve to avoid re-fetching items that were included inline.
return await this.secureResolve(item, sentFrom, allowAnonymousItems);
} else if (allowAnonymousItems) {
return await this.resolveAnonymous(item);
} else {
// ID is required if we have neither sentFrom not allowAnonymousItems
const id = getApId(item);
return await this.resolve(id);
}
})));
destination.push(...batch);
};
/**
* Securely resolves an AP object or URL that has been sent from another instance.
* An input object is trusted if and only if its ID matches the authority of sentFromUri.
* In all other cases, the object is re-fetched from remote by input string or object ID.
* @param input The input object or URL to resolve
* @param sentFromUri The URL where this object originated. This MUST be accurate - all security checks depend on this value!
* @param allowAnonymous If true, anonymous objects are allowed and will have their ID set to sentFromUri. If false (default) then anonymous objects will be rejected with an error.
*/
@bindThis
public async secureResolve(input: ApObject, sentFromUri: string): Promise<IObjectWithId> {
public async secureResolve(input: string | IObject | [string | IObject], sentFromUri: string, allowAnonymous?: boolean): Promise<IObjectWithId> {
// Unpack arrays to get the value element.
const value = fromTuple(input);
if (value == null) {
throw new IdentifiableError('20058164-9de1-4573-8715-425753a21c1d', 'Cannot resolve null input');
// If anonymous input is allowed, then any object is automatically valid if we set the ID.
// We can short-circuit here and avoid un-necessary checks.
if (allowAnonymous && typeof(value) === 'object' && value.id == null) {
value.id = sentFromUri;
return value as IObjectWithId;
}
// This will throw if the input has no ID, which is good because we can't verify an anonymous object anyway.
const id = getApId(value);
// This ensures the input has a string ID, protecting against type confusion and rejecting anonymous objects.
const id = getApId(value, sentFromUri);
// Check if we can use the provided object as-is.
// Our security requires that the object ID matches the host authority that sent it, otherwise it can't be trusted.
@ -100,28 +197,52 @@ export class Resolver {
}
// If the checks didn't pass, then we must fetch the object and use that.
return await this.resolve(id);
return await this.resolve(id, allowAnonymous);
}
public async resolve(value: string | [string]): Promise<IObjectWithId>;
public async resolve(value: string | IObject | [string | IObject]): Promise<IObject>;
/**
* Resolves an anonymous object.
* The returned value will not have any ID present.
* If one is provided in the response, it will be removed automatically.
*/
@bindThis
public async resolve(value: string | IObject | [string | IObject]): Promise<IObject> {
public async resolveAnonymous(value: string | IObject | [string | IObject]): Promise<IAnonymousObject> {
value = fromTuple(value);
const object = await this.resolve(value);
object.id = undefined;
return object as IAnonymousObject;
}
public async resolve(value: string | [string], allowAnonymous?: boolean): Promise<IObjectWithId>;
public async resolve(value: string | IObjectWithId | [string | IObjectWithId], allowAnonymous?: boolean): Promise<IObjectWithId>;
public async resolve(value: string | IObject | [string | IObject], allowAnonymous?: boolean): Promise<IObject>;
/**
* Resolves a URL or object to an AP object.
* Tuples are expanded to their first element before anything else, and non-string inputs are returned as-is.
* Otherwise, the string URL is fetched and validated to represent a valid ActivityPub object.
* @param value The input value to resolve
* @param allowAnonymous Determines what to do if a response object lacks an ID field. If false (default), then an exception is thrown. If true, then the ID is populated from the final response URL.
*/
@bindThis
public async resolve(value: string | IObject | [string | IObject], allowAnonymous = false): Promise<IObject> {
value = fromTuple(value);
// TODO try and remove this eventually, as it's a major security foot-gun
if (typeof value !== 'string') {
return value;
}
const host = this.utilityService.extractDbHost(value);
if (this.config.activityLogging.enabled && !this.utilityService.isSelfHost(host)) {
return await this._resolveLogged(value, host);
return await this._resolveLogged(value, host, allowAnonymous);
} else {
return await this._resolve(value, host);
return await this._resolve(value, host, allowAnonymous);
}
}
private async _resolveLogged(requestUri: string, host: string): Promise<IObjectWithId> {
private async _resolveLogged(requestUri: string, host: string, allowAnonymous: boolean): Promise<IObjectWithId> {
const startTime = process.hrtime.bigint();
const log = await this.apLogService.createFetchLog({
@ -130,7 +251,7 @@ export class Resolver {
});
try {
const result = await this._resolve(requestUri, host, log);
const result = await this._resolve(requestUri, host, allowAnonymous, log);
log.accepted = true;
log.result = 'ok';
@ -150,20 +271,20 @@ export class Resolver {
}
}
private async _resolve(value: string, host: string, log?: SkApFetchLog): Promise<IObjectWithId> {
private async _resolve(value: string, host: string, allowAnonymous: boolean, log?: SkApFetchLog): Promise<IObjectWithId> {
if (value.includes('#')) {
// URLs with fragment parts cannot be resolved correctly because
// the fragment part does not get transmitted over HTTP(S).
// Avoid strange behaviour by not trying to resolve these at all.
throw new IdentifiableError('b94fd5b1-0e3b-4678-9df2-dad4cd515ab2', `cannot resolve URL with fragment: ${value}`);
throw new IdentifiableError('b94fd5b1-0e3b-4678-9df2-dad4cd515ab2', `failed to resolve ${value}: URL contains fragment`);
}
if (this.history.has(value)) {
throw new IdentifiableError('0dc86cf6-7cd6-4e56-b1e6-5903d62d7ea5', `cannot resolve already resolved URL: ${value}`);
throw new IdentifiableError('0dc86cf6-7cd6-4e56-b1e6-5903d62d7ea5', `failed to resolve ${value}: recursive resolution blocked`);
}
if (this.history.size > this.recursionLimit) {
throw new IdentifiableError('d592da9f-822f-4d91-83d7-4ceefabcf3d2', `hit recursion limit: ${value}`);
throw new IdentifiableError('d592da9f-822f-4d91-83d7-4ceefabcf3d2', `failed to resolve ${value}: hit recursion limit`);
}
this.history.add(value);
@ -173,7 +294,7 @@ export class Resolver {
}
if (!this.utilityService.isFederationAllowedHost(host)) {
throw new IdentifiableError('09d79f9e-64f1-4316-9cfa-e75c4d091574', `cannot fetch AP object ${value}: blocked instance ${host}`);
throw new IdentifiableError('09d79f9e-64f1-4316-9cfa-e75c4d091574', `failed to resolve ${value}: instance ${host} is blocked`);
}
if (this.config.signToActivityPubGet && !this.user) {
@ -181,8 +302,8 @@ export class Resolver {
}
const object = (this.user
? await this.apRequestService.signedGet(value, this.user)
: await this.httpRequestService.getActivityJson(value));
? await this.apRequestService.signedGet(value, this.user, allowAnonymous)
: await this.httpRequestService.getActivityJson(value, false, allowAnonymous));
if (log) {
const { object: objectOnly, context, contextHash } = extractObjectContext(object);
@ -203,12 +324,12 @@ export class Resolver {
!(object['@context'] as unknown[]).includes('https://www.w3.org/ns/activitystreams') :
object['@context'] !== 'https://www.w3.org/ns/activitystreams'
) {
throw new IdentifiableError('72180409-793c-4973-868e-5a118eb5519b', `invalid AP object ${value}: does not have ActivityStreams context`);
throw new IdentifiableError('72180409-793c-4973-868e-5a118eb5519b', `failed to resolve ${value}: response does not have ActivityStreams context`);
}
// The object ID is already validated to match the final URL's authority by signedGet / getActivityJson.
// We only need to validate that it also matches the original URL's authority, in case of redirects.
const objectId = getApId(object);
const objectId = getApId(object, value);
// We allow some limited cross-domain redirects, which means the host may have changed during fetch.
// Additional checks are needed to validate the scope of cross-domain redirects.
@ -219,21 +340,22 @@ export class Resolver {
// Check if the redirect bounce from [allowed domain] to [blocked domain].
if (!this.utilityService.isFederationAllowedHost(finalHost)) {
throw new IdentifiableError('0a72bf24-2d9b-4f1d-886b-15aaa31adeda', `cannot fetch AP object ${value}: redirected to blocked instance ${finalHost}`);
throw new IdentifiableError('0a72bf24-2d9b-4f1d-886b-15aaa31adeda', `failed to resolve ${value}: redirected to blocked instance ${finalHost}`);
}
}
return object;
}
// TODO try to remove this, as it creates a large attack surface
@bindThis
private resolveLocal(url: string): Promise<IObjectWithId> {
const parsed = this.apDbResolverService.parseUri(url);
if (!parsed.local) throw new IdentifiableError('02b40cd0-fa92-4b0c-acc9-fb2ada952ab8', `resolveLocal - not a local URL: ${url}`);
if (!parsed.local) throw new IdentifiableError('02b40cd0-fa92-4b0c-acc9-fb2ada952ab8', `failed to resolve local ${url}: not a local URL`);
switch (parsed.type) {
case 'notes':
return this.notesRepository.findOneByOrFail({ id: parsed.id })
return this.notesRepository.findOneByOrFail({ id: parsed.id, userHost: IsNull() })
.then(async note => {
const author = await this.usersRepository.findOneByOrFail({ id: note.userId });
if (parsed.rest === 'activity') {
@ -244,22 +366,26 @@ export class Resolver {
}
}) as Promise<IObjectWithId>;
case 'users':
return this.usersRepository.findOneByOrFail({ id: parsed.id })
return this.usersRepository.findOneByOrFail({ id: parsed.id, host: IsNull() })
.then(user => this.apRendererService.renderPerson(user as MiLocalUser));
case 'questions':
// Polls are indexed by the note they are attached to.
return Promise.all([
this.notesRepository.findOneByOrFail({ id: parsed.id }),
this.pollsRepository.findOneByOrFail({ noteId: parsed.id }),
this.notesRepository.findOneByOrFail({ id: parsed.id, userHost: IsNull() }),
this.pollsRepository.findOneByOrFail({ noteId: parsed.id, userHost: IsNull() }),
])
.then(([note, poll]) => this.apRendererService.renderQuestion({ id: note.userId }, note, poll)) as Promise<IObjectWithId>;
case 'likes':
return this.noteReactionsRepository.findOneByOrFail({ id: parsed.id }).then(async reaction =>
this.apRendererService.addContext(await this.apRendererService.renderLike(reaction, { uri: null })));
return this.noteReactionsRepository.findOneOrFail({ where: { id: parsed.id }, relations: { user: true } }).then(async reaction => {
if (reaction.user?.host != null) {
throw new IdentifiableError('02b40cd0-fa92-4b0c-acc9-fb2ada952ab8', `failed to resolve local ${url}: not a local reaction`);
}
return this.apRendererService.addContext(await this.apRendererService.renderLike(reaction, { uri: null }));
});
case 'follows':
return this.followRequestsRepository.findOneBy({ id: parsed.id })
.then(async followRequest => {
if (followRequest == null) throw new IdentifiableError('a9d946e5-d276-47f8-95fb-f04230289bb0', `resolveLocal - invalid follow request ID ${parsed.id}: ${url}`);
if (followRequest == null) throw new IdentifiableError('a9d946e5-d276-47f8-95fb-f04230289bb0', `failed to resolve local ${url}: invalid follow request ID`);
const [follower, followee] = await Promise.all([
this.usersRepository.findOneBy({
id: followRequest.followerId,
@ -271,12 +397,12 @@ export class Resolver {
}),
]);
if (follower == null || followee == null) {
throw new IdentifiableError('06ae3170-1796-4d93-a697-2611ea6d83b6', `resolveLocal - follower or followee does not exist: ${url}`);
throw new IdentifiableError('06ae3170-1796-4d93-a697-2611ea6d83b6', `failed to resolve local ${url}: follower or followee does not exist`);
}
return this.apRendererService.addContext(this.apRendererService.renderFollow(follower as MiLocalUser | MiRemoteUser, followee as MiLocalUser | MiRemoteUser, url));
});
default:
throw new IdentifiableError('7a5d2fc0-94bc-4db6-b8b8-1bf24a2e23d0', `resolveLocal: type ${parsed.type} unhandled: ${url}`);
throw new IdentifiableError('7a5d2fc0-94bc-4db6-b8b8-1bf24a2e23d0', `failed to resolve local ${url}: unsupported type ${parsed.type}`);
}
}
}

8
packages/backend/src/core/activitypub/ApUtilityService.ts
View file

@ -24,7 +24,7 @@ export class ApUtilityService {
public assertIdMatchesUrlAuthority(object: IObject, url: string): void {
// This throws if the ID is missing or invalid, but that's ok.
// Anonymous objects are impossible to verify, so we don't allow fetching them.
const id = getApId(object);
const id = getApId(object, url);
// Make sure the object ID matches the final URL (which is where it actually exists).
// The caller (ApResolverService) will verify the ID against the original / entry URL, which ensures that all three match.
@ -80,7 +80,6 @@ export class ApUtilityService {
/**
* Verifies that a provided URL is in a format acceptable for federation.
* @throws {IdentifiableError} If URL cannot be parsed
* @throws {IdentifiableError} If URL contains a fragment
* @throws {IdentifiableError} If URL is not HTTPS
*/
public assertApUrl(url: string | URL): void {
@ -93,11 +92,6 @@ export class ApUtilityService {
}
}
// Hash component breaks federation
if (url.hash) {
throw new IdentifiableError('0bedd29b-e3bf-4604-af51-d3352e2518af', `invalid AP url ${url}: contains a fragment (#)`);
}
// Must be HTTPS
if (!this.checkHttps(url)) {
throw new IdentifiableError('0bedd29b-e3bf-4604-af51-d3352e2518af', `invalid AP url ${url}: unsupported protocol ${url.protocol}`);

3
packages/backend/src/core/activitypub/JsonLdService.ts
View file

@ -8,6 +8,7 @@ import { Injectable } from '@nestjs/common';
import { UnrecoverableError } from 'bullmq';
import { HttpRequestService } from '@/core/HttpRequestService.js';
import { bindThis } from '@/decorators.js';
import { StatusError } from '@/misc/status-error.js';
import { CONTEXT, PRELOADED_CONTEXTS } from './misc/contexts.js';
import { validateContentTypeSetAsJsonLD } from './misc/validator.js';
import type { JsonLdDocument } from 'jsonld';
@ -149,7 +150,7 @@ class JsonLd {
},
).then(res => {
if (!res.ok) {
throw new Error(`JSON-LD fetch failed with ${res.status} ${res.statusText}: ${url}`);
throw new StatusError(`failed to fetch JSON-LD from ${url}`, res.status, res.statusText);
} else {
return res.json();
}

4
packages/backend/src/core/activitypub/misc/contexts.ts
View file

@ -546,6 +546,10 @@ const extension_context_definition = {
featured: 'toot:featured',
discoverable: 'toot:discoverable',
indexable: 'toot:indexable',
attributionDomains: {
'@id': 'toot:attributionDomains',
'@type': '@id',
},
// schema
schema: 'http://schema.org#',
PropertyValue: 'schema:PropertyValue',

11
packages/backend/src/core/activitypub/misc/validator.ts
View file

@ -3,15 +3,14 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { IdentifiableError } from '@/misc/identifiable-error.js';
import type { Response } from 'node-fetch';
// TODO throw identifiable or unrecoverable errors
export function validateContentTypeSetAsActivityPub(response: Response): void {
const contentType = (response.headers.get('content-type') ?? '').toLowerCase();
if (contentType === '') {
throw new Error(`invalid content type of AP response - no content-type header: ${response.url}`);
throw new IdentifiableError('d09dc850-b76c-4f45-875a-7389339d78b8', `invalid AP response from ${response.url}: no content-type header`, true);
}
if (
contentType.startsWith('application/activity+json') ||
@ -19,7 +18,7 @@ export function validateContentTypeSetAsActivityPub(response: Response): void {
) {
return;
}
throw new Error(`invalid content type of AP response - content type is not application/activity+json or application/ld+json: ${response.url}`);
throw new IdentifiableError('dc110060-a5f2-461d-808b-39c62702ca64', `invalid AP response from ${response.url}: content type "${contentType}" is not application/activity+json or application/ld+json`);
}
const plusJsonSuffixRegex = /^\s*(application|text)\/[a-zA-Z0-9\.\-\+]+\+json\s*(;|$)/;
@ -28,7 +27,7 @@ export function validateContentTypeSetAsJsonLD(response: Response): void {
const contentType = (response.headers.get('content-type') ?? '').toLowerCase();
if (contentType === '') {
throw new Error(`invalid content type of JSON LD - no content-type header: ${response.url}`);
throw new IdentifiableError('45793ab7-7648-4886-b503-429f8a0d0f73', `invalid AP response from ${response.url}: no content-type header`, true);
}
if (
contentType.startsWith('application/ld+json') ||
@ -37,5 +36,5 @@ export function validateContentTypeSetAsJsonLD(response: Response): void {
) {
return;
}
throw new Error(`invalid content type of JSON LD - content type is not application/ld+json or application/json: ${response.url}`);
throw new IdentifiableError('4bf8f36b-4d33-4ac9-ad76-63fa11f354e9', `invalid AP response from ${response.url}: content type "${contentType}" is not application/ld+json or application/json`);
}

4
packages/backend/src/core/activitypub/models/ApImageService.ts
View file

@ -18,7 +18,7 @@ import type { Config } from '@/config.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import { ApResolverService } from '../ApResolverService.js';
import { ApLoggerService } from '../ApLoggerService.js';
import { isDocument, type IObject } from '../type.js';
import { getNullableApId, isDocument, type IObject } from '../type.js';
@Injectable()
export class ApImageService {
@ -48,7 +48,7 @@ export class ApImageService {
public async createImage(actor: MiRemoteUser, value: string | IObject): Promise<MiDriveFile | null> {
// 投稿者が凍結されていたらスキップ
if (actor.isSuspended) {
throw new IdentifiableError('85ab9bd7-3a41-4530-959d-f07073900109', `actor has been suspended: ${actor.uri}`);
throw new IdentifiableError('85ab9bd7-3a41-4530-959d-f07073900109', `failed to create image ${getNullableApId(value)}: actor ${actor.id} has been suspended`);
}
const image = await this.apResolverService.createResolver().resolve(value);

101
packages/backend/src/core/activitypub/models/ApNoteService.ts
View file

@ -26,6 +26,7 @@ import { bindThis } from '@/decorators.js';
import { checkHttps } from '@/misc/check-https.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import { isRetryableError } from '@/misc/is-retryable-error.js';
import { renderInlineError } from '@/misc/render-inline-error.js';
import { getOneApId, getApId, validPost, isEmoji, getApType, isApObject, isDocument, IApDocument } from '../type.js';
import { ApLoggerService } from '../ApLoggerService.js';
import { ApMfmService } from '../ApMfmService.js';
@ -100,29 +101,29 @@ export class ApNoteService {
const apType = getApType(object);
if (apType == null || !validPost.includes(apType)) {
return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', `invalid Note: invalid object type ${apType ?? 'undefined'}`);
return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', `invalid Note from ${uri}: invalid object type ${apType ?? 'undefined'}`);
}
if (object.id && this.utilityService.extractDbHost(object.id) !== expectHost) {
return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', `invalid Note: id has different host. expected: ${expectHost}, actual: ${this.utilityService.extractDbHost(object.id)}`);
return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', `invalid Note from ${uri}: id has different host. expected: ${expectHost}, actual: ${this.utilityService.extractDbHost(object.id)}`);
}
const actualHost = object.attributedTo && this.utilityService.extractDbHost(getOneApId(object.attributedTo));
if (object.attributedTo && actualHost !== expectHost) {
return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', `invalid Note: attributedTo has different host. expected: ${expectHost}, actual: ${actualHost}`);
return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', `invalid Note from ${uri}: attributedTo has different host. expected: ${expectHost}, actual: ${actualHost}`);
}
if (object.published && !this.idService.isSafeT(new Date(object.published).valueOf())) {
return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', 'invalid Note: published timestamp is malformed');
return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', 'invalid Note from ${uri}: published timestamp is malformed');
}
if (actor) {
const attribution = (object.attributedTo) ? getOneApId(object.attributedTo) : actor.uri;
if (attribution !== actor.uri) {
return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', `invalid Note: attribution does not match the actor that send it. attribution: ${attribution}, actor: ${actor.uri}`);
return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', `invalid Note from ${uri}: attribution does not match the actor that send it. attribution: ${attribution}, actor: ${actor.uri}`);
}
if (user && attribution !== user.uri) {
return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', `invalid Note: updated attribution does not match original attribution. updated attribution: ${user.uri}, original attribution: ${attribution}`);
return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', `invalid Note from ${uri}: updated attribution does not match original attribution. updated attribution: ${user.uri}, original attribution: ${attribution}`);
}
}
@ -161,7 +162,7 @@ export class ApNoteService {
const entryUri = getApId(value);
const err = this.validateNote(object, entryUri, actor);
if (err) {
this.logger.error(err.message, {
this.logger.error(`Error creating note: ${renderInlineError(err)}`, {
resolver: { history: resolver.getHistory() },
value,
object,
@ -174,11 +175,11 @@ export class ApNoteService {
this.logger.debug(`Note fetched: ${JSON.stringify(note, null, 2)}`);
if (note.id == null) {
throw new UnrecoverableError(`Refusing to create note without id: ${entryUri}`);
throw new UnrecoverableError(`failed to create note ${entryUri}: missing ID`);
}
if (!checkHttps(note.id)) {
throw new UnrecoverableError(`unexpected schema of note.id ${note.id} in ${entryUri}`);
throw new UnrecoverableError(`failed to create note ${entryUri}: unexpected schema`);
}
const url = this.apUtilityService.findBestObjectUrl(note);
@ -187,7 +188,7 @@ export class ApNoteService {
// 投稿者をフェッチ
if (note.attributedTo == null) {
throw new UnrecoverableError(`invalid note.attributedTo ${note.attributedTo} in ${entryUri}`);
throw new UnrecoverableError(`failed to create note: ${entryUri}: missing attributedTo`);
}
const uri = getOneApId(note.attributedTo);
@ -196,7 +197,7 @@ export class ApNoteService {
// eslint-disable-next-line no-param-reassign
actor ??= await this.apPersonService.fetchPerson(uri) as MiRemoteUser | undefined;
if (actor && actor.isSuspended) {
throw new IdentifiableError('85ab9bd7-3a41-4530-959d-f07073900109', `actor ${uri} has been suspended: ${entryUri}`);
throw new IdentifiableError('85ab9bd7-3a41-4530-959d-f07073900109', `failed to create note ${entryUri}: actor ${uri} has been suspended`);
}
const apMentions = await this.apMentionService.extractApMentions(note.tag, resolver);
@ -223,7 +224,7 @@ export class ApNoteService {
*/
const hasProhibitedWords = this.noteCreateService.checkProhibitedWordsContain({ cw, text, pollChoices: poll?.choices });
if (hasProhibitedWords) {
throw new IdentifiableError('689ee33f-f97c-479a-ac49-1b9f8140af99', `Note contains prohibited words: ${entryUri}`);
throw new IdentifiableError('689ee33f-f97c-479a-ac49-1b9f8140af99', `failed to create note ${entryUri}: contains prohibited words`);
}
//#endregion
@ -232,7 +233,7 @@ export class ApNoteService {
// 解決した投稿者が凍結されていたらスキップ
if (actor.isSuspended) {
throw new IdentifiableError('85ab9bd7-3a41-4530-959d-f07073900109', `actor has been suspended: ${entryUri}`);
throw new IdentifiableError('85ab9bd7-3a41-4530-959d-f07073900109', `failed to create note ${entryUri}: actor ${actor.id} has been suspended`);
}
const noteAudience = await this.apAudienceService.parseAudience(actor, note.to, note.cc, resolver);
@ -269,15 +270,15 @@ export class ApNoteService {
? await this.resolveNote(note.inReplyTo, { resolver })
.then(x => {
if (x == null) {
this.logger.warn('Specified inReplyTo, but not found');
throw new Error(`could not fetch inReplyTo ${note.inReplyTo} for note ${entryUri}`);
this.logger.warn(`Specified inReplyTo "${note.inReplyTo}", but not found`);
throw new IdentifiableError('1ebf0a96-2769-4973-a6c2-3dcbad409dff', `failed to create note ${entryUri}: could not fetch inReplyTo ${note.inReplyTo}`, true);
}
return x;
})
.catch(async err => {
this.logger.warn(`error ${err.statusCode ?? err} fetching inReplyTo ${note.inReplyTo} for note ${entryUri}`);
throw err;
this.logger.warn(`error ${renderInlineError(err)} fetching inReplyTo ${note.inReplyTo} for note ${entryUri}`);
throw new IdentifiableError('1ebf0a96-2769-4973-a6c2-3dcbad409dff', `failed to create note ${entryUri}: could not fetch inReplyTo ${note.inReplyTo}`, true, err);
})
: null;
@ -285,6 +286,13 @@ export class ApNoteService {
const quote = await this.getQuote(note, entryUri, resolver);
const processErrors = quote === null ? ['quoteUnavailable'] : null;
if (reply && reply.userHost == null && reply.localOnly) {
throw new IdentifiableError('12e23cec-edd9-442b-aa48-9c21f0c3b215', 'Cannot reply to local-only note');
}
if (quote && quote.userHost == null && quote.localOnly) {
throw new IdentifiableError('12e23cec-edd9-442b-aa48-9c21f0c3b215', 'Cannot quote a local-only note');
}
// vote
if (reply && reply.hasPoll) {
const poll = await this.pollsRepository.findOneByOrFail({ noteId: reply.id });
@ -341,7 +349,7 @@ export class ApNoteService {
this.logger.info('The note is already inserted while creating itself, reading again');
const duplicate = await this.fetchNote(value);
if (!duplicate) {
throw new Error(`The note creation failed with duplication error even when there is no duplication: ${entryUri}`);
throw new IdentifiableError('39c328e1-e829-458b-bfc9-65dcd513d1f8', `failed to create note ${entryUri}: the note creation failed with duplication error even when there is no duplication. This is likely a bug.`);
}
return duplicate;
}
@ -355,45 +363,39 @@ export class ApNoteService {
const noteUri = getApId(value);
// URIがこのサーバーを指しているならスキップ
if (noteUri.startsWith(this.config.url + '/')) throw new UnrecoverableError(`uri points local: ${noteUri}`);
if (this.utilityService.isUriLocal(noteUri)) {
throw new UnrecoverableError(`failed to update note ${noteUri}: uri is local`);
}
//#region このサーバーに既に登録されているか
const updatedNote = await this.notesRepository.findOneBy({ uri: noteUri });
if (updatedNote == null) throw new Error(`Note is not registered (no note): ${noteUri}`);
if (updatedNote == null) throw new UnrecoverableError(`failed to update note ${noteUri}: note does not exist`);
const user = await this.usersRepository.findOneBy({ id: updatedNote.userId }) as MiRemoteUser | null;
if (user == null) throw new Error(`Note is not registered (no user): ${noteUri}`);
if (user == null) throw new UnrecoverableError(`failed to update note ${noteUri}: user does not exist`);
// eslint-disable-next-line no-param-reassign
if (resolver == null) resolver = this.apResolverService.createResolver();
resolver ??= this.apResolverService.createResolver();
const object = await resolver.resolve(value);
const entryUri = getApId(value);
const err = this.validateNote(object, entryUri, actor, user);
if (err) {
this.logger.error(err.message, {
resolver: { history: resolver.getHistory() },
value,
object,
});
this.logger.error(`Failed to update note ${noteUri}: ${renderInlineError(err)}`);
throw err;
}
// `validateNote` checks that the actor and user are one and the same
// eslint-disable-next-line no-param-reassign
actor ??= user;
const note = object as IPost;
this.logger.debug(`Note fetched: ${JSON.stringify(note, null, 2)}`);
if (note.id == null) {
throw new UnrecoverableError(`Refusing to update note without id: ${noteUri}`);
throw new UnrecoverableError(`failed to update note ${entryUri}: missing ID`);
}
if (!checkHttps(note.id)) {
throw new UnrecoverableError(`unexpected schema of note.id ${note.id} in ${noteUri}`);
throw new UnrecoverableError(`failed to update note ${entryUri}: unexpected schema`);
}
const url = this.apUtilityService.findBestObjectUrl(note);
@ -401,7 +403,7 @@ export class ApNoteService {
this.logger.info(`Creating the Note: ${note.id}`);
if (actor.isSuspended) {
throw new IdentifiableError('85ab9bd7-3a41-4530-959d-f07073900109', `actor ${actor.id} has been suspended: ${noteUri}`);
throw new IdentifiableError('85ab9bd7-3a41-4530-959d-f07073900109', `failed to update note ${entryUri}: actor ${actor.id} has been suspended`);
}
const apMentions = await this.apMentionService.extractApMentions(note.tag, resolver);
@ -428,7 +430,7 @@ export class ApNoteService {
*/
const hasProhibitedWords = this.noteCreateService.checkProhibitedWordsContain({ cw, text, pollChoices: poll?.choices });
if (hasProhibitedWords) {
throw new IdentifiableError('689ee33f-f97c-479a-ac49-1b9f8140af99', `Note contains prohibited words: ${noteUri}`);
throw new IdentifiableError('689ee33f-f97c-479a-ac49-1b9f8140af99', `failed to update note ${noteUri}: contains prohibited words`);
}
//#endregion
@ -466,15 +468,15 @@ export class ApNoteService {
? await this.resolveNote(note.inReplyTo, { resolver })
.then(x => {
if (x == null) {
this.logger.warn('Specified inReplyTo, but not found');
throw new Error(`could not fetch inReplyTo ${note.inReplyTo} for note ${entryUri}`);
this.logger.warn(`Specified inReplyTo "${note.inReplyTo}", but not found`);
throw new IdentifiableError('1ebf0a96-2769-4973-a6c2-3dcbad409dff', `failed to update note ${entryUri}: could not fetch inReplyTo ${note.inReplyTo}`, true);
}
return x;
})
.catch(async err => {
this.logger.warn(`error ${err.statusCode ?? err} fetching inReplyTo ${note.inReplyTo} for note ${entryUri}`);
throw err;
this.logger.warn(`error ${renderInlineError(err)} fetching inReplyTo ${note.inReplyTo} for note ${entryUri}`);
throw new IdentifiableError('1ebf0a96-2769-4973-a6c2-3dcbad409dff', `failed to update note ${entryUri}: could not fetch inReplyTo ${note.inReplyTo}`, true, err);
})
: null;
@ -482,6 +484,10 @@ export class ApNoteService {
const quote = await this.getQuote(note, entryUri, resolver);
const processErrors = quote === null ? ['quoteUnavailable'] : null;
if (quote && quote.userHost == null && quote.localOnly) {
throw new IdentifiableError('12e23cec-edd9-442b-aa48-9c21f0c3b215', 'Cannot quote a local-only note');
}
// vote
if (reply && reply.hasPoll) {
const poll = await this.pollsRepository.findOneByOrFail({ noteId: reply.id });
@ -538,7 +544,7 @@ export class ApNoteService {
this.logger.info('The note is already inserted while creating itself, reading again');
const duplicate = await this.fetchNote(value);
if (!duplicate) {
throw new Error(`The note creation failed with duplication error even when there is no duplication: ${noteUri}`);
throw new IdentifiableError('39c328e1-e829-458b-bfc9-65dcd513d1f8', `failed to update note ${entryUri}: the note update failed with duplication error even when there is no duplication. This is likely a bug.`);
}
return duplicate;
}
@ -555,8 +561,7 @@ export class ApNoteService {
const uri = getApId(value);
if (!this.utilityService.isFederationAllowedUri(uri)) {
// TODO convert to identifiable error
throw new StatusError(`blocked host: ${uri}`, 451, 'blocked host');
throw new IdentifiableError('04620a7e-044e-45ce-b72c-10e1bdc22e69', `failed to resolve note ${uri}: host is blocked`);
}
//#region このサーバーに既に登録されていたらそれを返す
@ -566,8 +571,7 @@ export class ApNoteService {
// Bail if local URI doesn't exist
if (this.utilityService.isUriLocal(uri)) {
// TODO convert to identifiable error
throw new StatusError(`cannot resolve local note: ${uri}`, 400, 'cannot resolve local note');
throw new IdentifiableError('cbac7358-23f2-4c70-833e-cffb4bf77913', `failed to resolve note ${uri}: URL is local and does not exist`);
}
const unlock = await this.appLockService.getApLock(uri);
@ -674,18 +678,13 @@ export class ApNoteService {
const quote = await this.resolveNote(uri, { resolver });
if (quote == null) {
this.logger.warn(`Failed to resolve quote "${uri}" for note "${entryUri}": request error`);
this.logger.warn(`Failed to resolve quote "${uri}" for note "${entryUri}": fetch failed`);
return false;
}
return quote;
} catch (e) {
if (e instanceof Error) {
this.logger.warn(`Failed to resolve quote "${uri}" for note "${entryUri}":`, e);
} else {
this.logger.warn(`Failed to resolve quote "${uri}" for note "${entryUri}": ${e}`);
}
this.logger.warn(`Failed to resolve quote "${uri}" for note "${entryUri}": ${renderInlineError(e)}`);
return isRetryableError(e);
}
};

170
packages/backend/src/core/activitypub/models/ApPersonService.ts
View file

@ -7,7 +7,6 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
import promiseLimit from 'promise-limit';
import { DataSource } from 'typeorm';
import { ModuleRef } from '@nestjs/core';
import { AbortError } from 'node-fetch';
import { UnrecoverableError } from 'bullmq';
import { DI } from '@/di-symbols.js';
import type { FollowingsRepository, InstancesRepository, MiMeta, UserProfilesRepository, UserPublickeysRepository, UsersRepository } from '@/models/_.js';
@ -44,6 +43,8 @@ import { AppLockService } from '@/core/AppLockService.js';
import { MemoryKVCache } from '@/misc/cache.js';
import { HttpRequestService } from '@/core/HttpRequestService.js';
import { verifyFieldLinks } from '@/misc/verify-field-link.js';
import { isRetryableError } from '@/misc/is-retryable-error.js';
import { renderInlineError } from '@/misc/render-inline-error.js';
import { getApId, getApType, isActor, isCollection, isCollectionOrOrderedCollection, isPropertyValue } from '../type.js';
import { extractApHashtags } from './tag.js';
import type { OnModuleInit } from '@nestjs/common';
@ -54,6 +55,7 @@ import type { ApLoggerService } from '../ApLoggerService.js';
import type { ApImageService } from './ApImageService.js';
import type { IActor, ICollection, IObject, IOrderedCollection } from '../type.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
const nameLength = 128;
const summaryLength = 2048;
@ -157,21 +159,21 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
const expectHost = this.utilityService.punyHostPSLDomain(uri);
if (!isActor(x)) {
throw new UnrecoverableError(`invalid Actor type '${x.type}' in ${uri}`);
throw new UnrecoverableError(`invalid Actor ${uri}: unknown type '${x.type}'`);
}
if (!(typeof x.id === 'string' && x.id.length > 0)) {
throw new UnrecoverableError(`invalid Actor ${uri} - wrong id type`);
throw new UnrecoverableError(`invalid Actor ${uri}: wrong id type`);
}
if (!(typeof x.inbox === 'string' && x.inbox.length > 0)) {
throw new UnrecoverableError(`invalid Actor ${uri} - wrong inbox type`);
throw new UnrecoverableError(`invalid Actor ${uri}: wrong inbox type`);
}
this.apUtilityService.assertApUrl(x.inbox);
const inboxHost = this.utilityService.punyHostPSLDomain(x.inbox);
if (inboxHost !== expectHost) {
throw new UnrecoverableError(`invalid Actor ${uri} - wrong inbox ${inboxHost}`);
throw new UnrecoverableError(`invalid Actor ${uri}: wrong inbox host ${inboxHost}`);
}
const sharedInboxObject = x.sharedInbox ?? (x.endpoints ? x.endpoints.sharedInbox : undefined);
@ -179,7 +181,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
const sharedInbox = getApId(sharedInboxObject);
this.apUtilityService.assertApUrl(sharedInbox);
if (!(typeof sharedInbox === 'string' && sharedInbox.length > 0 && this.utilityService.punyHostPSLDomain(sharedInbox) === expectHost)) {
throw new UnrecoverableError(`invalid Actor ${uri} - wrong shared inbox ${sharedInbox}`);
throw new UnrecoverableError(`invalid Actor ${uri}: wrong shared inbox ${sharedInbox}`);
}
}
@ -190,7 +192,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
if (typeof collectionUri === 'string' && collectionUri.length > 0) {
this.apUtilityService.assertApUrl(collectionUri);
if (this.utilityService.punyHostPSLDomain(collectionUri) !== expectHost) {
throw new UnrecoverableError(`invalid Actor ${uri} - wrong ${collection} ${collectionUri}`);
throw new UnrecoverableError(`invalid Actor ${uri}: wrong ${collection} host ${collectionUri}`);
}
} else if (collectionUri != null) {
throw new UnrecoverableError(`invalid Actor ${uri}: wrong ${collection} type`);
@ -199,7 +201,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
}
if (!(typeof x.preferredUsername === 'string' && x.preferredUsername.length > 0 && x.preferredUsername.length <= 128 && /^\w([\w-.]*\w)?$/.test(x.preferredUsername))) {
throw new UnrecoverableError(`invalid Actor ${uri} - wrong username`);
throw new UnrecoverableError(`invalid Actor ${uri}: wrong username`);
}
// These fields are only informational, and some AP software allows these
@ -207,7 +209,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
// we can at least see these users and their activities.
if (x.name) {
if (!(typeof x.name === 'string' && x.name.length > 0)) {
throw new UnrecoverableError(`invalid Actor ${uri} - wrong name`);
throw new UnrecoverableError(`invalid Actor ${uri}: wrong name`);
}
x.name = truncate(x.name, nameLength);
} else if (x.name === '') {
@ -216,24 +218,24 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
}
if (x.summary) {
if (!(typeof x.summary === 'string' && x.summary.length > 0)) {
throw new UnrecoverableError(`invalid Actor ${uri} - wrong summary`);
throw new UnrecoverableError(`invalid Actor ${uri}: wrong summary`);
}
x.summary = truncate(x.summary, summaryLength);
}
const idHost = this.utilityService.punyHostPSLDomain(x.id);
if (idHost !== expectHost) {
throw new UnrecoverableError(`invalid Actor ${uri} - wrong id ${x.id}`);
throw new UnrecoverableError(`invalid Actor ${uri}: wrong id ${x.id}`);
}
if (x.publicKey) {
if (typeof x.publicKey.id !== 'string') {
throw new UnrecoverableError(`invalid Actor ${uri} - wrong publicKey.id type`);
throw new UnrecoverableError(`invalid Actor ${uri}: wrong publicKey.id type`);
}
const publicKeyIdHost = this.utilityService.punyHostPSLDomain(x.publicKey.id);
if (publicKeyIdHost !== expectHost) {
throw new UnrecoverableError(`invalid Actor ${uri} - wrong publicKey.id ${x.publicKey.id}`);
throw new UnrecoverableError(`invalid Actor ${uri}: wrong publicKey.id ${x.publicKey.id}`);
}
}
@ -271,8 +273,6 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
}
private async resolveAvatarAndBanner(user: MiRemoteUser, icon: any, image: any, bgimg: any): Promise<Partial<Pick<MiRemoteUser, 'avatarId' | 'bannerId' | 'backgroundId' | 'avatarUrl' | 'bannerUrl' | 'backgroundUrl' | 'avatarBlurhash' | 'bannerBlurhash' | 'backgroundBlurhash'>>> {
if (user == null) throw new Error('failed to create user: user is null');
const [avatar, banner, background] = await Promise.all([icon, image, bgimg].map(img => {
// icon and image may be arrays
// see https://www.w3.org/TR/activitystreams-vocabulary/#dfn-icon
@ -325,12 +325,11 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
*/
@bindThis
public async createPerson(uri: string, resolver?: Resolver): Promise<MiRemoteUser> {
if (typeof uri !== 'string') throw new UnrecoverableError(`uri is not string: ${uri}`);
if (typeof uri !== 'string') throw new UnrecoverableError(`failed to create user ${uri}: input is not string`);
const host = this.utilityService.punyHost(uri);
if (host === this.utilityService.toPuny(this.config.host)) {
// TODO convert to unrecoverable error
throw new StatusError(`cannot resolve local user: ${uri}`, 400, 'cannot resolve local user');
throw new UnrecoverableError(`failed to create user ${uri}: URI is local`);
}
return await this._createPerson(uri, resolver);
@ -340,8 +339,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
const uri = getApId(value);
const host = this.utilityService.punyHost(uri);
// eslint-disable-next-line no-param-reassign
if (resolver == null) resolver = this.apResolverService.createResolver();
resolver ??= this.apResolverService.createResolver();
const object = await resolver.resolve(value);
const person = this.validateActor(object, uri);
@ -356,14 +354,16 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
const [followingVisibility, followersVisibility] = await Promise.all(
[
this.isPublicCollection(person.following, resolver),
this.isPublicCollection(person.followers, resolver),
this.isPublicCollection(person.following, resolver, uri),
this.isPublicCollection(person.followers, resolver, uri),
].map((p): Promise<'public' | 'private'> => p
.then(isPublic => isPublic ? 'public' : 'private')
.catch(err => {
if (!(err instanceof StatusError) || err.isRetryable) {
this.logger.error('error occurred while fetching following/followers collection', { stack: err });
// Permanent error implies hidden or inaccessible, which is a normal thing.
if (isRetryableError(err)) {
this.logger.error(`error occurred while fetching following/followers collection: ${renderInlineError(err)}`);
}
return 'private';
}),
),
@ -372,7 +372,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
const bday = person['vcard:bday']?.match(/^\d{4}-\d{2}-\d{2}/);
if (person.id == null) {
throw new UnrecoverableError(`Refusing to create person without id: ${uri}`);
throw new UnrecoverableError(`failed to create user ${uri}: missing ID`);
}
const url = this.apUtilityService.findBestObjectUrl(person);
@ -387,16 +387,27 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
const emojis = await this.apNoteService.extractEmojis(person.tag ?? [], host)
.then(_emojis => _emojis.map(emoji => emoji.name))
.catch(err => {
this.logger.error('error occurred while fetching user emojis', { stack: err });
// Permanent error implies hidden or inaccessible, which is a normal thing.
if (isRetryableError(err)) {
this.logger.error(`error occurred while fetching user emojis: ${renderInlineError(err)}`);
}
return [];
});
//#endregion
//#region resolve counts
const _resolver = resolver ?? this.apResolverService.createResolver();
const outboxcollection = await _resolver.resolveCollection(person.outbox).catch(() => { return null; });
const followerscollection = await _resolver.resolveCollection(person.followers!).catch(() => { return null; });
const followingcollection = await _resolver.resolveCollection(person.following!).catch(() => { return null; });
const outboxCollection = person.outbox
? await resolver.resolveCollection(person.outbox, true, uri).catch(() => { return null; })
: null;
const followersCollection = person.followers
? await resolver.resolveCollection(person.followers, true, uri).catch(() => { return null; })
: null;
const followingCollection = person.following
? await resolver.resolveCollection(person.following, true, uri).catch(() => { return null; })
: null;
// Register the instance first, to avoid FK errors
await this.federatedInstanceService.fetchOrRegister(host);
try {
// Start transaction
@ -423,9 +434,9 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
host,
inbox: person.inbox,
sharedInbox: person.sharedInbox ?? person.endpoints?.sharedInbox ?? null,
notesCount: outboxcollection?.totalItems ?? 0,
followersCount: followerscollection?.totalItems ?? 0,
followingCount: followingcollection?.totalItems ?? 0,
notesCount: outboxCollection?.totalItems ?? 0,
followersCount: followersCollection?.totalItems ?? 0,
followingCount: followingCollection?.totalItems ?? 0,
followersUri: person.followers ? getApId(person.followers) : undefined,
featured: person.featured ? getApId(person.featured) : undefined,
uri: person.id,
@ -437,6 +448,11 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
makeNotesFollowersOnlyBefore: (person as any).makeNotesFollowersOnlyBefore ?? null,
makeNotesHiddenBefore: (person as any).makeNotesHiddenBefore ?? null,
emojis,
attributionDomains: Array.isArray(person.attributionDomains)
? person.attributionDomains
.filter((a: unknown) => typeof(a) === 'string' && a.length > 0 && a.length <= 128)
.slice(0, 32)
: [],
})) as MiRemoteUser;
let _description: string | null = null;
@ -480,7 +496,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
user = u as MiRemoteUser;
publicKey = await this.userPublickeysRepository.findOneBy({ userId: user.id });
} else {
this.logger.error(e instanceof Error ? e : new Error(e as string));
this.logger.error('Error creating Person:', e instanceof Error ? e : new Error(e as string));
throw e;
}
}
@ -520,11 +536,19 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
// Register to the cache
this.cacheService.uriPersonCache.set(user.uri, user);
} catch (err) {
this.logger.error('error occurred while fetching user avatar/banner', { stack: err });
// Permanent error implies hidden or inaccessible, which is a normal thing.
if (isRetryableError(err)) {
this.logger.error(`error occurred while fetching user avatar/banner: ${renderInlineError(err)}`);
}
}
//#endregion
await this.updateFeatured(user.id, resolver).catch(err => this.logger.error(err));
await this.updateFeatured(user.id, resolver).catch(err => {
// Permanent error implies hidden or inaccessible, which is a normal thing.
if (isRetryableError(err)) {
this.logger.error(`Error updating featured notes: ${renderInlineError(err)}`);
}
});
return user;
}
@ -541,7 +565,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
*/
@bindThis
public async updatePerson(uri: string, resolver?: Resolver | null, hint?: IObject, movePreventUris: string[] = []): Promise<string | void> {
if (typeof uri !== 'string') throw new UnrecoverableError('uri is not string');
if (typeof uri !== 'string') throw new UnrecoverableError(`failed to update user ${uri}: input is not string`);
// URIがこのサーバーを指しているならスキップ
if (this.utilityService.isUriLocal(uri)) return;
@ -561,8 +585,11 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
this.logger.info(`Updating the Person: ${person.id}`);
// カスタム絵文字取得
const emojis = await this.apNoteService.extractEmojis(person.tag ?? [], exist.host).catch(e => {
this.logger.info(`extractEmojis: ${e}`);
const emojis = await this.apNoteService.extractEmojis(person.tag ?? [], exist.host).catch(err => {
// Permanent error implies hidden or inaccessible, which is a normal thing.
if (isRetryableError(err)) {
this.logger.error(`error occurred while fetching user emojis: ${renderInlineError(err)}`);
}
return [];
});
@ -574,16 +601,18 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
const [followingVisibility, followersVisibility] = await Promise.all(
[
this.isPublicCollection(person.following, resolver),
this.isPublicCollection(person.followers, resolver),
this.isPublicCollection(person.following, resolver, exist.uri),
this.isPublicCollection(person.followers, resolver, exist.uri),
].map((p): Promise<'public' | 'private' | undefined> => p
.then(isPublic => isPublic ? 'public' : 'private')
.catch(err => {
if (!(err instanceof StatusError) || err.isRetryable) {
this.logger.error('error occurred while fetching following/followers collection', { stack: err });
// Permanent error implies hidden or inaccessible, which is a normal thing.
if (isRetryableError(err)) {
this.logger.error(`error occurred while fetching following/followers collection: ${renderInlineError(err)}`);
// Do not update the visibility on transient errors.
return undefined;
}
return 'private';
}),
),
@ -592,7 +621,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
const bday = person['vcard:bday']?.match(/^\d{4}-\d{2}-\d{2}/);
if (person.id == null) {
throw new UnrecoverableError(`Refusing to update person without id: ${uri}`);
throw new UnrecoverableError(`failed to update user ${uri}: missing ID`);
}
const url = this.apUtilityService.findBestObjectUrl(person);
@ -620,7 +649,20 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
// We use "!== false" to handle incorrect types, missing / null values, and "default to true" logic.
hideOnlineStatus: person.hideOnlineStatus !== false,
isExplorable: person.discoverable !== false,
...(await this.resolveAvatarAndBanner(exist, person.icon, person.image, person.backgroundUrl).catch(() => ({}))),
attributionDomains: Array.isArray(person.attributionDomains)
? person.attributionDomains
.filter((a: unknown) => typeof(a) === 'string' && a.length > 0 && a.length <= 128)
.slice(0, 32)
: [],
...(await this.resolveAvatarAndBanner(exist, person.icon, person.image, person.backgroundUrl).catch(err => {
// Permanent error implies hidden or inaccessible, which is a normal thing.
if (isRetryableError(err)) {
this.logger.error(`error occurred while fetching user avatar/banner: ${renderInlineError(err)}`);
}
// Can't return null or destructuring operator will break
return {};
})),
} as Partial<MiRemoteUser> & Pick<MiRemoteUser, 'isBot' | 'isCat' | 'speakAsCat' | 'isLocked' | 'movedToUri' | 'alsoKnownAs' | 'isExplorable'>;
const moving = ((): boolean => {
@ -704,7 +746,12 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
{ followerSharedInbox: person.sharedInbox ?? person.endpoints?.sharedInbox ?? null },
);
await this.updateFeatured(exist.id, resolver).catch(err => this.logger.error(err));
await this.updateFeatured(exist.id, resolver).catch(err => {
// Permanent error implies hidden or inaccessible, which is a normal thing.
if (isRetryableError(err)) {
this.logger.error(`Error updating featured notes: ${renderInlineError(err)}`);
}
});
const updated = { ...exist, ...updates };
@ -743,8 +790,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
const uri = getApId(value);
if (!this.utilityService.isFederationAllowedUri(uri)) {
// TODO convert to identifiable error
throw new StatusError(`blocked host: ${uri}`, 451, 'blocked host');
throw new IdentifiableError('590719b3-f51f-48a9-8e7d-6f559ad00e5d', `failed to resolve person ${uri}: host is blocked`);
}
//#region このサーバーに既に登録されていたらそれを返す
@ -754,8 +800,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
// Bail if local URI doesn't exist
if (this.utilityService.isUriLocal(uri)) {
// TODO convert to identifiable error
throw new StatusError(`cannot resolve local person: ${uri}`, 400, 'cannot resolve local person');
throw new IdentifiableError('efb573fd-6b9e-4912-9348-a02f5603df4f', `failed to resolve person ${uri}: URL is local and does not exist`);
}
const unlock = await this.appLockService.getApLock(uri);
@ -799,16 +844,17 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
const _resolver = resolver ?? this.apResolverService.createResolver();
// Resolve to (Ordered)Collection Object
const collection = await _resolver.resolveCollection(user.featured).catch(err => {
if (err instanceof AbortError || err instanceof StatusError) {
this.logger.warn(`Failed to update featured notes: ${err.name}: ${err.message}`);
} else {
this.logger.error('Failed to update featured notes:', err);
const collection = user.featured ? await _resolver.resolveCollection(user.featured, true, user.uri).catch(err => {
// Permanent error implies hidden or inaccessible, which is a normal thing.
if (isRetryableError(err)) {
this.logger.warn(`Failed to update featured notes: ${renderInlineError(err)}`);
}
});
return null;
}) : null;
if (!collection) return;
if (!isCollectionOrOrderedCollection(collection)) throw new UnrecoverableError(`featured ${user.featured} is not Collection or OrderedCollection in ${user.uri}`);
if (!isCollectionOrOrderedCollection(collection)) throw new UnrecoverableError(`failed to update user ${user.uri}: featured ${user.featured} is not Collection or OrderedCollection`);
// Resolve to Object(may be Note) arrays
const unresolvedItems = isCollection(collection) ? collection.items : collection.orderedItems;
@ -891,11 +937,13 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
}
@bindThis
private async isPublicCollection(collection: string | ICollection | IOrderedCollection | undefined, resolver: Resolver): Promise<boolean> {
private async isPublicCollection(collection: string | ICollection | IOrderedCollection | undefined, resolver: Resolver, sentFrom: string): Promise<boolean> {
if (collection) {
const resolved = await resolver.resolveCollection(collection);
if (resolved.first || (resolved as ICollection).items || (resolved as IOrderedCollection).orderedItems) {
return true;
const resolved = await resolver.resolveCollection(collection, true, sentFrom).catch(() => null);
if (resolved) {
if (resolved.first || (resolved as ICollection).items || (resolved as IOrderedCollection).orderedItems) {
return true;
}
}
}

1
packages/backend/src/core/activitypub/models/ApQuestionService.ts
View file

@ -93,7 +93,6 @@ export class ApQuestionService {
// eslint-disable-next-line no-param-reassign
if (resolver == null) resolver = this.apResolverService.createResolver();
const question = await resolver.resolve(value);
this.logger.debug(`fetched question: ${JSON.stringify(question, null, 2)}`);
if (!isQuestion(question)) throw new UnrecoverableError(`object ${getApType(question)} is not a Question: ${uri}`);

93
packages/backend/src/core/activitypub/type.ts
View file

@ -43,6 +43,18 @@ export interface IObjectWithId extends IObject {
id: string;
}
export function isObjectWithId(object: IObject): object is IObjectWithId {
return typeof(object.id) === 'string';
}
export interface IAnonymousObject extends IObject {
id: undefined;
}
export function isAnonymousObject(object: IObject): object is IAnonymousObject {
return object.id === undefined;
}
/**
* Get array of ActivityStreams Objects id
*/
@ -63,24 +75,34 @@ export function getOneApId(value: ApObject): string {
/**
* Get ActivityStreams Object id
*/
export function getApId(value: string | IObject | [string | IObject]): string {
// eslint-disable-next-line no-param-reassign
value = fromTuple(value);
export function getApId(value: string | IObject | [string | IObject], sourceForLogs?: string): string {
const id = getNullableApId(value);
if (typeof value === 'string') return value;
if (typeof value.id === 'string') return value.id;
throw new IdentifiableError('ad2dc287-75c1-44c4-839d-3d2e64576675', `invalid AP object ${value}: missing id`);
if (id == null) {
const message = sourceForLogs
? `invalid AP object ${value} (sent from ${sourceForLogs}): missing id`
: `invalid AP object ${value}: missing id`;
throw new IdentifiableError('ad2dc287-75c1-44c4-839d-3d2e64576675', message);
}
return id;
}
/**
* Get ActivityStreams Object id, or null if not present
*/
export function getNullableApId(value: string | IObject | [string | IObject]): string | null {
// eslint-disable-next-line no-param-reassign
value = fromTuple(value);
export function getNullableApId(source: string | IObject | [string | IObject]): string | null {
const value: unknown = fromTuple(source);
if (value != null) {
if (typeof value === 'string') {
return value;
}
if (typeof (value) === 'object' && 'id' in value && typeof (value.id) === 'string') {
return value.id;
}
}
if (typeof value === 'string') return value;
if (typeof value.id === 'string') return value.id;
return null;
}
@ -125,48 +147,46 @@ export interface IActivity extends IObject {
};
}
export interface ICollection extends IObject {
export interface CollectionBase extends IObject {
totalItems?: number;
first?: IObject | string;
last?: IObject | string;
current?: IObject | string;
partOf?: IObject | string;
next?: IObject | string;
prev?: IObject | string;
items?: ApObject;
orderedItems?: ApObject;
}
export interface ICollection extends CollectionBase {
type: 'Collection';
totalItems: number;
first?: IObject | string;
last?: IObject | string;
current?: IObject | string;
items?: ApObject;
orderedItems?: undefined;
}
export interface IOrderedCollection extends IObject {
export interface IOrderedCollection extends CollectionBase {
type: 'OrderedCollection';
totalItems: number;
first?: IObject | string;
last?: IObject | string;
current?: IObject | string;
items?: undefined;
orderedItems?: ApObject;
}
export interface ICollectionPage extends IObject {
export interface ICollectionPage extends CollectionBase {
type: 'CollectionPage';
totalItems: number;
first?: IObject | string;
last?: IObject | string;
current?: IObject | string;
partOf?: IObject | string;
next?: IObject | string;
prev?: IObject | string;
items?: ApObject;
orderedItems?: undefined;
}
export interface IOrderedCollectionPage extends IObject {
export interface IOrderedCollectionPage extends CollectionBase {
type: 'OrderedCollectionPage';
totalItems: number;
first?: IObject | string;
last?: IObject | string;
current?: IObject | string;
partOf?: IObject | string;
next?: IObject | string;
prev?: IObject | string;
items?: undefined;
orderedItems?: ApObject;
}
export type AnyCollection = ICollection | IOrderedCollection | ICollectionPage | IOrderedCollectionPage;
export const validPost = ['Note', 'Question', 'Article', 'Audio', 'Document', 'Image', 'Page', 'Video', 'Event'];
export const isPost = (object: IObject): object is IPost => {
@ -255,6 +275,7 @@ export interface IActor extends IObject {
enableRss?: boolean;
listenbrainz?: string;
backgroundUrl?: string;
attributionDomains?: string[];
}
export const isCollection = (object: IObject): object is ICollection =>
@ -269,7 +290,7 @@ export const isCollectionPage = (object: IObject): object is ICollectionPage =>
export const isOrderedCollectionPage = (object: IObject): object is IOrderedCollectionPage =>
getApType(object) === 'OrderedCollectionPage';
export const isCollectionOrOrderedCollection = (object: IObject): object is ICollection | IOrderedCollection =>
export const isCollectionOrOrderedCollection = (object: IObject): object is AnyCollection =>
isCollection(object) || isOrderedCollection(object) || isCollectionPage(object) || isOrderedCollectionPage(object);
export interface IApPropertyValue extends IObject {

23
packages/backend/src/core/chart/charts/federation.ts
View file

@ -44,10 +44,6 @@ export default class FederationChart extends Chart<typeof schema> { // eslint-di
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
const suspendedInstancesQuery = this.instancesRepository.createQueryBuilder('instance')
.select('instance.host')
.where('instance.suspensionState != \'none\'');
const pubsubSubQuery = this.followingsRepository.createQueryBuilder('f')
.select('f.followerHost')
.where('f.followerHost IS NOT NULL');
@ -64,22 +60,25 @@ export default class FederationChart extends Chart<typeof schema> { // eslint-di
this.followingsRepository.createQueryBuilder('following')
.select('COUNT(DISTINCT following.followeeHost)')
.where('following.followeeHost IS NOT NULL')
.andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || following.followeeHost) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)')
.andWhere(`following.followeeHost NOT IN (${ suspendedInstancesQuery.getQuery() })`)
.innerJoin('following.followeeInstance', 'followeeInstance')
.andWhere('followeeInstance.suspensionState = \'none\'')
.andWhere('followeeInstance.isBlocked = false')
.getRawOne()
.then(x => parseInt(x.count, 10)),
this.followingsRepository.createQueryBuilder('following')
.select('COUNT(DISTINCT following.followerHost)')
.where('following.followerHost IS NOT NULL')
.andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || following.followerHost) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)')
.andWhere(`following.followerHost NOT IN (${ suspendedInstancesQuery.getQuery() })`)
.innerJoin('following.followerInstance', 'followerInstance')
.andWhere('followerInstance.isBlocked = false')
.andWhere('followerInstance.suspensionState = \'none\'')
.getRawOne()
.then(x => parseInt(x.count, 10)),
this.followingsRepository.createQueryBuilder('following')
.select('COUNT(DISTINCT following.followeeHost)')
.where('following.followeeHost IS NOT NULL')
.andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || following.followeeHost) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)')
.andWhere(`following.followeeHost NOT IN (${ suspendedInstancesQuery.getQuery() })`)
.innerJoin('following.followeeInstance', 'followeeInstance')
.andWhere('followeeInstance.isBlocked = false')
.andWhere('followeeInstance.suspensionState = \'none\'')
.andWhere(`following.followeeHost IN (${ pubsubSubQuery.getQuery() })`)
.setParameters(pubsubSubQuery.getParameters())
.getRawOne()
@ -87,7 +86,7 @@ export default class FederationChart extends Chart<typeof schema> { // eslint-di
this.instancesRepository.createQueryBuilder('instance')
.select('COUNT(instance.id)')
.where(`instance.host IN (${ subInstancesQuery.getQuery() })`)
.andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || instance.host) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)')
.andWhere('instance.isBlocked = false')
.andWhere('instance.suspensionState = \'none\'')
.andWhere('instance.isNotResponding = false')
.getRawOne()
@ -95,7 +94,7 @@ export default class FederationChart extends Chart<typeof schema> { // eslint-di
this.instancesRepository.createQueryBuilder('instance')
.select('COUNT(instance.id)')
.where(`instance.host IN (${ pubInstancesQuery.getQuery() })`)
.andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || instance.host) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)')
.andWhere('instance.isBlocked = false')
.andWhere('instance.suspensionState = \'none\'')
.andWhere('instance.isNotResponding = false')
.getRawOne()

36
packages/backend/src/core/entities/AbuseUserReportEntityService.ts
View file

@ -5,13 +5,14 @@
import { Inject, Injectable } from '@nestjs/common';
import { DI } from '@/di-symbols.js';
import type { AbuseUserReportsRepository } from '@/models/_.js';
import type { AbuseUserReportsRepository, InstancesRepository, MiInstance, MiUser } from '@/models/_.js';
import { awaitAll } from '@/misc/prelude/await-all.js';
import type { MiAbuseUserReport } from '@/models/AbuseUserReport.js';
import { bindThis } from '@/decorators.js';
import { IdService } from '@/core/IdService.js';
import type { Packed } from '@/misc/json-schema.js';
import { UserEntityService } from './UserEntityService.js';
import { InstanceEntityService } from './InstanceEntityService.js';
@Injectable()
export class AbuseUserReportEntityService {
@ -19,6 +20,10 @@ export class AbuseUserReportEntityService {
@Inject(DI.abuseUserReportsRepository)
private abuseUserReportsRepository: AbuseUserReportsRepository,
@Inject(DI.instancesRepository)
private instancesRepository: InstancesRepository,
private readonly instanceEntityService: InstanceEntityService,
private userEntityService: UserEntityService,
private idService: IdService,
) {
@ -30,11 +35,14 @@ export class AbuseUserReportEntityService {
hint?: {
packedReporter?: Packed<'UserDetailedNotMe'>,
packedTargetUser?: Packed<'UserDetailedNotMe'>,
packedTargetInstance?: Packed<'FederationInstance'>,
packedAssignee?: Packed<'UserDetailedNotMe'>,
},
me?: MiUser | null,
) {
const report = typeof src === 'object' ? src : await this.abuseUserReportsRepository.findOneByOrFail({ id: src });
// noinspection ES6MissingAwait
return await awaitAll({
id: report.id,
createdAt: this.idService.parse(report.id).date.toISOString(),
@ -43,13 +51,22 @@ export class AbuseUserReportEntityService {
reporterId: report.reporterId,
targetUserId: report.targetUserId,
assigneeId: report.assigneeId,
reporter: hint?.packedReporter ?? this.userEntityService.pack(report.reporter ?? report.reporterId, null, {
reporter: hint?.packedReporter ?? this.userEntityService.pack(report.reporter ?? report.reporterId, me, {
schema: 'UserDetailedNotMe',
}),
targetUser: hint?.packedTargetUser ?? this.userEntityService.pack(report.targetUser ?? report.targetUserId, null, {
targetUser: hint?.packedTargetUser ?? this.userEntityService.pack(report.targetUser ?? report.targetUserId, me, {
schema: 'UserDetailedNotMe',
}),
assignee: report.assigneeId ? hint?.packedAssignee ?? this.userEntityService.pack(report.assignee ?? report.assigneeId, null, {
// return hint, or pack by relation, or fetch and pack by id, or null
targetInstance: hint?.packedTargetInstance ?? (
report.targetUserInstance
? this.instanceEntityService.pack(report.targetUserInstance, me)
: report.targetUserHost
? this.instancesRepository.findOneBy({ host: report.targetUserHost }).then(instance => instance
? this.instanceEntityService.pack(instance, me)
: null)
: null),
assignee: report.assigneeId ? hint?.packedAssignee ?? this.userEntityService.pack(report.assignee ?? report.assigneeId, me, {
schema: 'UserDetailedNotMe',
}) : null,
forwarded: report.forwarded,
@ -61,21 +78,28 @@ export class AbuseUserReportEntityService {
@bindThis
public async packMany(
reports: MiAbuseUserReport[],
me?: MiUser | null,
) {
const _reporters = reports.map(({ reporter, reporterId }) => reporter ?? reporterId);
const _targetUsers = reports.map(({ targetUser, targetUserId }) => targetUser ?? targetUserId);
const _assignees = reports.map(({ assignee, assigneeId }) => assignee ?? assigneeId).filter(x => x != null);
const _userMap = await this.userEntityService.packMany(
[..._reporters, ..._targetUsers, ..._assignees],
null,
me,
{ schema: 'UserDetailedNotMe' },
).then(users => new Map(users.map(u => [u.id, u])));
const _targetInstances = reports
.map(({ targetUserInstance, targetUserHost }) => targetUserInstance ?? targetUserHost)
.filter((i): i is MiInstance | string => i != null);
const _instanceMap = await this.instanceEntityService.packMany(await this.instanceEntityService.fetchInstancesByHost(_targetInstances), me)
.then(instances => new Map(instances.map(i => [i.host, i])));
return Promise.all(
reports.map(report => {
const packedReporter = _userMap.get(report.reporterId);
const packedTargetUser = _userMap.get(report.targetUserId);
const packedTargetInstance = report.targetUserHost ? _instanceMap.get(report.targetUserHost) : undefined;
const packedAssignee = report.assigneeId != null ? _userMap.get(report.assigneeId) : undefined;
return this.pack(report, { packedReporter, packedTargetUser, packedAssignee });
return this.pack(report, { packedReporter, packedTargetUser, packedAssignee, packedTargetInstance }, me);
}),
);
}

36
packages/backend/src/core/entities/InstanceEntityService.ts
View file

@ -4,6 +4,7 @@
*/
import { Inject, Injectable } from '@nestjs/common';
import { In } from 'typeorm';
import type { Packed } from '@/misc/json-schema.js';
import type { MiInstance } from '@/models/Instance.js';
import { bindThis } from '@/decorators.js';
@ -11,7 +12,7 @@ import { UtilityService } from '@/core/UtilityService.js';
import { RoleService } from '@/core/RoleService.js';
import { MiUser } from '@/models/User.js';
import { DI } from '@/di-symbols.js';
import { MiMeta } from '@/models/_.js';
import type { InstancesRepository, MiMeta } from '@/models/_.js';
@Injectable()
export class InstanceEntityService {
@ -19,6 +20,9 @@ export class InstanceEntityService {
@Inject(DI.meta)
private meta: MiMeta,
@Inject(DI.instancesRepository)
private readonly instancesRepository: InstancesRepository,
private roleService: RoleService,
private utilityService: UtilityService,
@ -44,7 +48,7 @@ export class InstanceEntityService {
isNotResponding: instance.isNotResponding,
isSuspended: instance.suspensionState !== 'none' || Boolean(softwareSuspended),
suspensionState: instance.suspensionState === 'none' && softwareSuspended ? 'softwareSuspended' : instance.suspensionState,
isBlocked: this.utilityService.isBlockedHost(this.meta.blockedHosts, instance.host),
isBlocked: instance.isBlocked,
softwareName: instance.softwareName,
softwareVersion: instance.softwareVersion,
openRegistrations: instance.openRegistrations,
@ -52,8 +56,8 @@ export class InstanceEntityService {
description: instance.description,
maintainerName: instance.maintainerName,
maintainerEmail: instance.maintainerEmail,
isSilenced: this.utilityService.isSilencedHost(this.meta.silencedHosts, instance.host),
isMediaSilenced: this.utilityService.isMediaSilencedHost(this.meta.mediaSilencedHosts, instance.host),
isSilenced: instance.isSilenced,
isMediaSilenced: instance.isMediaSilenced,
iconUrl: instance.iconUrl,
faviconUrl: instance.faviconUrl,
themeColor: instance.themeColor,
@ -63,6 +67,7 @@ export class InstanceEntityService {
rejectReports: instance.rejectReports,
rejectQuotes: instance.rejectQuotes,
moderationNote: iAmModerator ? instance.moderationNote : null,
isBubbled: this.utilityService.isBubbledHost(instance.host),
};
}
@ -73,5 +78,28 @@ export class InstanceEntityService {
) {
return Promise.all(instances.map(x => this.pack(x, me)));
}
@bindThis
public async fetchInstancesByHost(instances: (MiInstance | MiInstance['host'])[]): Promise<MiInstance[]> {
const result: MiInstance[] = [];
const toFetch: string[] = [];
for (const instance of instances) {
if (typeof(instance) === 'string') {
toFetch.push(instance);
} else {
result.push(instance);
}
}
if (toFetch.length > 0) {
const fetched = await this.instancesRepository.findBy({
host: In(toFetch),
});
result.push(...fetched);
}
return result;
}
}

9
packages/backend/src/core/entities/NoteEntityService.ts
View file

@ -17,6 +17,7 @@ import { DebounceLoader } from '@/misc/loader.js';
import { IdService } from '@/core/IdService.js';
import { ReactionsBufferingService } from '@/core/ReactionsBufferingService.js';
import { isPackedPureRenote } from '@/misc/is-renote.js';
import type { Config } from '@/config.js';
import type { OnModuleInit } from '@nestjs/common';
import type { CacheService } from '../CacheService.js';
import type { CustomEmojiService } from '../CustomEmojiService.js';
@ -92,6 +93,9 @@ export class NoteEntityService implements OnModuleInit {
@Inject(DI.channelsRepository)
private channelsRepository: ChannelsRepository,
@Inject(DI.config)
private readonly config: Config,
//private userEntityService: UserEntityService,
//private driveFileEntityService: DriveFileEntityService,
//private customEmojiService: CustomEmojiService,
@ -680,4 +684,9 @@ export class NoteEntityService implements OnModuleInit {
return map;
}, {} as Record<string, string | undefined>);
}
@bindThis
public genLocalNoteUri(noteId: string): string {
return `${this.config.url}/notes/${noteId}`;
}
}

38
packages/backend/src/core/entities/UserEntityService.ts
View file

@ -487,7 +487,10 @@ export class UserEntityService implements OnModuleInit {
includeSecrets: false,
}, options);
const user = typeof src === 'object' ? src : await this.usersRepository.findOneByOrFail({ id: src });
const user = typeof src === 'object' ? src : await this.usersRepository.findOneOrFail({
where: { id: src },
relations: { userProfile: true },
});
// migration
if (user.avatarId != null && user.avatarUrl === null) {
@ -521,7 +524,7 @@ export class UserEntityService implements OnModuleInit {
const iAmModerator = me ? await this.roleService.isModerator(me as MiUser) : false;
const profile = isDetailed
? (opts.userProfile ?? await this.userProfilesRepository.findOneByOrFail({ userId: user.id }))
? (opts.userProfile ?? user.userProfile ?? await this.userProfilesRepository.findOneByOrFail({ userId: user.id }))
: null;
let relation: UserRelation | null = null;
@ -556,7 +559,7 @@ export class UserEntityService implements OnModuleInit {
}
}
const mastoapi = !isDetailed ? opts.userProfile ?? await this.userProfilesRepository.findOneByOrFail({ userId: user.id }) : null;
const mastoapi = !isDetailed ? opts.userProfile ?? user.userProfile ?? await this.userProfilesRepository.findOneByOrFail({ userId: user.id }) : null;
const followingCount = profile == null ? null :
(profile.followingVisibility === 'public') || isMe || iAmModerator ? user.followingCount :
@ -603,19 +606,21 @@ export class UserEntityService implements OnModuleInit {
enableRss: user.enableRss,
mandatoryCW: user.mandatoryCW,
rejectQuotes: user.rejectQuotes,
attributionDomains: user.attributionDomains,
isSilenced: user.isSilenced || this.roleService.getUserPolicies(user.id).then(r => !r.canPublicNote),
speakAsCat: user.speakAsCat ?? false,
approved: user.approved,
requireSigninToViewContents: user.requireSigninToViewContents === false ? undefined : true,
makeNotesFollowersOnlyBefore: user.makeNotesFollowersOnlyBefore ?? undefined,
makeNotesHiddenBefore: user.makeNotesHiddenBefore ?? undefined,
instance: user.host ? this.federatedInstanceService.federatedInstanceCache.fetch(user.host).then(instance => instance ? {
instance: user.host ? this.federatedInstanceService.fetch(user.host).then(instance => instance ? {
name: instance.name,
softwareName: instance.softwareName,
softwareVersion: instance.softwareVersion,
iconUrl: instance.iconUrl,
faviconUrl: instance.faviconUrl,
themeColor: instance.themeColor,
isSilenced: instance.isSilenced,
} : undefined) : undefined,
followersCount: followersCount ?? 0,
followingCount: followingCount ?? 0,
@ -783,8 +788,13 @@ export class UserEntityService implements OnModuleInit {
const _users = users.filter((user): user is MiUser => typeof user !== 'string');
if (_users.length !== users.length) {
_users.push(
...await this.usersRepository.findBy({
id: In(users.filter((user): user is string => typeof user === 'string')),
...await this.usersRepository.find({
where: {
id: In(users.filter((user): user is string => typeof user === 'string')),
},
relations: {
userProfile: true,
},
}),
);
}
@ -798,8 +808,20 @@ export class UserEntityService implements OnModuleInit {
let pinNotes: Map<MiUser['id'], MiUserNotePining[]> = new Map();
if (options?.schema !== 'UserLite') {
profilesMap = await this.userProfilesRepository.findBy({ userId: In(_userIds) })
.then(profiles => new Map(profiles.map(p => [p.userId, p])));
const _profiles: MiUserProfile[] = [];
const _profilesToFetch: string[] = [];
for (const user of _users) {
if (user.userProfile) {
_profiles.push(user.userProfile);
} else {
_profilesToFetch.push(user.id);
}
}
if (_profilesToFetch.length > 0) {
const fetched = await this.userProfilesRepository.findBy({ userId: In(_profilesToFetch) });
_profiles.push(...fetched);
}
profilesMap = new Map(_profiles.map(p => [p.userId, p]));
const meId = me ? me.id : null;
if (meId) {

1
packages/backend/src/env.ts
View file

@ -11,6 +11,7 @@ const envOption = {
verbose: false,
withLogTime: false,
quiet: false,
hideWorkerId: false,
};
for (const key of Object.keys(envOption) as (keyof typeof envOption)[]) {

4
packages/backend/src/logger.ts
View file

@ -71,7 +71,9 @@ export default class Logger {
level === 'info' ? message :
null;
let log = `${l} ${worker}\t[${contexts.join(' ')}]\t${m}`;
let log = envOption.hideWorkerId
? `${l}\t[${contexts.join(' ')}]\t\t${m}`
: `${l} ${worker}\t[${contexts.join(' ')}]\t\t${m}`;
if (envOption.withLogTime) log = chalk.gray(time) + ' ' + log;
const args: unknown[] = [important ? chalk.bold(log) : log];

2
packages/backend/src/misc/FileWriterStream.ts
View file

@ -21,7 +21,7 @@ export class FileWriterStream extends WritableStream<Uint8Array> {
write: async (chunk, controller) => {
if (file === null) {
controller.error();
throw new Error();
throw new Error('file is null');
}
await file.write(chunk);

9
packages/backend/src/misc/cache.ts
View file

@ -308,8 +308,17 @@ export class MemoryKVCache<T> {
}
}
/**
* Removes all entries from the cache, but does not dispose it.
*/
@bindThis
public clear(): void {
this.cache.clear();
}
@bindThis
public dispose(): void {
this.clear();
clearInterval(this.gcIntervalHandle);
}

102
packages/backend/src/misc/diff-arrays.ts Normal file
View file

@ -0,0 +1,102 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export interface DiffResult<T> {
added: T[];
removed: T[];
}
/**
* Calculates the difference between two snapshots of data.
* Null, undefined, and empty arrays are supported, and duplicate values are ignored.
* Result sets are de-duplicated, and will be empty if no data was added or removed (respectively).
* The inputs are treated as un-ordered, so a re-ordering of the same data will NOT be considered a change.
* @param dataBefore Array containing data before the change
* @param dataAfter Array containing data after the change
*/
export function diffArrays<T>(dataBefore: T[] | null | undefined, dataAfter: T[] | null | undefined): DiffResult<T> {
const before = dataBefore ? new Set(dataBefore) : null;
const after = dataAfter ? new Set(dataAfter) : null;
// data before AND after => changed
if (before?.size && after?.size) {
const added: T[] = [];
const removed: T[] = [];
for (const host of before) {
// before and NOT after => removed
// delete operation removes duplicates to speed up the "after" loop
if (!after.delete(host)) {
removed.push(host);
}
}
for (const host of after) {
// after and NOT before => added
if (!before.has(host)) {
added.push(host);
}
}
return { added, removed };
}
// data ONLY before => all removed
if (before?.size) {
return { added: [], removed: Array.from(before) };
}
// data ONLY after => all added
if (after?.size) {
return { added: Array.from(after), removed: [] };
}
// data NEITHER before nor after => no change
return { added: [], removed: [] };
}
/**
* Checks for any difference between two snapshots of data.
* Null, undefined, and empty arrays are supported, and duplicate values are ignored.
* The inputs are treated as un-ordered, so a re-ordering of the same data will NOT be considered a change.
* @param dataBefore Array containing data before the change
* @param dataAfter Array containing data after the change
*/
export function diffArraysSimple<T>(dataBefore: T[] | null | undefined, dataAfter: T[] | null | undefined): boolean {
const before = dataBefore ? new Set(dataBefore) : null;
const after = dataAfter ? new Set(dataAfter) : null;
if (before?.size && after?.size) {
// different size => changed
if (before.size !== after.size) return true;
// removed => changed
for (const host of before) {
// delete operation removes duplicates to speed up the "after" loop
if (!after.delete(host)) {
return true;
}
}
// added => changed
for (const host of after) {
if (!before.has(host)) {
return true;
}
}
// identical values => no change
return false;
}
// before and NOT after => change
if (before?.size) return true;
// after and NOT before => change
if (after?.size) return true;
// NEITHER before nor after => no change
return false;
}

4
packages/backend/src/misc/fastify-reply-error.ts
View file

@ -8,8 +8,8 @@ export class FastifyReplyError extends Error {
public message: string;
public statusCode: number;
constructor(statusCode: number, message: string) {
super(message);
constructor(statusCode: number, message: string, cause?: unknown) {
super(message, cause ? { cause } : undefined);
this.message = message;
this.statusCode = statusCode;
}

3
packages/backend/src/misc/id/aid.ts
View file

@ -8,6 +8,7 @@
import * as crypto from 'node:crypto';
import { parseBigInt36 } from '@/misc/bigint.js';
import { IdentifiableError } from '../identifiable-error.js';
export const aidRegExp = /^[0-9a-z]{10}$/;
@ -26,7 +27,7 @@ function getNoise(): string {
}
export function genAid(t: number): string {
if (isNaN(t)) throw new Error('Failed to create AID: Invalid Date');
if (isNaN(t)) throw new IdentifiableError('6b73b7d5-9d2b-48b4-821c-ef955efe80ad', 'Failed to create AID: Invalid Date');
counter++;
return getTime(t) + getNoise();
}

3
packages/backend/src/misc/id/aidx.ts
View file

@ -10,6 +10,7 @@
import { customAlphabet } from 'nanoid';
import { parseBigInt36 } from '@/misc/bigint.js';
import { IdentifiableError } from '../identifiable-error.js';
export const aidxRegExp = /^[0-9a-z]{16}$/;
@ -34,7 +35,7 @@ function getNoise(): string {
}
export function genAidx(t: number): string {
if (isNaN(t)) throw new Error('Failed to create AIDX: Invalid Date');
if (isNaN(t)) throw new IdentifiableError('6b73b7d5-9d2b-48b4-821c-ef955efe80ad', 'Failed to create AIDX: Invalid Date');
counter++;
return getTime(t) + nodeId + getNoise();
}

4
packages/backend/src/misc/identifiable-error.ts
View file

@ -15,8 +15,8 @@ export class IdentifiableError extends Error {
*/
public readonly isRetryable: boolean;
constructor(id: string, message?: string, isRetryable = false) {
super(message);
constructor(id: string, message?: string, isRetryable = false, cause?: unknown) {
super(message, cause ? { cause } : undefined);
this.message = message ?? '';
this.id = id;
this.isRetryable = isRetryable;

16
packages/backend/src/misc/is-retryable-error.ts
View file

@ -3,20 +3,34 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { AbortError } from 'node-fetch';
import { AbortError, FetchError } from 'node-fetch';
import { UnrecoverableError } from 'bullmq';
import { StatusError } from '@/misc/status-error.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import { CaptchaError, captchaErrorCodes } from '@/core/CaptchaService.js';
import { FastifyReplyError } from '@/misc/fastify-reply-error.js';
import { ConflictError } from '@/server/SkRateLimiterService.js';
/**
* Returns false if the provided value represents a "permanent" error that cannot be retried.
* Returns true if the error is retryable, unknown (as all errors are retryable by default), or not an error object.
*/
export function isRetryableError(e: unknown): boolean {
if (e instanceof AggregateError) return e.errors.every(inner => isRetryableError(inner));
if (e instanceof StatusError) return e.isRetryable;
if (e instanceof IdentifiableError) return e.isRetryable;
if (e instanceof CaptchaError) {
if (e.code === captchaErrorCodes.verificationFailed) return false;
if (e.code === captchaErrorCodes.invalidParameters) return false;
if (e.code === captchaErrorCodes.invalidProvider) return false;
return true;
}
if (e instanceof FastifyReplyError) return false;
if (e instanceof ConflictError) return true;
if (e instanceof UnrecoverableError) return false;
if (e instanceof AbortError) return true;
if (e instanceof FetchError) return true;
if (e instanceof SyntaxError) return false;
if (e instanceof Error) return e.name === 'AbortError';
return true;
}

60
packages/backend/src/misc/render-full-error.ts Normal file
View file

@ -0,0 +1,60 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
import * as Bull from 'bullmq';
import { AbortError, FetchError } from 'node-fetch';
import { StatusError } from '@/misc/status-error.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import { renderInlineError } from '@/misc/render-inline-error.js';
import { CaptchaError, captchaErrorCodes } from '@/core/CaptchaService.js';
export function renderFullError(e?: unknown): unknown {
if (e === undefined) return 'undefined';
if (e === null) return 'null';
if (e instanceof Error) {
if (isSimpleError(e)) {
return renderInlineError(e);
}
const data: ErrorData = {};
if (e.stack) data.stack = e.stack;
if (e.message) data.message = e.message;
if (e.name) data.name = e.name;
// mix "cause" and "errors"
if (e instanceof AggregateError && e.errors.length > 0) {
const causes = e.errors.map(inner => renderFullError(inner));
if (e.cause) {
causes.push(renderFullError(e.cause));
}
data.cause = causes;
} else if (e.cause) {
data.cause = renderFullError(e.cause);
}
return data;
}
return e;
}
function isSimpleError(e: Error): boolean {
if (e instanceof Bull.UnrecoverableError) return true;
if (e instanceof AbortError || e.name === 'AbortError') return true;
if (e instanceof FetchError || e.name === 'FetchError') return true;
if (e instanceof StatusError) return true;
if (e instanceof IdentifiableError) return true;
if (e instanceof FetchError) return true;
if (e instanceof CaptchaError && e.code !== captchaErrorCodes.unknown) return true;
return false;
}
interface ErrorData {
stack?: Error['stack'];
message?: Error['message'];
name?: Error['name'];
cause?: Error['cause'] | Error['cause'][];
}

75
packages/backend/src/misc/render-inline-error.ts Normal file
View file

@ -0,0 +1,75 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { IdentifiableError } from '@/misc/identifiable-error.js';
import { StatusError } from '@/misc/status-error.js';
import { CaptchaError } from '@/core/CaptchaService.js';
export function renderInlineError(err: unknown): string {
const parts: string[] = [];
renderTo(err, parts);
return parts.join('');
}
function renderTo(err: unknown, parts: string[]): void {
parts.push(printError(err));
if (err instanceof AggregateError) {
for (let i = 0; i < err.errors.length; i++) {
parts.push(` [${i + 1}/${err.errors.length}]: `);
renderTo(err.errors[i], parts);
}
}
if (err instanceof Error) {
if (err.cause) {
parts.push(' [caused by]: ');
renderTo(err.cause, parts);
// const cause = renderInlineError(err.cause);
// parts.push(' [caused by]: ', cause);
}
}
}
function printError(err: unknown): string {
if (err === undefined) return 'undefined';
if (err === null) return 'null';
if (err instanceof IdentifiableError) {
if (err.message) {
return `${err.name} ${err.id}: ${err.message}`;
} else {
return `${err.name} ${err.id}`;
}
}
if (err instanceof StatusError) {
if (err.message) {
return `${err.name} ${err.statusCode}: ${err.message}`;
} else if (err.statusMessage) {
return `${err.name} ${err.statusCode}: ${err.statusMessage}`;
} else {
return `${err.name} ${err.statusCode}`;
}
}
if (err instanceof CaptchaError) {
if (err.code.description) {
return `${err.name} ${err.code.description}: ${err.message}`;
} else {
return `${err.name}: ${err.message}`;
}
}
if (err instanceof Error) {
if (err.message) {
return `${err.name}: ${err.message}`;
} else {
return err.name;
}
}
return String(err);
}

4
packages/backend/src/misc/status-error.ts
View file

@ -9,8 +9,8 @@ export class StatusError extends Error {
public isClientError: boolean;
public isRetryable: boolean;
constructor(message: string, statusCode: number, statusMessage?: string) {
super(message);
constructor(message: string, statusCode: number, statusMessage?: string, cause?: unknown) {
super(message, cause ? { cause } : undefined);
this.name = 'StatusError';
this.statusCode = statusCode;
this.statusMessage = statusMessage;

21
packages/backend/src/models/AbuseUserReport.ts
View file

@ -4,6 +4,7 @@
*/
import { PrimaryColumn, Entity, Index, JoinColumn, Column, ManyToOne } from 'typeorm';
import { MiInstance } from '@/models/Instance.js';
import { id } from './util/id.js';
import { MiUser } from './User.js';
@ -88,11 +89,31 @@ export class MiAbuseUserReport {
})
public targetUserHost: string | null;
@ManyToOne(() => MiInstance, {
// TODO create a foreign key constraint after hazelnoot/labs/persisted-instance-blocks is merged
createForeignKeyConstraints: false,
})
@JoinColumn({
name: 'targetUserHost',
referencedColumnName: 'host',
})
public targetUserInstance: MiInstance | null;
@Index()
@Column('varchar', {
length: 128, nullable: true,
comment: '[Denormalized]',
})
public reporterHost: string | null;
@ManyToOne(() => MiInstance, {
// TODO create a foreign key constraint after hazelnoot/labs/persisted-instance-blocks is merged
createForeignKeyConstraints: false,
})
@JoinColumn({
name: 'reporterHost',
referencedColumnName: 'host',
})
public reporterInstance: MiInstance | null;
//#endregion
}

21
packages/backend/src/models/Following.ts
View file

@ -4,6 +4,7 @@
*/
import { PrimaryColumn, Entity, Index, JoinColumn, Column, ManyToOne } from 'typeorm';
import { MiInstance } from '@/models/Instance.js';
import { id } from './util/id.js';
import { MiUser } from './User.js';
@ -66,6 +67,16 @@ export class MiFollowing {
})
public followerHost: string | null;
@ManyToOne(() => MiInstance, {
onDelete: 'CASCADE',
})
@JoinColumn({
name: 'followerHost',
foreignKeyConstraintName: 'FK_following_followerHost',
referencedColumnName: 'host',
})
public followerInstance: MiInstance | null;
@Column('varchar', {
length: 512, nullable: true,
comment: '[Denormalized]',
@ -85,6 +96,16 @@ export class MiFollowing {
})
public followeeHost: string | null;
@ManyToOne(() => MiInstance, {
onDelete: 'CASCADE',
})
@JoinColumn({
name: 'followeeHost',
foreignKeyConstraintName: 'FK_following_followeeHost',
referencedColumnName: 'host',
})
public followeeInstance: MiInstance | null;
@Column('varchar', {
length: 512, nullable: true,
comment: '[Denormalized]',

51
packages/backend/src/models/Instance.ts
View file

@ -6,6 +6,7 @@
import { Entity, PrimaryColumn, Index, Column } from 'typeorm';
import { id } from './util/id.js';
@Index('IDX_instance_host_key', { synchronize: false }) // ((lower(reverse("host"::text)) || '.'::text)
@Entity('instance')
export class MiInstance {
@PrimaryColumn(id())
@ -98,6 +99,56 @@ export class MiInstance {
})
public suspensionState: 'none' | 'manuallySuspended' | 'goneSuspended' | 'autoSuspendedForNotResponding';
/**
* True if this instance is blocked from federation.
*/
@Column('boolean', {
nullable: false,
default: false,
comment: 'True if this instance is blocked from federation.',
})
public isBlocked: boolean;
/**
* True if this instance is allow-listed.
*/
@Column('boolean', {
nullable: false,
default: false,
comment: 'True if this instance is allow-listed.',
})
public isAllowListed: boolean;
/**
* True if this instance is part of the local bubble.
*/
@Column('boolean', {
nullable: false,
default: false,
comment: 'True if this instance is part of the local bubble.',
})
public isBubbled: boolean;
/**
* True if this instance is silenced.
*/
@Column('boolean', {
nullable: false,
default: false,
comment: 'True if this instance is silenced.',
})
public isSilenced: boolean;
/**
* True if this instance is media-silenced.
*/
@Column('boolean', {
nullable: false,
default: false,
comment: 'True if this instance is media-silenced.',
})
public isMediaSilenced: boolean;
@Column('varchar', {
length: 64, nullable: true,
comment: 'The software of the Instance.',

36
packages/backend/src/models/Note.ts
View file

@ -5,6 +5,7 @@
import { Entity, Index, JoinColumn, Column, PrimaryColumn, ManyToOne } from 'typeorm';
import { noteVisibilities } from '@/types.js';
import { MiInstance } from '@/models/Instance.js';
import { id } from './util/id.js';
import { MiUser } from './User.js';
import { MiChannel } from './Channel.js';
@ -21,6 +22,8 @@ import type { MiDriveFile } from './DriveFile.js';
// because it will always run CREATE INDEX in transaction based on decorators.
// Not appending `{ concurrent: true }` to `@Index` will not cause any problem in production,
@Index('IDX_724b311e6f883751f261ebe378', ['userId', 'id'])
@Index('IDX_note_userHost_id', { synchronize: false }) // (userHost, id desc)
@Index('IDX_note_for_timelines', { synchronize: false }) // (id desc, channelId, visibility, userHost)
@Entity('note')
export class MiNote {
@PrimaryColumn(id())
@ -140,6 +143,7 @@ export class MiNote {
})
public uri: string | null;
@Index('IDX_note_url')
@Column('varchar', {
length: 512, nullable: true,
comment: 'The human readable url of a note. it will be null when the note is local.',
@ -225,13 +229,22 @@ export class MiNote {
public processErrors: string[] | null;
//#region Denormalized fields
@Index()
@Column('varchar', {
length: 128, nullable: true,
comment: '[Denormalized]',
})
public userHost: string | null;
@ManyToOne(() => MiInstance, {
onDelete: 'CASCADE',
})
@JoinColumn({
name: 'userHost',
foreignKeyConstraintName: 'FK_note_userHost',
referencedColumnName: 'host',
})
public userInstance: MiInstance | null;
@Column({
...id(),
nullable: true,
@ -245,6 +258,16 @@ export class MiNote {
})
public replyUserHost: string | null;
@ManyToOne(() => MiInstance, {
onDelete: 'CASCADE',
})
@JoinColumn({
name: 'replyUserHost',
foreignKeyConstraintName: 'FK_note_replyUserHost',
referencedColumnName: 'host',
})
public replyUserInstance: MiInstance | null;
@Column({
...id(),
nullable: true,
@ -258,6 +281,17 @@ export class MiNote {
})
public renoteUserHost: string | null;
@ManyToOne(() => MiInstance, {
onDelete: 'CASCADE',
})
@JoinColumn({
name: 'renoteUserHost',
foreignKeyConstraintName: 'FK_note_renoteUserHost',
referencedColumnName: 'host',
})
public renoteUserInstance: MiInstance | null;
//#endregion
constructor(data: Partial<MiNote>) {
if (data == null) return;

23
packages/backend/src/models/User.ts
View file

@ -3,10 +3,12 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Entity, Column, Index, OneToOne, JoinColumn, PrimaryColumn } from 'typeorm';
import { Entity, Column, Index, OneToOne, JoinColumn, PrimaryColumn, ManyToOne } from 'typeorm';
import { type UserUnsignedFetchOption, userUnsignedFetchOptions } from '@/const.js';
import { MiInstance } from '@/models/Instance.js';
import { id } from './util/id.js';
import { MiDriveFile } from './DriveFile.js';
import type { MiUserProfile } from './UserProfile.js';
@Entity('user')
@Index(['usernameLower', 'host'], { unique: true })
@ -292,6 +294,16 @@ export class MiUser {
})
public host: string | null;
@ManyToOne(() => MiInstance, {
onDelete: 'CASCADE',
})
@JoinColumn({
name: 'host',
foreignKeyConstraintName: 'FK_user_host',
referencedColumnName: 'host',
})
public instance: MiInstance | null;
@Column('varchar', {
length: 512, nullable: true,
comment: 'The inbox URL of the User. It will be null if the origin of the user is local.',
@ -378,6 +390,15 @@ export class MiUser {
})
public allowUnsignedFetch: UserUnsignedFetchOption;
@Column('text', {
name: 'attributionDomains',
array: true, default: '{}',
})
public attributionDomains: string[];
@OneToOne('user_profile', (profile: MiUserProfile) => profile.user)
public userProfile: MiUserProfile | null;
constructor(data: Partial<MiUser>) {
if (data == null) return;

2
packages/backend/src/models/UserProfile.ts
View file

@ -17,7 +17,7 @@ export class MiUserProfile {
@PrimaryColumn(id())
public userId: MiUser['id'];
@OneToOne(type => MiUser, {
@OneToOne(() => MiUser, user => user.userProfile, {
onDelete: 'CASCADE',
})
@JoinColumn()

4
packages/backend/src/models/json-schema/federation-instance.ts
View file

@ -135,5 +135,9 @@ export const packedFederationInstanceSchema = {
type: 'string',
optional: true, nullable: true,
},
isBubbled: {
type: 'boolean',
optional: false, nullable: false,
},
},
} as const;

12
packages/backend/src/models/json-schema/user.ts
View file

@ -214,6 +214,10 @@ export const packedUserLiteSchema = {
type: 'string',
nullable: true, optional: false,
},
isSilenced: {
type: 'boolean',
nullable: false, optional: false,
},
},
},
followersCount: {
@ -262,6 +266,14 @@ export const packedUserLiteSchema = {
},
},
},
attributionDomains: {
type: 'array',
nullable: false, optional: false,
items: {
type: 'string',
nullable: false, optional: false,
},
},
},
} as const;

57
packages/backend/src/postgres.ts
View file

@ -98,9 +98,12 @@ pg.types.setTypeParser(20, Number);
export const dbLogger = new MisskeyLogger('db');
const sqlLogger = dbLogger.createSubLogger('sql', 'gray');
const sqlMigrateLogger = sqlLogger.createSubLogger('migrate');
const sqlSchemaLogger = sqlLogger.createSubLogger('schema');
export type LoggerProps = {
disableQueryTruncation?: boolean;
enableQueryLogging?: boolean;
enableQueryParamLogging?: boolean;
printReplicationMode?: boolean,
};
@ -112,7 +115,7 @@ function highlightSql(sql: string) {
}
function truncateSql(sql: string) {
return sql.length > 100 ? `${sql.substring(0, 100)}...` : sql;
return sql.length > 100 ? `${sql.substring(0, 100)} [truncated]` : sql;
}
function stringifyParameter(param: any) {
@ -136,13 +139,16 @@ class MyCustomLogger implements Logger {
modded = truncateSql(modded);
}
return highlightSql(modded);
return this.props.enableQueryLogging ? highlightSql(modded) : modded;
}
@bindThis
private transformParameters(parameters?: any[]) {
if (this.props.enableQueryParamLogging && parameters && parameters.length > 0) {
return parameters.map(stringifyParameter);
return parameters.reduce((params, p, i) => {
params[`$${i + 1}`] = stringifyParameter(p);
return params;
}, {} as Record<string, string>);
}
return undefined;
@ -150,10 +156,13 @@ class MyCustomLogger implements Logger {
@bindThis
public logQuery(query: string, parameters?: any[], queryRunner?: QueryRunner) {
if (!this.props.enableQueryLogging) return;
const prefix = (this.props.printReplicationMode && queryRunner)
? `[${queryRunner.getReplicationMode()}] `
: undefined;
sqlLogger.info(this.transformQueryLog(query, { prefix }), this.transformParameters(parameters));
const transformed = this.transformQueryLog(query, { prefix });
sqlLogger.debug(`Query run: ${transformed}`, this.transformParameters(parameters));
}
@bindThis
@ -161,7 +170,8 @@ class MyCustomLogger implements Logger {
const prefix = (this.props.printReplicationMode && queryRunner)
? `[${queryRunner.getReplicationMode()}] `
: undefined;
sqlLogger.error(this.transformQueryLog(query, { prefix }), this.transformParameters(parameters));
const transformed = this.transformQueryLog(query, { prefix });
sqlLogger.error(`Query error (${error}): ${transformed}`, this.transformParameters(parameters));
}
@bindThis
@ -169,22 +179,32 @@ class MyCustomLogger implements Logger {
const prefix = (this.props.printReplicationMode && queryRunner)
? `[${queryRunner.getReplicationMode()}] `
: undefined;
sqlLogger.warn(this.transformQueryLog(query, { prefix }), this.transformParameters(parameters));
const transformed = this.transformQueryLog(query, { prefix });
sqlLogger.warn(`Query is slow (${time}ms): ${transformed}`, this.transformParameters(parameters));
}
@bindThis
public logSchemaBuild(message: string) {
sqlLogger.info(message);
sqlSchemaLogger.debug(message);
}
@bindThis
public log(message: string) {
sqlLogger.info(message);
public log(level: 'log' | 'info' | 'warn', message: string) {
switch (level) {
case 'log':
case 'info': {
sqlLogger.info(message);
break;
}
case 'warn': {
sqlLogger.warn(message);
}
}
}
@bindThis
public logMigration(message: string) {
sqlLogger.info(message);
sqlMigrateLogger.debug(message);
}
}
@ -306,7 +326,7 @@ export function createPostgresDataSource(config: Config) {
} : {}),
synchronize: process.env.NODE_ENV === 'test',
dropSchema: process.env.NODE_ENV === 'test',
cache: !config.db.disableCache && process.env.NODE_ENV !== 'test' ? { // dbをcloseしても何故かredisのコネクションが内部的に残り続けるようで、テストの際に支障が出るため無効にする(キャッシュも含めてテストしたいため本当は有効にしたいが...)
cache: config.db.disableCache === false && process.env.NODE_ENV !== 'test' ? { // dbをcloseしても何故かredisのコネクションが内部的に残り続けるようで、テストの際に支障が出るため無効にする(キャッシュも含めてテストしたいため本当は有効にしたいが...)
type: 'ioredis',
options: {
...config.redis,
@ -314,14 +334,13 @@ export function createPostgresDataSource(config: Config) {
},
} : false,
logging: log,
logger: log
? new MyCustomLogger({
disableQueryTruncation: config.logging?.sql?.disableQueryTruncation,
enableQueryParamLogging: config.logging?.sql?.enableQueryParamLogging,
printReplicationMode: !!config.dbReplications,
})
: undefined,
maxQueryExecutionTime: 300,
logger: new MyCustomLogger({
disableQueryTruncation: config.logging?.sql?.disableQueryTruncation,
enableQueryLogging: log,
enableQueryParamLogging: config.logging?.sql?.enableQueryParamLogging,
printReplicationMode: !!config.dbReplications,
}),
maxQueryExecutionTime: config.db.slowQueryThreshold,
entities: entities,
migrations: ['../../migration/*.js'],
});

125
packages/backend/src/queue/QueueProcessorService.ts
View file

@ -11,7 +11,7 @@ import { DI } from '@/di-symbols.js';
import type Logger from '@/logger.js';
import { bindThis } from '@/decorators.js';
import { CheckModeratorsActivityProcessorService } from '@/queue/processors/CheckModeratorsActivityProcessorService.js';
import { StatusError } from '@/misc/status-error.js';
import { renderFullError } from '@/misc/render-full-error.js';
import { UserWebhookDeliverProcessorService } from './processors/UserWebhookDeliverProcessorService.js';
import { SystemWebhookDeliverProcessorService } from './processors/SystemWebhookDeliverProcessorService.js';
import { EndedPollNotificationProcessorService } from './processors/EndedPollNotificationProcessorService.js';
@ -73,7 +73,9 @@ function getJobInfo(job: Bull.Job | undefined, increment = false): string {
const currentAttempts = job.attemptsMade + (increment ? 1 : 0);
const maxAttempts = job.opts.attempts ?? 0;
return `id=${job.id} attempts=${currentAttempts}/${maxAttempts} age=${formated}`;
return job.name
? `id=${job.id} attempts=${currentAttempts}/${maxAttempts} age=${formated} name=${job.name}`
: `id=${job.id} attempts=${currentAttempts}/${maxAttempts} age=${formated}`;
}
@Injectable()
@ -134,35 +136,6 @@ export class QueueProcessorService implements OnApplicationShutdown {
) {
this.logger = this.queueLoggerService.logger;
function renderError(e?: Error) {
// 何故かeがundefinedで来ることがある
if (!e) return '?';
if (e instanceof Bull.UnrecoverableError || e.name === 'AbortError' || e instanceof StatusError) {
return `${e.name}: ${e.message}`;
}
return {
stack: e.stack,
message: e.message,
name: e.name,
};
}
function renderJob(job?: Bull.Job) {
if (!job) return '?';
const info: Record<string, string> = {
info: getJobInfo(job),
data: job.data,
};
if (job.name) info.name = job.name;
if (job.failedReason) info.failedReason = job.failedReason;
return info;
}
//#region system
{
const processer = (job: Bull.Job) => {
@ -196,7 +169,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
.on('active', (job) => logger.debug(`active id=${job.id}`))
.on('completed', (job, result) => logger.debug(`completed(${result}) id=${job.id}`))
.on('failed', (job, err: Error) => {
logger.error(`failed(${err.name}: ${err.message}) id=${job?.id ?? '?'}`, { job: renderJob(job), e: renderError(err) });
this.logError(logger, err, job);
if (config.sentryForBackend) {
Sentry.captureMessage(`Queue: System: ${job?.name ?? '?'}: ${err.name}: ${err.message}`, {
level: 'error',
@ -204,7 +177,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
});
}
})
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
.on('error', (err: Error) => this.logError(logger, err))
.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
}
//#endregion
@ -261,7 +234,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
.on('active', (job) => logger.debug(`active id=${job.id}`))
.on('completed', (job, result) => logger.debug(`completed(${result}) id=${job.id}`))
.on('failed', (job, err) => {
logger.error(`failed(${err.name}: ${err.message}) id=${job?.id ?? '?'}`, { job: renderJob(job), e: renderError(err) });
this.logError(logger, err, job);
if (config.sentryForBackend) {
Sentry.captureMessage(`Queue: DB: ${job?.name ?? '?'}: ${err.name}: ${err.message}`, {
level: 'error',
@ -269,7 +242,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
});
}
})
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
.on('error', (err: Error) => this.logError(logger, err))
.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
}
//#endregion
@ -301,7 +274,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
.on('active', (job) => logger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`))
.on('completed', (job, result) => logger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`))
.on('failed', (job, err) => {
logger.error(`failed(${err.name}: ${err.message}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`);
this.logError(logger, err, job);
if (config.sentryForBackend) {
Sentry.captureMessage(`Queue: Deliver: ${err.name}: ${err.message}`, {
level: 'error',
@ -309,7 +282,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
});
}
})
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
.on('error', (err: Error) => this.logError(logger, err))
.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
}
//#endregion
@ -341,7 +314,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
.on('active', (job) => logger.debug(`active ${getJobInfo(job, true)}`))
.on('completed', (job, result) => logger.debug(`completed(${result}) ${getJobInfo(job, true)}`))
.on('failed', (job, err) => {
logger.error(`failed(${err.name}: ${err.message}) ${getJobInfo(job)} activity=${job ? (job.data.activity ? job.data.activity.id : 'none') : '-'}`, { job: renderJob(job), e: renderError(err) });
this.logError(logger, err, job);
if (config.sentryForBackend) {
Sentry.captureMessage(`Queue: Inbox: ${err.name}: ${err.message}`, {
level: 'error',
@ -349,7 +322,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
});
}
})
.on('error', (err: Error) => logger.error('inbox error:', renderError(err)))
.on('error', (err: Error) => this.logError(logger, err))
.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
}
//#endregion
@ -381,7 +354,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
.on('active', (job) => logger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`))
.on('completed', (job, result) => logger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`))
.on('failed', (job, err) => {
logger.error(`failed(${err.name}: ${err.message}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`);
this.logError(logger, err, job);
if (config.sentryForBackend) {
Sentry.captureMessage(`Queue: UserWebhookDeliver: ${err.name}: ${err.message}`, {
level: 'error',
@ -389,7 +362,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
});
}
})
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
.on('error', (err: Error) => this.logError(logger, err))
.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
}
//#endregion
@ -421,7 +394,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
.on('active', (job) => logger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`))
.on('completed', (job, result) => logger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`))
.on('failed', (job, err) => {
logger.error(`failed(${err.name}: ${err.message}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`);
this.logError(logger, err, job);
if (config.sentryForBackend) {
Sentry.captureMessage(`Queue: SystemWebhookDeliver: ${err.name}: ${err.message}`, {
level: 'error',
@ -429,7 +402,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
});
}
})
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
.on('error', (err: Error) => this.logError(logger, err))
.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
}
//#endregion
@ -468,7 +441,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
.on('active', (job) => logger.debug(`active id=${job.id}`))
.on('completed', (job, result) => logger.debug(`completed(${result}) id=${job.id}`))
.on('failed', (job, err) => {
logger.error(`failed(${err.name}: ${err.message}) id=${job?.id ?? '?'}`, { job: renderJob(job), e: renderError(err) });
this.logError(logger, err, job);
if (config.sentryForBackend) {
Sentry.captureMessage(`Queue: Relationship: ${job?.name ?? '?'}: ${err.name}: ${err.message}`, {
level: 'error',
@ -476,7 +449,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
});
}
})
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
.on('error', (err: Error) => this.logError(logger, err))
.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
}
//#endregion
@ -509,7 +482,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
.on('active', (job) => logger.debug(`active id=${job.id}`))
.on('completed', (job, result) => logger.debug(`completed(${result}) id=${job.id}`))
.on('failed', (job, err) => {
logger.error(`failed(${err.name}: ${err.message}) id=${job?.id ?? '?'}`, { job: renderJob(job), e: renderError(err) });
this.logError(logger, err, job);
if (config.sentryForBackend) {
Sentry.captureMessage(`Queue: ObjectStorage: ${job?.name ?? '?'}: ${err.name}: ${err.message}`, {
level: 'error',
@ -517,13 +490,15 @@ export class QueueProcessorService implements OnApplicationShutdown {
});
}
})
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
.on('error', (err: Error) => this.logError(logger, err))
.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
}
//#endregion
//#region ended poll notification
{
const logger = this.logger.createSubLogger('endedPollNotification');
this.endedPollNotificationQueueWorker = new Bull.Worker(QUEUE.ENDED_POLL_NOTIFICATION, (job) => {
if (this.config.sentryForBackend) {
return Sentry.startSpan({ name: 'Queue: EndedPollNotification' }, () => this.endedPollNotificationProcessorService.process(job));
@ -534,19 +509,75 @@ export class QueueProcessorService implements OnApplicationShutdown {
...baseWorkerOptions(this.config, QUEUE.ENDED_POLL_NOTIFICATION),
autorun: false,
});
this.endedPollNotificationQueueWorker
.on('active', (job) => logger.debug(`active id=${job.id}`))
.on('completed', (job, result) => logger.debug(`completed(${result}) id=${job.id}`))
.on('failed', (job, err) => {
this.logError(logger, err, job);
if (config.sentryForBackend) {
Sentry.captureMessage(`Queue: EndedPollNotification: ${job?.name ?? '?'}: ${err.name}: ${err.message}`, {
level: 'error',
extra: { job, err },
});
}
})
.on('error', (err: Error) => this.logError(logger, err))
.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
}
//#endregion
//#region schedule note post
{
const logger = this.logger.createSubLogger('scheduleNotePost');
this.schedulerNotePostQueueWorker = new Bull.Worker(QUEUE.SCHEDULE_NOTE_POST, (job) => this.scheduleNotePostProcessorService.process(job), {
...baseWorkerOptions(this.config, QUEUE.SCHEDULE_NOTE_POST),
autorun: false,
});
this.schedulerNotePostQueueWorker
.on('active', (job) => logger.debug(`active id=${job.id}`))
.on('completed', (job, result) => logger.debug(`completed(${result}) id=${job.id}`))
.on('failed', (job, err) => {
this.logError(logger, err, job);
if (config.sentryForBackend) {
Sentry.captureMessage(`Queue: ${QUEUE.SCHEDULE_NOTE_POST}: ${job?.name ?? '?'}: ${err.name}: ${err.message}`, {
level: 'error',
extra: { job, err },
});
}
})
.on('error', (err: Error) => this.logError(logger, err))
.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
}
//#endregion
}
private logError(logger: Logger, err: unknown, job?: Bull.Job | null): void {
const parts: string[] = [];
// Render job
if (job) {
parts.push('job [');
parts.push(getJobInfo(job));
parts.push('] failed: ');
} else {
parts.push('job failed: ');
}
// Render error
const fullError = renderFullError(err);
const errorText = typeof(fullError) === 'string' ? fullError : undefined;
if (errorText) {
parts.push(errorText);
} else if (job?.failedReason) {
parts.push(job.failedReason);
}
const message = parts.join('');
const data = typeof(fullError) !== 'string' ? { err: fullError } : undefined;
logger.error(message, data);
}
@bindThis
public async start(): Promise<void> {
await Promise.all([

4
packages/backend/src/queue/processors/AggregateRetentionProcessorService.ts
View file

@ -62,7 +62,7 @@ export class AggregateRetentionProcessorService {
});
} catch (err) {
if (isDuplicateKeyValueError(err)) {
this.logger.succ('Skip because it has already been processed by another worker.');
this.logger.debug('Skip because it has already been processed by another worker.');
return;
}
throw err;
@ -87,6 +87,6 @@ export class AggregateRetentionProcessorService {
});
}
this.logger.succ('Retention aggregated.');
this.logger.info('Retention aggregated.');
}
}

2
packages/backend/src/queue/processors/BakeBufferedReactionsProcessorService.ts
View file

@ -37,6 +37,6 @@ export class BakeBufferedReactionsProcessorService {
await this.reactionsBufferingService.bake();
this.logger.succ('All buffered reactions baked.');
this.logger.info('All buffered reactions baked.');
}
}

2
packages/backend/src/queue/processors/CheckExpiredMutingsProcessorService.ts
View file

@ -41,6 +41,6 @@ export class CheckExpiredMutingsProcessorService {
await this.userMutingService.unmute(expired);
}
this.logger.succ('All expired mutings checked.');
this.logger.info('All expired mutings checked.');
}
}

6
packages/backend/src/queue/processors/CheckModeratorsActivityProcessorService.ts
View file

@ -98,16 +98,16 @@ export class CheckModeratorsActivityProcessorService {
@bindThis
public async process(): Promise<void> {
this.logger.info('start.');
this.logger.debug('start.');
const meta = await this.metaService.fetch(false);
if (!meta.disableRegistration) {
await this.processImpl();
} else {
this.logger.info('is already invitation only.');
this.logger.debug('is already invitation only.');
}
this.logger.succ('finish.');
this.logger.debug('finish.');
}
@bindThis

2
packages/backend/src/queue/processors/CleanChartsProcessorService.ts
View file

@ -62,6 +62,6 @@ export class CleanChartsProcessorService {
await this.perUserDriveChart.clean();
await this.apRequestChart.clean();
this.logger.succ('All charts successfully cleaned.');
this.logger.info('All charts successfully cleaned.');
}
}

Some files were not shown because too many files have changed in this diff Show more