diff --git a/.config/ci.yml b/.config/ci.yml index 5657b8beae..5fcf78b737 100644 --- a/.config/ci.yml +++ b/.config/ci.yml @@ -115,8 +115,14 @@ db: user: postgres pass: ci - # Whether disable Caching queries - #disableCache: true + ## Log a warning to the server console if any query takes longer than this to complete. + ## Measured in milliseconds; set to 0 to disable. (default: 300) + #slowQueryThreshold: 300 + + # If false, then query results will be cached in redis. + # If true (default), then queries will not be cached. + # This will reduce database load at the cost of increased Redis traffic and risk of bugs and unpredictable behavior. + #disableCache: false # Extra Connection options #extra: diff --git a/.config/cypress-devcontainer.yml b/.config/cypress-devcontainer.yml index ca62616462..97263da68f 100644 --- a/.config/cypress-devcontainer.yml +++ b/.config/cypress-devcontainer.yml @@ -57,8 +57,14 @@ db: user: postgres pass: postgres - # Whether disable Caching queries - #disableCache: true + ## Log a warning to the server console if any query takes longer than this to complete. + ## Measured in milliseconds; set to 0 to disable. (default: 300) + #slowQueryThreshold: 300 + + # If false, then query results will be cached in redis. + # If true (default), then queries will not be cached. + # This will reduce database load at the cost of increased Redis traffic and risk of bugs and unpredictable behavior. + #disableCache: false # Extra Connection options #extra: diff --git a/.config/docker_example.yml b/.config/docker_example.yml index df5d77a97f..3aaa56e333 100644 --- a/.config/docker_example.yml +++ b/.config/docker_example.yml @@ -118,8 +118,14 @@ db: user: example-misskey-user pass: example-misskey-pass - # Whether disable Caching queries - #disableCache: true + ## Log a warning to the server console if any query takes longer than this to complete. + ## Measured in milliseconds; set to 0 to disable. (default: 300) + #slowQueryThreshold: 300 + + # If false, then query results will be cached in redis. + # If true (default), then queries will not be cached. + # This will reduce database load at the cost of increased Redis traffic and risk of bugs and unpredictable behavior. + #disableCache: false # Extra Connection options #extra: diff --git a/.config/example.yml b/.config/example.yml index 9e2b6d6da5..8cac42c050 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -121,8 +121,14 @@ db: user: sharkey pass: example-misskey-pass - # Whether disable Caching queries - #disableCache: true + ## Log a warning to the server console if any query takes longer than this to complete. + ## Measured in milliseconds; set to 0 to disable. (default: 300) + #slowQueryThreshold: 300 + + # If false, then query results will be cached in redis. + # If true (default), then queries will not be cached. + # This will reduce database load at the cost of increased Redis traffic and risk of bugs and unpredictable behavior. + #disableCache: false # Extra Connection options #extra: diff --git a/.gitignore b/.gitignore index b07d195a3f..ea761882da 100644 --- a/.gitignore +++ b/.gitignore @@ -87,3 +87,7 @@ vite.config.local-dev.ts.timestamp-* # Sharkey /packages/megalodon/lib + +# TypeScript +.tsbuildinfo +*.tsbuildinfo diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c8a58fadab..ef08d5275f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -690,7 +690,7 @@ seems to do a decent job) * re-generate locales (`pnpm run build-assets`) and commit * build the frontend: `rm -rf built/; NODE_ENV=development pnpm --filter=frontend --filter=frontend-embed --filter=frontend-shared build` (the `development` tells it to keep some of the original filenames in the built files) * make sure there aren't any new `ti-*` classes (Tabler Icons), and replace them with appropriate `ph-*` ones (Phosphor Icons) in [`vite.replaceicons.ts`](packages/frontend/vite.replaceIcons.ts). - * This command should show you want to change: `grep -ohrP '(?<=["'\'']ti )(ti-(?!fw)[\w\-]+)' --exclude \*.map -- built/ | sort -u`. + * This command should show you want to change: `grep -ohrP '(?<=["'\''](ti )?)(ti-(?!fw)[\w\-]+)' --exclude \*.map -- built/ | sort -u`. * NOTE: `ti-fw` is a special class that's defined by Misskey, leave it alone. * After every change, re-build the frontend and check again, until there are no more `ti-*` classes in the built files. * Commit! diff --git a/cypress/tsconfig.json b/cypress/tsconfig.json index 6fe7f32cc4..4435a4fda8 100644 --- a/cypress/tsconfig.json +++ b/cypress/tsconfig.json @@ -2,7 +2,8 @@ "compilerOptions": { "lib": ["dom", "es5"], "target": "es5", - "types": ["cypress", "node"] + "types": ["cypress", "node"], + "incremental": true }, "include": ["./**/*.ts"] } diff --git a/locales/index.d.ts b/locales/index.d.ts index b446fbb8f6..382cd5a114 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -12524,6 +12524,14 @@ export interface Locale extends ILocale { * Displays content centered. */ "centerDescription": string; + /** + * Unix Time + */ + "unixtime": string; + /** + * Displays a timestamp in the viewer's current timezone. + */ + "unixtimeDescription": string; /** * Code (Inline) */ @@ -13101,6 +13109,26 @@ export interface Locale extends ILocale { * Users popular on {name} */ "popularUsersLocal": ParameterizedString<"name">; + /** + * Polls trending on {name} + */ + "pollsOnLocal": ParameterizedString<"name">; + /** + * Polls trending on the global network + */ + "pollsOnRemote": string; + /** + * Polls that have ended recently + */ + "pollsExpired": string; + /** + * Trending polls are disabled on this instance. + */ + "trendingPollsDisabled": string; + /** + * Please log in to view trending polls. + */ + "trendingPollsDisabledLogIn": string; /** * Silenced */ @@ -13161,6 +13189,106 @@ export interface Locale extends ILocale { * Timeout in milliseconds for translation API requests. */ "translationTimeoutCaption": string; + /** + * Staff notes + */ + "staffNotes": string; + /** + * Icon of {name} + */ + "instanceIconAlt": ParameterizedString<"name">; + /** + * Attribution Domains + */ + "attributionDomains": string; + /** + * A list of domains whose content can be attributed to you on link previews, separated by new-line. Any subdomain will also be valid. The following needs to be on the webpage: + */ + "attributionDomainsDescription": string; + /** + * Written by {user} + */ + "writtenBy": ParameterizedString<"user">; + /** + * Following (Pub) + */ + "followingPub": string; + /** + * Followers (Sub) + */ + "followersSub": string; + /** + * Well-known resources + */ + "wellKnownResources": string; + /** + * Last posted: {at} + */ + "lastPosted": ParameterizedString<"at">; + /** + * NSFW + */ + "nsfw": string; + /** + * Raw + */ + "raw": string; + /** + * CW + */ + "cw": string; + /** + * Media Silenced + */ + "mediaSilenced": string; + /** + * Bubble + */ + "bubble": string; + /** + * Verified + */ + "verified": string; + /** + * Not Verified + */ + "notVerified": string; + /** + * Hibernated + */ + "hibernated": string; + /** + * When replying to a post with a Content Warning, automatically use the same CW for the reply. + */ + "keepCwDescription": string; + /** + * Disabled (do not copy CWs) + */ + "keepCwDisabled": string; + /** + * Enabled (copy CWs verbatim) + */ + "keepCwEnabled": string; + /** + * Enabled (copy CW and prepend "RE:") + */ + "keepCwPrependRe": string; + /** + * Note controls + */ + "noteFooterLabel": string; + /** + * Packed user data in its raw form. Most of these fields are public and visible to all users. + */ + "rawUserDescription": string; + /** + * Extended user data in its raw form. These fields are private and can only be accessed by moderators. + */ + "rawInfoDescription": string; + /** + * ActivityPub user data in its raw form. These fields are public and accessible to other instances. + */ + "rawApDescription": string; } declare const locales: { [lang: string]: Locale; diff --git a/packages/backend/migration/1747938628395-add-missing-indexes.js b/packages/backend/migration/1747938628395-add-missing-indexes.js index 745b39c855..0229a6c898 100644 --- a/packages/backend/migration/1747938628395-add-missing-indexes.js +++ b/packages/backend/migration/1747938628395-add-missing-indexes.js @@ -4,19 +4,35 @@ */ export class AddMissingIndexes1747938628395 { - name = 'AddMissingIndexes1747938628395' + name = 'AddMissingIndexes1747938628395' - async up(queryRunner) { - await queryRunner.query(`CREATE INDEX "IDX_58699f75b9cf904f5f007909cb" ON "user_profile" ("birthday") `); - await queryRunner.query(`CREATE INDEX "IDX_021015e6683570ae9f6b0c62be" ON "user_list_membership" ("userId") `); - await queryRunner.query(`CREATE INDEX "IDX_cddcaf418dc4d392ecfcca842a" ON "user_list_membership" ("userListId") `); - await queryRunner.query(`CREATE UNIQUE INDEX "IDX_e4f3094c43f2d665e6030b0337" ON "user_list_membership" ("userId", "userListId") `); - } + async up(queryRunner) { + // Some instances have duplicate list entries + await queryRunner.query(` + DELETE FROM "user_list_membership" + WHERE "id" NOT IN ( + SELECT MIN("id") + FROM "user_list_membership" + GROUP BY "userId", "userListId" + )`); - async down(queryRunner) { - await queryRunner.query(`DROP INDEX "public"."IDX_e4f3094c43f2d665e6030b0337"`); - await queryRunner.query(`DROP INDEX "public"."IDX_cddcaf418dc4d392ecfcca842a"`); - await queryRunner.query(`DROP INDEX "public"."IDX_021015e6683570ae9f6b0c62be"`); - await queryRunner.query(`DROP INDEX "public"."IDX_58699f75b9cf904f5f007909cb"`); - } + // Some instances already have these indexes, for an unknown reason + await queryRunner.query(`DROP INDEX IF EXISTS "public"."IDX_e4f3094c43f2d665e6030b0337"`); + await queryRunner.query(`DROP INDEX IF EXISTS "public"."IDX_cddcaf418dc4d392ecfcca842a"`); + await queryRunner.query(`DROP INDEX IF EXISTS "public"."IDX_021015e6683570ae9f6b0c62be"`); + await queryRunner.query(`DROP INDEX IF EXISTS "public"."IDX_58699f75b9cf904f5f007909cb"`); + + // Now the actual migration + await queryRunner.query(`CREATE INDEX "IDX_58699f75b9cf904f5f007909cb" ON "user_profile" ("birthday") `); + await queryRunner.query(`CREATE INDEX "IDX_021015e6683570ae9f6b0c62be" ON "user_list_membership" ("userId") `); + await queryRunner.query(`CREATE INDEX "IDX_cddcaf418dc4d392ecfcca842a" ON "user_list_membership" ("userListId") `); + await queryRunner.query(`CREATE UNIQUE INDEX "IDX_e4f3094c43f2d665e6030b0337" ON "user_list_membership" ("userId", "userListId") `); + } + + async down(queryRunner) { + await queryRunner.query(`DROP INDEX "public"."IDX_e4f3094c43f2d665e6030b0337"`); + await queryRunner.query(`DROP INDEX "public"."IDX_cddcaf418dc4d392ecfcca842a"`); + await queryRunner.query(`DROP INDEX "public"."IDX_021015e6683570ae9f6b0c62be"`); + await queryRunner.query(`DROP INDEX "public"."IDX_58699f75b9cf904f5f007909cb"`); + } } diff --git a/packages/backend/migration/1748096357260-AddAttributionDomains.js b/packages/backend/migration/1748096357260-AddAttributionDomains.js new file mode 100644 index 0000000000..0a9679bccd --- /dev/null +++ b/packages/backend/migration/1748096357260-AddAttributionDomains.js @@ -0,0 +1,16 @@ +/* + * SPDX-FileCopyrightText: piuvas and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class AddAttributionDomains1748096357260 { + name = 'AddAttributionDomains1748096357260' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "user" ADD "attributionDomains" text array NOT NULL DEFAULT '{}'`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "user" DROP COLUMN "attributionDomains"`); + } +} diff --git a/packages/backend/migration/1748104955717-index_IDX_instance_host_key.js b/packages/backend/migration/1748104955717-index_IDX_instance_host_key.js new file mode 100644 index 0000000000..139eae740f --- /dev/null +++ b/packages/backend/migration/1748104955717-index_IDX_instance_host_key.js @@ -0,0 +1,16 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class IndexIDXInstanceHostKey1748104955717 { + name = 'IndexIDXInstanceHostKey1748104955717' + + async up(queryRunner) { + await queryRunner.query(`CREATE UNIQUE INDEX "IDX_instance_host_key" ON "instance" (((lower(reverse("host")) || '.')::text) text_pattern_ops)`); + } + + async down(queryRunner) { + await queryRunner.query(`DROP INDEX "IDX_instance_host_key"`); + } +} diff --git a/packages/backend/migration/1748105111513-add_instance_block_columns.js b/packages/backend/migration/1748105111513-add_instance_block_columns.js new file mode 100644 index 0000000000..6e3d78d5e8 --- /dev/null +++ b/packages/backend/migration/1748105111513-add_instance_block_columns.js @@ -0,0 +1,85 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +/** + * @typedef {import('typeorm').MigrationInterface} MigrationInterface + * @typedef {{ blockedHosts: string[], silencedHosts: string[], mediaSilencedHosts: string[], federationHosts: string[], bubbleInstances: string[] }} Meta + */ + +/** + * @class + * @implements {MigrationInterface} + */ +export class AddInstanceBlockColumns1748105111513 { + name = 'AddInstanceBlockColumns1748105111513' + + async up(queryRunner) { + // Schema migration + await queryRunner.query(`ALTER TABLE "instance" ADD "isBlocked" boolean NOT NULL DEFAULT false`); + await queryRunner.query(`COMMENT ON COLUMN "instance"."isBlocked" IS 'True if this instance is blocked from federation.'`); + await queryRunner.query(`ALTER TABLE "instance" ADD "isAllowListed" boolean NOT NULL DEFAULT false`); + await queryRunner.query(`COMMENT ON COLUMN "instance"."isAllowListed" IS 'True if this instance is allow-listed.'`); + await queryRunner.query(`ALTER TABLE "instance" ADD "isBubbled" boolean NOT NULL DEFAULT false`); + await queryRunner.query(`COMMENT ON COLUMN "instance"."isBubbled" IS 'True if this instance is part of the local bubble.'`); + await queryRunner.query(`ALTER TABLE "instance" ADD "isSilenced" boolean NOT NULL DEFAULT false`); + await queryRunner.query(`COMMENT ON COLUMN "instance"."isSilenced" IS 'True if this instance is silenced.'`); + await queryRunner.query(`ALTER TABLE "instance" ADD "isMediaSilenced" boolean NOT NULL DEFAULT false`); + await queryRunner.query(`COMMENT ON COLUMN "instance"."isMediaSilenced" IS 'True if this instance is media-silenced.'`); + + // Data migration + /** @type {Meta[]} */ + const metas = await queryRunner.query(`SELECT "blockedHosts", "silencedHosts", "mediaSilencedHosts", "federationHosts", "bubbleInstances" FROM "meta"`); + if (metas.length > 0) { + /** @type {Meta} */ + const meta = metas[0]; + + // Blocked hosts + if (meta.blockedHosts.length > 0) { + const patterns = buildPatterns(meta.blockedHosts); + await queryRunner.query(`UPDATE "instance" SET "isBlocked" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]); + } + + // Silenced hosts + if (meta.silencedHosts.length > 0) { + const patterns = buildPatterns(meta.silencedHosts); + await queryRunner.query(`UPDATE "instance" SET "isSilenced" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]); + } + + // Media silenced hosts + if (meta.mediaSilencedHosts.length > 0) { + const patterns = buildPatterns(meta.mediaSilencedHosts); + await queryRunner.query(`UPDATE "instance" SET "isMediaSilenced" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]); + } + + // Allow-listed hosts + if (meta.federationHosts.length > 0) { + const patterns = buildPatterns(meta.federationHosts); + await queryRunner.query(`UPDATE "instance" SET "isAllowListed" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]); + } + + // Bubbled hosts + if (meta.bubbleInstances.length > 0) { + const patterns = buildPatterns(meta.bubbleInstances); + await queryRunner.query(`UPDATE "instance" SET "isBubbled" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]); + } + } + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isMediaSilenced"`); + await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isSilenced"`); + await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isBubbled"`); + await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isAllowListed"`); + await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isBlocked"`); + } +} + +/** + * @param {string[]} input + * @returns {string[]} + */ +function buildPatterns(input) { + return input.map(i => i.toLowerCase().split('').reverse().join('') + '.%'); +} diff --git a/packages/backend/migration/1748128176881-add_instance_foreign_keys.js b/packages/backend/migration/1748128176881-add_instance_foreign_keys.js new file mode 100644 index 0000000000..2c2383c50f --- /dev/null +++ b/packages/backend/migration/1748128176881-add_instance_foreign_keys.js @@ -0,0 +1,44 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +/** + * @typedef {import('typeorm').MigrationInterface} MigrationInterface + */ + +/** + * @class + * @implements {MigrationInterface} + */ +export class AddInstanceForeignKeys1748128176881 { + name = 'AddInstanceForeignKeys1748128176881' + + async up(queryRunner) { + // Fix-up: Some older instances have users without a matching instance entry + await queryRunner.query(` + INSERT INTO "instance" ("id", "host", "firstRetrievedAt") + SELECT + MIN("id"), + "host", + COALESCE(MIN("lastFetchedAt"), CURRENT_TIMESTAMP) + FROM "user" + WHERE + "host" IS NOT NULL AND + NOT EXISTS (select 1 from "instance" where "instance"."host" = "user"."host") + GROUP BY "host" + `); + + await queryRunner.query(`ALTER TABLE "user" ADD CONSTRAINT "FK_user_host" FOREIGN KEY ("host") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`); + await queryRunner.query(`ALTER TABLE "note" ADD CONSTRAINT "FK_note_userHost" FOREIGN KEY ("userHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`); + await queryRunner.query(`ALTER TABLE "note" ADD CONSTRAINT "FK_note_replyUserHost" FOREIGN KEY ("replyUserHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`); + await queryRunner.query(`ALTER TABLE "note" ADD CONSTRAINT "FK_note_renoteUserHost" FOREIGN KEY ("renoteUserHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "note" DROP CONSTRAINT "FK_note_renoteUserHost"`); + await queryRunner.query(`ALTER TABLE "note" DROP CONSTRAINT "FK_note_replyUserHost"`); + await queryRunner.query(`ALTER TABLE "note" DROP CONSTRAINT "FK_note_userHost"`); + await queryRunner.query(`ALTER TABLE "user" DROP CONSTRAINT "FK_user_host"`); + } +} diff --git a/packages/backend/migration/1748137683887-add_instance_foreign_keys_to_following.js b/packages/backend/migration/1748137683887-add_instance_foreign_keys_to_following.js new file mode 100644 index 0000000000..8f4a977ff5 --- /dev/null +++ b/packages/backend/migration/1748137683887-add_instance_foreign_keys_to_following.js @@ -0,0 +1,26 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +/** + * @typedef {import('typeorm').MigrationInterface} MigrationInterface + */ + +/** + * @class + * @implements {MigrationInterface} + */ +export class AddInstanceForeignKeysToFollowing1748137683887 { + name = 'AddInstanceForeignKeysToFollowing1748137683887' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "following" ADD CONSTRAINT "FK_following_followerHost" FOREIGN KEY ("followerHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`); + await queryRunner.query(`ALTER TABLE "following" ADD CONSTRAINT "FK_following_followeeHost" FOREIGN KEY ("followeeHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "following" DROP CONSTRAINT "FK_following_followeeHost"`); + await queryRunner.query(`ALTER TABLE "following" DROP CONSTRAINT "FK_following_followerHost"`); + } +} diff --git a/packages/backend/migration/1748191631151-analyze_instance-user-note-following.js b/packages/backend/migration/1748191631151-analyze_instance-user-note-following.js new file mode 100644 index 0000000000..f03a60980b --- /dev/null +++ b/packages/backend/migration/1748191631151-analyze_instance-user-note-following.js @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +/** + * @typedef {import('typeorm').MigrationInterface} MigrationInterface + */ + +/** + * @class + * @implements {MigrationInterface} + */ +export class AnalyzeInstanceUserNoteFollowing1748191631151 { + name = 'AnalyzeInstanceUserNoteFollowing1748191631151' + + async up(queryRunner) { + // Refresh statistics for tables impacted by new indexes. + // This helps the query planner to efficiently use them without waiting for the next full vacuum. + await queryRunner.query(`ANALYZE "instance", "user", "following", "note"`); + } + + async down(queryRunner) { + } +} diff --git a/packages/backend/migration/1748990452958-replace_note-userHost_index.js b/packages/backend/migration/1748990452958-replace_note-userHost_index.js new file mode 100644 index 0000000000..55aadd8136 --- /dev/null +++ b/packages/backend/migration/1748990452958-replace_note-userHost_index.js @@ -0,0 +1,22 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class ReplaceNoteUserHostIndex1748990452958 { + name = 'ReplaceNoteUserHostIndex1748990452958' + + async up(queryRunner) { + await queryRunner.query(`DROP INDEX "public"."IDX_7125a826ab192eb27e11d358a5"`); + await queryRunner.query(` + create index "IDX_note_userHost_id" + on "note" ("userHost", "id" desc) + nulls not distinct`); + await queryRunner.query(`comment on index "IDX_note_userHost_id" is 'User host with ID included'`); + } + + async down(queryRunner) { + await queryRunner.query(`drop index if exists "IDX_note_userHost_id"`); + await queryRunner.query(`CREATE INDEX "IDX_7125a826ab192eb27e11d358a5" ON "note" ("userHost") `); + } +} diff --git a/packages/backend/migration/1748990662839-fix-IDX_instance_host_key.js b/packages/backend/migration/1748990662839-fix-IDX_instance_host_key.js new file mode 100644 index 0000000000..fc6d303743 --- /dev/null +++ b/packages/backend/migration/1748990662839-fix-IDX_instance_host_key.js @@ -0,0 +1,22 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class FixIDXInstanceHostKey1748990662839 { + async up(queryRunner) { + // must include host for index-only scans: https://www.postgresql.org/docs/current/indexes-index-only-scans.html + await queryRunner.query(`DROP INDEX "public"."IDX_instance_host_key"`); + await queryRunner.query(` + create index "IDX_instance_host_key" + on "instance" ((lower(reverse("host"::text)) || '.'::text) text_pattern_ops) + include ("host") + `); + await queryRunner.query(`comment on index "IDX_instance_host_key" is 'Expression index for finding instances by base domain'`); + } + + async down(queryRunner) { + await queryRunner.query(`DROP INDEX "public"."IDX_instance_host_key"`); + await queryRunner.query(`CREATE UNIQUE INDEX "IDX_instance_host_key" ON "instance" (((lower(reverse("host")) || '.')::text) text_pattern_ops)`); + } +} diff --git a/packages/backend/migration/1748991828473-create-IDX_note_for_timelines.js b/packages/backend/migration/1748991828473-create-IDX_note_for_timelines.js new file mode 100644 index 0000000000..2ea7fe95d2 --- /dev/null +++ b/packages/backend/migration/1748991828473-create-IDX_note_for_timelines.js @@ -0,0 +1,19 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class CreateIDXNoteForTimelines1748991828473 { + async up(queryRunner) { + await queryRunner.query(` + create index "IDX_note_for_timelines" + on "note" ("id" desc, "channelId", "visibility", "userHost") + include ("userId", "userHost", "replyId", "replyUserId", "replyUserHost", "renoteId", "renoteUserId", "renoteUserHost") + NULLS NOT DISTINCT`); + await queryRunner.query(`comment on index "IDX_note_for_timelines" is 'Covering index for timeline queries'`); + } + + async down(queryRunner) { + await queryRunner.query(`DROP INDEX "IDX_note_for_timelines"`); + } +} diff --git a/packages/backend/migration/1748992017688-create-IDX_instance_host_filters.js b/packages/backend/migration/1748992017688-create-IDX_instance_host_filters.js new file mode 100644 index 0000000000..76cf16a6de --- /dev/null +++ b/packages/backend/migration/1748992017688-create-IDX_instance_host_filters.js @@ -0,0 +1,17 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class CreateIDXInstanceHostFilters1748992017688 { + async up(queryRunner) { + await queryRunner.query(` + create index "IDX_instance_host_filters" + on "instance" ("host", "isBlocked", "isSilenced", "isMediaSilenced", "isAllowListed", "isBubbled", "suspensionState")`); + await queryRunner.query(`comment on index "IDX_instance_host_filters" is 'Covering index for host filter queries'`); + } + + async down(queryRunner) { + await queryRunner.query(`DROP INDEX "IDX_instance_host_filters"`); + } +} diff --git a/packages/backend/migration/1748992128683-create-statistics.js b/packages/backend/migration/1748992128683-create-statistics.js new file mode 100644 index 0000000000..5d08868536 --- /dev/null +++ b/packages/backend/migration/1748992128683-create-statistics.js @@ -0,0 +1,28 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class CreateStatistics1748992128683 { + async up(queryRunner) { + await queryRunner.query(`CREATE STATISTICS "STTS_instance_isBlocked_isBubbled" (mcv) ON "isBlocked", "isBubbled" FROM "instance"`); + await queryRunner.query(`CREATE STATISTICS "STTS_instance_isBlocked_isSilenced" (mcv) ON "isBlocked", "isSilenced" FROM "instance"`); + await queryRunner.query(`CREATE STATISTICS "STTS_note_replyId_replyUserId_replyUserHost" (dependencies) ON "replyId", "replyUserId", "replyUserHost" FROM "note"`) + await queryRunner.query(`CREATE STATISTICS "STTS_note_renoteId_renoteUserId_renoteUserHost" (dependencies) ON "renoteId", "renoteUserId", "renoteUserHost" FROM "note"`); + await queryRunner.query(`CREATE STATISTICS "STTS_note_userId_userHost" (mcv) ON "userId", "userHost" FROM "note"`); + await queryRunner.query(`CREATE STATISTICS "STTS_note_replyUserId_replyUserHost" (mcv) ON "replyUserId", "replyUserHost" FROM "note"`); + await queryRunner.query(`CREATE STATISTICS "STTS_note_renoteUserId_renoteUserHost" (mcv) ON "renoteUserId", "renoteUserHost" FROM "note"`); + await queryRunner.query(`ANALYZE "note", "instance"`); + } + + async down(queryRunner) { + await queryRunner.query(`DROP STATISTICS "STTS_instance_isBlocked_isBubbled"`); + await queryRunner.query(`DROP STATISTICS "STTS_instance_isBlocked_isSilenced"`); + await queryRunner.query(`DROP STATISTICS "STTS_note_replyId_replyUserId_replyUserHost"`); + await queryRunner.query(`DROP STATISTICS "STTS_note_renoteId_renoteUserId_renoteUserHost"`); + await queryRunner.query(`DROP STATISTICS "STTS_note_userId_userHost"`); + await queryRunner.query(`DROP STATISTICS "STTS_note_replyUserId_replyUserHost"`); + await queryRunner.query(`DROP STATISTICS "STTS_note_renoteUserId_renoteUserHost"`); + await queryRunner.query(`ANALYZE "note", "instance"`); + } +} diff --git a/packages/backend/migration/1749097536193-fix-IDX_note_for_timeline.js b/packages/backend/migration/1749097536193-fix-IDX_note_for_timeline.js new file mode 100644 index 0000000000..9a651e5871 --- /dev/null +++ b/packages/backend/migration/1749097536193-fix-IDX_note_for_timeline.js @@ -0,0 +1,28 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class FixIDXNoteForTimeline1749097536193 { + async up(queryRunner) { + await queryRunner.query('drop index "IDX_note_for_timelines"'); + await queryRunner.query(` + create index "IDX_note_for_timelines" + on "note" ("id" desc, "channelId", "visibility", "userHost") + include ("userId", "replyId", "replyUserId", "replyUserHost", "renoteId", "renoteUserId", "renoteUserHost", "threadId") + NULLS NOT DISTINCT + `); + await queryRunner.query(`comment on index "IDX_note_for_timelines" is 'Covering index for timeline queries'`); + } + + async down(queryRunner) { + await queryRunner.query('drop index "IDX_note_for_timelines"'); + await queryRunner.query(` + create index "IDX_note_for_timelines" + on "note" ("id" desc, "channelId", "visibility", "userHost") + include ("userId", "userHost", "replyId", "replyUserId", "replyUserHost", "renoteId", "renoteUserId", "renoteUserHost") + NULLS NOT DISTINCT + `); + await queryRunner.query(`comment on index "IDX_note_for_timelines" is 'Covering index for timeline queries'`); + } +} diff --git a/packages/backend/migration/1749229288946-create-IDX_note_url.js b/packages/backend/migration/1749229288946-create-IDX_note_url.js new file mode 100644 index 0000000000..4b2fc25cf7 --- /dev/null +++ b/packages/backend/migration/1749229288946-create-IDX_note_url.js @@ -0,0 +1,16 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class CreateIDXNoteUrl1749229288946 { + name = 'CreateIDXNoteUrl1749229288946' + + async up(queryRunner) { + await queryRunner.query(`CREATE INDEX IF NOT EXISTS "IDX_note_url" ON "note" ("url") `); + } + + async down(queryRunner) { + await queryRunner.query(`DROP INDEX "public"."IDX_note_url"`); + } +} diff --git a/packages/backend/migration/1749267016885-remove-IDX_instance_host_filters.js b/packages/backend/migration/1749267016885-remove-IDX_instance_host_filters.js new file mode 100644 index 0000000000..d0a4e4f91e --- /dev/null +++ b/packages/backend/migration/1749267016885-remove-IDX_instance_host_filters.js @@ -0,0 +1,17 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class RemoveIDXInstanceHostFilters1749267016885 { + async up(queryRunner) { + await queryRunner.query(`DROP INDEX IF EXISTS "IDX_instance_host_filters"`); + } + + async down(queryRunner) { + await queryRunner.query(` + create index "IDX_instance_host_filters" + on "instance" ("host", "isBlocked", "isSilenced", "isMediaSilenced", "isAllowListed", "isBubbled", "suspensionState")`); + await queryRunner.query(`comment on index "IDX_instance_host_filters" is 'Covering index for host filter queries'`); + } +} diff --git a/packages/backend/package.json b/packages/backend/package.json index 8738fb5da5..8e92748884 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -10,6 +10,9 @@ "start": "node ./built/boot/entry.js", "start:test": "cross-env NODE_ENV=test node ./built/boot/entry.js", "migrate": "pnpm typeorm migration:run -d ormconfig.js", + "migrate:revert": "pnpm typeorm migration:revert -d ormconfig.js", + "migrate:generate": "pnpm typeorm migration:generate -d ormconfig.js", + "migrate:create": "pnpm typeorm migration:create", "revert": "pnpm typeorm migration:revert -d ormconfig.js", "check:connect": "node ./scripts/check_connect.js", "build": "swc src -d built -D --strip-leading-paths", diff --git a/packages/backend/src/boot/entry.ts b/packages/backend/src/boot/entry.ts index 735a0f4666..e52d77ab9b 100644 --- a/packages/backend/src/boot/entry.ts +++ b/packages/backend/src/boot/entry.ts @@ -9,6 +9,7 @@ import cluster from 'node:cluster'; import { EventEmitter } from 'node:events'; +import { inspect } from 'node:util'; import chalk from 'chalk'; import Xev from 'xev'; import Logger from '@/logger.js'; @@ -53,15 +54,22 @@ async function main() { // Display detail of unhandled promise rejection if (!envOption.quiet) { - process.on('unhandledRejection', console.dir); + process.on('unhandledRejection', e => { + try { + logger.error('Unhandled rejection:', inspect(e)); + } catch { + console.error('Unhandled rejection:', inspect(e)); + } + }); } // Display detail of uncaught exception process.on('uncaughtException', err => { try { - logger.error(err); - console.trace(err); - } catch { } + logger.error('Uncaught exception:', err); + } catch { + console.error('Uncaught exception:', err); + } }); // Dying away... diff --git a/packages/backend/src/boot/master.ts b/packages/backend/src/boot/master.ts index 538c529106..a90228eabc 100644 --- a/packages/backend/src/boot/master.ts +++ b/packages/backend/src/boot/master.ts @@ -74,7 +74,7 @@ export async function masterMain() { process.exit(1); } - bootLogger.succ('Sharkey initialized'); + bootLogger.info('Sharkey initialized'); if (config.sentryForBackend) { Sentry.init({ @@ -140,10 +140,10 @@ export async function masterMain() { } if (envOption.onlyQueue) { - bootLogger.succ('Queue started', null, true); + bootLogger.info('Queue started', null, true); } else { const addressString = net.isIPv6(config.address) ? `[${config.address}]` : config.address; - bootLogger.succ(config.socket ? `Now listening on socket ${config.socket} on ${config.url}` : `Now listening on ${addressString}:${config.port} on ${config.url}`, null, true); + bootLogger.info(config.socket ? `Now listening on socket ${config.socket} on ${config.url}` : `Now listening on ${addressString}:${config.port} on ${config.url}`, null, true); } } @@ -172,7 +172,7 @@ function loadConfigBoot(): Config { config = loadConfig(); } catch (exception) { if (typeof exception === 'string') { - configLogger.error(exception); + configLogger.error('Exception loading config:', exception); process.exit(1); } else if ((exception as any).code === 'ENOENT') { configLogger.error('Configuration file not found', null, true); @@ -181,7 +181,7 @@ function loadConfigBoot(): Config { throw exception; } - configLogger.succ('Loaded'); + configLogger.info('Loaded'); return config; } @@ -195,7 +195,7 @@ async function connectDb(): Promise { dbLogger.info('Connecting...'); await initDb(); const v = await db.query('SHOW server_version').then(x => x[0].server_version); - dbLogger.succ(`Connected: v${v}`); + dbLogger.info(`Connected: v${v}`); } catch (err) { dbLogger.error('Cannot connect', null, true); dbLogger.error(err); @@ -211,7 +211,7 @@ async function spawnWorkers(limit = 1) { bootLogger.info(`Starting ${workers} worker${workers === 1 ? '' : 's'}...`); await Promise.all([...Array(workers)].map(spawnWorker)); - bootLogger.succ('All workers started'); + bootLogger.info('All workers started'); } function spawnWorker(): Promise { diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 8507420839..c2e7efd456 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -9,6 +9,7 @@ import { dirname, resolve } from 'node:path'; import * as yaml from 'js-yaml'; import { globSync } from 'glob'; import ipaddr from 'ipaddr.js'; +import Logger from './logger.js'; import type * as Sentry from '@sentry/node'; import type * as SentryVue from '@sentry/vue'; import type { RedisOptions } from 'ioredis'; @@ -40,6 +41,7 @@ type Source = { db?: string; user?: string; pass?: string; + slowQueryThreshold?: number; disableCache?: boolean; extra?: { [x: string]: string }; }; @@ -155,6 +157,8 @@ type Source = { } }; +const configLogger = new Logger('config'); + export type PrivateNetworkSource = string | { network?: string, ports?: number[] }; export type PrivateNetwork = { @@ -192,7 +196,7 @@ export function parsePrivateNetworks(patterns: PrivateNetworkSource[] | undefine } } - console.warn('[config] Skipping invalid entry in allowedPrivateNetworks: ', e); + configLogger.warn('Skipping invalid entry in allowedPrivateNetworks: ', e); return null; }) .filter(p => p != null); @@ -222,6 +226,7 @@ export type Config = { db: string; user: string; pass: string; + slowQueryThreshold?: number; disableCache?: boolean; extra?: { [x: string]: string }; }; @@ -375,11 +380,14 @@ export function loadConfig(): Config { if (configFiles.length === 0 && !process.env['MK_WARNED_ABOUT_CONFIG']) { - console.log('No config files loaded, check if this is intentional'); + configLogger.warn('No config files loaded, check if this is intentional'); process.env['MK_WARNED_ABOUT_CONFIG'] = '1'; } - const config = configFiles.map(path => fs.readFileSync(path, 'utf-8')) + const config = configFiles.map(path => { + configLogger.info(`Reading configuration from ${path}`); + return fs.readFileSync(path, 'utf-8'); + }) .map(contents => yaml.load(contents) as Source) .reduce( (acc: Source, cur: Source) => Object.assign(acc, cur), @@ -405,6 +413,10 @@ export function loadConfig(): Config { const internalMediaProxy = `${scheme}://${host}/proxy`; const redis = convertRedisOptions(config.redis, host); + // nullish => 300 (default) + // 0 => undefined (disabled) + const slowQueryThreshold = (config.db.slowQueryThreshold ?? 300) || undefined; + return { version, publishTarballInsteadOfProvideRepositoryUrl: !!config.publishTarballInsteadOfProvideRepositoryUrl, @@ -423,7 +435,7 @@ export function loadConfig(): Config { apiUrl: `${scheme}://${host}/api`, authUrl: `${scheme}://${host}/auth`, driveUrl: `${scheme}://${host}/files`, - db: { ...config.db, db: dbDb, user: dbUser, pass: dbPass }, + db: { ...config.db, db: dbDb, user: dbUser, pass: dbPass, slowQueryThreshold }, dbReplications: config.dbReplications, dbSlaves: config.dbSlaves, fulltextSearch: config.fulltextSearch, @@ -496,6 +508,10 @@ export function loadConfig(): Config { } function tryCreateUrl(url: string) { + if (!url) { + throw new Error('Failed to load: no "url" property found in config. Please check the value of "MISSKEY_CONFIG_DIR" and "MISSKEY_CONFIG_YML", and verify that all configuration files are correct.'); + } + try { return new URL(url); } catch (e) { @@ -627,7 +643,7 @@ function applyEnvOverrides(config: Source) { // these are all the settings that can be overridden _apply_top([['url', 'port', 'address', 'socket', 'chmodSocket', 'disableHsts', 'id', 'dbReplications', 'websocketCompression']]); - _apply_top(['db', ['host', 'port', 'db', 'user', 'pass', 'disableCache']]); + _apply_top(['db', ['host', 'port', 'db', 'user', 'pass', 'slowQueryThreshold', 'disableCache']]); _apply_top(['dbSlaves', Array.from((config.dbSlaves ?? []).keys()), ['host', 'port', 'db', 'user', 'pass']]); _apply_top([ ['redis', 'redisForPubsub', 'redisForJobQueue', 'redisForTimelines', 'redisForReactions', 'redisForRateLimit'], diff --git a/packages/backend/src/core/AbuseReportService.ts b/packages/backend/src/core/AbuseReportService.ts index 846d2c8ebd..bccb9f86f6 100644 --- a/packages/backend/src/core/AbuseReportService.ts +++ b/packages/backend/src/core/AbuseReportService.ts @@ -13,6 +13,7 @@ import { QueueService } from '@/core/QueueService.js'; import { ApRendererService } from '@/core/activitypub/ApRendererService.js'; import { ModerationLogService } from '@/core/ModerationLogService.js'; import { SystemAccountService } from '@/core/SystemAccountService.js'; +import { IdentifiableError } from '@/misc/identifiable-error.js'; import { IdService } from './IdService.js'; @Injectable() @@ -125,11 +126,11 @@ export class AbuseReportService { const report = await this.abuseUserReportsRepository.findOneByOrFail({ id: reportId }); if (report.targetUserHost == null) { - throw new Error('The target user host is null.'); + throw new IdentifiableError('0b1ce202-b2c1-4ee4-8af4-2742a51b383d', 'The target user host is null.'); } if (report.forwarded) { - throw new Error('The report has already been forwarded.'); + throw new IdentifiableError('5c008bdf-f0e8-4154-9f34-804e114516d7', 'The report has already been forwarded.'); } await this.abuseUserReportsRepository.update(report.id, { diff --git a/packages/backend/src/core/BunnyService.ts b/packages/backend/src/core/BunnyService.ts index bdb5bba3dd..c9f8a427f5 100644 --- a/packages/backend/src/core/BunnyService.ts +++ b/packages/backend/src/core/BunnyService.ts @@ -80,15 +80,15 @@ export class BunnyService { }); req.on('error', (error) => { - this.bunnyCdnLogger.error(error); + this.bunnyCdnLogger.error('Unhandled error', error); data.destroy(); - throw new IdentifiableError('689ee33f-f97c-479a-ac49-1b9f8140bf91', 'An error has occured during the connectiong to BunnyCDN'); + throw new IdentifiableError('689ee33f-f97c-479a-ac49-1b9f8140bf91', 'An error has occurred while connecting to BunnyCDN', true, error); }); data.pipe(req).on('finish', () => { data.destroy(); }); - + // wait till stream gets destroyed upon finish of piping to prevent the UI from showing the upload as success way too early await finished(data); } diff --git a/packages/backend/src/core/CaptchaService.ts b/packages/backend/src/core/CaptchaService.ts index 13200bf7b3..c526a80aeb 100644 --- a/packages/backend/src/core/CaptchaService.ts +++ b/packages/backend/src/core/CaptchaService.ts @@ -54,7 +54,7 @@ export class CaptchaError extends Error { public readonly cause?: unknown; constructor(code: CaptchaErrorCode, message: string, cause?: unknown) { - super(message); + super(message, cause ? { cause } : undefined); this.code = code; this.cause = cause; this.name = 'CaptchaError'; @@ -117,7 +117,7 @@ export class CaptchaService { } const result = await this.getCaptchaResponse('https://www.recaptcha.net/recaptcha/api/siteverify', secret, response).catch(err => { - throw new CaptchaError(captchaErrorCodes.requestFailed, `recaptcha-request-failed: ${err}`); + throw new CaptchaError(captchaErrorCodes.requestFailed, `recaptcha-request-failed: ${err}`, err); }); if (result.success !== true) { @@ -133,7 +133,7 @@ export class CaptchaService { } const result = await this.getCaptchaResponse('https://hcaptcha.com/siteverify', secret, response).catch(err => { - throw new CaptchaError(captchaErrorCodes.requestFailed, `hcaptcha-request-failed: ${err}`); + throw new CaptchaError(captchaErrorCodes.requestFailed, `hcaptcha-request-failed: ${err}`, err); }); if (result.success !== true) { @@ -209,7 +209,7 @@ export class CaptchaService { } const result = await this.getCaptchaResponse('https://challenges.cloudflare.com/turnstile/v0/siteverify', secret, response).catch(err => { - throw new CaptchaError(captchaErrorCodes.requestFailed, `turnstile-request-failed: ${err}`); + throw new CaptchaError(captchaErrorCodes.requestFailed, `turnstile-request-failed: ${err}`, err); }); if (result.success !== true) { @@ -386,7 +386,7 @@ export class CaptchaService { this.logger.info(err); const error = err instanceof CaptchaError ? err - : new CaptchaError(captchaErrorCodes.unknown, `unknown error: ${err}`); + : new CaptchaError(captchaErrorCodes.unknown, `unknown error: ${err}`, err); return { success: false, error, diff --git a/packages/backend/src/core/DownloadService.ts b/packages/backend/src/core/DownloadService.ts index 26e60e00b3..cb5bdb6cb7 100644 --- a/packages/backend/src/core/DownloadService.ts +++ b/packages/backend/src/core/DownloadService.ts @@ -18,6 +18,7 @@ import { LoggerService } from '@/core/LoggerService.js'; import type Logger from '@/logger.js'; import { bindThis } from '@/decorators.js'; +import { renderInlineError } from '@/misc/render-inline-error.js'; @Injectable() export class DownloadService { @@ -37,7 +38,7 @@ export class DownloadService { public async downloadUrl(url: string, path: string, options: { timeout?: number, operationTimeout?: number, maxSize?: number } = {} ): Promise<{ filename: string; }> { - this.logger.info(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`); + this.logger.debug(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`); const timeout = options.timeout ?? 30 * 1000; const operationTimeout = options.operationTimeout ?? 60 * 1000; @@ -86,7 +87,7 @@ export class DownloadService { filename = parsed.parameters.filename; } } catch (e) { - this.logger.warn(`Failed to parse content-disposition: ${contentDisposition}`, { stack: e }); + this.logger.warn(`Failed to parse content-disposition ${contentDisposition}: ${renderInlineError(e)}`); } } }).on('downloadProgress', (progress: Got.Progress) => { @@ -100,13 +101,17 @@ export class DownloadService { await stream.pipeline(req, fs.createWriteStream(path)); } catch (e) { if (e instanceof Got.HTTPError) { - throw new StatusError(`${e.response.statusCode} ${e.response.statusMessage}`, e.response.statusCode, e.response.statusMessage); - } else { + throw new StatusError(`download error from ${url}`, e.response.statusCode, e.response.statusMessage, e); + } else if (e instanceof Got.RequestError || e instanceof Got.AbortError) { + throw new Error(String(e), { cause: e }); + } else if (e instanceof Error) { throw e; + } else { + throw new Error(String(e), { cause: e }); } } - this.logger.succ(`Download finished: ${chalk.cyan(url)}`); + this.logger.info(`Download finished: ${chalk.cyan(url)}`); return { filename, @@ -118,7 +123,7 @@ export class DownloadService { // Create temp file const [path, cleanup] = await createTemp(); - this.logger.info(`text file: Temp file is ${path}`); + this.logger.debug(`text file: Temp file is ${path}`); try { // write content at URL to temp file diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts index 82c447baaa..1f15b16617 100644 --- a/packages/backend/src/core/DriveService.ts +++ b/packages/backend/src/core/DriveService.ts @@ -45,6 +45,7 @@ import { isMimeImage } from '@/misc/is-mime-image.js'; import { ModerationLogService } from '@/core/ModerationLogService.js'; import { UtilityService } from '@/core/UtilityService.js'; import { BunnyService } from '@/core/BunnyService.js'; +import { renderInlineError } from '@/misc/render-inline-error.js'; import { LoggerService } from './LoggerService.js'; type AddFileArgs = { @@ -159,6 +160,14 @@ export class DriveService { // thunbnail, webpublic を必要なら生成 const alts = await this.generateAlts(path, type, !file.uri); + if (type && type.startsWith('video/')) { + try { + await this.videoProcessingService.webOptimizeVideo(path, type); + } catch (err) { + this.registerLogger.warn(`Video optimization failed: ${err instanceof Error ? err.message : String(err)}`, { error: err }); + } + } + if (this.meta.useObjectStorage) { //#region ObjectStorage params let [ext] = (name.match(/\.([a-zA-Z0-9_-]+)$/) ?? ['']); @@ -194,7 +203,7 @@ export class DriveService { //#endregion //#region Uploads - this.registerLogger.info(`uploading original: ${key}`); + this.registerLogger.debug(`uploading original: ${key}`); const uploads = [ this.upload(key, fs.createReadStream(path), type, null, name), ]; @@ -203,7 +212,7 @@ export class DriveService { webpublicKey = `${prefix}webpublic-${randomUUID()}.${alts.webpublic.ext}`; webpublicUrl = `${ baseUrl }/${ webpublicKey }`; - this.registerLogger.info(`uploading webpublic: ${webpublicKey}`); + this.registerLogger.debug(`uploading webpublic: ${webpublicKey}`); uploads.push(this.upload(webpublicKey, alts.webpublic.data, alts.webpublic.type, alts.webpublic.ext, name)); } @@ -211,7 +220,7 @@ export class DriveService { thumbnailKey = `${prefix}thumbnail-${randomUUID()}.${alts.thumbnail.ext}`; thumbnailUrl = `${ baseUrl }/${ thumbnailKey }`; - this.registerLogger.info(`uploading thumbnail: ${thumbnailKey}`); + this.registerLogger.debug(`uploading thumbnail: ${thumbnailKey}`); uploads.push(this.upload(thumbnailKey, alts.thumbnail.data, alts.thumbnail.type, alts.thumbnail.ext, `${name}.thumbnail`)); } @@ -255,11 +264,11 @@ export class DriveService { const [url, thumbnailUrl, webpublicUrl] = await Promise.all(promises); if (thumbnailUrl) { - this.registerLogger.info(`thumbnail stored: ${thumbnailAccessKey}`); + this.registerLogger.debug(`thumbnail stored: ${thumbnailAccessKey}`); } if (webpublicUrl) { - this.registerLogger.info(`web stored: ${webpublicAccessKey}`); + this.registerLogger.debug(`web stored: ${webpublicAccessKey}`); } file.storedInternal = true; @@ -303,7 +312,7 @@ export class DriveService { thumbnail, }; } catch (err) { - this.registerLogger.warn(`GenerateVideoThumbnail failed: ${err}`); + this.registerLogger.warn(`GenerateVideoThumbnail failed: ${renderInlineError(err)}`); return { webpublic: null, thumbnail: null, @@ -336,7 +345,7 @@ export class DriveService { metadata.height && metadata.height <= 2048 ); } catch (err) { - this.registerLogger.warn(`sharp failed: ${err}`); + this.registerLogger.warn(`sharp failed: ${renderInlineError(err)}`); return { webpublic: null, thumbnail: null, @@ -347,7 +356,7 @@ export class DriveService { let webpublic: IImage | null = null; if (generateWeb && !satisfyWebpublic && !isAnimated) { - this.registerLogger.info('creating web image'); + this.registerLogger.debug('creating web image'); try { if (['image/jpeg', 'image/webp', 'image/avif'].includes(type)) { @@ -361,9 +370,9 @@ export class DriveService { this.registerLogger.warn('web image not created (an error occurred)', err as Error); } } else { - if (satisfyWebpublic) this.registerLogger.info('web image not created (original satisfies webpublic)'); - else if (isAnimated) this.registerLogger.info('web image not created (animated image)'); - else this.registerLogger.info('web image not created (from remote)'); + if (satisfyWebpublic) this.registerLogger.debug('web image not created (original satisfies webpublic)'); + else if (isAnimated) this.registerLogger.debug('web image not created (animated image)'); + else this.registerLogger.debug('web image not created (from remote)'); } // #endregion webpublic @@ -490,7 +499,6 @@ export class DriveService { }: AddFileArgs): Promise { const userRoleNSFW = user && (await this.roleService.getUserPolicies(user.id)).alwaysMarkNsfw; const info = await this.fileInfoService.getFileInfo(path); - this.registerLogger.info(`${JSON.stringify(info)}`); // detect name const detectedName = correctFilename( @@ -500,6 +508,8 @@ export class DriveService { ext ?? info.type.ext, ); + this.registerLogger.debug(`Detected file info: ${JSON.stringify(info)}`); + if (user && !force) { // Check if there is a file with the same hash const matched = await this.driveFilesRepository.findOneBy({ @@ -508,7 +518,7 @@ export class DriveService { }); if (matched) { - this.registerLogger.info(`file with same hash is found: ${matched.id}`); + this.registerLogger.debug(`file with same hash is found: ${matched.id}`); if (sensitive && !matched.isSensitive) { // The file is federated as sensitive for this time, but was federated as non-sensitive before. // Therefore, update the file to sensitive. @@ -636,14 +646,14 @@ export class DriveService { } catch (err) { // duplicate key error (when already registered) if (isDuplicateKeyValueError(err)) { - this.registerLogger.info(`already registered ${file.uri}`); + this.registerLogger.debug(`already registered ${file.uri}`); file = await this.driveFilesRepository.findOneBy({ uri: file.uri!, userId: user ? user.id : IsNull(), }) as MiDriveFile; } else { - this.registerLogger.error(err as Error); + this.registerLogger.error('Error in drive register', err as Error); throw err; } } @@ -651,7 +661,7 @@ export class DriveService { file = await (this.save(file, path, detectedName, info)); } - this.registerLogger.succ(`drive file has been created ${file.id}`); + this.registerLogger.info(`Created file ${file.id} (${detectedName}) of type ${info.type.mime} for user ${user?.id ?? ''}`); if (user) { this.driveFileEntityService.pack(file, { self: true }).then(packedFile => { @@ -884,13 +894,10 @@ export class DriveService { } const driveFile = await this.addFile({ user, path, name, comment, folderId, force, isLink, url, uri, sensitive, requestIp, requestHeaders }); - this.downloaderLogger.succ(`Got: ${driveFile.id}`); + this.downloaderLogger.debug(`Upload succeeded: created file ${driveFile.id}`); return driveFile!; } catch (err) { - this.downloaderLogger.error(`Failed to create drive file: ${err}`, { - url: url, - e: err, - }); + this.downloaderLogger.error(`Failed to create drive file from ${url}: ${renderInlineError(err)}`); throw err; } finally { cleanup(); diff --git a/packages/backend/src/core/FanoutTimelineEndpointService.ts b/packages/backend/src/core/FanoutTimelineEndpointService.ts index 38371d2a07..40e2af1487 100644 --- a/packages/backend/src/core/FanoutTimelineEndpointService.ts +++ b/packages/backend/src/core/FanoutTimelineEndpointService.ts @@ -137,10 +137,10 @@ export class FanoutTimelineEndpointService { const parentFilter = filter; filter = (note) => { if (!ps.ignoreAuthorFromInstanceBlock) { - if (this.utilityService.isBlockedHost(this.meta.blockedHosts, note.userHost)) return false; + if (note.userInstance?.isBlocked) return false; } - if (note.userId !== note.renoteUserId && this.utilityService.isBlockedHost(this.meta.blockedHosts, note.renoteUserHost)) return false; - if (note.userId !== note.replyUserId && this.utilityService.isBlockedHost(this.meta.blockedHosts, note.replyUserHost)) return false; + if (note.userId !== note.renoteUserId && note.renoteUserInstance?.isBlocked) return false; + if (note.userId !== note.replyUserId && note.replyUserInstance?.isBlocked) return false; return parentFilter(note); }; @@ -212,7 +212,10 @@ export class FanoutTimelineEndpointService { .leftJoinAndSelect('note.renote', 'renote') .leftJoinAndSelect('reply.user', 'replyUser') .leftJoinAndSelect('renote.user', 'renoteUser') - .leftJoinAndSelect('note.channel', 'channel'); + .leftJoinAndSelect('note.channel', 'channel') + .leftJoinAndSelect('note.userInstance', 'userInstance') + .leftJoinAndSelect('note.replyUserInstance', 'replyUserInstance') + .leftJoinAndSelect('note.renoteUserInstance', 'renoteUserInstance'); const notes = (await query.getMany()).filter(noteFilter); diff --git a/packages/backend/src/core/FederatedInstanceService.ts b/packages/backend/src/core/FederatedInstanceService.ts index 3f7ed99348..34df10f0ff 100644 --- a/packages/backend/src/core/FederatedInstanceService.ts +++ b/packages/backend/src/core/FederatedInstanceService.ts @@ -5,23 +5,24 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import * as Redis from 'ioredis'; -import { QueryFailedError } from 'typeorm'; -import type { InstancesRepository } from '@/models/_.js'; +import type { InstancesRepository, MiMeta } from '@/models/_.js'; import type { MiInstance } from '@/models/Instance.js'; -import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js'; +import { MemoryKVCache } from '@/misc/cache.js'; import { IdService } from '@/core/IdService.js'; import { DI } from '@/di-symbols.js'; import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; -import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js'; +import type { GlobalEvents } from '@/core/GlobalEventService.js'; +import { Serialized } from '@/types.js'; +import { diffArrays, diffArraysSimple } from '@/misc/diff-arrays.js'; @Injectable() export class FederatedInstanceService implements OnApplicationShutdown { - public federatedInstanceCache: RedisKVCache; + private readonly federatedInstanceCache: MemoryKVCache; constructor( - @Inject(DI.redis) - private redisClient: Redis.Redis, + @Inject(DI.redisForSub) + private redisForSub: Redis.Redis, @Inject(DI.instancesRepository) private instancesRepository: InstancesRepository, @@ -29,67 +30,46 @@ export class FederatedInstanceService implements OnApplicationShutdown { private utilityService: UtilityService, private idService: IdService, ) { - this.federatedInstanceCache = new RedisKVCache(this.redisClient, 'federatedInstance', { - lifetime: 1000 * 60 * 30, // 30m - memoryCacheLifetime: 1000 * 60 * 3, // 3m - fetcher: (key) => this.instancesRepository.findOneBy({ host: key }), - toRedisConverter: (value) => JSON.stringify(value), - fromRedisConverter: (value) => { - const parsed = JSON.parse(value); - if (parsed == null) return null; - return { - ...parsed, - firstRetrievedAt: new Date(parsed.firstRetrievedAt), - latestRequestReceivedAt: parsed.latestRequestReceivedAt ? new Date(parsed.latestRequestReceivedAt) : null, - infoUpdatedAt: parsed.infoUpdatedAt ? new Date(parsed.infoUpdatedAt) : null, - notRespondingSince: parsed.notRespondingSince ? new Date(parsed.notRespondingSince) : null, - }; - }, - }); + this.federatedInstanceCache = new MemoryKVCache(1000 * 60 * 3); // 3m + this.redisForSub.on('message', this.onMessage); } @bindThis public async fetchOrRegister(host: string): Promise { host = this.utilityService.toPuny(host); - const cached = await this.federatedInstanceCache.get(host); + const cached = this.federatedInstanceCache.get(host); if (cached) return cached; - const index = await this.instancesRepository.findOneBy({ host }); - + let index = await this.instancesRepository.findOneBy({ host }); if (index == null) { - let i; - try { - i = await this.instancesRepository.insertOne({ + await this.instancesRepository.createQueryBuilder('instance') + .insert() + .values({ id: this.idService.gen(), host, firstRetrievedAt: new Date(), - }); - } catch (e: unknown) { - if (e instanceof QueryFailedError) { - if (isDuplicateKeyValueError(e)) { - i = await this.instancesRepository.findOneBy({ host }); - } - } + isBlocked: this.utilityService.isBlockedHost(host), + isSilenced: this.utilityService.isSilencedHost(host), + isMediaSilenced: this.utilityService.isMediaSilencedHost(host), + isAllowListed: this.utilityService.isAllowListedHost(host), + isBubbled: this.utilityService.isBubbledHost(host), + }) + .orIgnore() + .execute(); - if (i == null) { - throw e; - } - } - - this.federatedInstanceCache.set(host, i); - return i; - } else { - this.federatedInstanceCache.set(host, index); - return index; + index = await this.instancesRepository.findOneByOrFail({ host }); } + + this.federatedInstanceCache.set(host, index); + return index; } @bindThis public async fetch(host: string): Promise { host = this.utilityService.toPuny(host); - const cached = await this.federatedInstanceCache.get(host); + const cached = this.federatedInstanceCache.get(host); if (cached !== undefined) return cached; const index = await this.instancesRepository.findOneBy({ host }); @@ -117,8 +97,35 @@ export class FederatedInstanceService implements OnApplicationShutdown { this.federatedInstanceCache.set(result.host, result); } + private syncCache(before: Serialized, after: Serialized): void { + const changed = + diffArraysSimple(before?.blockedHosts, after.blockedHosts) || + diffArraysSimple(before?.silencedHosts, after.silencedHosts) || + diffArraysSimple(before?.mediaSilencedHosts, after.mediaSilencedHosts) || + diffArraysSimple(before?.federationHosts, after.federationHosts) || + diffArraysSimple(before?.bubbleInstances, after.bubbleInstances); + + if (changed) { + // We have to clear the whole thing, otherwise subdomains won't be synced. + this.federatedInstanceCache.clear(); + } + } + + @bindThis + private async onMessage(_: string, data: string): Promise { + const obj = JSON.parse(data); + + if (obj.channel === 'internal') { + const { type, body } = obj.message as GlobalEvents['internal']['payload']; + if (type === 'metaUpdated') { + this.syncCache(body.before, body.after); + } + } + } + @bindThis public dispose(): void { + this.redisForSub.off('message', this.onMessage); this.federatedInstanceCache.dispose(); } diff --git a/packages/backend/src/core/FetchInstanceMetadataService.ts b/packages/backend/src/core/FetchInstanceMetadataService.ts index 980f1fcacf..9bfd7381f1 100644 --- a/packages/backend/src/core/FetchInstanceMetadataService.ts +++ b/packages/backend/src/core/FetchInstanceMetadataService.ts @@ -15,6 +15,7 @@ import { LoggerService } from '@/core/LoggerService.js'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { bindThis } from '@/decorators.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; +import { renderInlineError } from '@/misc/render-inline-error.js'; import type { CheerioAPI } from 'cheerio'; type NodeInfo = { @@ -90,7 +91,7 @@ export class FetchInstanceMetadataService { } } - this.logger.info(`Fetching metadata of ${instance.host} ...`); + this.logger.debug(`Fetching metadata of ${instance.host} ...`); const [info, dom, manifest] = await Promise.all([ this.fetchNodeinfo(instance).catch(() => null), @@ -106,7 +107,7 @@ export class FetchInstanceMetadataService { this.getDescription(info, dom, manifest).catch(() => null), ]); - this.logger.succ(`Successfuly fetched metadata of ${instance.host}`); + this.logger.debug(`Successfuly fetched metadata of ${instance.host}`); const updates = { infoUpdatedAt: new Date(), @@ -128,9 +129,9 @@ export class FetchInstanceMetadataService { await this.federatedInstanceService.update(instance.id, updates); - this.logger.succ(`Successfuly updated metadata of ${instance.host}`); + this.logger.info(`Successfully updated metadata of ${instance.host}`); } catch (e) { - this.logger.error(`Failed to update metadata of ${instance.host}: ${e}`); + this.logger.error(`Failed to update metadata of ${instance.host}: ${renderInlineError(e)}`); } finally { await this.unlock(host); } @@ -138,7 +139,7 @@ export class FetchInstanceMetadataService { @bindThis private async fetchNodeinfo(instance: MiInstance): Promise { - this.logger.info(`Fetching nodeinfo of ${instance.host} ...`); + this.logger.debug(`Fetching nodeinfo of ${instance.host} ...`); try { const wellknown = await this.httpRequestService.getJson('https://' + instance.host + '/.well-known/nodeinfo') @@ -170,11 +171,11 @@ export class FetchInstanceMetadataService { throw err.statusCode ?? err.message; }); - this.logger.succ(`Successfuly fetched nodeinfo of ${instance.host}`); + this.logger.debug(`Successfuly fetched nodeinfo of ${instance.host}`); return info as NodeInfo; } catch (err) { - this.logger.error(`Failed to fetch nodeinfo of ${instance.host}: ${err}`); + this.logger.warn(`Failed to fetch nodeinfo of ${instance.host}: ${renderInlineError(err)}`); throw err; } @@ -182,7 +183,7 @@ export class FetchInstanceMetadataService { @bindThis private async fetchDom(instance: MiInstance): Promise { - this.logger.info(`Fetching HTML of ${instance.host} ...`); + this.logger.debug(`Fetching HTML of ${instance.host} ...`); const url = 'https://' + instance.host; diff --git a/packages/backend/src/core/FileInfoService.ts b/packages/backend/src/core/FileInfoService.ts index cc66e9fe3a..98fbfe5f23 100644 --- a/packages/backend/src/core/FileInfoService.ts +++ b/packages/backend/src/core/FileInfoService.ts @@ -46,11 +46,13 @@ const TYPE_SVG = { @Injectable() export class FileInfoService { private logger: Logger; + private ffprobeLogger: Logger; constructor( private loggerService: LoggerService, ) { this.logger = this.loggerService.getLogger('file-info'); + this.ffprobeLogger = this.logger.createSubLogger('ffprobe'); } /** @@ -162,20 +164,19 @@ export class FileInfoService { */ @bindThis private hasVideoTrackOnVideoFile(path: string): Promise { - const sublogger = this.logger.createSubLogger('ffprobe'); - sublogger.info(`Checking the video file. File path: ${path}`); + this.ffprobeLogger.debug(`Checking the video file. File path: ${path}`); return new Promise((resolve) => { try { FFmpeg.ffprobe(path, (err, metadata) => { if (err) { - sublogger.warn(`Could not check the video file. Returns true. File path: ${path}`, err); + this.ffprobeLogger.warn(`Could not check the video file. Returns true. File path: ${path}`, err); resolve(true); return; } resolve(metadata.streams.some((stream) => stream.codec_type === 'video')); }); } catch (err) { - sublogger.warn(`Could not check the video file. Returns true. File path: ${path}`, err as Error); + this.ffprobeLogger.warn(`Could not check the video file. Returns true. File path: ${path}`, err as Error); resolve(true); } }); diff --git a/packages/backend/src/core/HttpRequestService.ts b/packages/backend/src/core/HttpRequestService.ts index 5c271b81e3..151097095d 100644 --- a/packages/backend/src/core/HttpRequestService.ts +++ b/packages/backend/src/core/HttpRequestService.ts @@ -235,7 +235,7 @@ export class HttpRequestService { } @bindThis - public async getActivityJson(url: string, isLocalAddressAllowed = false): Promise { + public async getActivityJson(url: string, isLocalAddressAllowed = false, allowAnonymous = false): Promise { this.apUtilityService.assertApUrl(url); const res = await this.send(url, { @@ -255,7 +255,11 @@ export class HttpRequestService { // Make sure the object ID matches the final URL (which is where it actually exists). // The caller (ApResolverService) will verify the ID against the original / entry URL, which ensures that all three match. - this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url); + if (allowAnonymous && activity.id == null) { + activity.id = res.url; + } else { + this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url); + } return activity as IObjectWithId; } @@ -327,7 +331,7 @@ export class HttpRequestService { }); if (!res.ok && extra.throwErrorWhenResponseNotOk) { - throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText); + throw new StatusError(`request error from ${url}`, res.status, res.statusText); } if (res.ok) { diff --git a/packages/backend/src/core/LatestNoteService.ts b/packages/backend/src/core/LatestNoteService.ts index c379805506..63f973c6c6 100644 --- a/packages/backend/src/core/LatestNoteService.ts +++ b/packages/backend/src/core/LatestNoteService.ts @@ -7,6 +7,7 @@ import { DI } from '@/di-symbols.js'; import type { LatestNotesRepository, NotesRepository } from '@/models/_.js'; import { LoggerService } from '@/core/LoggerService.js'; import Logger from '@/logger.js'; +import { QueryService } from './QueryService.js'; @Injectable() export class LatestNoteService { @@ -14,11 +15,12 @@ export class LatestNoteService { constructor( @Inject(DI.notesRepository) - private notesRepository: NotesRepository, + private readonly notesRepository: NotesRepository, @Inject(DI.latestNotesRepository) - private latestNotesRepository: LatestNotesRepository, + private readonly latestNotesRepository: LatestNotesRepository, + private readonly queryService: QueryService, loggerService: LoggerService, ) { this.logger = loggerService.getLogger('LatestNoteService'); @@ -91,7 +93,7 @@ export class LatestNoteService { // Find the newest remaining note for the user. // We exclude DMs and pure renotes. - const nextLatest = await this.notesRepository + const query = this.notesRepository .createQueryBuilder('note') .select() .where({ @@ -106,18 +108,11 @@ export class LatestNoteService { ? Not(null) : null, }) - .andWhere(` - ( - note."renoteId" IS NULL - OR note.text IS NOT NULL - OR note.cw IS NOT NULL - OR note."replyId" IS NOT NULL - OR note."hasPoll" - OR note."fileIds" != '{}' - ) - `) - .orderBy({ id: 'DESC' }) - .getOne(); + .orderBy({ id: 'DESC' }); + + this.queryService.andIsNotRenote(query, 'note'); + + const nextLatest = await query.getOne(); if (!nextLatest) return; // Record it as the latest diff --git a/packages/backend/src/core/MetaService.ts b/packages/backend/src/core/MetaService.ts index 40e7439f5f..07f82dc23e 100644 --- a/packages/backend/src/core/MetaService.ts +++ b/packages/backend/src/core/MetaService.ts @@ -4,7 +4,7 @@ */ import { Inject, Injectable } from '@nestjs/common'; -import { DataSource } from 'typeorm'; +import { DataSource, EntityManager } from 'typeorm'; import * as Redis from 'ioredis'; import { DI } from '@/di-symbols.js'; import { MiMeta } from '@/models/Meta.js'; @@ -12,6 +12,9 @@ import { GlobalEventService } from '@/core/GlobalEventService.js'; import { bindThis } from '@/decorators.js'; import type { GlobalEvents } from '@/core/GlobalEventService.js'; import { FeaturedService } from '@/core/FeaturedService.js'; +import { MiInstance } from '@/models/Instance.js'; +import { diffArrays } from '@/misc/diff-arrays.js'; +import type { MetasRepository } from '@/models/_.js'; import type { OnApplicationShutdown } from '@nestjs/common'; @Injectable() @@ -26,6 +29,9 @@ export class MetaService implements OnApplicationShutdown { @Inject(DI.db) private db: DataSource, + @Inject(DI.metasRepository) + private readonly metasRepository: MetasRepository, + private featuredService: FeaturedService, private globalEventService: GlobalEventService, ) { @@ -67,35 +73,35 @@ export class MetaService implements OnApplicationShutdown { public async fetch(noCache = false): Promise { if (!noCache && this.cache) return this.cache; - return await this.db.transaction(async transactionalEntityManager => { - // 過去のバグでレコードが複数出来てしまっている可能性があるので新しいIDを優先する - const metas = await transactionalEntityManager.find(MiMeta, { - order: { + // 過去のバグでレコードが複数出来てしまっている可能性があるので新しいIDを優先する + let meta = await this.metasRepository.createQueryBuilder('meta') + .select() + .orderBy({ + id: 'DESC', + }) + .limit(1) + .getOne(); + + if (!meta) { + await this.metasRepository.createQueryBuilder('meta') + .insert() + .values({ + id: 'x', + }) + .orIgnore() + .execute(); + + meta = await this.metasRepository.createQueryBuilder('meta') + .select() + .orderBy({ id: 'DESC', - }, - }); + }) + .limit(1) + .getOneOrFail(); + } - const meta = metas[0]; - - if (meta) { - this.cache = meta; - return meta; - } else { - // metaが空のときfetchMetaが同時に呼ばれるとここが同時に呼ばれてしまうことがあるのでフェイルセーフなupsertを使う - const saved = await transactionalEntityManager - .upsert( - MiMeta, - { - id: 'x', - }, - ['id'], - ) - .then((x) => transactionalEntityManager.findOneByOrFail(MiMeta, x.identifiers[0])); - - this.cache = saved; - return saved; - } - }); + this.cache = meta; + return meta; } @bindThis @@ -103,7 +109,7 @@ export class MetaService implements OnApplicationShutdown { let before: MiMeta | undefined; const updated = await this.db.transaction(async transactionalEntityManager => { - const metas = await transactionalEntityManager.find(MiMeta, { + const metas: (MiMeta | undefined)[] = await transactionalEntityManager.find(MiMeta, { order: { id: 'DESC', }, @@ -126,6 +132,10 @@ export class MetaService implements OnApplicationShutdown { }, }); + // Propagate changes to blockedHosts, silencedHosts, mediaSilencedHosts, federationInstances, and bubbleInstances to the relevant instance rows + // Do this inside the transaction to avoid potential race condition (when an instance gets registered while we're updating). + await this.persistBlocks(transactionalEntityManager, before ?? {}, afters[0]); + return afters[0]; }); @@ -159,4 +169,49 @@ export class MetaService implements OnApplicationShutdown { public onApplicationShutdown(signal?: string | undefined): void { this.dispose(); } + + private async persistBlocks(tem: EntityManager, before: Partial, after: Partial): Promise { + await this.persistBlock(tem, before.blockedHosts, after.blockedHosts, 'isBlocked'); + await this.persistBlock(tem, before.silencedHosts, after.silencedHosts, 'isSilenced'); + await this.persistBlock(tem, before.mediaSilencedHosts, after.mediaSilencedHosts, 'isMediaSilenced'); + await this.persistBlock(tem, before.federationHosts, after.federationHosts, 'isAllowListed'); + await this.persistBlock(tem, before.bubbleInstances, after.bubbleInstances, 'isBubbled'); + } + + private async persistBlock(tem: EntityManager, before: string[] | undefined, after: string[] | undefined, field: keyof MiInstance): Promise { + const { added, removed } = diffArrays(before, after); + + if (removed.length > 0) { + await this.updateInstancesByHost(tem, field, false, removed); + } + + if (added.length > 0) { + await this.updateInstancesByHost(tem, field, true, added); + } + } + + private async updateInstancesByHost(tem: EntityManager, field: keyof MiInstance, value: boolean, hosts: string[]): Promise { + // Use non-array queries when possible, as they are indexed and can be much faster. + if (hosts.length === 1) { + const pattern = genHostPattern(hosts[0]); + await tem + .createQueryBuilder(MiInstance, 'instance') + .update() + .set({ [field]: value }) + .where('(lower(reverse("host")) || \'.\') LIKE :pattern', { pattern }) + .execute(); + } else if (hosts.length > 1) { + const patterns = hosts.map(host => genHostPattern(host)); + await tem + .createQueryBuilder(MiInstance, 'instance') + .update() + .set({ [field]: value }) + .where('(lower(reverse("host")) || \'.\') LIKE ANY (:patterns)', { patterns }) + .execute(); + } + } +} + +function genHostPattern(host: string): string { + return host.toLowerCase().split('').reverse().join('') + '.%'; } diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 097d657ba3..4dceb6e953 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -296,7 +296,7 @@ export class NoteCreateService implements OnApplicationShutdown { case 'followers': // 他人のfollowers noteはreject if (data.renote.userId !== user.id) { - throw new Error('Renote target is not public or home'); + throw new IdentifiableError('b6352a84-e5cd-4b05-a26c-63437a6b98ba', 'Renote target is not public or home'); } // Renote対象がfollowersならfollowersにする @@ -304,7 +304,7 @@ export class NoteCreateService implements OnApplicationShutdown { break; case 'specified': // specified / direct noteはreject - throw new Error('Renote target is not public or home'); + throw new IdentifiableError('b6352a84-e5cd-4b05-a26c-63437a6b98ba', 'Renote target is not public or home'); } } @@ -317,7 +317,7 @@ export class NoteCreateService implements OnApplicationShutdown { if (data.renote.userId !== user.id) { const blocked = await this.userBlockingService.checkBlocked(data.renote.userId, user.id); if (blocked) { - throw new Error('blocked'); + throw new IdentifiableError('b6352a84-e5cd-4b05-a26c-63437a6b98ba', 'Renote target is blocked'); } } } @@ -489,10 +489,10 @@ export class NoteCreateService implements OnApplicationShutdown { // should really not happen, but better safe than sorry if (data.reply?.id === insert.id) { - throw new Error('A note can\'t reply to itself'); + throw new IdentifiableError('ea93b7c2-3d6c-4e10-946b-00d50b1a75cb', 'A note can\'t reply to itself'); } if (data.renote?.id === insert.id) { - throw new Error('A note can\'t renote itself'); + throw new IdentifiableError('ea93b7c2-3d6c-4e10-946b-00d50b1a75cb', 'A note can\'t renote itself'); } if (data.uri != null) insert.uri = data.uri; @@ -549,8 +549,6 @@ export class NoteCreateService implements OnApplicationShutdown { throw err; } - console.error(e); - throw e; } } diff --git a/packages/backend/src/core/NoteEditService.ts b/packages/backend/src/core/NoteEditService.ts index 58233b90ee..34af1c76dd 100644 --- a/packages/backend/src/core/NoteEditService.ts +++ b/packages/backend/src/core/NoteEditService.ts @@ -309,7 +309,7 @@ export class NoteEditService implements OnApplicationShutdown { if (this.isRenote(data)) { if (data.renote.id === oldnote.id) { - throw new UnrecoverableError(`edit failed for ${oldnote.id}: cannot renote itself`); + throw new IdentifiableError('ea93b7c2-3d6c-4e10-946b-00d50b1a75cb', `edit failed for ${oldnote.id}: cannot renote itself`); } switch (data.renote.visibility) { @@ -325,7 +325,7 @@ export class NoteEditService implements OnApplicationShutdown { case 'followers': // 他人のfollowers noteはreject if (data.renote.userId !== user.id) { - throw new Error('Renote target is not public or home'); + throw new IdentifiableError('b6352a84-e5cd-4b05-a26c-63437a6b98ba', 'Renote target is not public or home'); } // Renote対象がfollowersならfollowersにする @@ -333,7 +333,7 @@ export class NoteEditService implements OnApplicationShutdown { break; case 'specified': // specified / direct noteはreject - throw new Error('Renote target is not public or home'); + throw new IdentifiableError('b6352a84-e5cd-4b05-a26c-63437a6b98ba', 'Renote target is not public or home'); } } diff --git a/packages/backend/src/core/NotePiningService.ts b/packages/backend/src/core/NotePiningService.ts index 86f1a62d4a..a678108189 100644 --- a/packages/backend/src/core/NotePiningService.ts +++ b/packages/backend/src/core/NotePiningService.ts @@ -61,7 +61,7 @@ export class NotePiningService { }); if (note == null) { - throw new IdentifiableError('70c4e51f-5bea-449c-a030-53bee3cce202', 'No such note.'); + throw new IdentifiableError('70c4e51f-5bea-449c-a030-53bee3cce202', `Note ${noteId} does not exist`); } await this.db.transaction(async tem => { @@ -102,7 +102,7 @@ export class NotePiningService { }); if (note == null) { - throw new IdentifiableError('b302d4cf-c050-400a-bbb3-be208681f40c', 'No such note.'); + throw new IdentifiableError('b302d4cf-c050-400a-bbb3-be208681f40c', `Note ${noteId} does not exist`); } this.userNotePiningsRepository.delete({ diff --git a/packages/backend/src/core/QueryService.ts b/packages/backend/src/core/QueryService.ts index e4bf97c107..e2e019bc1b 100644 --- a/packages/backend/src/core/QueryService.ts +++ b/packages/backend/src/core/QueryService.ts @@ -4,13 +4,14 @@ */ import { Inject, Injectable } from '@nestjs/common'; -import { Brackets, ObjectLiteral } from 'typeorm'; +import { Brackets, Not, WhereExpressionBuilder } from 'typeorm'; import { DI } from '@/di-symbols.js'; import type { MiUser } from '@/models/User.js'; -import type { UserProfilesRepository, FollowingsRepository, ChannelFollowingsRepository, BlockingsRepository, NoteThreadMutingsRepository, MutingsRepository, RenoteMutingsRepository, MiMeta } from '@/models/_.js'; +import { MiInstance } from '@/models/Instance.js'; +import type { UserProfilesRepository, FollowingsRepository, ChannelFollowingsRepository, BlockingsRepository, NoteThreadMutingsRepository, MutingsRepository, RenoteMutingsRepository, MiMeta, InstancesRepository } from '@/models/_.js'; import { bindThis } from '@/decorators.js'; import { IdService } from '@/core/IdService.js'; -import type { SelectQueryBuilder } from 'typeorm'; +import type { SelectQueryBuilder, ObjectLiteral } from 'typeorm'; @Injectable() export class QueryService { @@ -36,6 +37,9 @@ export class QueryService { @Inject(DI.renoteMutingsRepository) private renoteMutingsRepository: RenoteMutingsRepository, + @Inject(DI.instancesRepository) + private readonly instancesRepository: InstancesRepository, + @Inject(DI.meta) private meta: MiMeta, @@ -79,219 +83,484 @@ export class QueryService { // ここでいうBlockedは被Blockedの意 @bindThis - public generateBlockedUserQueryForNotes(q: SelectQueryBuilder, me: { id: MiUser['id'] }): void { - const blockingQuery = this.blockingsRepository.createQueryBuilder('blocking') - .select('blocking.blockerId') - .where('blocking.blockeeId = :blockeeId', { blockeeId: me.id }); - + public generateBlockedUserQueryForNotes(q: SelectQueryBuilder, me: { id: MiUser['id'] }): SelectQueryBuilder { // 投稿の作者にブロックされていない かつ // 投稿の返信先の作者にブロックされていない かつ // 投稿の引用元の作者にブロックされていない - q - .andWhere(`note.userId NOT IN (${ blockingQuery.getQuery() })`) - .andWhere(new Brackets(qb => { - qb - .where('note.replyUserId IS NULL') - .orWhere(`note.replyUserId NOT IN (${ blockingQuery.getQuery() })`); - })) - .andWhere(new Brackets(qb => { - qb - .where('note.renoteUserId IS NULL') - .orWhere(`note.renoteUserId NOT IN (${ blockingQuery.getQuery() })`); - })); - - q.setParameters(blockingQuery.getParameters()); + return this + .andNotBlockingUser(q, 'note.userId', ':meId') + .andWhere(new Brackets(qb => this + .orNotBlockingUser(qb, 'note.replyUserId', ':meId') + .orWhere('note.replyUserId IS NULL'))) + .andWhere(new Brackets(qb => this + .orNotBlockingUser(qb, 'note.renoteUserId', ':meId') + .orWhere('note.renoteUserId IS NULL'))) + .setParameters({ meId: me.id }); } @bindThis - public generateBlockQueryForUsers(q: SelectQueryBuilder, me: { id: MiUser['id'] }): void { - const blockingQuery = this.blockingsRepository.createQueryBuilder('blocking') - .select('blocking.blockeeId') - .where('blocking.blockerId = :blockerId', { blockerId: me.id }); - - const blockedQuery = this.blockingsRepository.createQueryBuilder('blocking') - .select('blocking.blockerId') - .where('blocking.blockeeId = :blockeeId', { blockeeId: me.id }); - - q.andWhere(`user.id NOT IN (${ blockingQuery.getQuery() })`); - q.setParameters(blockingQuery.getParameters()); - - q.andWhere(`user.id NOT IN (${ blockedQuery.getQuery() })`); - q.setParameters(blockedQuery.getParameters()); + public generateBlockQueryForUsers(q: SelectQueryBuilder, me: { id: MiUser['id'] }): SelectQueryBuilder { + this.andNotBlockingUser(q, ':meId', 'user.id'); + this.andNotBlockingUser(q, 'user.id', ':me.id'); + return q.setParameters({ meId: me.id }); } @bindThis - public generateMutedNoteThreadQuery(q: SelectQueryBuilder, me: { id: MiUser['id'] }): void { - const mutedQuery = this.noteThreadMutingsRepository.createQueryBuilder('threadMuted') - .select('threadMuted.threadId') - .where('threadMuted.userId = :userId', { userId: me.id }); - - q.andWhere(`note.id NOT IN (${ mutedQuery.getQuery() })`); - q.andWhere(new Brackets(qb => { - qb - .where('note.threadId IS NULL') - .orWhere(`note.threadId NOT IN (${ mutedQuery.getQuery() })`); - })); - - q.setParameters(mutedQuery.getParameters()); + public generateMutedNoteThreadQuery(q: SelectQueryBuilder, me: { id: MiUser['id'] }): SelectQueryBuilder { + return this + .andNotMutingThread(q, ':meId', 'note.id') + .andWhere(new Brackets(qb => this + .orNotMutingThread(qb, ':meId', 'note.threadId') + .orWhere('note.threadId IS NULL'))) + .setParameters({ meId: me.id }); } @bindThis - public generateMutedUserQueryForNotes(q: SelectQueryBuilder, me: { id: MiUser['id'] }, exclude?: { id: MiUser['id'] }): void { - const mutingQuery = this.mutingsRepository.createQueryBuilder('muting') - .select('muting.muteeId') - .where('muting.muterId = :muterId', { muterId: me.id }); - - if (exclude) { - mutingQuery.andWhere('muting.muteeId != :excludeId', { excludeId: exclude.id }); - } - - const mutingInstanceQuery = this.userProfilesRepository.createQueryBuilder('user_profile') - .select('user_profile.mutedInstances') - .where('user_profile.userId = :muterId', { muterId: me.id }); - + public generateMutedUserQueryForNotes(q: SelectQueryBuilder, me: { id: MiUser['id'] }, exclude?: { id: MiUser['id'] }): SelectQueryBuilder { // 投稿の作者をミュートしていない かつ // 投稿の返信先の作者をミュートしていない かつ // 投稿の引用元の作者をミュートしていない - q - .andWhere(`note.userId NOT IN (${ mutingQuery.getQuery() })`) - .andWhere(new Brackets(qb => { - qb - .where('note.replyUserId IS NULL') - .orWhere(`note.replyUserId NOT IN (${ mutingQuery.getQuery() })`); - })) - .andWhere(new Brackets(qb => { - qb - .where('note.renoteUserId IS NULL') - .orWhere(`note.renoteUserId NOT IN (${ mutingQuery.getQuery() })`); - })) + return this + .andNotMutingUser(q, ':meId', 'note.userId', exclude) + .andWhere(new Brackets(qb => this + .orNotMutingUser(qb, ':meId', 'note.replyUserId', exclude) + .orWhere('note.replyUserId IS NULL'))) + .andWhere(new Brackets(qb => this + .orNotMutingUser(qb, ':meId', 'note.renoteUserId', exclude) + .orWhere('note.renoteUserId IS NULL'))) + // TODO exclude should also pass a host to skip these instances // mute instances - .andWhere(new Brackets(qb => { - qb - .andWhere('note.userHost IS NULL') - .orWhere(`NOT ((${ mutingInstanceQuery.getQuery() })::jsonb ? note.userHost)`); - })) - .andWhere(new Brackets(qb => { - qb - .where('note.replyUserHost IS NULL') - .orWhere(`NOT ((${ mutingInstanceQuery.getQuery() })::jsonb ? note.replyUserHost)`); - })) - .andWhere(new Brackets(qb => { - qb - .where('note.renoteUserHost IS NULL') - .orWhere(`NOT ((${ mutingInstanceQuery.getQuery() })::jsonb ? note.renoteUserHost)`); - })); - - q.setParameters(mutingQuery.getParameters()); - q.setParameters(mutingInstanceQuery.getParameters()); + .andWhere(new Brackets(qb => this + .andNotMutingInstance(qb, ':meId', 'note.userHost') + .orWhere('note.userHost IS NULL'))) + .andWhere(new Brackets(qb => this + .orNotMutingInstance(qb, ':meId', 'note.replyUserHost') + .orWhere('note.replyUserHost IS NULL'))) + .andWhere(new Brackets(qb => this + .orNotMutingInstance(qb, ':meId', 'note.renoteUserHost') + .orWhere('note.renoteUserHost IS NULL'))) + .setParameters({ meId: me.id }); } @bindThis - public generateMutedUserQueryForUsers(q: SelectQueryBuilder, me: { id: MiUser['id'] }): void { - const mutingQuery = this.mutingsRepository.createQueryBuilder('muting') - .select('muting.muteeId') - .where('muting.muterId = :muterId', { muterId: me.id }); - - q.andWhere(`user.id NOT IN (${ mutingQuery.getQuery() })`); - - q.setParameters(mutingQuery.getParameters()); + public generateMutedUserQueryForUsers(q: SelectQueryBuilder, me: { id: MiUser['id'] }): SelectQueryBuilder { + return this + .andNotMutingUser(q, ':meId', 'user.id') + .setParameters({ meId: me.id }); } + // This intentionally skips isSuspended, isDeleted, makeNotesFollowersOnlyBefore, makeNotesHiddenBefore, and requireSigninToViewContents. + // NoteEntityService checks these automatically and calls hideNote() to hide them without breaking threads. + // For moderation purposes, you can set isSilenced to forcibly hide existing posts by a user. @bindThis - public generateVisibilityQuery(q: SelectQueryBuilder, me?: { id: MiUser['id'] } | null): void { + public generateVisibilityQuery(q: SelectQueryBuilder, me?: { id: MiUser['id'] } | null): SelectQueryBuilder { // This code must always be synchronized with the checks in Notes.isVisibleForMe. - if (me == null) { - q.andWhere(new Brackets(qb => { - qb - .where('note.visibility = \'public\'') - .orWhere('note.visibility = \'home\''); - })); - } else { - const followingQuery = this.followingsRepository.createQueryBuilder('following') - .select('following.followeeId') - .where('following.followerId = :meId'); + return q.andWhere(new Brackets(qb => { + // Public post + qb.orWhere('note.visibility = \'public\'') + .orWhere('note.visibility = \'home\''); - q.andWhere(new Brackets(qb => { + if (me != null) { qb - // 公開投稿である - .where(new Brackets(qb => { - qb - .where('note.visibility = \'public\'') - .orWhere('note.visibility = \'home\''); - })) - // または 自分自身 - .orWhere('note.userId = :meId') - // または 自分宛て + // My post + .orWhere(':meId = note.userId') + // Reply to me + .orWhere(':meId = note.replyUserId') + // DM to me .orWhere(':meIdAsList <@ note.visibleUserIds') - .orWhere(new Brackets(qb => { - qb - // または フォロワー宛ての投稿であり、 - .where('note.visibility = \'followers\'') - .andWhere(new Brackets(qb => { - qb - // 自分がフォロワーである - .where(`note.userId IN (${ followingQuery.getQuery() })`) - // または 自分の投稿へのリプライ - .orWhere('note.replyUserId = :meId') - .orWhere(':meIdAsList <@ note.mentions'); - })); - })); + // Mentions me + .orWhere(':meIdAsList <@ note.mentions') + // Followers-only post + .orWhere(new Brackets(qb => this + .andFollowingUser(qb, ':meId', 'note.userId') + .andWhere('note.visibility = \'followers\''))); + + q.setParameters({ meId: me.id, meIdAsList: [me.id] }); + } + })); + } + + @bindThis + public generateMutedUserRenotesQueryForNotes(q: SelectQueryBuilder, me: { id: MiUser['id'] }): SelectQueryBuilder { + return q + .andWhere(new Brackets(qb => this + .orNotMutingRenote(qb, ':meId', 'note.userId') + .orWhere('note.renoteId IS NULL') + .orWhere('note.text IS NOT NULL') + .orWhere('note.cw IS NOT NULL') + .orWhere('note.replyId IS NOT NULL') + .orWhere('note.hasPoll = true') + .orWhere('note.fileIds != \'{}\''))) + .setParameters({ meId: me.id }); + } + + @bindThis + public generateExcludedRenotesQueryForNotes(q: Q): Q { + return this.andIsNotRenote(q, 'note'); + } + + @bindThis + public generateBlockedHostQueryForNote(q: SelectQueryBuilder, excludeAuthor?: boolean): SelectQueryBuilder { + const checkFor = (key: 'user' | 'replyUser' | 'renoteUser') => this + .leftJoinInstance(q, `note.${key}Instance`, `${key}Instance`) + .andWhere(new Brackets(qb => { + qb + .orWhere(`"${key}Instance" IS NULL`) // local + .orWhere(`"${key}Instance"."isBlocked" = false`); // not blocked + + if (excludeAuthor) { + qb.orWhere(`note.userId = note.${key}Id`); // author + } })); - q.setParameters({ meId: me.id, meIdAsList: [me.id] }); + if (!excludeAuthor) { + checkFor('user'); } + checkFor('replyUser'); + checkFor('renoteUser'); + + return q; } @bindThis - public generateMutedUserRenotesQueryForNotes(q: SelectQueryBuilder, me: { id: MiUser['id'] }): void { + public generateSilencedUserQueryForNotes(q: SelectQueryBuilder, me?: { id: MiUser['id'] } | null): SelectQueryBuilder { + if (!me) { + return q.andWhere('user.isSilenced = false'); + } + + return this + .leftJoinInstance(q, 'note.userInstance', 'userInstance') + .andWhere(new Brackets(qb => this + // case 1: we are following the user + .orFollowingUser(qb, ':meId', 'note.userId') + // case 2: user not silenced AND instance not silenced + .orWhere(new Brackets(qbb => qbb + .andWhere(new Brackets(qbbb => qbbb + .orWhere('"userInstance"."isSilenced" = false') + .orWhere('"userInstance" IS NULL'))) + .andWhere('user.isSilenced = false'))))) + .setParameters({ meId: me.id }); + } + + /** + * Left-joins an instance in to the query with a given alias and optional condition. + * These calls are de-duplicated - multiple uses of the same alias are skipped. + */ + @bindThis + public leftJoinInstance(q: SelectQueryBuilder, relation: string | typeof MiInstance, alias: string, condition?: string): SelectQueryBuilder { + // Skip if it's already joined, otherwise we'll get an error + if (!q.expressionMap.joinAttributes.some(j => j.alias.name === alias)) { + q.leftJoin(relation, alias, condition); + } + + return q; + } + + /** + * Adds OR condition that noteProp (note ID) refers to a quote. + * The prop should be an expression, not a raw value. + */ + @bindThis + public orIsQuote(q: Q, noteProp: string): Q { + return this.addIsQuote(q, noteProp, 'orWhere'); + } + + /** + * Adds AND condition that noteProp (note ID) refers to a quote. + * The prop should be an expression, not a raw value. + */ + @bindThis + public andIsQuote(q: Q, noteProp: string): Q { + return this.addIsQuote(q, noteProp, 'andWhere'); + } + + private addIsQuote(q: Q, noteProp: string, join: 'andWhere' | 'orWhere'): Q { + return q[join](new Brackets(qb => qb + .andWhere(`${noteProp}.renoteId IS NOT NULL`) + .andWhere(new Brackets(qbb => qbb + .orWhere(`${noteProp}.text IS NOT NULL`) + .orWhere(`${noteProp}.cw IS NOT NULL`) + .orWhere(`${noteProp}.replyId IS NOT NULL`) + .orWhere(`${noteProp}.hasPoll = true`) + .orWhere(`${noteProp}.fileIds != '{}'`))))); + } + + /** + * Adds OR condition that noteProp (note ID) does not refer to a quote. + * The prop should be an expression, not a raw value. + */ + @bindThis + public orIsNotQuote(q: Q, noteProp: string): Q { + return this.addIsNotQuote(q, noteProp, 'orWhere'); + } + + /** + * Adds AND condition that noteProp (note ID) does not refer to a quote. + * The prop should be an expression, not a raw value. + */ + @bindThis + public andIsNotQuote(q: Q, noteProp: string): Q { + return this.addIsNotQuote(q, noteProp, 'andWhere'); + } + + private addIsNotQuote(q: Q, noteProp: string, join: 'andWhere' | 'orWhere'): Q { + return q[join](new Brackets(qb => qb + .orWhere(`${noteProp}.renoteId IS NULL`) + .orWhere(new Brackets(qb => qb + .andWhere(`${noteProp}.text IS NULL`) + .andWhere(`${noteProp}.cw IS NULL`) + .andWhere(`${noteProp}.replyId IS NULL`) + .andWhere(`${noteProp}.hasPoll = false`) + .andWhere(`${noteProp}.fileIds = '{}'`))))); + } + + /** + * Adds OR condition that noteProp (note ID) refers to a renote. + * The prop should be an expression, not a raw value. + */ + @bindThis + public orIsRenote(q: Q, noteProp: string): Q { + return this.addIsRenote(q, noteProp, 'orWhere'); + } + + /** + * Adds AND condition that noteProp (note ID) refers to a renote. + * The prop should be an expression, not a raw value. + */ + @bindThis + public andIsRenote(q: Q, noteProp: string): Q { + return this.addIsRenote(q, noteProp, 'andWhere'); + } + + private addIsRenote(q: Q, noteProp: string, join: 'andWhere' | 'orWhere'): Q { + return q[join](new Brackets(qb => qb + .andWhere(`${noteProp}.renoteId IS NOT NULL`) + .andWhere(`${noteProp}.text IS NULL`) + .andWhere(`${noteProp}.cw IS NULL`) + .andWhere(`${noteProp}.replyId IS NULL`) + .andWhere(`${noteProp}.hasPoll = false`) + .andWhere(`${noteProp}.fileIds = '{}'`))); + } + + /** + * Adds OR condition that noteProp (note ID) does not refer to a renote. + * The prop should be an expression, not a raw value. + */ + @bindThis + public orIsNotRenote(q: Q, noteProp: string): Q { + return this.addIsNotRenote(q, noteProp, 'orWhere'); + } + + /** + * Adds AND condition that noteProp (note ID) does not refer to a renote. + * The prop should be an expression, not a raw value. + */ + @bindThis + public andIsNotRenote(q: Q, noteProp: string): Q { + return this.addIsNotRenote(q, noteProp, 'andWhere'); + } + + private addIsNotRenote(q: Q, noteProp: string, join: 'andWhere' | 'orWhere'): Q { + return q[join](new Brackets(qb => qb + .orWhere(`${noteProp}.renoteId IS NULL`) + .orWhere(`${noteProp}.text IS NOT NULL`) + .orWhere(`${noteProp}.cw IS NOT NULL`) + .orWhere(`${noteProp}.replyId IS NOT NULL`) + .orWhere(`${noteProp}.hasPoll = true`) + .orWhere(`${noteProp}.fileIds != '{}'`))); + } + + /** + * Adds OR condition that followerProp (user ID) is following followeeProp (user ID). + * Both props should be expressions, not raw values. + */ + @bindThis + public orFollowingUser(q: Q, followerProp: string, followeeProp: string): Q { + return this.addFollowingUser(q, followerProp, followeeProp, 'orWhere'); + } + + /** + * Adds AND condition that followerProp (user ID) is following followeeProp (user ID). + * Both props should be expressions, not raw values. + */ + @bindThis + public andFollowingUser(q: Q, followerProp: string, followeeProp: string): Q { + return this.addFollowingUser(q, followerProp, followeeProp, 'andWhere'); + } + + private addFollowingUser(q: Q, followerProp: string, followeeProp: string, join: 'andWhere' | 'orWhere'): Q { + const followingQuery = this.followingsRepository.createQueryBuilder('following') + .select('1') + .andWhere(`following.followerId = ${followerProp}`) + .andWhere(`following.followeeId = ${followeeProp}`); + + return q[join](`EXISTS (${followingQuery.getQuery()})`, followingQuery.getParameters()); + }; + + /** + * Adds OR condition that followerProp (user ID) is following followeeProp (channel ID). + * Both props should be expressions, not raw values. + */ + @bindThis + public orFollowingChannel(q: Q, followerProp: string, followeeProp: string): Q { + return this.addFollowingChannel(q, followerProp, followeeProp, 'orWhere'); + } + + /** + * Adds AND condition that followerProp (user ID) is following followeeProp (channel ID). + * Both props should be expressions, not raw values. + */ + @bindThis + public andFollowingChannel(q: Q, followerProp: string, followeeProp: string): Q { + return this.addFollowingChannel(q, followerProp, followeeProp, 'andWhere'); + } + + private addFollowingChannel(q: Q, followerProp: string, followeeProp: string, join: 'andWhere' | 'orWhere'): Q { + const followingQuery = this.channelFollowingsRepository.createQueryBuilder('following') + .select('1') + .andWhere(`following.followerId = ${followerProp}`) + .andWhere(`following.followeeId = ${followeeProp}`); + + return q[join](`EXISTS (${followingQuery.getQuery()})`, followingQuery.getParameters()); + } + + /** + * Adds OR condition that blockerProp (user ID) is not blocking blockeeProp (user ID). + * Both props should be expressions, not raw values. + */ + @bindThis + public orNotBlockingUser(q: Q, blockerProp: string, blockeeProp: string): Q { + return this.excludeBlockingUser(q, blockerProp, blockeeProp, 'orWhere'); + } + + /** + * Adds AND condition that blockerProp (user ID) is not blocking blockeeProp (user ID). + * Both props should be expressions, not raw values. + */ + @bindThis + public andNotBlockingUser(q: Q, blockerProp: string, blockeeProp: string): Q { + return this.excludeBlockingUser(q, blockerProp, blockeeProp, 'andWhere'); + } + + private excludeBlockingUser(q: Q, blockerProp: string, blockeeProp: string, join: 'andWhere' | 'orWhere'): Q { + const blockingQuery = this.blockingsRepository.createQueryBuilder('blocking') + .select('1') + .andWhere(`blocking.blockerId = ${blockerProp}`) + .andWhere(`blocking.blockeeId = ${blockeeProp}`); + + return q[join](`NOT EXISTS (${blockingQuery.getQuery()})`, blockingQuery.getParameters()); + }; + + /** + * Adds OR condition that muterProp (user ID) is not muting muteeProp (user ID). + * Both props should be expressions, not raw values. + */ + @bindThis + public orNotMutingUser(q: Q, muterProp: string, muteeProp: string, exclude?: { id: MiUser['id'] }): Q { + return this.excludeMutingUser(q, muterProp, muteeProp, 'orWhere', exclude); + } + + /** + * Adds AND condition that muterProp (user ID) is not muting muteeProp (user ID). + * Both props should be expressions, not raw values. + */ + @bindThis + public andNotMutingUser(q: Q, muterProp: string, muteeProp: string, exclude?: { id: MiUser['id'] }): Q { + return this.excludeMutingUser(q, muterProp, muteeProp, 'andWhere', exclude); + } + + private excludeMutingUser(q: Q, muterProp: string, muteeProp: string, join: 'andWhere' | 'orWhere', exclude?: { id: MiUser['id'] }): Q { + const mutingQuery = this.mutingsRepository.createQueryBuilder('muting') + .select('1') + .andWhere(`muting.muterId = ${muterProp}`) + .andWhere(`muting.muteeId = ${muteeProp}`); + + if (exclude) { + mutingQuery.andWhere({ muteeId: Not(exclude.id) }); + } + + return q[join](`NOT EXISTS (${mutingQuery.getQuery()})`, mutingQuery.getParameters()); + } + + /** + * Adds OR condition that muterProp (user ID) is not muting renotes by muteeProp (user ID). + * Both props should be expressions, not raw values. + */ + @bindThis + public orNotMutingRenote(q: Q, muterProp: string, muteeProp: string): Q { + return this.excludeMutingRenote(q, muterProp, muteeProp, 'orWhere'); + } + + /** + * Adds AND condition that muterProp (user ID) is not muting renotes by muteeProp (user ID). + * Both props should be expressions, not raw values. + */ + @bindThis + public andNotMutingRenote(q: Q, muterProp: string, muteeProp: string): Q { + return this.excludeMutingRenote(q, muterProp, muteeProp, 'andWhere'); + } + + private excludeMutingRenote(q: Q, muterProp: string, muteeProp: string, join: 'andWhere' | 'orWhere'): Q { const mutingQuery = this.renoteMutingsRepository.createQueryBuilder('renote_muting') - .select('renote_muting.muteeId') - .where('renote_muting.muterId = :muterId', { muterId: me.id }); + .select('1') + .andWhere(`renote_muting.muterId = ${muterProp}`) + .andWhere(`renote_muting.muteeId = ${muteeProp}`); - q.andWhere(new Brackets(qb => { - qb - .where(new Brackets(qb => { - qb.where('note.renoteId IS NOT NULL'); - qb.andWhere('note.text IS NULL'); - qb.andWhere(`note.userId NOT IN (${ mutingQuery.getQuery() })`); - })) - .orWhere('note.renoteId IS NULL') - .orWhere('note.text IS NOT NULL'); - })); + return q[join](`NOT EXISTS (${mutingQuery.getQuery()})`, mutingQuery.getParameters()); + }; - q.setParameters(mutingQuery.getParameters()); + /** + * Adds OR condition that muterProp (user ID) is not muting muteeProp (instance host). + * Both props should be expressions, not raw values. + */ + @bindThis + public orNotMutingInstance(q: Q, muterProp: string, muteeProp: string): Q { + return this.excludeMutingInstance(q, muterProp, muteeProp, 'orWhere'); } + /** + * Adds AND condition that muterProp (user ID) is not muting muteeProp (instance host). + * Both props should be expressions, not raw values. + */ @bindThis - public generateBlockedHostQueryForNote(q: SelectQueryBuilder, excludeAuthor?: boolean): void { - let nonBlockedHostQuery: (part: string) => string; - if (this.meta.blockedHosts.length === 0) { - nonBlockedHostQuery = () => '1=1'; - } else { - nonBlockedHostQuery = (match: string) => `('.' || ${match}) NOT ILIKE ALL(select '%.' || x from (select unnest("blockedHosts") as x from "meta") t)`; - } + public andNotMutingInstance(q: Q, muterProp: string, muteeProp: string): Q { + return this.excludeMutingInstance(q, muterProp, muteeProp, 'andWhere'); + } - if (excludeAuthor) { - const instanceSuspension = (user: string) => new Brackets(qb => qb - .where(`note.${user}Id IS NULL`) // no corresponding user - .orWhere(`note.userId = note.${user}Id`) - .orWhere(`note.${user}Host IS NULL`) // local - .orWhere(nonBlockedHostQuery(`note.${user}Host`))); + private excludeMutingInstance(q: Q, muterProp: string, muteeProp: string, join: 'andWhere' | 'orWhere'): Q { + const mutingInstanceQuery = this.userProfilesRepository.createQueryBuilder('user_profile') + .select('1') + .andWhere(`user_profile.userId = ${muterProp}`) + .andWhere(`"user_profile"."mutedInstances"::jsonb ? ${muteeProp}`); - q - .andWhere(instanceSuspension('replyUser')) - .andWhere(instanceSuspension('renoteUser')); - } else { - const instanceSuspension = (user: string) => new Brackets(qb => qb - .where(`note.${user}Id IS NULL`) // no corresponding user - .orWhere(`note.${user}Host IS NULL`) // local - .orWhere(nonBlockedHostQuery(`note.${user}Host`))); + return q[join](`NOT EXISTS (${mutingInstanceQuery.getQuery()})`, mutingInstanceQuery.getParameters()); + } - q - .andWhere(instanceSuspension('user')) - .andWhere(instanceSuspension('replyUser')) - .andWhere(instanceSuspension('renoteUser')); - } + /** + * Adds OR condition that muterProp (user ID) is not muting muteeProp (note ID). + * Both props should be expressions, not raw values. + */ + @bindThis + public orNotMutingThread(q: Q, muterProp: string, muteeProp: string): Q { + return this.excludeMutingThread(q, muterProp, muteeProp, 'orWhere'); + } + + /** + * Adds AND condition that muterProp (user ID) is not muting muteeProp (note ID). + * Both props should be expressions, not raw values. + */ + @bindThis + public andNotMutingThread(q: Q, muterProp: string, muteeProp: string): Q { + return this.excludeMutingThread(q, muterProp, muteeProp, 'andWhere'); + } + + private excludeMutingThread(q: Q, muterProp: string, muteeProp: string, join: 'andWhere' | 'orWhere'): Q { + const threadMutedQuery = this.noteThreadMutingsRepository.createQueryBuilder('threadMuted') + .select('1') + .andWhere(`threadMuted.userId = ${muterProp}`) + .andWhere(`threadMuted.threadId = ${muteeProp}`); + + return q[join](`NOT EXISTS (${threadMutedQuery.getQuery()})`, threadMutedQuery.getParameters()); } // Requirements: user replyUser renoteUser must be joined diff --git a/packages/backend/src/core/ReactionService.ts b/packages/backend/src/core/ReactionService.ts index f05ee2ee73..c23bb51178 100644 --- a/packages/backend/src/core/ReactionService.ts +++ b/packages/backend/src/core/ReactionService.ts @@ -10,7 +10,7 @@ import { IdentifiableError } from '@/misc/identifiable-error.js'; import type { MiRemoteUser, MiUser } from '@/models/User.js'; import type { MiNote } from '@/models/Note.js'; import { IdService } from '@/core/IdService.js'; -import type { MiNoteReaction } from '@/models/NoteReaction.js'; +import { MiNoteReaction } from '@/models/NoteReaction.js'; import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import { NotificationService } from '@/core/NotificationService.js'; @@ -31,6 +31,7 @@ import { isQuote, isRenote } from '@/misc/is-renote.js'; import { ReactionsBufferingService } from '@/core/ReactionsBufferingService.js'; import { PER_NOTE_REACTION_USER_PAIR_CACHE_MAX } from '@/const.js'; import { CacheService } from '@/core/CacheService.js'; +import type { DataSource } from 'typeorm'; const FALLBACK = '\u2764'; @@ -89,6 +90,9 @@ export class ReactionService { @Inject(DI.emojisRepository) private emojisRepository: EmojisRepository, + @Inject(DI.db) + private readonly db: DataSource, + private utilityService: UtilityService, private customEmojiService: CustomEmojiService, private roleService: RoleService, @@ -113,7 +117,7 @@ export class ReactionService { if (note.userId !== user.id) { const blocked = await this.userBlockingService.checkBlocked(note.userId, user.id); if (blocked) { - throw new IdentifiableError('e70412a4-7197-4726-8e74-f3e0deb92aa7'); + throw new IdentifiableError('e70412a4-7197-4726-8e74-f3e0deb92aa7', 'Note not accessible for you.'); } } @@ -176,26 +180,28 @@ export class ReactionService { reaction, }; - try { - await this.noteReactionsRepository.insert(record); - } catch (e) { - if (isDuplicateKeyValueError(e)) { - const exists = await this.noteReactionsRepository.findOneByOrFail({ - noteId: note.id, - userId: user.id, - }); + const result = await this.db.transaction(async tem => { + await tem.createQueryBuilder(MiNoteReaction, 'noteReaction') + .insert() + .values(record) + .orIgnore() + .execute(); - if (exists.reaction !== reaction) { - // 別のリアクションがすでにされていたら置き換える - await this.delete(user, note); - await this.noteReactionsRepository.insert(record); - } else { - // 同じリアクションがすでにされていたらエラー - throw new IdentifiableError('51c42bb4-931a-456b-bff7-e5a8a70dd298'); - } - } else { - throw e; + return await tem.createQueryBuilder(MiNoteReaction, 'noteReaction') + .select() + .where({ noteId: note.id, userId: user.id }) + .getOneOrFail(); + }); + + if (result.id !== record.id) { + // Conflict with the same ID => nothing to do. + if (result.reaction === record.reaction) { + return; } + + // 別のリアクションがすでにされていたら置き換える + await this.delete(user, note); + await this.noteReactionsRepository.insert(record); } // Increment reactions count @@ -316,14 +322,14 @@ export class ReactionService { }); if (exist == null) { - throw new IdentifiableError('60527ec9-b4cb-4a88-a6bd-32d3ad26817d', 'not reacted'); + throw new IdentifiableError('60527ec9-b4cb-4a88-a6bd-32d3ad26817d', 'reaction does not exist'); } // Delete reaction const result = await this.noteReactionsRepository.delete(exist.id); if (result.affected !== 1) { - throw new IdentifiableError('60527ec9-b4cb-4a88-a6bd-32d3ad26817d', 'not reacted'); + throw new IdentifiableError('60527ec9-b4cb-4a88-a6bd-32d3ad26817d', 'reaction does not exist'); } // Decrement reactions count diff --git a/packages/backend/src/core/RemoteUserResolveService.ts b/packages/backend/src/core/RemoteUserResolveService.ts index a2f1b73cdb..4dbc9d6a36 100644 --- a/packages/backend/src/core/RemoteUserResolveService.ts +++ b/packages/backend/src/core/RemoteUserResolveService.ts @@ -3,7 +3,6 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { URL } from 'node:url'; import { Inject, Injectable } from '@nestjs/common'; import chalk from 'chalk'; import { IsNull } from 'typeorm'; @@ -18,6 +17,7 @@ import { RemoteLoggerService } from '@/core/RemoteLoggerService.js'; import { ApDbResolverService } from '@/core/activitypub/ApDbResolverService.js'; import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js'; import { bindThis } from '@/decorators.js'; +import { renderInlineError } from '@/misc/render-inline-error.js'; @Injectable() export class RemoteUserResolveService { @@ -44,27 +44,13 @@ export class RemoteUserResolveService { const usernameLower = username.toLowerCase(); if (host == null) { - this.logger.info(`return local user: ${usernameLower}`); - return await this.usersRepository.findOneBy({ usernameLower, host: IsNull() }).then(u => { - if (u == null) { - throw new Error('user not found'); - } else { - return u; - } - }) as MiLocalUser; + return await this.usersRepository.findOneByOrFail({ usernameLower, host: IsNull() }) as MiLocalUser; } host = this.utilityService.toPuny(host); if (host === this.utilityService.toPuny(this.config.host)) { - this.logger.info(`return local user: ${usernameLower}`); - return await this.usersRepository.findOneBy({ usernameLower, host: IsNull() }).then(u => { - if (u == null) { - throw new Error('user not found'); - } else { - return u; - } - }) as MiLocalUser; + return await this.usersRepository.findOneByOrFail({ usernameLower, host: IsNull() }) as MiLocalUser; } const user = await this.usersRepository.findOneBy({ usernameLower, host }) as MiRemoteUser | null; @@ -82,7 +68,7 @@ export class RemoteUserResolveService { .getUserFromApId(self.href) .then((u) => { if (u == null) { - throw new Error('local user not found'); + throw new Error(`local user not found: ${self.href}`); } else { return u; } @@ -90,7 +76,7 @@ export class RemoteUserResolveService { } } - this.logger.succ(`return new remote user: ${chalk.magenta(acctLower)}`); + this.logger.info(`Fetching new remote user ${chalk.magenta(acctLower)} from ${self.href}`); return await this.apPersonService.createPerson(self.href); } @@ -101,18 +87,16 @@ export class RemoteUserResolveService { lastFetchedAt: new Date(), }); - this.logger.info(`try resync: ${acctLower}`); const self = await this.resolveSelf(acctLower); if (user.uri !== self.href) { // if uri mismatch, Fix (user@host <=> AP's Person id(RemoteUser.uri)) mapping. - this.logger.info(`uri missmatch: ${acctLower}`); - this.logger.info(`recovery missmatch uri for (username=${username}, host=${host}) from ${user.uri} to ${self.href}`); + this.logger.warn(`Detected URI mismatch for ${acctLower}`); // validate uri - const uri = new URL(self.href); - if (uri.hostname !== host) { - throw new Error('Invalid uri'); + const uriHost = this.utilityService.extractDbHost(self.href); + if (uriHost !== host) { + throw new Error(`Failed to correct URI for ${acctLower}: new URI ${self.href} has different host from previous URI ${user.uri}`); } await this.usersRepository.update({ @@ -121,37 +105,28 @@ export class RemoteUserResolveService { }, { uri: self.href, }); - } else { - this.logger.info(`uri is fine: ${acctLower}`); } + this.logger.info(`Corrected URI for ${acctLower} from ${user.uri} to ${self.href}`); + await this.apPersonService.updatePerson(self.href); - this.logger.info(`return resynced remote user: ${acctLower}`); - return await this.usersRepository.findOneBy({ uri: self.href }).then(u => { - if (u == null) { - throw new Error('user not found'); - } else { - return u as MiLocalUser | MiRemoteUser; - } - }); + return await this.usersRepository.findOneByOrFail({ uri: self.href }) as MiLocalUser | MiRemoteUser; } - this.logger.info(`return existing remote user: ${acctLower}`); return user; } @bindThis private async resolveSelf(acctLower: string): Promise { - this.logger.info(`WebFinger for ${chalk.yellow(acctLower)}`); const finger = await this.webfingerService.webfinger(acctLower).catch(err => { - this.logger.error(`Failed to WebFinger for ${chalk.yellow(acctLower)}: ${ err.statusCode ?? err.message }`); - throw new Error(`Failed to WebFinger for ${acctLower}: ${ err.statusCode ?? err.message }`); + this.logger.error(`Failed to WebFinger for ${chalk.yellow(acctLower)}: ${renderInlineError(err)}`); + throw new Error(`Failed to WebFinger for ${acctLower}: error thrown`, { cause: err }); }); const self = finger.links.find(link => link.rel != null && link.rel.toLowerCase() === 'self'); if (!self) { this.logger.error(`Failed to WebFinger for ${chalk.yellow(acctLower)}: self link not found`); - throw new Error('self link not found'); + throw new Error(`Failed to WebFinger for ${acctLower}: self link not found`); } return self; } diff --git a/packages/backend/src/core/ReversiService.ts b/packages/backend/src/core/ReversiService.ts index 8c0a8f6cc7..b57ab6d9cb 100644 --- a/packages/backend/src/core/ReversiService.ts +++ b/packages/backend/src/core/ReversiService.ts @@ -587,6 +587,8 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit { lastActiveDate: parsed.user1.lastActiveDate != null ? new Date(parsed.user1.lastActiveDate) : null, lastFetchedAt: parsed.user1.lastFetchedAt != null ? new Date(parsed.user1.lastFetchedAt) : null, movedAt: parsed.user1.movedAt != null ? new Date(parsed.user1.movedAt) : null, + instance: null, + userProfile: null, } : null, user2: parsed.user2 != null ? { ...parsed.user2, @@ -597,6 +599,8 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit { lastActiveDate: parsed.user2.lastActiveDate != null ? new Date(parsed.user2.lastActiveDate) : null, lastFetchedAt: parsed.user2.lastFetchedAt != null ? new Date(parsed.user2.lastFetchedAt) : null, movedAt: parsed.user2.movedAt != null ? new Date(parsed.user2.movedAt) : null, + instance: null, + userProfile: null, } : null, }; } else { diff --git a/packages/backend/src/core/RoleService.ts b/packages/backend/src/core/RoleService.ts index d3c458eec7..b250eeee21 100644 --- a/packages/backend/src/core/RoleService.ts +++ b/packages/backend/src/core/RoleService.ts @@ -86,10 +86,10 @@ export const DEFAULT_POLICIES: RolePolicies = { canManageCustomEmojis: false, canManageAvatarDecorations: false, canSearchNotes: false, - canUseTranslator: true, + canUseTranslator: false, canHideAds: false, driveCapacityMb: 100, - maxFileSizeMb: 10, + maxFileSizeMb: 25, alwaysMarkNsfw: false, canUpdateBioMedia: true, pinLimit: 5, diff --git a/packages/backend/src/core/UtilityService.ts b/packages/backend/src/core/UtilityService.ts index 31371d867b..5de76e00a6 100644 --- a/packages/backend/src/core/UtilityService.ts +++ b/packages/backend/src/core/UtilityService.ts @@ -51,22 +51,49 @@ export class UtilityService { return regexp.test(email); } + public isBlockedHost(host: string | null): boolean; + public isBlockedHost(blockedHosts: string[], host: string | null): boolean; @bindThis - public isBlockedHost(blockedHosts: string[], host: string | null): boolean { + public isBlockedHost(blockedHostsOrHost: string[] | string | null, host?: string | null): boolean { + const blockedHosts = Array.isArray(blockedHostsOrHost) ? blockedHostsOrHost : this.meta.blockedHosts; + host = Array.isArray(blockedHostsOrHost) ? host : blockedHostsOrHost; + if (host == null) return false; return blockedHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`)); } + public isSilencedHost(host: string | null): boolean; + public isSilencedHost(silencedHosts: string[], host: string | null): boolean; @bindThis - public isSilencedHost(silencedHosts: string[] | undefined, host: string | null): boolean { - if (!silencedHosts || host == null) return false; + public isSilencedHost(silencedHostsOrHost: string[] | string | null, host?: string | null): boolean { + const silencedHosts = Array.isArray(silencedHostsOrHost) ? silencedHostsOrHost : this.meta.silencedHosts; + host = Array.isArray(silencedHostsOrHost) ? host : silencedHostsOrHost; + + if (host == null) return false; return silencedHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`)); } + public isMediaSilencedHost(host: string | null): boolean; + public isMediaSilencedHost(silencedHosts: string[], host: string | null): boolean; @bindThis - public isMediaSilencedHost(silencedHosts: string[] | undefined, host: string | null): boolean { - if (!silencedHosts || host == null) return false; - return silencedHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`)); + public isMediaSilencedHost(mediaSilencedHostsOrHost: string[] | string | null, host?: string | null): boolean { + const mediaSilencedHosts = Array.isArray(mediaSilencedHostsOrHost) ? mediaSilencedHostsOrHost : this.meta.mediaSilencedHosts; + host = Array.isArray(mediaSilencedHostsOrHost) ? host : mediaSilencedHostsOrHost; + + if (host == null) return false; + return mediaSilencedHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`)); + } + + @bindThis + public isAllowListedHost(host: string | null): boolean { + if (host == null) return false; + return this.meta.federationHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`)); + } + + @bindThis + public isBubbledHost(host: string | null): boolean { + if (host == null) return false; + return this.meta.bubbleInstances.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`)); } @bindThis diff --git a/packages/backend/src/core/VideoProcessingService.ts b/packages/backend/src/core/VideoProcessingService.ts index 747fe4fc7e..3e4fd6a4b0 100644 --- a/packages/backend/src/core/VideoProcessingService.ts +++ b/packages/backend/src/core/VideoProcessingService.ts @@ -3,24 +3,41 @@ * SPDX-License-Identifier: AGPL-3.0-only */ +import fs from 'node:fs/promises'; import { Inject, Injectable } from '@nestjs/common'; import FFmpeg from 'fluent-ffmpeg'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import { ImageProcessingService } from '@/core/ImageProcessingService.js'; import type { IImage } from '@/core/ImageProcessingService.js'; -import { createTempDir } from '@/misc/create-temp.js'; +import { createTemp, createTempDir } from '@/misc/create-temp.js'; import { bindThis } from '@/decorators.js'; import { appendQuery, query } from '@/misc/prelude/url.js'; +import { LoggerService } from '@/core/LoggerService.js'; +import type Logger from '@/logger.js'; + +// faststart is only supported for MP4, M4A, M4W and MOV files (the MOV family). +// WebM (and Matroska) files always support faststart-like behavior. +const supportedMimeTypes = new Map([ + ['video/mp4', 'mp4'], + ['video/m4a', 'mp4'], + ['video/m4v', 'mp4'], + ['video/quicktime', 'mov'], +]); @Injectable() export class VideoProcessingService { + private readonly logger: Logger; + constructor( @Inject(DI.config) private config: Config, private imageProcessingService: ImageProcessingService, + + private loggerService: LoggerService, ) { + this.logger = this.loggerService.getLogger('video-processing'); } @bindThis @@ -60,5 +77,50 @@ export class VideoProcessingService { }), ); } + + /** + * Optimize video for web playback by adding faststart flag. + * This allows the video to start playing before it is fully downloaded. + * The original file is modified in-place. + * @param source Path to the video file + * @param mimeType The MIME type of the video + * @returns Promise that resolves when optimization is complete + */ + @bindThis + public async webOptimizeVideo(source: string, mimeType: string): Promise { + const outputFormat = supportedMimeTypes.get(mimeType); + if (!outputFormat) { + this.logger.debug(`Skipping web optimization for unsupported MIME type: ${mimeType}`); + return; + } + + const [tempPath, cleanup] = await createTemp(); + + try { + await new Promise((resolve, reject) => { + FFmpeg(source) + .format(outputFormat) // Specify output format + .addOutputOptions('-c copy') // Copy streams without re-encoding + .addOutputOptions('-movflags +faststart') + .on('error', reject) + .on('end', async () => { + try { + // Replace original file with optimized version + await fs.copyFile(tempPath, source); + this.logger.info(`Web-optimized video: ${source}`); + resolve(); + } catch (copyError) { + reject(copyError); + } + }) + .save(tempPath); + }); + } catch (error) { + this.logger.warn(`Failed to web-optimize video: ${source}`, { error }); + throw error; + } finally { + cleanup(); + } + } } diff --git a/packages/backend/src/core/WebAuthnService.ts b/packages/backend/src/core/WebAuthnService.ts index 372e1e2ab7..afd1d68ce4 100644 --- a/packages/backend/src/core/WebAuthnService.ts +++ b/packages/backend/src/core/WebAuthnService.ts @@ -17,6 +17,8 @@ import type { Config } from '@/config.js'; import { bindThis } from '@/decorators.js'; import { MiUser } from '@/models/_.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; +import { LoggerService } from '@/core/LoggerService.js'; +import Logger from '@/logger.js'; import type { AuthenticationResponseJSON, AuthenticatorTransportFuture, @@ -28,6 +30,8 @@ import type { @Injectable() export class WebAuthnService { + private readonly logger: Logger; + constructor( @Inject(DI.config) private config: Config, @@ -40,7 +44,9 @@ export class WebAuthnService { @Inject(DI.userSecurityKeysRepository) private userSecurityKeysRepository: UserSecurityKeysRepository, + loggerService: LoggerService, ) { + this.logger = loggerService.getLogger('web-authn'); } @bindThis @@ -114,8 +120,8 @@ export class WebAuthnService { requireUserVerification: true, }); } catch (error) { - console.error(error); - throw new IdentifiableError('5c1446f8-8ca7-4d31-9f39-656afe9c5d87', 'verification failed'); + this.logger.error(error as Error, 'Error authenticating webauthn'); + throw new IdentifiableError('5c1446f8-8ca7-4d31-9f39-656afe9c5d87', 'verification failed', true, error); } const { verified } = verification; @@ -221,7 +227,7 @@ export class WebAuthnService { requireUserVerification: true, }); } catch (error) { - throw new IdentifiableError('b18c89a7-5b5e-4cec-bb5b-0419f332d430', `verification failed: ${error}`); + throw new IdentifiableError('b18c89a7-5b5e-4cec-bb5b-0419f332d430', `verification failed`, true, error); } const { verified, authenticationInfo } = verification; @@ -301,8 +307,8 @@ export class WebAuthnService { requireUserVerification: true, }); } catch (error) { - console.error(error); - throw new IdentifiableError('b18c89a7-5b5e-4cec-bb5b-0419f332d430', 'verification failed'); + this.logger.error(error as Error, 'Error authenticating webauthn'); + throw new IdentifiableError('b18c89a7-5b5e-4cec-bb5b-0419f332d430', 'verification failed', true, error); } const { verified, authenticationInfo } = verification; diff --git a/packages/backend/src/core/WebfingerService.ts b/packages/backend/src/core/WebfingerService.ts index f57e7a2c1f..664963f3a3 100644 --- a/packages/backend/src/core/WebfingerService.ts +++ b/packages/backend/src/core/WebfingerService.ts @@ -9,6 +9,7 @@ import { XMLParser } from 'fast-xml-parser'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { bindThis } from '@/decorators.js'; import type Logger from '@/logger.js'; +import { renderInlineError } from '@/misc/render-inline-error.js'; import { RemoteLoggerService } from './RemoteLoggerService.js'; export type ILink = { @@ -109,7 +110,7 @@ export class WebfingerService { const template = (hostMeta['XRD']['Link'] as Array).filter(p => p['@_rel'] === 'lrdd')[0]['@_template']; return template.indexOf('{uri}') < 0 ? null : template; } catch (err) { - this.logger.error(`error while request host-meta for ${url}: ${err}`); + this.logger.error(`error while request host-meta for ${url}: ${renderInlineError(err)}`); return null; } } diff --git a/packages/backend/src/core/WebhookTestService.ts b/packages/backend/src/core/WebhookTestService.ts index e9be72c8a4..aee16a74bb 100644 --- a/packages/backend/src/core/WebhookTestService.ts +++ b/packages/backend/src/core/WebhookTestService.ts @@ -63,6 +63,7 @@ function generateDummyUser(override?: Partial): MiUser { emojis: [], score: 0, host: null, + instance: null, inbox: null, sharedInbox: null, featured: null, @@ -76,6 +77,8 @@ function generateDummyUser(override?: Partial): MiUser { mandatoryCW: null, rejectQuotes: false, allowUnsignedFetch: 'staff', + userProfile: null, + attributionDomains: [], ...override, }; } @@ -114,10 +117,13 @@ function generateDummyNote(override?: Partial): MiNote { channelId: null, channel: null, userHost: null, + userInstance: null, replyUserId: null, replyUserHost: null, + replyUserInstance: null, renoteUserId: null, renoteUserHost: null, + renoteUserInstance: null, updatedAt: null, processErrors: [], ...override, @@ -358,8 +364,10 @@ export class WebhookTestService { id: 'dummy-abuse-report1', targetUserId: 'dummy-target-user', targetUser: null, + targetUserInstance: null, reporterId: 'dummy-reporter-user', reporter: null, + reporterInstance: null, assigneeId: null, assignee: null, resolved: false, @@ -451,6 +459,7 @@ export class WebhookTestService { isAdmin: false, isModerator: false, isSystem: false, + instance: undefined, ...override, }; } diff --git a/packages/backend/src/core/activitypub/ApDbResolverService.ts b/packages/backend/src/core/activitypub/ApDbResolverService.ts index d8aa80f5b7..e9e0dde9cd 100644 --- a/packages/backend/src/core/activitypub/ApDbResolverService.ts +++ b/packages/backend/src/core/activitypub/ApDbResolverService.ts @@ -165,18 +165,23 @@ export class ApDbResolverService implements OnApplicationShutdown { */ @bindThis public async refetchPublicKeyForApId(user: MiRemoteUser): Promise { - this.apLoggerService.logger.debug('Re-fetching public key for user', { userId: user.id, uri: user.uri }); + this.apLoggerService.logger.debug(`Updating public key for user ${user.id} (${user.uri})`); + + const oldKey = await this.apPersonService.findPublicKeyByUserId(user.id); await this.apPersonService.updatePerson(user.uri); + const newKey = await this.apPersonService.findPublicKeyByUserId(user.id); - const key = await this.apPersonService.findPublicKeyByUserId(user.id); - - if (key) { - this.apLoggerService.logger.info('Re-fetched public key for user', { userId: user.id, uri: user.uri }); + if (newKey) { + if (oldKey && newKey.keyPem === oldKey.keyPem) { + this.apLoggerService.logger.debug(`Public key is up-to-date for user ${user.id} (${user.uri})`); + } else { + this.apLoggerService.logger.info(`Updated public key for user ${user.id} (${user.uri})`); + } } else { - this.apLoggerService.logger.warn('Failed to re-fetch key for user', { userId: user.id, uri: user.uri }); + this.apLoggerService.logger.warn(`Failed to update public key for user ${user.id} (${user.uri})`); } - return key; + return newKey ?? oldKey; } @bindThis diff --git a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts index eaa592b9e0..746af41f55 100644 --- a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts +++ b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts @@ -57,7 +57,7 @@ class DeliverManager { ) { // 型で弾いてはいるが一応ローカルユーザーかチェック // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - if (actor.host != null) throw new Error('actor.host must be null'); + if (actor.host != null) throw new Error(`deliver failed for ${actor.id}: host is not null`); // パフォーマンス向上のためキューに突っ込むのはidのみに絞る this.actor = { @@ -124,12 +124,13 @@ class DeliverManager { select: { followerSharedInbox: true, followerInbox: true, + followerId: true, }, }); for (const following of followers) { const inbox = following.followerSharedInbox ?? following.followerInbox; - if (inbox === null) throw new UnrecoverableError(`inbox is null: following ${following.id}`); + if (inbox === null) throw new UnrecoverableError(`deliver failed for ${this.actor.id}: follower ${following.followerId} inbox is null`); inboxes.set(inbox, following.followerSharedInbox != null); } } diff --git a/packages/backend/src/core/activitypub/ApInboxService.ts b/packages/backend/src/core/activitypub/ApInboxService.ts index b8526a972c..b384ec58c5 100644 --- a/packages/backend/src/core/activitypub/ApInboxService.ts +++ b/packages/backend/src/core/activitypub/ApInboxService.ts @@ -32,11 +32,12 @@ import { AbuseReportService } from '@/core/AbuseReportService.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; import { fromTuple } from '@/misc/from-tuple.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; +import { renderInlineError } from '@/misc/render-inline-error.js'; import InstanceChart from '@/core/chart/charts/instance.js'; import FederationChart from '@/core/chart/charts/federation.js'; import { FetchInstanceMetadataService } from '@/core/FetchInstanceMetadataService.js'; import { UpdateInstanceQueue } from '@/core/UpdateInstanceQueue.js'; -import { getApHrefNullable, getApId, getApIds, getApType, getNullableApId, isAccept, isActor, isAdd, isAnnounce, isApObject, isBlock, isCollection, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isDislike, isMove, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost, isActivity, IObjectWithId } from './type.js'; +import { getApHrefNullable, getApId, getApIds, getApType, getNullableApId, isAccept, isActor, isAdd, isAnnounce, isApObject, isBlock, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isDislike, isMove, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost, isActivity, IObjectWithId } from './type.js'; import { ApNoteService } from './models/ApNoteService.js'; import { ApLoggerService } from './ApLoggerService.js'; import { ApDbResolverService } from './ApDbResolverService.js'; @@ -106,25 +107,29 @@ export class ApInboxService { let result = undefined as string | void; if (isCollectionOrOrderedCollection(activity)) { const results = [] as [string, string | void][]; - // eslint-disable-next-line no-param-reassign resolver ??= this.apResolverService.createResolver(); - const items = toArray(isCollection(activity) ? activity.items : activity.orderedItems); - if (items.length >= resolver.getRecursionLimit()) { - throw new Error(`skipping activity: collection would surpass recursion limit: ${this.utilityService.extractDbHost(actor.uri)}`); - } - - for (const item of items) { - const act = await resolver.resolve(item); - if (act.id == null || this.utilityService.extractDbHost(act.id) !== this.utilityService.extractDbHost(actor.uri)) { - this.logger.debug('skipping activity: activity id is null or mismatching'); - continue; + const items = await resolver.resolveCollectionItems(activity); + for (let i = 0; i < items.length; i++) { + const act = items[i]; + if (act.id != null) { + if (this.utilityService.extractDbHost(act.id) !== this.utilityService.extractDbHost(actor.uri)) { + this.logger.warn('skipping activity: activity id mismatch'); + continue; + } + } else { + // Activity ID should only be string or undefined. + act.id = undefined; } + + const id = getNullableApId(act) ?? `${getNullableApId(activity)}#${i}`; + try { - results.push([getApId(item), await this.performOneActivity(actor, act, resolver)]); + const result = await this.performOneActivity(actor, act, resolver); + results.push([id, result]); } catch (err) { if (err instanceof Error || typeof err === 'string') { - this.logger.error(err); + this.logger.error(`Unhandled error in activity ${id}:`, err); } else { throw err; } @@ -144,7 +149,8 @@ export class ApInboxService { if (actor.lastFetchedAt == null || Date.now() - actor.lastFetchedAt.getTime() > 1000 * 60 * 60 * 24) { setImmediate(() => { // 同一ユーザーの情報を再度処理するので、使用済みのresolverを再利用してはいけない - this.apPersonService.updatePerson(actor.uri); + this.apPersonService.updatePerson(actor.uri) + .catch(err => this.logger.error(`Failed to update person: ${renderInlineError(err)}`)); }); } } @@ -217,6 +223,10 @@ export class ApInboxService { const note = await this.apNoteService.resolveNote(object, { resolver }); if (!note) return `skip: target note not found ${targetUri}`; + if (note.userHost == null && note.localOnly) { + throw new IdentifiableError('12e23cec-edd9-442b-aa48-9c21f0c3b215', 'Cannot react to local-only note'); + } + await this.apNoteService.extractEmojis(activity.tag ?? [], actor.host).catch(() => null); try { @@ -246,7 +256,7 @@ export class ApInboxService { resolver ??= this.apResolverService.createResolver(); const object = await resolver.resolve(activity.object).catch(err => { - this.logger.error(`Resolution failed: ${err}`); + this.logger.error(`Resolution failed: ${renderInlineError(err)}`); throw err; }); @@ -319,7 +329,7 @@ export class ApInboxService { if (targetUri.startsWith('bear:')) return 'skip: bearcaps url not supported.'; const target = await resolver.secureResolve(activityObject, uri).catch(e => { - this.logger.error(`Resolution failed: ${e}`); + this.logger.error(`Resolution failed: ${renderInlineError(e)}`); throw e; }); @@ -350,27 +360,19 @@ export class ApInboxService { } // Announce対象をresolve - let renote; - try { - // The target ID is verified by secureResolve, so we know it shares host authority with the actor who sent it. - // This means we can pass that ID to resolveNote and avoid an extra fetch, which will fail if the note is private. - renote = await this.apNoteService.resolveNote(target, { resolver, sentFrom: getApId(target) }); - if (renote == null) return 'announce target is null'; - } catch (err) { - // 対象が4xxならスキップ - if (err instanceof StatusError) { - if (!err.isRetryable) { - return `skip: ignored announce target ${target.id} - ${err.statusCode}`; - } - return `Error in announce target ${target.id} - ${err.statusCode}`; - } - throw err; - } + // The target ID is verified by secureResolve, so we know it shares host authority with the actor who sent it. + // This means we can pass that ID to resolveNote and avoid an extra fetch, which will fail if the note is private. + const renote = await this.apNoteService.resolveNote(target, { resolver, sentFrom: getApId(target) }); + if (renote == null) return 'announce target is null'; if (!await this.noteEntityService.isVisibleForMe(renote, actor.id)) { return 'skip: invalid actor for this activity'; } + if (renote.userHost == null && renote.localOnly) { + throw new IdentifiableError('12e23cec-edd9-442b-aa48-9c21f0c3b215', 'Cannot renote a local-only note'); + } + this.logger.info(`Creating the (Re)Note: ${uri}`); const activityAudience = await this.apAudienceService.parseAudience(actor, activity.to, activity.cc, resolver); @@ -443,9 +445,11 @@ export class ApInboxService { setImmediate(() => { // Don't re-use the resolver, or it may throw recursion errors. // Instead, create a new resolver with an appropriately-reduced recursion limit. - this.apPersonService.updatePerson(actor.uri, this.apResolverService.createResolver({ + const subResolver = this.apResolverService.createResolver({ recursionLimit: resolver.getRecursionLimit() - resolver.getHistory().length, - })); + }); + this.apPersonService.updatePerson(actor.uri, subResolver) + .catch(err => this.logger.error(`Failed to update person: ${renderInlineError(err)}`)); }); } }); @@ -500,7 +504,7 @@ export class ApInboxService { resolver ??= this.apResolverService.createResolver(); const object = await resolver.resolve(activityObject).catch(e => { - this.logger.error(`Resolution failed: ${e}`); + this.logger.error(`Resolution failed: ${renderInlineError(e)}`); throw e; }); @@ -537,12 +541,6 @@ export class ApInboxService { await this.apNoteService.createNote(note, actor, resolver, silent); return 'ok'; - } catch (err) { - if (err instanceof StatusError && !err.isRetryable) { - return `skip: ${err.statusCode}`; - } else { - throw err; - } } finally { unlock(); } @@ -675,7 +673,7 @@ export class ApInboxService { resolver ??= this.apResolverService.createResolver(); const object = await resolver.resolve(activity.object).catch(e => { - this.logger.error(`Resolution failed: ${e}`); + this.logger.error(`Resolution failed: ${renderInlineError(e)}`); throw e; }); @@ -747,7 +745,7 @@ export class ApInboxService { resolver ??= this.apResolverService.createResolver(); const object = await resolver.resolve(activity.object).catch(e => { - this.logger.error(`Resolution failed: ${e}`); + this.logger.error(`Resolution failed: ${renderInlineError(e)}`); throw e; }); @@ -879,7 +877,7 @@ export class ApInboxService { resolver ??= this.apResolverService.createResolver(); const object = await resolver.resolve(activity.object).catch(e => { - this.logger.error(`Resolution failed: ${e}`); + this.logger.error(`Resolution failed: ${renderInlineError(e)}`); throw e; }); diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts index f41eeba39f..46a78687f3 100644 --- a/packages/backend/src/core/activitypub/ApRendererService.ts +++ b/packages/backend/src/core/activitypub/ApRendererService.ts @@ -613,6 +613,7 @@ export class ApRendererService { enableRss: user.enableRss, speakAsCat: user.speakAsCat, attachment: attachment.length ? attachment : undefined, + attributionDomains: user.attributionDomains, }; if (user.movedToUri) { diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index b665b51700..4c7cac2169 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -184,10 +184,11 @@ export class ApRequestService { * Get AP object with http-signature * @param user http-signature user * @param url URL to fetch - * @param followAlternate + * @param allowAnonymous If a fetched object lacks an ID, then it will be auto-generated from the final URL. (default: false) + * @param followAlternate Whether to resolve HTML responses to their referenced canonical AP endpoint. (default: true) */ @bindThis - public async signedGet(url: string, user: { id: MiUser['id'] }, followAlternate?: boolean): Promise { + public async signedGet(url: string, user: { id: MiUser['id'] }, allowAnonymous = false, followAlternate?: boolean): Promise { this.apUtilityService.assertApUrl(url); const _followAlternate = followAlternate ?? true; @@ -258,7 +259,7 @@ export class ApRequestService { if (alternate) { const href = alternate.getAttribute('href'); if (href && this.apUtilityService.haveSameAuthority(url, href)) { - return await this.signedGet(href, user, false); + return await this.signedGet(href, user, allowAnonymous, false); } } } catch { @@ -275,7 +276,11 @@ export class ApRequestService { // Make sure the object ID matches the final URL (which is where it actually exists). // The caller (ApResolverService) will verify the ID against the original / entry URL, which ensures that all three match. - this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url); + if (allowAnonymous && activity.id == null) { + activity.id = res.url; + } else { + this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url); + } return activity as IObjectWithId; } diff --git a/packages/backend/src/core/activitypub/ApResolverService.ts b/packages/backend/src/core/activitypub/ApResolverService.ts index 5e58f848c0..201920612c 100644 --- a/packages/backend/src/core/activitypub/ApResolverService.ts +++ b/packages/backend/src/core/activitypub/ApResolverService.ts @@ -5,6 +5,7 @@ import { Inject, Injectable } from '@nestjs/common'; import { IsNull, Not } from 'typeorm'; +import promiseLimit from 'promise-limit'; import type { MiLocalUser, MiRemoteUser } from '@/models/User.js'; import type { NotesRepository, PollsRepository, NoteReactionsRepository, UsersRepository, FollowRequestsRepository, MiMeta, SkApFetchLog } from '@/models/_.js'; import type { Config } from '@/config.js'; @@ -19,11 +20,12 @@ import { ApLogService, calculateDurationSince, extractObjectContext } from '@/co import { ApUtilityService } from '@/core/activitypub/ApUtilityService.js'; import { SystemAccountService } from '@/core/SystemAccountService.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; -import { getApId, getNullableApId, IObjectWithId, isCollectionOrOrderedCollection } from './type.js'; +import { toArray } from '@/misc/prelude/array.js'; +import { AnyCollection, getApId, getNullableApId, IObjectWithId, isCollection, isCollectionOrOrderedCollection, isCollectionPage, isOrderedCollection, isOrderedCollectionPage } from './type.js'; import { ApDbResolverService } from './ApDbResolverService.js'; import { ApRendererService } from './ApRendererService.js'; import { ApRequestService } from './ApRequestService.js'; -import type { IObject, ICollection, IOrderedCollection, ApObject } from './type.js'; +import type { IObject, ApObject, IAnonymousObject } from './type.js'; export class Resolver { private history: Set; @@ -63,34 +65,129 @@ export class Resolver { return this.recursionLimit; } + public async resolveCollection(value: string | IObjectWithId, allowAnonymous?: boolean, sentFromUri?: string): Promise; + public async resolveCollection(value: string | IObject, allowAnonymous: boolean | undefined, sentFromUri: string): Promise; + public async resolveCollection(value: string | IObject, allowAnonymous?: boolean, sentFromUri?: string): Promise; @bindThis - public async resolveCollection(value: string | IObject): Promise { + public async resolveCollection(value: string | IObject, allowAnonymous?: boolean, sentFromUri?: string): Promise { const collection = typeof value === 'string' - ? await this.resolve(value) - : value; + ? sentFromUri + ? await this.secureResolve(value, sentFromUri, allowAnonymous) + : await this.resolve(value, allowAnonymous) + : value; // TODO try and remove this eventually, as it's a major security foot-gun if (isCollectionOrOrderedCollection(collection)) { return collection; } else { - throw new IdentifiableError('f100eccf-f347-43fb-9b45-96a0831fb635', `unrecognized collection type: ${collection.type}`); + throw new IdentifiableError('f100eccf-f347-43fb-9b45-96a0831fb635', `collection ${getApId(value)} has unsupported type: ${collection.type}`); } } + public async resolveCollectionItems(collection: IAnonymousObject, limit?: number | null, allowAnonymousItems?: true, concurrency?: number): Promise; + public async resolveCollectionItems(collection: string | IObjectWithId, limit?: number | null, allowAnonymousItems?: boolean, concurrency?: number): Promise; + public async resolveCollectionItems(collection: string | IObject, limit?: number | null, allowAnonymousItems?: boolean, concurrency?: number): Promise; + /** + * Recursively resolves items from a collection. + * Stops when reaching the resolution limit or an optional item limit - whichever is lower. + * This method supports Collection, OrderedCollection, and individual pages of either type. + * Malformed collections (mixing Ordered and un-Ordered types) are also supported. + * @param collection Collection to resolve from - can be a URL or object of any supported collection type. + * @param limit Maximum number of items to resolve. If null or undefined (default), then items will be resolved until reaching the recursion limit. + * @param allowAnonymousItems If true, collection items can be anonymous (lack an ID). If false (default), then an error is thrown when reaching an item without ID. + * @param concurrency Maximum number of items to resolve at once. (default: 4) + */ + @bindThis + public async resolveCollectionItems(collection: string | IObject, limit?: number | null, allowAnonymousItems?: boolean, concurrency = 4): Promise { + const resolvedItems: IObject[] = []; + + // This is pulled up to avoid code duplication below + const iterate = async(items: ApObject, current: AnyCollection) => { + const sentFrom = current.id; + const itemArr = toArray(items); + const itemLimit = limit ?? Number.MAX_SAFE_INTEGER; + const allowAnonymous = allowAnonymousItems ?? false; + await this.resolveItemArray(itemArr, sentFrom, itemLimit, concurrency, allowAnonymous, resolvedItems); + }; + + let current: AnyCollection | null = await this.resolveCollection(collection); + do { + // Iterate all items in the current page + if (current.items) { + await iterate(current.items, current); + } + if (current.orderedItems) { + await iterate(current.orderedItems, current); + } + + if (this.history.size >= this.recursionLimit) { + // Stop when we reach the fetch limit + current = null; + } else if (limit != null && resolvedItems.length >= limit) { + // Stop when we reach the item limit + current = null; + } else if (isCollection(current) || isOrderedCollection(current)) { + // Continue to first page + current = current.first ? await this.resolveCollection(current.first, true, current.id) : null; + } else if (isCollectionPage(current) || isOrderedCollectionPage(current)) { + // Continue to next page + current = current.next ? await this.resolveCollection(current.next, true, current.id) : null; + } else { + // Stop in all other conditions + current = null; + } + } while (current != null); + + return resolvedItems; + } + + private async resolveItemArray(source: (string | IObject)[], sentFrom: undefined, itemLimit: number, concurrency: number, allowAnonymousItems: true, destination: IAnonymousObject[]): Promise; + private async resolveItemArray(source: (string | IObject)[], sentFrom: string, itemLimit: number, concurrency: number, allowAnonymousItems: boolean, destination: IObjectWithId[]): Promise; + private async resolveItemArray(source: (string | IObject)[], sentFrom: string | undefined, itemLimit: number, concurrency: number, allowAnonymousItems: boolean, destination: IObject[]): Promise; + private async resolveItemArray(source: (string | IObject)[], sentFrom: string | undefined, itemLimit: number, concurrency: number, allowAnonymousItems: boolean, destination: IObject[]): Promise { + const recursionLimit = this.recursionLimit - this.history.size; + const batchLimit = Math.min(source.length, recursionLimit, itemLimit); + + const limiter = promiseLimit(concurrency); + const batch = await Promise.all(source + .slice(0, batchLimit) + .map(item => limiter(async () => { + if (sentFrom) { + // Use secureResolve to avoid re-fetching items that were included inline. + return await this.secureResolve(item, sentFrom, allowAnonymousItems); + } else if (allowAnonymousItems) { + return await this.resolveAnonymous(item); + } else { + // ID is required if we have neither sentFrom not allowAnonymousItems + const id = getApId(item); + return await this.resolve(id); + } + }))); + + destination.push(...batch); + }; + /** * Securely resolves an AP object or URL that has been sent from another instance. * An input object is trusted if and only if its ID matches the authority of sentFromUri. * In all other cases, the object is re-fetched from remote by input string or object ID. + * @param input The input object or URL to resolve + * @param sentFromUri The URL where this object originated. This MUST be accurate - all security checks depend on this value! + * @param allowAnonymous If true, anonymous objects are allowed and will have their ID set to sentFromUri. If false (default) then anonymous objects will be rejected with an error. */ @bindThis - public async secureResolve(input: ApObject, sentFromUri: string): Promise { + public async secureResolve(input: string | IObject | [string | IObject], sentFromUri: string, allowAnonymous?: boolean): Promise { // Unpack arrays to get the value element. const value = fromTuple(input); - if (value == null) { - throw new IdentifiableError('20058164-9de1-4573-8715-425753a21c1d', 'Cannot resolve null input'); + + // If anonymous input is allowed, then any object is automatically valid if we set the ID. + // We can short-circuit here and avoid un-necessary checks. + if (allowAnonymous && typeof(value) === 'object' && value.id == null) { + value.id = sentFromUri; + return value as IObjectWithId; } - // This will throw if the input has no ID, which is good because we can't verify an anonymous object anyway. - const id = getApId(value); + // This ensures the input has a string ID, protecting against type confusion and rejecting anonymous objects. + const id = getApId(value, sentFromUri); // Check if we can use the provided object as-is. // Our security requires that the object ID matches the host authority that sent it, otherwise it can't be trusted. @@ -100,28 +197,52 @@ export class Resolver { } // If the checks didn't pass, then we must fetch the object and use that. - return await this.resolve(id); + return await this.resolve(id, allowAnonymous); } - public async resolve(value: string | [string]): Promise; - public async resolve(value: string | IObject | [string | IObject]): Promise; + /** + * Resolves an anonymous object. + * The returned value will not have any ID present. + * If one is provided in the response, it will be removed automatically. + */ @bindThis - public async resolve(value: string | IObject | [string | IObject]): Promise { + public async resolveAnonymous(value: string | IObject | [string | IObject]): Promise { value = fromTuple(value); + const object = await this.resolve(value); + object.id = undefined; + + return object as IAnonymousObject; + } + + public async resolve(value: string | [string], allowAnonymous?: boolean): Promise; + public async resolve(value: string | IObjectWithId | [string | IObjectWithId], allowAnonymous?: boolean): Promise; + public async resolve(value: string | IObject | [string | IObject], allowAnonymous?: boolean): Promise; + /** + * Resolves a URL or object to an AP object. + * Tuples are expanded to their first element before anything else, and non-string inputs are returned as-is. + * Otherwise, the string URL is fetched and validated to represent a valid ActivityPub object. + * @param value The input value to resolve + * @param allowAnonymous Determines what to do if a response object lacks an ID field. If false (default), then an exception is thrown. If true, then the ID is populated from the final response URL. + */ + @bindThis + public async resolve(value: string | IObject | [string | IObject], allowAnonymous = false): Promise { + value = fromTuple(value); + + // TODO try and remove this eventually, as it's a major security foot-gun if (typeof value !== 'string') { return value; } const host = this.utilityService.extractDbHost(value); if (this.config.activityLogging.enabled && !this.utilityService.isSelfHost(host)) { - return await this._resolveLogged(value, host); + return await this._resolveLogged(value, host, allowAnonymous); } else { - return await this._resolve(value, host); + return await this._resolve(value, host, allowAnonymous); } } - private async _resolveLogged(requestUri: string, host: string): Promise { + private async _resolveLogged(requestUri: string, host: string, allowAnonymous: boolean): Promise { const startTime = process.hrtime.bigint(); const log = await this.apLogService.createFetchLog({ @@ -130,7 +251,7 @@ export class Resolver { }); try { - const result = await this._resolve(requestUri, host, log); + const result = await this._resolve(requestUri, host, allowAnonymous, log); log.accepted = true; log.result = 'ok'; @@ -150,20 +271,20 @@ export class Resolver { } } - private async _resolve(value: string, host: string, log?: SkApFetchLog): Promise { + private async _resolve(value: string, host: string, allowAnonymous: boolean, log?: SkApFetchLog): Promise { if (value.includes('#')) { // URLs with fragment parts cannot be resolved correctly because // the fragment part does not get transmitted over HTTP(S). // Avoid strange behaviour by not trying to resolve these at all. - throw new IdentifiableError('b94fd5b1-0e3b-4678-9df2-dad4cd515ab2', `cannot resolve URL with fragment: ${value}`); + throw new IdentifiableError('b94fd5b1-0e3b-4678-9df2-dad4cd515ab2', `failed to resolve ${value}: URL contains fragment`); } if (this.history.has(value)) { - throw new IdentifiableError('0dc86cf6-7cd6-4e56-b1e6-5903d62d7ea5', `cannot resolve already resolved URL: ${value}`); + throw new IdentifiableError('0dc86cf6-7cd6-4e56-b1e6-5903d62d7ea5', `failed to resolve ${value}: recursive resolution blocked`); } if (this.history.size > this.recursionLimit) { - throw new IdentifiableError('d592da9f-822f-4d91-83d7-4ceefabcf3d2', `hit recursion limit: ${value}`); + throw new IdentifiableError('d592da9f-822f-4d91-83d7-4ceefabcf3d2', `failed to resolve ${value}: hit recursion limit`); } this.history.add(value); @@ -173,7 +294,7 @@ export class Resolver { } if (!this.utilityService.isFederationAllowedHost(host)) { - throw new IdentifiableError('09d79f9e-64f1-4316-9cfa-e75c4d091574', `cannot fetch AP object ${value}: blocked instance ${host}`); + throw new IdentifiableError('09d79f9e-64f1-4316-9cfa-e75c4d091574', `failed to resolve ${value}: instance ${host} is blocked`); } if (this.config.signToActivityPubGet && !this.user) { @@ -181,8 +302,8 @@ export class Resolver { } const object = (this.user - ? await this.apRequestService.signedGet(value, this.user) - : await this.httpRequestService.getActivityJson(value)); + ? await this.apRequestService.signedGet(value, this.user, allowAnonymous) + : await this.httpRequestService.getActivityJson(value, false, allowAnonymous)); if (log) { const { object: objectOnly, context, contextHash } = extractObjectContext(object); @@ -203,12 +324,12 @@ export class Resolver { !(object['@context'] as unknown[]).includes('https://www.w3.org/ns/activitystreams') : object['@context'] !== 'https://www.w3.org/ns/activitystreams' ) { - throw new IdentifiableError('72180409-793c-4973-868e-5a118eb5519b', `invalid AP object ${value}: does not have ActivityStreams context`); + throw new IdentifiableError('72180409-793c-4973-868e-5a118eb5519b', `failed to resolve ${value}: response does not have ActivityStreams context`); } // The object ID is already validated to match the final URL's authority by signedGet / getActivityJson. // We only need to validate that it also matches the original URL's authority, in case of redirects. - const objectId = getApId(object); + const objectId = getApId(object, value); // We allow some limited cross-domain redirects, which means the host may have changed during fetch. // Additional checks are needed to validate the scope of cross-domain redirects. @@ -219,21 +340,22 @@ export class Resolver { // Check if the redirect bounce from [allowed domain] to [blocked domain]. if (!this.utilityService.isFederationAllowedHost(finalHost)) { - throw new IdentifiableError('0a72bf24-2d9b-4f1d-886b-15aaa31adeda', `cannot fetch AP object ${value}: redirected to blocked instance ${finalHost}`); + throw new IdentifiableError('0a72bf24-2d9b-4f1d-886b-15aaa31adeda', `failed to resolve ${value}: redirected to blocked instance ${finalHost}`); } } return object; } + // TODO try to remove this, as it creates a large attack surface @bindThis private resolveLocal(url: string): Promise { const parsed = this.apDbResolverService.parseUri(url); - if (!parsed.local) throw new IdentifiableError('02b40cd0-fa92-4b0c-acc9-fb2ada952ab8', `resolveLocal - not a local URL: ${url}`); + if (!parsed.local) throw new IdentifiableError('02b40cd0-fa92-4b0c-acc9-fb2ada952ab8', `failed to resolve local ${url}: not a local URL`); switch (parsed.type) { case 'notes': - return this.notesRepository.findOneByOrFail({ id: parsed.id }) + return this.notesRepository.findOneByOrFail({ id: parsed.id, userHost: IsNull() }) .then(async note => { const author = await this.usersRepository.findOneByOrFail({ id: note.userId }); if (parsed.rest === 'activity') { @@ -244,22 +366,26 @@ export class Resolver { } }) as Promise; case 'users': - return this.usersRepository.findOneByOrFail({ id: parsed.id }) + return this.usersRepository.findOneByOrFail({ id: parsed.id, host: IsNull() }) .then(user => this.apRendererService.renderPerson(user as MiLocalUser)); case 'questions': // Polls are indexed by the note they are attached to. return Promise.all([ - this.notesRepository.findOneByOrFail({ id: parsed.id }), - this.pollsRepository.findOneByOrFail({ noteId: parsed.id }), + this.notesRepository.findOneByOrFail({ id: parsed.id, userHost: IsNull() }), + this.pollsRepository.findOneByOrFail({ noteId: parsed.id, userHost: IsNull() }), ]) .then(([note, poll]) => this.apRendererService.renderQuestion({ id: note.userId }, note, poll)) as Promise; case 'likes': - return this.noteReactionsRepository.findOneByOrFail({ id: parsed.id }).then(async reaction => - this.apRendererService.addContext(await this.apRendererService.renderLike(reaction, { uri: null }))); + return this.noteReactionsRepository.findOneOrFail({ where: { id: parsed.id }, relations: { user: true } }).then(async reaction => { + if (reaction.user?.host != null) { + throw new IdentifiableError('02b40cd0-fa92-4b0c-acc9-fb2ada952ab8', `failed to resolve local ${url}: not a local reaction`); + } + return this.apRendererService.addContext(await this.apRendererService.renderLike(reaction, { uri: null })); + }); case 'follows': return this.followRequestsRepository.findOneBy({ id: parsed.id }) .then(async followRequest => { - if (followRequest == null) throw new IdentifiableError('a9d946e5-d276-47f8-95fb-f04230289bb0', `resolveLocal - invalid follow request ID ${parsed.id}: ${url}`); + if (followRequest == null) throw new IdentifiableError('a9d946e5-d276-47f8-95fb-f04230289bb0', `failed to resolve local ${url}: invalid follow request ID`); const [follower, followee] = await Promise.all([ this.usersRepository.findOneBy({ id: followRequest.followerId, @@ -271,12 +397,12 @@ export class Resolver { }), ]); if (follower == null || followee == null) { - throw new IdentifiableError('06ae3170-1796-4d93-a697-2611ea6d83b6', `resolveLocal - follower or followee does not exist: ${url}`); + throw new IdentifiableError('06ae3170-1796-4d93-a697-2611ea6d83b6', `failed to resolve local ${url}: follower or followee does not exist`); } return this.apRendererService.addContext(this.apRendererService.renderFollow(follower as MiLocalUser | MiRemoteUser, followee as MiLocalUser | MiRemoteUser, url)); }); default: - throw new IdentifiableError('7a5d2fc0-94bc-4db6-b8b8-1bf24a2e23d0', `resolveLocal: type ${parsed.type} unhandled: ${url}`); + throw new IdentifiableError('7a5d2fc0-94bc-4db6-b8b8-1bf24a2e23d0', `failed to resolve local ${url}: unsupported type ${parsed.type}`); } } } diff --git a/packages/backend/src/core/activitypub/ApUtilityService.ts b/packages/backend/src/core/activitypub/ApUtilityService.ts index c3958cdf42..227dc3b9b3 100644 --- a/packages/backend/src/core/activitypub/ApUtilityService.ts +++ b/packages/backend/src/core/activitypub/ApUtilityService.ts @@ -24,7 +24,7 @@ export class ApUtilityService { public assertIdMatchesUrlAuthority(object: IObject, url: string): void { // This throws if the ID is missing or invalid, but that's ok. // Anonymous objects are impossible to verify, so we don't allow fetching them. - const id = getApId(object); + const id = getApId(object, url); // Make sure the object ID matches the final URL (which is where it actually exists). // The caller (ApResolverService) will verify the ID against the original / entry URL, which ensures that all three match. @@ -80,7 +80,6 @@ export class ApUtilityService { /** * Verifies that a provided URL is in a format acceptable for federation. * @throws {IdentifiableError} If URL cannot be parsed - * @throws {IdentifiableError} If URL contains a fragment * @throws {IdentifiableError} If URL is not HTTPS */ public assertApUrl(url: string | URL): void { @@ -93,11 +92,6 @@ export class ApUtilityService { } } - // Hash component breaks federation - if (url.hash) { - throw new IdentifiableError('0bedd29b-e3bf-4604-af51-d3352e2518af', `invalid AP url ${url}: contains a fragment (#)`); - } - // Must be HTTPS if (!this.checkHttps(url)) { throw new IdentifiableError('0bedd29b-e3bf-4604-af51-d3352e2518af', `invalid AP url ${url}: unsupported protocol ${url.protocol}`); diff --git a/packages/backend/src/core/activitypub/JsonLdService.ts b/packages/backend/src/core/activitypub/JsonLdService.ts index 9d1e2e06cc..1db5df6ad9 100644 --- a/packages/backend/src/core/activitypub/JsonLdService.ts +++ b/packages/backend/src/core/activitypub/JsonLdService.ts @@ -8,6 +8,7 @@ import { Injectable } from '@nestjs/common'; import { UnrecoverableError } from 'bullmq'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { bindThis } from '@/decorators.js'; +import { StatusError } from '@/misc/status-error.js'; import { CONTEXT, PRELOADED_CONTEXTS } from './misc/contexts.js'; import { validateContentTypeSetAsJsonLD } from './misc/validator.js'; import type { JsonLdDocument } from 'jsonld'; @@ -149,7 +150,7 @@ class JsonLd { }, ).then(res => { if (!res.ok) { - throw new Error(`JSON-LD fetch failed with ${res.status} ${res.statusText}: ${url}`); + throw new StatusError(`failed to fetch JSON-LD from ${url}`, res.status, res.statusText); } else { return res.json(); } diff --git a/packages/backend/src/core/activitypub/misc/contexts.ts b/packages/backend/src/core/activitypub/misc/contexts.ts index 5c0b8ffcbb..cedd1d8dd5 100644 --- a/packages/backend/src/core/activitypub/misc/contexts.ts +++ b/packages/backend/src/core/activitypub/misc/contexts.ts @@ -546,6 +546,10 @@ const extension_context_definition = { featured: 'toot:featured', discoverable: 'toot:discoverable', indexable: 'toot:indexable', + attributionDomains: { + '@id': 'toot:attributionDomains', + '@type': '@id', + }, // schema schema: 'http://schema.org#', PropertyValue: 'schema:PropertyValue', diff --git a/packages/backend/src/core/activitypub/misc/validator.ts b/packages/backend/src/core/activitypub/misc/validator.ts index 0ff83659c1..5cd2ddf006 100644 --- a/packages/backend/src/core/activitypub/misc/validator.ts +++ b/packages/backend/src/core/activitypub/misc/validator.ts @@ -3,15 +3,14 @@ * SPDX-License-Identifier: AGPL-3.0-only */ +import { IdentifiableError } from '@/misc/identifiable-error.js'; import type { Response } from 'node-fetch'; -// TODO throw identifiable or unrecoverable errors - export function validateContentTypeSetAsActivityPub(response: Response): void { const contentType = (response.headers.get('content-type') ?? '').toLowerCase(); if (contentType === '') { - throw new Error(`invalid content type of AP response - no content-type header: ${response.url}`); + throw new IdentifiableError('d09dc850-b76c-4f45-875a-7389339d78b8', `invalid AP response from ${response.url}: no content-type header`, true); } if ( contentType.startsWith('application/activity+json') || @@ -19,7 +18,7 @@ export function validateContentTypeSetAsActivityPub(response: Response): void { ) { return; } - throw new Error(`invalid content type of AP response - content type is not application/activity+json or application/ld+json: ${response.url}`); + throw new IdentifiableError('dc110060-a5f2-461d-808b-39c62702ca64', `invalid AP response from ${response.url}: content type "${contentType}" is not application/activity+json or application/ld+json`); } const plusJsonSuffixRegex = /^\s*(application|text)\/[a-zA-Z0-9\.\-\+]+\+json\s*(;|$)/; @@ -28,7 +27,7 @@ export function validateContentTypeSetAsJsonLD(response: Response): void { const contentType = (response.headers.get('content-type') ?? '').toLowerCase(); if (contentType === '') { - throw new Error(`invalid content type of JSON LD - no content-type header: ${response.url}`); + throw new IdentifiableError('45793ab7-7648-4886-b503-429f8a0d0f73', `invalid AP response from ${response.url}: no content-type header`, true); } if ( contentType.startsWith('application/ld+json') || @@ -37,5 +36,5 @@ export function validateContentTypeSetAsJsonLD(response: Response): void { ) { return; } - throw new Error(`invalid content type of JSON LD - content type is not application/ld+json or application/json: ${response.url}`); + throw new IdentifiableError('4bf8f36b-4d33-4ac9-ad76-63fa11f354e9', `invalid AP response from ${response.url}: content type "${contentType}" is not application/ld+json or application/json`); } diff --git a/packages/backend/src/core/activitypub/models/ApImageService.ts b/packages/backend/src/core/activitypub/models/ApImageService.ts index 423044b985..7a16972ea4 100644 --- a/packages/backend/src/core/activitypub/models/ApImageService.ts +++ b/packages/backend/src/core/activitypub/models/ApImageService.ts @@ -18,7 +18,7 @@ import type { Config } from '@/config.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; import { ApResolverService } from '../ApResolverService.js'; import { ApLoggerService } from '../ApLoggerService.js'; -import { isDocument, type IObject } from '../type.js'; +import { getNullableApId, isDocument, type IObject } from '../type.js'; @Injectable() export class ApImageService { @@ -48,7 +48,7 @@ export class ApImageService { public async createImage(actor: MiRemoteUser, value: string | IObject): Promise { // 投稿者が凍結されていたらスキップ if (actor.isSuspended) { - throw new IdentifiableError('85ab9bd7-3a41-4530-959d-f07073900109', `actor has been suspended: ${actor.uri}`); + throw new IdentifiableError('85ab9bd7-3a41-4530-959d-f07073900109', `failed to create image ${getNullableApId(value)}: actor ${actor.id} has been suspended`); } const image = await this.apResolverService.createResolver().resolve(value); diff --git a/packages/backend/src/core/activitypub/models/ApNoteService.ts b/packages/backend/src/core/activitypub/models/ApNoteService.ts index 7811b81795..57d4303982 100644 --- a/packages/backend/src/core/activitypub/models/ApNoteService.ts +++ b/packages/backend/src/core/activitypub/models/ApNoteService.ts @@ -26,6 +26,7 @@ import { bindThis } from '@/decorators.js'; import { checkHttps } from '@/misc/check-https.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; import { isRetryableError } from '@/misc/is-retryable-error.js'; +import { renderInlineError } from '@/misc/render-inline-error.js'; import { getOneApId, getApId, validPost, isEmoji, getApType, isApObject, isDocument, IApDocument } from '../type.js'; import { ApLoggerService } from '../ApLoggerService.js'; import { ApMfmService } from '../ApMfmService.js'; @@ -100,29 +101,29 @@ export class ApNoteService { const apType = getApType(object); if (apType == null || !validPost.includes(apType)) { - return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', `invalid Note: invalid object type ${apType ?? 'undefined'}`); + return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', `invalid Note from ${uri}: invalid object type ${apType ?? 'undefined'}`); } if (object.id && this.utilityService.extractDbHost(object.id) !== expectHost) { - return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', `invalid Note: id has different host. expected: ${expectHost}, actual: ${this.utilityService.extractDbHost(object.id)}`); + return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', `invalid Note from ${uri}: id has different host. expected: ${expectHost}, actual: ${this.utilityService.extractDbHost(object.id)}`); } const actualHost = object.attributedTo && this.utilityService.extractDbHost(getOneApId(object.attributedTo)); if (object.attributedTo && actualHost !== expectHost) { - return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', `invalid Note: attributedTo has different host. expected: ${expectHost}, actual: ${actualHost}`); + return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', `invalid Note from ${uri}: attributedTo has different host. expected: ${expectHost}, actual: ${actualHost}`); } if (object.published && !this.idService.isSafeT(new Date(object.published).valueOf())) { - return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', 'invalid Note: published timestamp is malformed'); + return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', 'invalid Note from ${uri}: published timestamp is malformed'); } if (actor) { const attribution = (object.attributedTo) ? getOneApId(object.attributedTo) : actor.uri; if (attribution !== actor.uri) { - return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', `invalid Note: attribution does not match the actor that send it. attribution: ${attribution}, actor: ${actor.uri}`); + return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', `invalid Note from ${uri}: attribution does not match the actor that send it. attribution: ${attribution}, actor: ${actor.uri}`); } if (user && attribution !== user.uri) { - return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', `invalid Note: updated attribution does not match original attribution. updated attribution: ${user.uri}, original attribution: ${attribution}`); + return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', `invalid Note from ${uri}: updated attribution does not match original attribution. updated attribution: ${user.uri}, original attribution: ${attribution}`); } } @@ -161,7 +162,7 @@ export class ApNoteService { const entryUri = getApId(value); const err = this.validateNote(object, entryUri, actor); if (err) { - this.logger.error(err.message, { + this.logger.error(`Error creating note: ${renderInlineError(err)}`, { resolver: { history: resolver.getHistory() }, value, object, @@ -174,11 +175,11 @@ export class ApNoteService { this.logger.debug(`Note fetched: ${JSON.stringify(note, null, 2)}`); if (note.id == null) { - throw new UnrecoverableError(`Refusing to create note without id: ${entryUri}`); + throw new UnrecoverableError(`failed to create note ${entryUri}: missing ID`); } if (!checkHttps(note.id)) { - throw new UnrecoverableError(`unexpected schema of note.id ${note.id} in ${entryUri}`); + throw new UnrecoverableError(`failed to create note ${entryUri}: unexpected schema`); } const url = this.apUtilityService.findBestObjectUrl(note); @@ -187,7 +188,7 @@ export class ApNoteService { // 投稿者をフェッチ if (note.attributedTo == null) { - throw new UnrecoverableError(`invalid note.attributedTo ${note.attributedTo} in ${entryUri}`); + throw new UnrecoverableError(`failed to create note: ${entryUri}: missing attributedTo`); } const uri = getOneApId(note.attributedTo); @@ -196,7 +197,7 @@ export class ApNoteService { // eslint-disable-next-line no-param-reassign actor ??= await this.apPersonService.fetchPerson(uri) as MiRemoteUser | undefined; if (actor && actor.isSuspended) { - throw new IdentifiableError('85ab9bd7-3a41-4530-959d-f07073900109', `actor ${uri} has been suspended: ${entryUri}`); + throw new IdentifiableError('85ab9bd7-3a41-4530-959d-f07073900109', `failed to create note ${entryUri}: actor ${uri} has been suspended`); } const apMentions = await this.apMentionService.extractApMentions(note.tag, resolver); @@ -223,7 +224,7 @@ export class ApNoteService { */ const hasProhibitedWords = this.noteCreateService.checkProhibitedWordsContain({ cw, text, pollChoices: poll?.choices }); if (hasProhibitedWords) { - throw new IdentifiableError('689ee33f-f97c-479a-ac49-1b9f8140af99', `Note contains prohibited words: ${entryUri}`); + throw new IdentifiableError('689ee33f-f97c-479a-ac49-1b9f8140af99', `failed to create note ${entryUri}: contains prohibited words`); } //#endregion @@ -232,7 +233,7 @@ export class ApNoteService { // 解決した投稿者が凍結されていたらスキップ if (actor.isSuspended) { - throw new IdentifiableError('85ab9bd7-3a41-4530-959d-f07073900109', `actor has been suspended: ${entryUri}`); + throw new IdentifiableError('85ab9bd7-3a41-4530-959d-f07073900109', `failed to create note ${entryUri}: actor ${actor.id} has been suspended`); } const noteAudience = await this.apAudienceService.parseAudience(actor, note.to, note.cc, resolver); @@ -269,15 +270,15 @@ export class ApNoteService { ? await this.resolveNote(note.inReplyTo, { resolver }) .then(x => { if (x == null) { - this.logger.warn('Specified inReplyTo, but not found'); - throw new Error(`could not fetch inReplyTo ${note.inReplyTo} for note ${entryUri}`); + this.logger.warn(`Specified inReplyTo "${note.inReplyTo}", but not found`); + throw new IdentifiableError('1ebf0a96-2769-4973-a6c2-3dcbad409dff', `failed to create note ${entryUri}: could not fetch inReplyTo ${note.inReplyTo}`, true); } return x; }) .catch(async err => { - this.logger.warn(`error ${err.statusCode ?? err} fetching inReplyTo ${note.inReplyTo} for note ${entryUri}`); - throw err; + this.logger.warn(`error ${renderInlineError(err)} fetching inReplyTo ${note.inReplyTo} for note ${entryUri}`); + throw new IdentifiableError('1ebf0a96-2769-4973-a6c2-3dcbad409dff', `failed to create note ${entryUri}: could not fetch inReplyTo ${note.inReplyTo}`, true, err); }) : null; @@ -285,6 +286,13 @@ export class ApNoteService { const quote = await this.getQuote(note, entryUri, resolver); const processErrors = quote === null ? ['quoteUnavailable'] : null; + if (reply && reply.userHost == null && reply.localOnly) { + throw new IdentifiableError('12e23cec-edd9-442b-aa48-9c21f0c3b215', 'Cannot reply to local-only note'); + } + if (quote && quote.userHost == null && quote.localOnly) { + throw new IdentifiableError('12e23cec-edd9-442b-aa48-9c21f0c3b215', 'Cannot quote a local-only note'); + } + // vote if (reply && reply.hasPoll) { const poll = await this.pollsRepository.findOneByOrFail({ noteId: reply.id }); @@ -341,7 +349,7 @@ export class ApNoteService { this.logger.info('The note is already inserted while creating itself, reading again'); const duplicate = await this.fetchNote(value); if (!duplicate) { - throw new Error(`The note creation failed with duplication error even when there is no duplication: ${entryUri}`); + throw new IdentifiableError('39c328e1-e829-458b-bfc9-65dcd513d1f8', `failed to create note ${entryUri}: the note creation failed with duplication error even when there is no duplication. This is likely a bug.`); } return duplicate; } @@ -355,45 +363,39 @@ export class ApNoteService { const noteUri = getApId(value); // URIがこのサーバーを指しているならスキップ - if (noteUri.startsWith(this.config.url + '/')) throw new UnrecoverableError(`uri points local: ${noteUri}`); + if (this.utilityService.isUriLocal(noteUri)) { + throw new UnrecoverableError(`failed to update note ${noteUri}: uri is local`); + } //#region このサーバーに既に登録されているか const updatedNote = await this.notesRepository.findOneBy({ uri: noteUri }); - if (updatedNote == null) throw new Error(`Note is not registered (no note): ${noteUri}`); + if (updatedNote == null) throw new UnrecoverableError(`failed to update note ${noteUri}: note does not exist`); const user = await this.usersRepository.findOneBy({ id: updatedNote.userId }) as MiRemoteUser | null; - if (user == null) throw new Error(`Note is not registered (no user): ${noteUri}`); + if (user == null) throw new UnrecoverableError(`failed to update note ${noteUri}: user does not exist`); - // eslint-disable-next-line no-param-reassign - if (resolver == null) resolver = this.apResolverService.createResolver(); + resolver ??= this.apResolverService.createResolver(); const object = await resolver.resolve(value); const entryUri = getApId(value); const err = this.validateNote(object, entryUri, actor, user); if (err) { - this.logger.error(err.message, { - resolver: { history: resolver.getHistory() }, - value, - object, - }); + this.logger.error(`Failed to update note ${noteUri}: ${renderInlineError(err)}`); throw err; } // `validateNote` checks that the actor and user are one and the same - // eslint-disable-next-line no-param-reassign actor ??= user; const note = object as IPost; - this.logger.debug(`Note fetched: ${JSON.stringify(note, null, 2)}`); - if (note.id == null) { - throw new UnrecoverableError(`Refusing to update note without id: ${noteUri}`); + throw new UnrecoverableError(`failed to update note ${entryUri}: missing ID`); } if (!checkHttps(note.id)) { - throw new UnrecoverableError(`unexpected schema of note.id ${note.id} in ${noteUri}`); + throw new UnrecoverableError(`failed to update note ${entryUri}: unexpected schema`); } const url = this.apUtilityService.findBestObjectUrl(note); @@ -401,7 +403,7 @@ export class ApNoteService { this.logger.info(`Creating the Note: ${note.id}`); if (actor.isSuspended) { - throw new IdentifiableError('85ab9bd7-3a41-4530-959d-f07073900109', `actor ${actor.id} has been suspended: ${noteUri}`); + throw new IdentifiableError('85ab9bd7-3a41-4530-959d-f07073900109', `failed to update note ${entryUri}: actor ${actor.id} has been suspended`); } const apMentions = await this.apMentionService.extractApMentions(note.tag, resolver); @@ -428,7 +430,7 @@ export class ApNoteService { */ const hasProhibitedWords = this.noteCreateService.checkProhibitedWordsContain({ cw, text, pollChoices: poll?.choices }); if (hasProhibitedWords) { - throw new IdentifiableError('689ee33f-f97c-479a-ac49-1b9f8140af99', `Note contains prohibited words: ${noteUri}`); + throw new IdentifiableError('689ee33f-f97c-479a-ac49-1b9f8140af99', `failed to update note ${noteUri}: contains prohibited words`); } //#endregion @@ -466,15 +468,15 @@ export class ApNoteService { ? await this.resolveNote(note.inReplyTo, { resolver }) .then(x => { if (x == null) { - this.logger.warn('Specified inReplyTo, but not found'); - throw new Error(`could not fetch inReplyTo ${note.inReplyTo} for note ${entryUri}`); + this.logger.warn(`Specified inReplyTo "${note.inReplyTo}", but not found`); + throw new IdentifiableError('1ebf0a96-2769-4973-a6c2-3dcbad409dff', `failed to update note ${entryUri}: could not fetch inReplyTo ${note.inReplyTo}`, true); } return x; }) .catch(async err => { - this.logger.warn(`error ${err.statusCode ?? err} fetching inReplyTo ${note.inReplyTo} for note ${entryUri}`); - throw err; + this.logger.warn(`error ${renderInlineError(err)} fetching inReplyTo ${note.inReplyTo} for note ${entryUri}`); + throw new IdentifiableError('1ebf0a96-2769-4973-a6c2-3dcbad409dff', `failed to update note ${entryUri}: could not fetch inReplyTo ${note.inReplyTo}`, true, err); }) : null; @@ -482,6 +484,10 @@ export class ApNoteService { const quote = await this.getQuote(note, entryUri, resolver); const processErrors = quote === null ? ['quoteUnavailable'] : null; + if (quote && quote.userHost == null && quote.localOnly) { + throw new IdentifiableError('12e23cec-edd9-442b-aa48-9c21f0c3b215', 'Cannot quote a local-only note'); + } + // vote if (reply && reply.hasPoll) { const poll = await this.pollsRepository.findOneByOrFail({ noteId: reply.id }); @@ -538,7 +544,7 @@ export class ApNoteService { this.logger.info('The note is already inserted while creating itself, reading again'); const duplicate = await this.fetchNote(value); if (!duplicate) { - throw new Error(`The note creation failed with duplication error even when there is no duplication: ${noteUri}`); + throw new IdentifiableError('39c328e1-e829-458b-bfc9-65dcd513d1f8', `failed to update note ${entryUri}: the note update failed with duplication error even when there is no duplication. This is likely a bug.`); } return duplicate; } @@ -555,8 +561,7 @@ export class ApNoteService { const uri = getApId(value); if (!this.utilityService.isFederationAllowedUri(uri)) { - // TODO convert to identifiable error - throw new StatusError(`blocked host: ${uri}`, 451, 'blocked host'); + throw new IdentifiableError('04620a7e-044e-45ce-b72c-10e1bdc22e69', `failed to resolve note ${uri}: host is blocked`); } //#region このサーバーに既に登録されていたらそれを返す @@ -566,8 +571,7 @@ export class ApNoteService { // Bail if local URI doesn't exist if (this.utilityService.isUriLocal(uri)) { - // TODO convert to identifiable error - throw new StatusError(`cannot resolve local note: ${uri}`, 400, 'cannot resolve local note'); + throw new IdentifiableError('cbac7358-23f2-4c70-833e-cffb4bf77913', `failed to resolve note ${uri}: URL is local and does not exist`); } const unlock = await this.appLockService.getApLock(uri); @@ -674,18 +678,13 @@ export class ApNoteService { const quote = await this.resolveNote(uri, { resolver }); if (quote == null) { - this.logger.warn(`Failed to resolve quote "${uri}" for note "${entryUri}": request error`); + this.logger.warn(`Failed to resolve quote "${uri}" for note "${entryUri}": fetch failed`); return false; } return quote; } catch (e) { - if (e instanceof Error) { - this.logger.warn(`Failed to resolve quote "${uri}" for note "${entryUri}":`, e); - } else { - this.logger.warn(`Failed to resolve quote "${uri}" for note "${entryUri}": ${e}`); - } - + this.logger.warn(`Failed to resolve quote "${uri}" for note "${entryUri}": ${renderInlineError(e)}`); return isRetryableError(e); } }; diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index 5c6716a0b8..b7aa036068 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -7,7 +7,6 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import promiseLimit from 'promise-limit'; import { DataSource } from 'typeorm'; import { ModuleRef } from '@nestjs/core'; -import { AbortError } from 'node-fetch'; import { UnrecoverableError } from 'bullmq'; import { DI } from '@/di-symbols.js'; import type { FollowingsRepository, InstancesRepository, MiMeta, UserProfilesRepository, UserPublickeysRepository, UsersRepository } from '@/models/_.js'; @@ -44,6 +43,8 @@ import { AppLockService } from '@/core/AppLockService.js'; import { MemoryKVCache } from '@/misc/cache.js'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { verifyFieldLinks } from '@/misc/verify-field-link.js'; +import { isRetryableError } from '@/misc/is-retryable-error.js'; +import { renderInlineError } from '@/misc/render-inline-error.js'; import { getApId, getApType, isActor, isCollection, isCollectionOrOrderedCollection, isPropertyValue } from '../type.js'; import { extractApHashtags } from './tag.js'; import type { OnModuleInit } from '@nestjs/common'; @@ -54,6 +55,7 @@ import type { ApLoggerService } from '../ApLoggerService.js'; import type { ApImageService } from './ApImageService.js'; import type { IActor, ICollection, IObject, IOrderedCollection } from '../type.js'; +import { IdentifiableError } from '@/misc/identifiable-error.js'; const nameLength = 128; const summaryLength = 2048; @@ -157,21 +159,21 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { const expectHost = this.utilityService.punyHostPSLDomain(uri); if (!isActor(x)) { - throw new UnrecoverableError(`invalid Actor type '${x.type}' in ${uri}`); + throw new UnrecoverableError(`invalid Actor ${uri}: unknown type '${x.type}'`); } if (!(typeof x.id === 'string' && x.id.length > 0)) { - throw new UnrecoverableError(`invalid Actor ${uri} - wrong id type`); + throw new UnrecoverableError(`invalid Actor ${uri}: wrong id type`); } if (!(typeof x.inbox === 'string' && x.inbox.length > 0)) { - throw new UnrecoverableError(`invalid Actor ${uri} - wrong inbox type`); + throw new UnrecoverableError(`invalid Actor ${uri}: wrong inbox type`); } this.apUtilityService.assertApUrl(x.inbox); const inboxHost = this.utilityService.punyHostPSLDomain(x.inbox); if (inboxHost !== expectHost) { - throw new UnrecoverableError(`invalid Actor ${uri} - wrong inbox ${inboxHost}`); + throw new UnrecoverableError(`invalid Actor ${uri}: wrong inbox host ${inboxHost}`); } const sharedInboxObject = x.sharedInbox ?? (x.endpoints ? x.endpoints.sharedInbox : undefined); @@ -179,7 +181,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { const sharedInbox = getApId(sharedInboxObject); this.apUtilityService.assertApUrl(sharedInbox); if (!(typeof sharedInbox === 'string' && sharedInbox.length > 0 && this.utilityService.punyHostPSLDomain(sharedInbox) === expectHost)) { - throw new UnrecoverableError(`invalid Actor ${uri} - wrong shared inbox ${sharedInbox}`); + throw new UnrecoverableError(`invalid Actor ${uri}: wrong shared inbox ${sharedInbox}`); } } @@ -190,7 +192,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { if (typeof collectionUri === 'string' && collectionUri.length > 0) { this.apUtilityService.assertApUrl(collectionUri); if (this.utilityService.punyHostPSLDomain(collectionUri) !== expectHost) { - throw new UnrecoverableError(`invalid Actor ${uri} - wrong ${collection} ${collectionUri}`); + throw new UnrecoverableError(`invalid Actor ${uri}: wrong ${collection} host ${collectionUri}`); } } else if (collectionUri != null) { throw new UnrecoverableError(`invalid Actor ${uri}: wrong ${collection} type`); @@ -199,7 +201,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { } if (!(typeof x.preferredUsername === 'string' && x.preferredUsername.length > 0 && x.preferredUsername.length <= 128 && /^\w([\w-.]*\w)?$/.test(x.preferredUsername))) { - throw new UnrecoverableError(`invalid Actor ${uri} - wrong username`); + throw new UnrecoverableError(`invalid Actor ${uri}: wrong username`); } // These fields are only informational, and some AP software allows these @@ -207,7 +209,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { // we can at least see these users and their activities. if (x.name) { if (!(typeof x.name === 'string' && x.name.length > 0)) { - throw new UnrecoverableError(`invalid Actor ${uri} - wrong name`); + throw new UnrecoverableError(`invalid Actor ${uri}: wrong name`); } x.name = truncate(x.name, nameLength); } else if (x.name === '') { @@ -216,24 +218,24 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { } if (x.summary) { if (!(typeof x.summary === 'string' && x.summary.length > 0)) { - throw new UnrecoverableError(`invalid Actor ${uri} - wrong summary`); + throw new UnrecoverableError(`invalid Actor ${uri}: wrong summary`); } x.summary = truncate(x.summary, summaryLength); } const idHost = this.utilityService.punyHostPSLDomain(x.id); if (idHost !== expectHost) { - throw new UnrecoverableError(`invalid Actor ${uri} - wrong id ${x.id}`); + throw new UnrecoverableError(`invalid Actor ${uri}: wrong id ${x.id}`); } if (x.publicKey) { if (typeof x.publicKey.id !== 'string') { - throw new UnrecoverableError(`invalid Actor ${uri} - wrong publicKey.id type`); + throw new UnrecoverableError(`invalid Actor ${uri}: wrong publicKey.id type`); } const publicKeyIdHost = this.utilityService.punyHostPSLDomain(x.publicKey.id); if (publicKeyIdHost !== expectHost) { - throw new UnrecoverableError(`invalid Actor ${uri} - wrong publicKey.id ${x.publicKey.id}`); + throw new UnrecoverableError(`invalid Actor ${uri}: wrong publicKey.id ${x.publicKey.id}`); } } @@ -271,8 +273,6 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { } private async resolveAvatarAndBanner(user: MiRemoteUser, icon: any, image: any, bgimg: any): Promise>> { - if (user == null) throw new Error('failed to create user: user is null'); - const [avatar, banner, background] = await Promise.all([icon, image, bgimg].map(img => { // icon and image may be arrays // see https://www.w3.org/TR/activitystreams-vocabulary/#dfn-icon @@ -325,12 +325,11 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { */ @bindThis public async createPerson(uri: string, resolver?: Resolver): Promise { - if (typeof uri !== 'string') throw new UnrecoverableError(`uri is not string: ${uri}`); + if (typeof uri !== 'string') throw new UnrecoverableError(`failed to create user ${uri}: input is not string`); const host = this.utilityService.punyHost(uri); if (host === this.utilityService.toPuny(this.config.host)) { - // TODO convert to unrecoverable error - throw new StatusError(`cannot resolve local user: ${uri}`, 400, 'cannot resolve local user'); + throw new UnrecoverableError(`failed to create user ${uri}: URI is local`); } return await this._createPerson(uri, resolver); @@ -340,8 +339,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { const uri = getApId(value); const host = this.utilityService.punyHost(uri); - // eslint-disable-next-line no-param-reassign - if (resolver == null) resolver = this.apResolverService.createResolver(); + resolver ??= this.apResolverService.createResolver(); const object = await resolver.resolve(value); const person = this.validateActor(object, uri); @@ -356,14 +354,16 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { const [followingVisibility, followersVisibility] = await Promise.all( [ - this.isPublicCollection(person.following, resolver), - this.isPublicCollection(person.followers, resolver), + this.isPublicCollection(person.following, resolver, uri), + this.isPublicCollection(person.followers, resolver, uri), ].map((p): Promise<'public' | 'private'> => p .then(isPublic => isPublic ? 'public' : 'private') .catch(err => { - if (!(err instanceof StatusError) || err.isRetryable) { - this.logger.error('error occurred while fetching following/followers collection', { stack: err }); + // Permanent error implies hidden or inaccessible, which is a normal thing. + if (isRetryableError(err)) { + this.logger.error(`error occurred while fetching following/followers collection: ${renderInlineError(err)}`); } + return 'private'; }), ), @@ -372,7 +372,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { const bday = person['vcard:bday']?.match(/^\d{4}-\d{2}-\d{2}/); if (person.id == null) { - throw new UnrecoverableError(`Refusing to create person without id: ${uri}`); + throw new UnrecoverableError(`failed to create user ${uri}: missing ID`); } const url = this.apUtilityService.findBestObjectUrl(person); @@ -387,16 +387,27 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { const emojis = await this.apNoteService.extractEmojis(person.tag ?? [], host) .then(_emojis => _emojis.map(emoji => emoji.name)) .catch(err => { - this.logger.error('error occurred while fetching user emojis', { stack: err }); + // Permanent error implies hidden or inaccessible, which is a normal thing. + if (isRetryableError(err)) { + this.logger.error(`error occurred while fetching user emojis: ${renderInlineError(err)}`); + } return []; }); //#endregion //#region resolve counts - const _resolver = resolver ?? this.apResolverService.createResolver(); - const outboxcollection = await _resolver.resolveCollection(person.outbox).catch(() => { return null; }); - const followerscollection = await _resolver.resolveCollection(person.followers!).catch(() => { return null; }); - const followingcollection = await _resolver.resolveCollection(person.following!).catch(() => { return null; }); + const outboxCollection = person.outbox + ? await resolver.resolveCollection(person.outbox, true, uri).catch(() => { return null; }) + : null; + const followersCollection = person.followers + ? await resolver.resolveCollection(person.followers, true, uri).catch(() => { return null; }) + : null; + const followingCollection = person.following + ? await resolver.resolveCollection(person.following, true, uri).catch(() => { return null; }) + : null; + + // Register the instance first, to avoid FK errors + await this.federatedInstanceService.fetchOrRegister(host); try { // Start transaction @@ -423,9 +434,9 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { host, inbox: person.inbox, sharedInbox: person.sharedInbox ?? person.endpoints?.sharedInbox ?? null, - notesCount: outboxcollection?.totalItems ?? 0, - followersCount: followerscollection?.totalItems ?? 0, - followingCount: followingcollection?.totalItems ?? 0, + notesCount: outboxCollection?.totalItems ?? 0, + followersCount: followersCollection?.totalItems ?? 0, + followingCount: followingCollection?.totalItems ?? 0, followersUri: person.followers ? getApId(person.followers) : undefined, featured: person.featured ? getApId(person.featured) : undefined, uri: person.id, @@ -437,6 +448,11 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { makeNotesFollowersOnlyBefore: (person as any).makeNotesFollowersOnlyBefore ?? null, makeNotesHiddenBefore: (person as any).makeNotesHiddenBefore ?? null, emojis, + attributionDomains: Array.isArray(person.attributionDomains) + ? person.attributionDomains + .filter((a: unknown) => typeof(a) === 'string' && a.length > 0 && a.length <= 128) + .slice(0, 32) + : [], })) as MiRemoteUser; let _description: string | null = null; @@ -480,7 +496,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { user = u as MiRemoteUser; publicKey = await this.userPublickeysRepository.findOneBy({ userId: user.id }); } else { - this.logger.error(e instanceof Error ? e : new Error(e as string)); + this.logger.error('Error creating Person:', e instanceof Error ? e : new Error(e as string)); throw e; } } @@ -520,11 +536,19 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { // Register to the cache this.cacheService.uriPersonCache.set(user.uri, user); } catch (err) { - this.logger.error('error occurred while fetching user avatar/banner', { stack: err }); + // Permanent error implies hidden or inaccessible, which is a normal thing. + if (isRetryableError(err)) { + this.logger.error(`error occurred while fetching user avatar/banner: ${renderInlineError(err)}`); + } } //#endregion - await this.updateFeatured(user.id, resolver).catch(err => this.logger.error(err)); + await this.updateFeatured(user.id, resolver).catch(err => { + // Permanent error implies hidden or inaccessible, which is a normal thing. + if (isRetryableError(err)) { + this.logger.error(`Error updating featured notes: ${renderInlineError(err)}`); + } + }); return user; } @@ -541,7 +565,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { */ @bindThis public async updatePerson(uri: string, resolver?: Resolver | null, hint?: IObject, movePreventUris: string[] = []): Promise { - if (typeof uri !== 'string') throw new UnrecoverableError('uri is not string'); + if (typeof uri !== 'string') throw new UnrecoverableError(`failed to update user ${uri}: input is not string`); // URIがこのサーバーを指しているならスキップ if (this.utilityService.isUriLocal(uri)) return; @@ -561,8 +585,11 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { this.logger.info(`Updating the Person: ${person.id}`); // カスタム絵文字取得 - const emojis = await this.apNoteService.extractEmojis(person.tag ?? [], exist.host).catch(e => { - this.logger.info(`extractEmojis: ${e}`); + const emojis = await this.apNoteService.extractEmojis(person.tag ?? [], exist.host).catch(err => { + // Permanent error implies hidden or inaccessible, which is a normal thing. + if (isRetryableError(err)) { + this.logger.error(`error occurred while fetching user emojis: ${renderInlineError(err)}`); + } return []; }); @@ -574,16 +601,18 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { const [followingVisibility, followersVisibility] = await Promise.all( [ - this.isPublicCollection(person.following, resolver), - this.isPublicCollection(person.followers, resolver), + this.isPublicCollection(person.following, resolver, exist.uri), + this.isPublicCollection(person.followers, resolver, exist.uri), ].map((p): Promise<'public' | 'private' | undefined> => p .then(isPublic => isPublic ? 'public' : 'private') .catch(err => { - if (!(err instanceof StatusError) || err.isRetryable) { - this.logger.error('error occurred while fetching following/followers collection', { stack: err }); + // Permanent error implies hidden or inaccessible, which is a normal thing. + if (isRetryableError(err)) { + this.logger.error(`error occurred while fetching following/followers collection: ${renderInlineError(err)}`); // Do not update the visibility on transient errors. return undefined; } + return 'private'; }), ), @@ -592,7 +621,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { const bday = person['vcard:bday']?.match(/^\d{4}-\d{2}-\d{2}/); if (person.id == null) { - throw new UnrecoverableError(`Refusing to update person without id: ${uri}`); + throw new UnrecoverableError(`failed to update user ${uri}: missing ID`); } const url = this.apUtilityService.findBestObjectUrl(person); @@ -620,7 +649,20 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { // We use "!== false" to handle incorrect types, missing / null values, and "default to true" logic. hideOnlineStatus: person.hideOnlineStatus !== false, isExplorable: person.discoverable !== false, - ...(await this.resolveAvatarAndBanner(exist, person.icon, person.image, person.backgroundUrl).catch(() => ({}))), + attributionDomains: Array.isArray(person.attributionDomains) + ? person.attributionDomains + .filter((a: unknown) => typeof(a) === 'string' && a.length > 0 && a.length <= 128) + .slice(0, 32) + : [], + ...(await this.resolveAvatarAndBanner(exist, person.icon, person.image, person.backgroundUrl).catch(err => { + // Permanent error implies hidden or inaccessible, which is a normal thing. + if (isRetryableError(err)) { + this.logger.error(`error occurred while fetching user avatar/banner: ${renderInlineError(err)}`); + } + + // Can't return null or destructuring operator will break + return {}; + })), } as Partial & Pick; const moving = ((): boolean => { @@ -704,7 +746,12 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { { followerSharedInbox: person.sharedInbox ?? person.endpoints?.sharedInbox ?? null }, ); - await this.updateFeatured(exist.id, resolver).catch(err => this.logger.error(err)); + await this.updateFeatured(exist.id, resolver).catch(err => { + // Permanent error implies hidden or inaccessible, which is a normal thing. + if (isRetryableError(err)) { + this.logger.error(`Error updating featured notes: ${renderInlineError(err)}`); + } + }); const updated = { ...exist, ...updates }; @@ -743,8 +790,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { const uri = getApId(value); if (!this.utilityService.isFederationAllowedUri(uri)) { - // TODO convert to identifiable error - throw new StatusError(`blocked host: ${uri}`, 451, 'blocked host'); + throw new IdentifiableError('590719b3-f51f-48a9-8e7d-6f559ad00e5d', `failed to resolve person ${uri}: host is blocked`); } //#region このサーバーに既に登録されていたらそれを返す @@ -754,8 +800,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { // Bail if local URI doesn't exist if (this.utilityService.isUriLocal(uri)) { - // TODO convert to identifiable error - throw new StatusError(`cannot resolve local person: ${uri}`, 400, 'cannot resolve local person'); + throw new IdentifiableError('efb573fd-6b9e-4912-9348-a02f5603df4f', `failed to resolve person ${uri}: URL is local and does not exist`); } const unlock = await this.appLockService.getApLock(uri); @@ -799,16 +844,17 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { const _resolver = resolver ?? this.apResolverService.createResolver(); // Resolve to (Ordered)Collection Object - const collection = await _resolver.resolveCollection(user.featured).catch(err => { - if (err instanceof AbortError || err instanceof StatusError) { - this.logger.warn(`Failed to update featured notes: ${err.name}: ${err.message}`); - } else { - this.logger.error('Failed to update featured notes:', err); + const collection = user.featured ? await _resolver.resolveCollection(user.featured, true, user.uri).catch(err => { + // Permanent error implies hidden or inaccessible, which is a normal thing. + if (isRetryableError(err)) { + this.logger.warn(`Failed to update featured notes: ${renderInlineError(err)}`); } - }); + + return null; + }) : null; if (!collection) return; - if (!isCollectionOrOrderedCollection(collection)) throw new UnrecoverableError(`featured ${user.featured} is not Collection or OrderedCollection in ${user.uri}`); + if (!isCollectionOrOrderedCollection(collection)) throw new UnrecoverableError(`failed to update user ${user.uri}: featured ${user.featured} is not Collection or OrderedCollection`); // Resolve to Object(may be Note) arrays const unresolvedItems = isCollection(collection) ? collection.items : collection.orderedItems; @@ -891,11 +937,13 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { } @bindThis - private async isPublicCollection(collection: string | ICollection | IOrderedCollection | undefined, resolver: Resolver): Promise { + private async isPublicCollection(collection: string | ICollection | IOrderedCollection | undefined, resolver: Resolver, sentFrom: string): Promise { if (collection) { - const resolved = await resolver.resolveCollection(collection); - if (resolved.first || (resolved as ICollection).items || (resolved as IOrderedCollection).orderedItems) { - return true; + const resolved = await resolver.resolveCollection(collection, true, sentFrom).catch(() => null); + if (resolved) { + if (resolved.first || (resolved as ICollection).items || (resolved as IOrderedCollection).orderedItems) { + return true; + } } } diff --git a/packages/backend/src/core/activitypub/models/ApQuestionService.ts b/packages/backend/src/core/activitypub/models/ApQuestionService.ts index 335ca189ec..80900be2dc 100644 --- a/packages/backend/src/core/activitypub/models/ApQuestionService.ts +++ b/packages/backend/src/core/activitypub/models/ApQuestionService.ts @@ -93,7 +93,6 @@ export class ApQuestionService { // eslint-disable-next-line no-param-reassign if (resolver == null) resolver = this.apResolverService.createResolver(); const question = await resolver.resolve(value); - this.logger.debug(`fetched question: ${JSON.stringify(question, null, 2)}`); if (!isQuestion(question)) throw new UnrecoverableError(`object ${getApType(question)} is not a Question: ${uri}`); diff --git a/packages/backend/src/core/activitypub/type.ts b/packages/backend/src/core/activitypub/type.ts index 281733d484..cc7599d394 100644 --- a/packages/backend/src/core/activitypub/type.ts +++ b/packages/backend/src/core/activitypub/type.ts @@ -43,6 +43,18 @@ export interface IObjectWithId extends IObject { id: string; } +export function isObjectWithId(object: IObject): object is IObjectWithId { + return typeof(object.id) === 'string'; +} + +export interface IAnonymousObject extends IObject { + id: undefined; +} + +export function isAnonymousObject(object: IObject): object is IAnonymousObject { + return object.id === undefined; +} + /** * Get array of ActivityStreams Objects id */ @@ -63,24 +75,34 @@ export function getOneApId(value: ApObject): string { /** * Get ActivityStreams Object id */ -export function getApId(value: string | IObject | [string | IObject]): string { - // eslint-disable-next-line no-param-reassign - value = fromTuple(value); +export function getApId(value: string | IObject | [string | IObject], sourceForLogs?: string): string { + const id = getNullableApId(value); - if (typeof value === 'string') return value; - if (typeof value.id === 'string') return value.id; - throw new IdentifiableError('ad2dc287-75c1-44c4-839d-3d2e64576675', `invalid AP object ${value}: missing id`); + if (id == null) { + const message = sourceForLogs + ? `invalid AP object ${value} (sent from ${sourceForLogs}): missing id` + : `invalid AP object ${value}: missing id`; + throw new IdentifiableError('ad2dc287-75c1-44c4-839d-3d2e64576675', message); + } + + return id; } /** * Get ActivityStreams Object id, or null if not present */ -export function getNullableApId(value: string | IObject | [string | IObject]): string | null { - // eslint-disable-next-line no-param-reassign - value = fromTuple(value); +export function getNullableApId(source: string | IObject | [string | IObject]): string | null { + const value: unknown = fromTuple(source); + + if (value != null) { + if (typeof value === 'string') { + return value; + } + if (typeof (value) === 'object' && 'id' in value && typeof (value.id) === 'string') { + return value.id; + } + } - if (typeof value === 'string') return value; - if (typeof value.id === 'string') return value.id; return null; } @@ -125,48 +147,46 @@ export interface IActivity extends IObject { }; } -export interface ICollection extends IObject { +export interface CollectionBase extends IObject { + totalItems?: number; + first?: IObject | string; + last?: IObject | string; + current?: IObject | string; + partOf?: IObject | string; + next?: IObject | string; + prev?: IObject | string; + items?: ApObject; + orderedItems?: ApObject; +} + +export interface ICollection extends CollectionBase { type: 'Collection'; totalItems: number; - first?: IObject | string; - last?: IObject | string; - current?: IObject | string; items?: ApObject; + orderedItems?: undefined; } -export interface IOrderedCollection extends IObject { +export interface IOrderedCollection extends CollectionBase { type: 'OrderedCollection'; totalItems: number; - first?: IObject | string; - last?: IObject | string; - current?: IObject | string; + items?: undefined; orderedItems?: ApObject; } -export interface ICollectionPage extends IObject { +export interface ICollectionPage extends CollectionBase { type: 'CollectionPage'; - totalItems: number; - first?: IObject | string; - last?: IObject | string; - current?: IObject | string; - partOf?: IObject | string; - next?: IObject | string; - prev?: IObject | string; items?: ApObject; + orderedItems?: undefined; } -export interface IOrderedCollectionPage extends IObject { +export interface IOrderedCollectionPage extends CollectionBase { type: 'OrderedCollectionPage'; - totalItems: number; - first?: IObject | string; - last?: IObject | string; - current?: IObject | string; - partOf?: IObject | string; - next?: IObject | string; - prev?: IObject | string; + items?: undefined; orderedItems?: ApObject; } +export type AnyCollection = ICollection | IOrderedCollection | ICollectionPage | IOrderedCollectionPage; + export const validPost = ['Note', 'Question', 'Article', 'Audio', 'Document', 'Image', 'Page', 'Video', 'Event']; export const isPost = (object: IObject): object is IPost => { @@ -255,6 +275,7 @@ export interface IActor extends IObject { enableRss?: boolean; listenbrainz?: string; backgroundUrl?: string; + attributionDomains?: string[]; } export const isCollection = (object: IObject): object is ICollection => @@ -269,7 +290,7 @@ export const isCollectionPage = (object: IObject): object is ICollectionPage => export const isOrderedCollectionPage = (object: IObject): object is IOrderedCollectionPage => getApType(object) === 'OrderedCollectionPage'; -export const isCollectionOrOrderedCollection = (object: IObject): object is ICollection | IOrderedCollection => +export const isCollectionOrOrderedCollection = (object: IObject): object is AnyCollection => isCollection(object) || isOrderedCollection(object) || isCollectionPage(object) || isOrderedCollectionPage(object); export interface IApPropertyValue extends IObject { diff --git a/packages/backend/src/core/chart/charts/federation.ts b/packages/backend/src/core/chart/charts/federation.ts index bf702884ca..b6db6f5454 100644 --- a/packages/backend/src/core/chart/charts/federation.ts +++ b/packages/backend/src/core/chart/charts/federation.ts @@ -44,10 +44,6 @@ export default class FederationChart extends Chart { // eslint-di } protected async tickMinor(): Promise>> { - const suspendedInstancesQuery = this.instancesRepository.createQueryBuilder('instance') - .select('instance.host') - .where('instance.suspensionState != \'none\''); - const pubsubSubQuery = this.followingsRepository.createQueryBuilder('f') .select('f.followerHost') .where('f.followerHost IS NOT NULL'); @@ -64,22 +60,25 @@ export default class FederationChart extends Chart { // eslint-di this.followingsRepository.createQueryBuilder('following') .select('COUNT(DISTINCT following.followeeHost)') .where('following.followeeHost IS NOT NULL') - .andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || following.followeeHost) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)') - .andWhere(`following.followeeHost NOT IN (${ suspendedInstancesQuery.getQuery() })`) + .innerJoin('following.followeeInstance', 'followeeInstance') + .andWhere('followeeInstance.suspensionState = \'none\'') + .andWhere('followeeInstance.isBlocked = false') .getRawOne() .then(x => parseInt(x.count, 10)), this.followingsRepository.createQueryBuilder('following') .select('COUNT(DISTINCT following.followerHost)') .where('following.followerHost IS NOT NULL') - .andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || following.followerHost) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)') - .andWhere(`following.followerHost NOT IN (${ suspendedInstancesQuery.getQuery() })`) + .innerJoin('following.followerInstance', 'followerInstance') + .andWhere('followerInstance.isBlocked = false') + .andWhere('followerInstance.suspensionState = \'none\'') .getRawOne() .then(x => parseInt(x.count, 10)), this.followingsRepository.createQueryBuilder('following') .select('COUNT(DISTINCT following.followeeHost)') .where('following.followeeHost IS NOT NULL') - .andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || following.followeeHost) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)') - .andWhere(`following.followeeHost NOT IN (${ suspendedInstancesQuery.getQuery() })`) + .innerJoin('following.followeeInstance', 'followeeInstance') + .andWhere('followeeInstance.isBlocked = false') + .andWhere('followeeInstance.suspensionState = \'none\'') .andWhere(`following.followeeHost IN (${ pubsubSubQuery.getQuery() })`) .setParameters(pubsubSubQuery.getParameters()) .getRawOne() @@ -87,7 +86,7 @@ export default class FederationChart extends Chart { // eslint-di this.instancesRepository.createQueryBuilder('instance') .select('COUNT(instance.id)') .where(`instance.host IN (${ subInstancesQuery.getQuery() })`) - .andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || instance.host) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)') + .andWhere('instance.isBlocked = false') .andWhere('instance.suspensionState = \'none\'') .andWhere('instance.isNotResponding = false') .getRawOne() @@ -95,7 +94,7 @@ export default class FederationChart extends Chart { // eslint-di this.instancesRepository.createQueryBuilder('instance') .select('COUNT(instance.id)') .where(`instance.host IN (${ pubInstancesQuery.getQuery() })`) - .andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || instance.host) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)') + .andWhere('instance.isBlocked = false') .andWhere('instance.suspensionState = \'none\'') .andWhere('instance.isNotResponding = false') .getRawOne() diff --git a/packages/backend/src/core/entities/AbuseUserReportEntityService.ts b/packages/backend/src/core/entities/AbuseUserReportEntityService.ts index 70ead890ab..c1d877aa12 100644 --- a/packages/backend/src/core/entities/AbuseUserReportEntityService.ts +++ b/packages/backend/src/core/entities/AbuseUserReportEntityService.ts @@ -5,13 +5,14 @@ import { Inject, Injectable } from '@nestjs/common'; import { DI } from '@/di-symbols.js'; -import type { AbuseUserReportsRepository } from '@/models/_.js'; +import type { AbuseUserReportsRepository, InstancesRepository, MiInstance, MiUser } from '@/models/_.js'; import { awaitAll } from '@/misc/prelude/await-all.js'; import type { MiAbuseUserReport } from '@/models/AbuseUserReport.js'; import { bindThis } from '@/decorators.js'; import { IdService } from '@/core/IdService.js'; import type { Packed } from '@/misc/json-schema.js'; import { UserEntityService } from './UserEntityService.js'; +import { InstanceEntityService } from './InstanceEntityService.js'; @Injectable() export class AbuseUserReportEntityService { @@ -19,6 +20,10 @@ export class AbuseUserReportEntityService { @Inject(DI.abuseUserReportsRepository) private abuseUserReportsRepository: AbuseUserReportsRepository, + @Inject(DI.instancesRepository) + private instancesRepository: InstancesRepository, + + private readonly instanceEntityService: InstanceEntityService, private userEntityService: UserEntityService, private idService: IdService, ) { @@ -30,11 +35,14 @@ export class AbuseUserReportEntityService { hint?: { packedReporter?: Packed<'UserDetailedNotMe'>, packedTargetUser?: Packed<'UserDetailedNotMe'>, + packedTargetInstance?: Packed<'FederationInstance'>, packedAssignee?: Packed<'UserDetailedNotMe'>, }, + me?: MiUser | null, ) { const report = typeof src === 'object' ? src : await this.abuseUserReportsRepository.findOneByOrFail({ id: src }); + // noinspection ES6MissingAwait return await awaitAll({ id: report.id, createdAt: this.idService.parse(report.id).date.toISOString(), @@ -43,13 +51,22 @@ export class AbuseUserReportEntityService { reporterId: report.reporterId, targetUserId: report.targetUserId, assigneeId: report.assigneeId, - reporter: hint?.packedReporter ?? this.userEntityService.pack(report.reporter ?? report.reporterId, null, { + reporter: hint?.packedReporter ?? this.userEntityService.pack(report.reporter ?? report.reporterId, me, { schema: 'UserDetailedNotMe', }), - targetUser: hint?.packedTargetUser ?? this.userEntityService.pack(report.targetUser ?? report.targetUserId, null, { + targetUser: hint?.packedTargetUser ?? this.userEntityService.pack(report.targetUser ?? report.targetUserId, me, { schema: 'UserDetailedNotMe', }), - assignee: report.assigneeId ? hint?.packedAssignee ?? this.userEntityService.pack(report.assignee ?? report.assigneeId, null, { + // return hint, or pack by relation, or fetch and pack by id, or null + targetInstance: hint?.packedTargetInstance ?? ( + report.targetUserInstance + ? this.instanceEntityService.pack(report.targetUserInstance, me) + : report.targetUserHost + ? this.instancesRepository.findOneBy({ host: report.targetUserHost }).then(instance => instance + ? this.instanceEntityService.pack(instance, me) + : null) + : null), + assignee: report.assigneeId ? hint?.packedAssignee ?? this.userEntityService.pack(report.assignee ?? report.assigneeId, me, { schema: 'UserDetailedNotMe', }) : null, forwarded: report.forwarded, @@ -61,21 +78,28 @@ export class AbuseUserReportEntityService { @bindThis public async packMany( reports: MiAbuseUserReport[], + me?: MiUser | null, ) { const _reporters = reports.map(({ reporter, reporterId }) => reporter ?? reporterId); const _targetUsers = reports.map(({ targetUser, targetUserId }) => targetUser ?? targetUserId); const _assignees = reports.map(({ assignee, assigneeId }) => assignee ?? assigneeId).filter(x => x != null); const _userMap = await this.userEntityService.packMany( [..._reporters, ..._targetUsers, ..._assignees], - null, + me, { schema: 'UserDetailedNotMe' }, ).then(users => new Map(users.map(u => [u.id, u]))); + const _targetInstances = reports + .map(({ targetUserInstance, targetUserHost }) => targetUserInstance ?? targetUserHost) + .filter((i): i is MiInstance | string => i != null); + const _instanceMap = await this.instanceEntityService.packMany(await this.instanceEntityService.fetchInstancesByHost(_targetInstances), me) + .then(instances => new Map(instances.map(i => [i.host, i]))); return Promise.all( reports.map(report => { const packedReporter = _userMap.get(report.reporterId); const packedTargetUser = _userMap.get(report.targetUserId); + const packedTargetInstance = report.targetUserHost ? _instanceMap.get(report.targetUserHost) : undefined; const packedAssignee = report.assigneeId != null ? _userMap.get(report.assigneeId) : undefined; - return this.pack(report, { packedReporter, packedTargetUser, packedAssignee }); + return this.pack(report, { packedReporter, packedTargetUser, packedAssignee, packedTargetInstance }, me); }), ); } diff --git a/packages/backend/src/core/entities/InstanceEntityService.ts b/packages/backend/src/core/entities/InstanceEntityService.ts index 75aedd6af0..f1149fe113 100644 --- a/packages/backend/src/core/entities/InstanceEntityService.ts +++ b/packages/backend/src/core/entities/InstanceEntityService.ts @@ -4,6 +4,7 @@ */ import { Inject, Injectable } from '@nestjs/common'; +import { In } from 'typeorm'; import type { Packed } from '@/misc/json-schema.js'; import type { MiInstance } from '@/models/Instance.js'; import { bindThis } from '@/decorators.js'; @@ -11,7 +12,7 @@ import { UtilityService } from '@/core/UtilityService.js'; import { RoleService } from '@/core/RoleService.js'; import { MiUser } from '@/models/User.js'; import { DI } from '@/di-symbols.js'; -import { MiMeta } from '@/models/_.js'; +import type { InstancesRepository, MiMeta } from '@/models/_.js'; @Injectable() export class InstanceEntityService { @@ -19,6 +20,9 @@ export class InstanceEntityService { @Inject(DI.meta) private meta: MiMeta, + @Inject(DI.instancesRepository) + private readonly instancesRepository: InstancesRepository, + private roleService: RoleService, private utilityService: UtilityService, @@ -44,7 +48,7 @@ export class InstanceEntityService { isNotResponding: instance.isNotResponding, isSuspended: instance.suspensionState !== 'none' || Boolean(softwareSuspended), suspensionState: instance.suspensionState === 'none' && softwareSuspended ? 'softwareSuspended' : instance.suspensionState, - isBlocked: this.utilityService.isBlockedHost(this.meta.blockedHosts, instance.host), + isBlocked: instance.isBlocked, softwareName: instance.softwareName, softwareVersion: instance.softwareVersion, openRegistrations: instance.openRegistrations, @@ -52,8 +56,8 @@ export class InstanceEntityService { description: instance.description, maintainerName: instance.maintainerName, maintainerEmail: instance.maintainerEmail, - isSilenced: this.utilityService.isSilencedHost(this.meta.silencedHosts, instance.host), - isMediaSilenced: this.utilityService.isMediaSilencedHost(this.meta.mediaSilencedHosts, instance.host), + isSilenced: instance.isSilenced, + isMediaSilenced: instance.isMediaSilenced, iconUrl: instance.iconUrl, faviconUrl: instance.faviconUrl, themeColor: instance.themeColor, @@ -63,6 +67,7 @@ export class InstanceEntityService { rejectReports: instance.rejectReports, rejectQuotes: instance.rejectQuotes, moderationNote: iAmModerator ? instance.moderationNote : null, + isBubbled: this.utilityService.isBubbledHost(instance.host), }; } @@ -73,5 +78,28 @@ export class InstanceEntityService { ) { return Promise.all(instances.map(x => this.pack(x, me))); } + + @bindThis + public async fetchInstancesByHost(instances: (MiInstance | MiInstance['host'])[]): Promise { + const result: MiInstance[] = []; + + const toFetch: string[] = []; + for (const instance of instances) { + if (typeof(instance) === 'string') { + toFetch.push(instance); + } else { + result.push(instance); + } + } + + if (toFetch.length > 0) { + const fetched = await this.instancesRepository.findBy({ + host: In(toFetch), + }); + result.push(...fetched); + } + + return result; + } } diff --git a/packages/backend/src/core/entities/NoteEntityService.ts b/packages/backend/src/core/entities/NoteEntityService.ts index 6ada5463a3..d2373a70a2 100644 --- a/packages/backend/src/core/entities/NoteEntityService.ts +++ b/packages/backend/src/core/entities/NoteEntityService.ts @@ -17,6 +17,7 @@ import { DebounceLoader } from '@/misc/loader.js'; import { IdService } from '@/core/IdService.js'; import { ReactionsBufferingService } from '@/core/ReactionsBufferingService.js'; import { isPackedPureRenote } from '@/misc/is-renote.js'; +import type { Config } from '@/config.js'; import type { OnModuleInit } from '@nestjs/common'; import type { CacheService } from '../CacheService.js'; import type { CustomEmojiService } from '../CustomEmojiService.js'; @@ -92,6 +93,9 @@ export class NoteEntityService implements OnModuleInit { @Inject(DI.channelsRepository) private channelsRepository: ChannelsRepository, + @Inject(DI.config) + private readonly config: Config, + //private userEntityService: UserEntityService, //private driveFileEntityService: DriveFileEntityService, //private customEmojiService: CustomEmojiService, @@ -680,4 +684,9 @@ export class NoteEntityService implements OnModuleInit { return map; }, {} as Record); } + + @bindThis + public genLocalNoteUri(noteId: string): string { + return `${this.config.url}/notes/${noteId}`; + } } diff --git a/packages/backend/src/core/entities/UserEntityService.ts b/packages/backend/src/core/entities/UserEntityService.ts index 56506a5fa4..326baaefd4 100644 --- a/packages/backend/src/core/entities/UserEntityService.ts +++ b/packages/backend/src/core/entities/UserEntityService.ts @@ -487,7 +487,10 @@ export class UserEntityService implements OnModuleInit { includeSecrets: false, }, options); - const user = typeof src === 'object' ? src : await this.usersRepository.findOneByOrFail({ id: src }); + const user = typeof src === 'object' ? src : await this.usersRepository.findOneOrFail({ + where: { id: src }, + relations: { userProfile: true }, + }); // migration if (user.avatarId != null && user.avatarUrl === null) { @@ -521,7 +524,7 @@ export class UserEntityService implements OnModuleInit { const iAmModerator = me ? await this.roleService.isModerator(me as MiUser) : false; const profile = isDetailed - ? (opts.userProfile ?? await this.userProfilesRepository.findOneByOrFail({ userId: user.id })) + ? (opts.userProfile ?? user.userProfile ?? await this.userProfilesRepository.findOneByOrFail({ userId: user.id })) : null; let relation: UserRelation | null = null; @@ -556,7 +559,7 @@ export class UserEntityService implements OnModuleInit { } } - const mastoapi = !isDetailed ? opts.userProfile ?? await this.userProfilesRepository.findOneByOrFail({ userId: user.id }) : null; + const mastoapi = !isDetailed ? opts.userProfile ?? user.userProfile ?? await this.userProfilesRepository.findOneByOrFail({ userId: user.id }) : null; const followingCount = profile == null ? null : (profile.followingVisibility === 'public') || isMe || iAmModerator ? user.followingCount : @@ -603,19 +606,21 @@ export class UserEntityService implements OnModuleInit { enableRss: user.enableRss, mandatoryCW: user.mandatoryCW, rejectQuotes: user.rejectQuotes, + attributionDomains: user.attributionDomains, isSilenced: user.isSilenced || this.roleService.getUserPolicies(user.id).then(r => !r.canPublicNote), speakAsCat: user.speakAsCat ?? false, approved: user.approved, requireSigninToViewContents: user.requireSigninToViewContents === false ? undefined : true, makeNotesFollowersOnlyBefore: user.makeNotesFollowersOnlyBefore ?? undefined, makeNotesHiddenBefore: user.makeNotesHiddenBefore ?? undefined, - instance: user.host ? this.federatedInstanceService.federatedInstanceCache.fetch(user.host).then(instance => instance ? { + instance: user.host ? this.federatedInstanceService.fetch(user.host).then(instance => instance ? { name: instance.name, softwareName: instance.softwareName, softwareVersion: instance.softwareVersion, iconUrl: instance.iconUrl, faviconUrl: instance.faviconUrl, themeColor: instance.themeColor, + isSilenced: instance.isSilenced, } : undefined) : undefined, followersCount: followersCount ?? 0, followingCount: followingCount ?? 0, @@ -783,8 +788,13 @@ export class UserEntityService implements OnModuleInit { const _users = users.filter((user): user is MiUser => typeof user !== 'string'); if (_users.length !== users.length) { _users.push( - ...await this.usersRepository.findBy({ - id: In(users.filter((user): user is string => typeof user === 'string')), + ...await this.usersRepository.find({ + where: { + id: In(users.filter((user): user is string => typeof user === 'string')), + }, + relations: { + userProfile: true, + }, }), ); } @@ -798,8 +808,20 @@ export class UserEntityService implements OnModuleInit { let pinNotes: Map = new Map(); if (options?.schema !== 'UserLite') { - profilesMap = await this.userProfilesRepository.findBy({ userId: In(_userIds) }) - .then(profiles => new Map(profiles.map(p => [p.userId, p]))); + const _profiles: MiUserProfile[] = []; + const _profilesToFetch: string[] = []; + for (const user of _users) { + if (user.userProfile) { + _profiles.push(user.userProfile); + } else { + _profilesToFetch.push(user.id); + } + } + if (_profilesToFetch.length > 0) { + const fetched = await this.userProfilesRepository.findBy({ userId: In(_profilesToFetch) }); + _profiles.push(...fetched); + } + profilesMap = new Map(_profiles.map(p => [p.userId, p])); const meId = me ? me.id : null; if (meId) { diff --git a/packages/backend/src/env.ts b/packages/backend/src/env.ts index ba44cfa2e6..9a50eb8561 100644 --- a/packages/backend/src/env.ts +++ b/packages/backend/src/env.ts @@ -11,6 +11,7 @@ const envOption = { verbose: false, withLogTime: false, quiet: false, + hideWorkerId: false, }; for (const key of Object.keys(envOption) as (keyof typeof envOption)[]) { diff --git a/packages/backend/src/logger.ts b/packages/backend/src/logger.ts index b3735200eb..ca9b494ff2 100644 --- a/packages/backend/src/logger.ts +++ b/packages/backend/src/logger.ts @@ -71,7 +71,9 @@ export default class Logger { level === 'info' ? message : null; - let log = `${l} ${worker}\t[${contexts.join(' ')}]\t${m}`; + let log = envOption.hideWorkerId + ? `${l}\t[${contexts.join(' ')}]\t\t${m}` + : `${l} ${worker}\t[${contexts.join(' ')}]\t\t${m}`; if (envOption.withLogTime) log = chalk.gray(time) + ' ' + log; const args: unknown[] = [important ? chalk.bold(log) : log]; diff --git a/packages/backend/src/misc/FileWriterStream.ts b/packages/backend/src/misc/FileWriterStream.ts index 27c67cb5df..a61d949ef4 100644 --- a/packages/backend/src/misc/FileWriterStream.ts +++ b/packages/backend/src/misc/FileWriterStream.ts @@ -21,7 +21,7 @@ export class FileWriterStream extends WritableStream { write: async (chunk, controller) => { if (file === null) { controller.error(); - throw new Error(); + throw new Error('file is null'); } await file.write(chunk); diff --git a/packages/backend/src/misc/cache.ts b/packages/backend/src/misc/cache.ts index 48b8f43678..a6ab96c189 100644 --- a/packages/backend/src/misc/cache.ts +++ b/packages/backend/src/misc/cache.ts @@ -308,8 +308,17 @@ export class MemoryKVCache { } } + /** + * Removes all entries from the cache, but does not dispose it. + */ + @bindThis + public clear(): void { + this.cache.clear(); + } + @bindThis public dispose(): void { + this.clear(); clearInterval(this.gcIntervalHandle); } diff --git a/packages/backend/src/misc/diff-arrays.ts b/packages/backend/src/misc/diff-arrays.ts new file mode 100644 index 0000000000..b50ca1d4f7 --- /dev/null +++ b/packages/backend/src/misc/diff-arrays.ts @@ -0,0 +1,102 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export interface DiffResult { + added: T[]; + removed: T[]; +} + +/** + * Calculates the difference between two snapshots of data. + * Null, undefined, and empty arrays are supported, and duplicate values are ignored. + * Result sets are de-duplicated, and will be empty if no data was added or removed (respectively). + * The inputs are treated as un-ordered, so a re-ordering of the same data will NOT be considered a change. + * @param dataBefore Array containing data before the change + * @param dataAfter Array containing data after the change + */ +export function diffArrays(dataBefore: T[] | null | undefined, dataAfter: T[] | null | undefined): DiffResult { + const before = dataBefore ? new Set(dataBefore) : null; + const after = dataAfter ? new Set(dataAfter) : null; + + // data before AND after => changed + if (before?.size && after?.size) { + const added: T[] = []; + const removed: T[] = []; + + for (const host of before) { + // before and NOT after => removed + // delete operation removes duplicates to speed up the "after" loop + if (!after.delete(host)) { + removed.push(host); + } + } + + for (const host of after) { + // after and NOT before => added + if (!before.has(host)) { + added.push(host); + } + } + + return { added, removed }; + } + + // data ONLY before => all removed + if (before?.size) { + return { added: [], removed: Array.from(before) }; + } + + // data ONLY after => all added + if (after?.size) { + return { added: Array.from(after), removed: [] }; + } + + // data NEITHER before nor after => no change + return { added: [], removed: [] }; +} + +/** + * Checks for any difference between two snapshots of data. + * Null, undefined, and empty arrays are supported, and duplicate values are ignored. + * The inputs are treated as un-ordered, so a re-ordering of the same data will NOT be considered a change. + * @param dataBefore Array containing data before the change + * @param dataAfter Array containing data after the change + */ +export function diffArraysSimple(dataBefore: T[] | null | undefined, dataAfter: T[] | null | undefined): boolean { + const before = dataBefore ? new Set(dataBefore) : null; + const after = dataAfter ? new Set(dataAfter) : null; + + if (before?.size && after?.size) { + // different size => changed + if (before.size !== after.size) return true; + + // removed => changed + for (const host of before) { + // delete operation removes duplicates to speed up the "after" loop + if (!after.delete(host)) { + return true; + } + } + + // added => changed + for (const host of after) { + if (!before.has(host)) { + return true; + } + } + + // identical values => no change + return false; + } + + // before and NOT after => change + if (before?.size) return true; + + // after and NOT before => change + if (after?.size) return true; + + // NEITHER before nor after => no change + return false; +} diff --git a/packages/backend/src/misc/fastify-reply-error.ts b/packages/backend/src/misc/fastify-reply-error.ts index e6c4e78d2f..03109e8b96 100644 --- a/packages/backend/src/misc/fastify-reply-error.ts +++ b/packages/backend/src/misc/fastify-reply-error.ts @@ -8,8 +8,8 @@ export class FastifyReplyError extends Error { public message: string; public statusCode: number; - constructor(statusCode: number, message: string) { - super(message); + constructor(statusCode: number, message: string, cause?: unknown) { + super(message, cause ? { cause } : undefined); this.message = message; this.statusCode = statusCode; } diff --git a/packages/backend/src/misc/id/aid.ts b/packages/backend/src/misc/id/aid.ts index c0e8478db5..f0eba2d99c 100644 --- a/packages/backend/src/misc/id/aid.ts +++ b/packages/backend/src/misc/id/aid.ts @@ -8,6 +8,7 @@ import * as crypto from 'node:crypto'; import { parseBigInt36 } from '@/misc/bigint.js'; +import { IdentifiableError } from '../identifiable-error.js'; export const aidRegExp = /^[0-9a-z]{10}$/; @@ -26,7 +27,7 @@ function getNoise(): string { } export function genAid(t: number): string { - if (isNaN(t)) throw new Error('Failed to create AID: Invalid Date'); + if (isNaN(t)) throw new IdentifiableError('6b73b7d5-9d2b-48b4-821c-ef955efe80ad', 'Failed to create AID: Invalid Date'); counter++; return getTime(t) + getNoise(); } diff --git a/packages/backend/src/misc/id/aidx.ts b/packages/backend/src/misc/id/aidx.ts index 006673a6d0..d2bb566e35 100644 --- a/packages/backend/src/misc/id/aidx.ts +++ b/packages/backend/src/misc/id/aidx.ts @@ -10,6 +10,7 @@ import { customAlphabet } from 'nanoid'; import { parseBigInt36 } from '@/misc/bigint.js'; +import { IdentifiableError } from '../identifiable-error.js'; export const aidxRegExp = /^[0-9a-z]{16}$/; @@ -34,7 +35,7 @@ function getNoise(): string { } export function genAidx(t: number): string { - if (isNaN(t)) throw new Error('Failed to create AIDX: Invalid Date'); + if (isNaN(t)) throw new IdentifiableError('6b73b7d5-9d2b-48b4-821c-ef955efe80ad', 'Failed to create AIDX: Invalid Date'); counter++; return getTime(t) + nodeId + getNoise(); } diff --git a/packages/backend/src/misc/identifiable-error.ts b/packages/backend/src/misc/identifiable-error.ts index f5c3fcd6cb..56e13f2622 100644 --- a/packages/backend/src/misc/identifiable-error.ts +++ b/packages/backend/src/misc/identifiable-error.ts @@ -15,8 +15,8 @@ export class IdentifiableError extends Error { */ public readonly isRetryable: boolean; - constructor(id: string, message?: string, isRetryable = false) { - super(message); + constructor(id: string, message?: string, isRetryable = false, cause?: unknown) { + super(message, cause ? { cause } : undefined); this.message = message ?? ''; this.id = id; this.isRetryable = isRetryable; diff --git a/packages/backend/src/misc/is-retryable-error.ts b/packages/backend/src/misc/is-retryable-error.ts index 9bb8700c7a..63b561b280 100644 --- a/packages/backend/src/misc/is-retryable-error.ts +++ b/packages/backend/src/misc/is-retryable-error.ts @@ -3,20 +3,34 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { AbortError } from 'node-fetch'; +import { AbortError, FetchError } from 'node-fetch'; import { UnrecoverableError } from 'bullmq'; import { StatusError } from '@/misc/status-error.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; +import { CaptchaError, captchaErrorCodes } from '@/core/CaptchaService.js'; +import { FastifyReplyError } from '@/misc/fastify-reply-error.js'; +import { ConflictError } from '@/server/SkRateLimiterService.js'; /** * Returns false if the provided value represents a "permanent" error that cannot be retried. * Returns true if the error is retryable, unknown (as all errors are retryable by default), or not an error object. */ export function isRetryableError(e: unknown): boolean { + if (e instanceof AggregateError) return e.errors.every(inner => isRetryableError(inner)); if (e instanceof StatusError) return e.isRetryable; if (e instanceof IdentifiableError) return e.isRetryable; + if (e instanceof CaptchaError) { + if (e.code === captchaErrorCodes.verificationFailed) return false; + if (e.code === captchaErrorCodes.invalidParameters) return false; + if (e.code === captchaErrorCodes.invalidProvider) return false; + return true; + } + if (e instanceof FastifyReplyError) return false; + if (e instanceof ConflictError) return true; if (e instanceof UnrecoverableError) return false; if (e instanceof AbortError) return true; + if (e instanceof FetchError) return true; + if (e instanceof SyntaxError) return false; if (e instanceof Error) return e.name === 'AbortError'; return true; } diff --git a/packages/backend/src/misc/render-full-error.ts b/packages/backend/src/misc/render-full-error.ts new file mode 100644 index 0000000000..5f0a09bba9 --- /dev/null +++ b/packages/backend/src/misc/render-full-error.ts @@ -0,0 +1,60 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import * as Bull from 'bullmq'; +import { AbortError, FetchError } from 'node-fetch'; +import { StatusError } from '@/misc/status-error.js'; +import { IdentifiableError } from '@/misc/identifiable-error.js'; +import { renderInlineError } from '@/misc/render-inline-error.js'; +import { CaptchaError, captchaErrorCodes } from '@/core/CaptchaService.js'; + +export function renderFullError(e?: unknown): unknown { + if (e === undefined) return 'undefined'; + if (e === null) return 'null'; + + if (e instanceof Error) { + if (isSimpleError(e)) { + return renderInlineError(e); + } + + const data: ErrorData = {}; + if (e.stack) data.stack = e.stack; + if (e.message) data.message = e.message; + if (e.name) data.name = e.name; + + // mix "cause" and "errors" + if (e instanceof AggregateError && e.errors.length > 0) { + const causes = e.errors.map(inner => renderFullError(inner)); + if (e.cause) { + causes.push(renderFullError(e.cause)); + } + data.cause = causes; + } else if (e.cause) { + data.cause = renderFullError(e.cause); + } + + return data; + } + + return e; +} + +function isSimpleError(e: Error): boolean { + if (e instanceof Bull.UnrecoverableError) return true; + if (e instanceof AbortError || e.name === 'AbortError') return true; + if (e instanceof FetchError || e.name === 'FetchError') return true; + if (e instanceof StatusError) return true; + if (e instanceof IdentifiableError) return true; + if (e instanceof FetchError) return true; + if (e instanceof CaptchaError && e.code !== captchaErrorCodes.unknown) return true; + return false; +} + +interface ErrorData { + stack?: Error['stack']; + message?: Error['message']; + name?: Error['name']; + cause?: Error['cause'] | Error['cause'][]; +} diff --git a/packages/backend/src/misc/render-inline-error.ts b/packages/backend/src/misc/render-inline-error.ts new file mode 100644 index 0000000000..07f9f3068e --- /dev/null +++ b/packages/backend/src/misc/render-inline-error.ts @@ -0,0 +1,75 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { IdentifiableError } from '@/misc/identifiable-error.js'; +import { StatusError } from '@/misc/status-error.js'; +import { CaptchaError } from '@/core/CaptchaService.js'; + +export function renderInlineError(err: unknown): string { + const parts: string[] = []; + renderTo(err, parts); + return parts.join(''); +} + +function renderTo(err: unknown, parts: string[]): void { + parts.push(printError(err)); + + if (err instanceof AggregateError) { + for (let i = 0; i < err.errors.length; i++) { + parts.push(` [${i + 1}/${err.errors.length}]: `); + renderTo(err.errors[i], parts); + } + } + + if (err instanceof Error) { + if (err.cause) { + parts.push(' [caused by]: '); + renderTo(err.cause, parts); + // const cause = renderInlineError(err.cause); + // parts.push(' [caused by]: ', cause); + } + } +} + +function printError(err: unknown): string { + if (err === undefined) return 'undefined'; + if (err === null) return 'null'; + + if (err instanceof IdentifiableError) { + if (err.message) { + return `${err.name} ${err.id}: ${err.message}`; + } else { + return `${err.name} ${err.id}`; + } + } + + if (err instanceof StatusError) { + if (err.message) { + return `${err.name} ${err.statusCode}: ${err.message}`; + } else if (err.statusMessage) { + return `${err.name} ${err.statusCode}: ${err.statusMessage}`; + } else { + return `${err.name} ${err.statusCode}`; + } + } + + if (err instanceof CaptchaError) { + if (err.code.description) { + return `${err.name} ${err.code.description}: ${err.message}`; + } else { + return `${err.name}: ${err.message}`; + } + } + + if (err instanceof Error) { + if (err.message) { + return `${err.name}: ${err.message}`; + } else { + return err.name; + } + } + + return String(err); +} diff --git a/packages/backend/src/misc/status-error.ts b/packages/backend/src/misc/status-error.ts index c3533db607..4fd3bfcafb 100644 --- a/packages/backend/src/misc/status-error.ts +++ b/packages/backend/src/misc/status-error.ts @@ -9,8 +9,8 @@ export class StatusError extends Error { public isClientError: boolean; public isRetryable: boolean; - constructor(message: string, statusCode: number, statusMessage?: string) { - super(message); + constructor(message: string, statusCode: number, statusMessage?: string, cause?: unknown) { + super(message, cause ? { cause } : undefined); this.name = 'StatusError'; this.statusCode = statusCode; this.statusMessage = statusMessage; diff --git a/packages/backend/src/models/AbuseUserReport.ts b/packages/backend/src/models/AbuseUserReport.ts index d43ebf9342..8f8d759004 100644 --- a/packages/backend/src/models/AbuseUserReport.ts +++ b/packages/backend/src/models/AbuseUserReport.ts @@ -4,6 +4,7 @@ */ import { PrimaryColumn, Entity, Index, JoinColumn, Column, ManyToOne } from 'typeorm'; +import { MiInstance } from '@/models/Instance.js'; import { id } from './util/id.js'; import { MiUser } from './User.js'; @@ -88,11 +89,31 @@ export class MiAbuseUserReport { }) public targetUserHost: string | null; + @ManyToOne(() => MiInstance, { + // TODO create a foreign key constraint after hazelnoot/labs/persisted-instance-blocks is merged + createForeignKeyConstraints: false, + }) + @JoinColumn({ + name: 'targetUserHost', + referencedColumnName: 'host', + }) + public targetUserInstance: MiInstance | null; + @Index() @Column('varchar', { length: 128, nullable: true, comment: '[Denormalized]', }) public reporterHost: string | null; + + @ManyToOne(() => MiInstance, { + // TODO create a foreign key constraint after hazelnoot/labs/persisted-instance-blocks is merged + createForeignKeyConstraints: false, + }) + @JoinColumn({ + name: 'reporterHost', + referencedColumnName: 'host', + }) + public reporterInstance: MiInstance | null; //#endregion } diff --git a/packages/backend/src/models/Following.ts b/packages/backend/src/models/Following.ts index 62cbc29f26..0aa1b13976 100644 --- a/packages/backend/src/models/Following.ts +++ b/packages/backend/src/models/Following.ts @@ -4,6 +4,7 @@ */ import { PrimaryColumn, Entity, Index, JoinColumn, Column, ManyToOne } from 'typeorm'; +import { MiInstance } from '@/models/Instance.js'; import { id } from './util/id.js'; import { MiUser } from './User.js'; @@ -66,6 +67,16 @@ export class MiFollowing { }) public followerHost: string | null; + @ManyToOne(() => MiInstance, { + onDelete: 'CASCADE', + }) + @JoinColumn({ + name: 'followerHost', + foreignKeyConstraintName: 'FK_following_followerHost', + referencedColumnName: 'host', + }) + public followerInstance: MiInstance | null; + @Column('varchar', { length: 512, nullable: true, comment: '[Denormalized]', @@ -85,6 +96,16 @@ export class MiFollowing { }) public followeeHost: string | null; + @ManyToOne(() => MiInstance, { + onDelete: 'CASCADE', + }) + @JoinColumn({ + name: 'followeeHost', + foreignKeyConstraintName: 'FK_following_followeeHost', + referencedColumnName: 'host', + }) + public followeeInstance: MiInstance | null; + @Column('varchar', { length: 512, nullable: true, comment: '[Denormalized]', diff --git a/packages/backend/src/models/Instance.ts b/packages/backend/src/models/Instance.ts index c64ebb1b3b..0cde4b75fc 100644 --- a/packages/backend/src/models/Instance.ts +++ b/packages/backend/src/models/Instance.ts @@ -6,6 +6,7 @@ import { Entity, PrimaryColumn, Index, Column } from 'typeorm'; import { id } from './util/id.js'; +@Index('IDX_instance_host_key', { synchronize: false }) // ((lower(reverse("host"::text)) || '.'::text) @Entity('instance') export class MiInstance { @PrimaryColumn(id()) @@ -98,6 +99,56 @@ export class MiInstance { }) public suspensionState: 'none' | 'manuallySuspended' | 'goneSuspended' | 'autoSuspendedForNotResponding'; + /** + * True if this instance is blocked from federation. + */ + @Column('boolean', { + nullable: false, + default: false, + comment: 'True if this instance is blocked from federation.', + }) + public isBlocked: boolean; + + /** + * True if this instance is allow-listed. + */ + @Column('boolean', { + nullable: false, + default: false, + comment: 'True if this instance is allow-listed.', + }) + public isAllowListed: boolean; + + /** + * True if this instance is part of the local bubble. + */ + @Column('boolean', { + nullable: false, + default: false, + comment: 'True if this instance is part of the local bubble.', + }) + public isBubbled: boolean; + + /** + * True if this instance is silenced. + */ + @Column('boolean', { + nullable: false, + default: false, + comment: 'True if this instance is silenced.', + }) + public isSilenced: boolean; + + /** + * True if this instance is media-silenced. + */ + @Column('boolean', { + nullable: false, + default: false, + comment: 'True if this instance is media-silenced.', + }) + public isMediaSilenced: boolean; + @Column('varchar', { length: 64, nullable: true, comment: 'The software of the Instance.', diff --git a/packages/backend/src/models/Note.ts b/packages/backend/src/models/Note.ts index c4d5e794fa..b064e95493 100644 --- a/packages/backend/src/models/Note.ts +++ b/packages/backend/src/models/Note.ts @@ -5,6 +5,7 @@ import { Entity, Index, JoinColumn, Column, PrimaryColumn, ManyToOne } from 'typeorm'; import { noteVisibilities } from '@/types.js'; +import { MiInstance } from '@/models/Instance.js'; import { id } from './util/id.js'; import { MiUser } from './User.js'; import { MiChannel } from './Channel.js'; @@ -21,6 +22,8 @@ import type { MiDriveFile } from './DriveFile.js'; // because it will always run CREATE INDEX in transaction based on decorators. // Not appending `{ concurrent: true }` to `@Index` will not cause any problem in production, @Index('IDX_724b311e6f883751f261ebe378', ['userId', 'id']) +@Index('IDX_note_userHost_id', { synchronize: false }) // (userHost, id desc) +@Index('IDX_note_for_timelines', { synchronize: false }) // (id desc, channelId, visibility, userHost) @Entity('note') export class MiNote { @PrimaryColumn(id()) @@ -140,6 +143,7 @@ export class MiNote { }) public uri: string | null; + @Index('IDX_note_url') @Column('varchar', { length: 512, nullable: true, comment: 'The human readable url of a note. it will be null when the note is local.', @@ -225,13 +229,22 @@ export class MiNote { public processErrors: string[] | null; //#region Denormalized fields - @Index() @Column('varchar', { length: 128, nullable: true, comment: '[Denormalized]', }) public userHost: string | null; + @ManyToOne(() => MiInstance, { + onDelete: 'CASCADE', + }) + @JoinColumn({ + name: 'userHost', + foreignKeyConstraintName: 'FK_note_userHost', + referencedColumnName: 'host', + }) + public userInstance: MiInstance | null; + @Column({ ...id(), nullable: true, @@ -245,6 +258,16 @@ export class MiNote { }) public replyUserHost: string | null; + @ManyToOne(() => MiInstance, { + onDelete: 'CASCADE', + }) + @JoinColumn({ + name: 'replyUserHost', + foreignKeyConstraintName: 'FK_note_replyUserHost', + referencedColumnName: 'host', + }) + public replyUserInstance: MiInstance | null; + @Column({ ...id(), nullable: true, @@ -258,6 +281,17 @@ export class MiNote { }) public renoteUserHost: string | null; + @ManyToOne(() => MiInstance, { + onDelete: 'CASCADE', + }) + @JoinColumn({ + name: 'renoteUserHost', + foreignKeyConstraintName: 'FK_note_renoteUserHost', + referencedColumnName: 'host', + }) + public renoteUserInstance: MiInstance | null; + //#endregion + constructor(data: Partial) { if (data == null) return; diff --git a/packages/backend/src/models/User.ts b/packages/backend/src/models/User.ts index 46f8e84a94..f40bb41a22 100644 --- a/packages/backend/src/models/User.ts +++ b/packages/backend/src/models/User.ts @@ -3,10 +3,12 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { Entity, Column, Index, OneToOne, JoinColumn, PrimaryColumn } from 'typeorm'; +import { Entity, Column, Index, OneToOne, JoinColumn, PrimaryColumn, ManyToOne } from 'typeorm'; import { type UserUnsignedFetchOption, userUnsignedFetchOptions } from '@/const.js'; +import { MiInstance } from '@/models/Instance.js'; import { id } from './util/id.js'; import { MiDriveFile } from './DriveFile.js'; +import type { MiUserProfile } from './UserProfile.js'; @Entity('user') @Index(['usernameLower', 'host'], { unique: true }) @@ -292,6 +294,16 @@ export class MiUser { }) public host: string | null; + @ManyToOne(() => MiInstance, { + onDelete: 'CASCADE', + }) + @JoinColumn({ + name: 'host', + foreignKeyConstraintName: 'FK_user_host', + referencedColumnName: 'host', + }) + public instance: MiInstance | null; + @Column('varchar', { length: 512, nullable: true, comment: 'The inbox URL of the User. It will be null if the origin of the user is local.', @@ -378,6 +390,15 @@ export class MiUser { }) public allowUnsignedFetch: UserUnsignedFetchOption; + @Column('text', { + name: 'attributionDomains', + array: true, default: '{}', + }) + public attributionDomains: string[]; + + @OneToOne('user_profile', (profile: MiUserProfile) => profile.user) + public userProfile: MiUserProfile | null; + constructor(data: Partial) { if (data == null) return; diff --git a/packages/backend/src/models/UserProfile.ts b/packages/backend/src/models/UserProfile.ts index 6db555b060..156df3a50c 100644 --- a/packages/backend/src/models/UserProfile.ts +++ b/packages/backend/src/models/UserProfile.ts @@ -17,7 +17,7 @@ export class MiUserProfile { @PrimaryColumn(id()) public userId: MiUser['id']; - @OneToOne(type => MiUser, { + @OneToOne(() => MiUser, user => user.userProfile, { onDelete: 'CASCADE', }) @JoinColumn() diff --git a/packages/backend/src/models/json-schema/federation-instance.ts b/packages/backend/src/models/json-schema/federation-instance.ts index a4aa6be8bb..0e9ce9334e 100644 --- a/packages/backend/src/models/json-schema/federation-instance.ts +++ b/packages/backend/src/models/json-schema/federation-instance.ts @@ -135,5 +135,9 @@ export const packedFederationInstanceSchema = { type: 'string', optional: true, nullable: true, }, + isBubbled: { + type: 'boolean', + optional: false, nullable: false, + }, }, } as const; diff --git a/packages/backend/src/models/json-schema/user.ts b/packages/backend/src/models/json-schema/user.ts index 013d8949c8..23c8086fa6 100644 --- a/packages/backend/src/models/json-schema/user.ts +++ b/packages/backend/src/models/json-schema/user.ts @@ -214,6 +214,10 @@ export const packedUserLiteSchema = { type: 'string', nullable: true, optional: false, }, + isSilenced: { + type: 'boolean', + nullable: false, optional: false, + }, }, }, followersCount: { @@ -262,6 +266,14 @@ export const packedUserLiteSchema = { }, }, }, + attributionDomains: { + type: 'array', + nullable: false, optional: false, + items: { + type: 'string', + nullable: false, optional: false, + }, + }, }, } as const; diff --git a/packages/backend/src/postgres.ts b/packages/backend/src/postgres.ts index 632fd58927..45caec54ce 100644 --- a/packages/backend/src/postgres.ts +++ b/packages/backend/src/postgres.ts @@ -98,9 +98,12 @@ pg.types.setTypeParser(20, Number); export const dbLogger = new MisskeyLogger('db'); const sqlLogger = dbLogger.createSubLogger('sql', 'gray'); +const sqlMigrateLogger = sqlLogger.createSubLogger('migrate'); +const sqlSchemaLogger = sqlLogger.createSubLogger('schema'); export type LoggerProps = { disableQueryTruncation?: boolean; + enableQueryLogging?: boolean; enableQueryParamLogging?: boolean; printReplicationMode?: boolean, }; @@ -112,7 +115,7 @@ function highlightSql(sql: string) { } function truncateSql(sql: string) { - return sql.length > 100 ? `${sql.substring(0, 100)}...` : sql; + return sql.length > 100 ? `${sql.substring(0, 100)} [truncated]` : sql; } function stringifyParameter(param: any) { @@ -136,13 +139,16 @@ class MyCustomLogger implements Logger { modded = truncateSql(modded); } - return highlightSql(modded); + return this.props.enableQueryLogging ? highlightSql(modded) : modded; } @bindThis private transformParameters(parameters?: any[]) { if (this.props.enableQueryParamLogging && parameters && parameters.length > 0) { - return parameters.map(stringifyParameter); + return parameters.reduce((params, p, i) => { + params[`$${i + 1}`] = stringifyParameter(p); + return params; + }, {} as Record); } return undefined; @@ -150,10 +156,13 @@ class MyCustomLogger implements Logger { @bindThis public logQuery(query: string, parameters?: any[], queryRunner?: QueryRunner) { + if (!this.props.enableQueryLogging) return; + const prefix = (this.props.printReplicationMode && queryRunner) ? `[${queryRunner.getReplicationMode()}] ` : undefined; - sqlLogger.info(this.transformQueryLog(query, { prefix }), this.transformParameters(parameters)); + const transformed = this.transformQueryLog(query, { prefix }); + sqlLogger.debug(`Query run: ${transformed}`, this.transformParameters(parameters)); } @bindThis @@ -161,7 +170,8 @@ class MyCustomLogger implements Logger { const prefix = (this.props.printReplicationMode && queryRunner) ? `[${queryRunner.getReplicationMode()}] ` : undefined; - sqlLogger.error(this.transformQueryLog(query, { prefix }), this.transformParameters(parameters)); + const transformed = this.transformQueryLog(query, { prefix }); + sqlLogger.error(`Query error (${error}): ${transformed}`, this.transformParameters(parameters)); } @bindThis @@ -169,22 +179,32 @@ class MyCustomLogger implements Logger { const prefix = (this.props.printReplicationMode && queryRunner) ? `[${queryRunner.getReplicationMode()}] ` : undefined; - sqlLogger.warn(this.transformQueryLog(query, { prefix }), this.transformParameters(parameters)); + const transformed = this.transformQueryLog(query, { prefix }); + sqlLogger.warn(`Query is slow (${time}ms): ${transformed}`, this.transformParameters(parameters)); } @bindThis public logSchemaBuild(message: string) { - sqlLogger.info(message); + sqlSchemaLogger.debug(message); } @bindThis - public log(message: string) { - sqlLogger.info(message); + public log(level: 'log' | 'info' | 'warn', message: string) { + switch (level) { + case 'log': + case 'info': { + sqlLogger.info(message); + break; + } + case 'warn': { + sqlLogger.warn(message); + } + } } @bindThis public logMigration(message: string) { - sqlLogger.info(message); + sqlMigrateLogger.debug(message); } } @@ -306,7 +326,7 @@ export function createPostgresDataSource(config: Config) { } : {}), synchronize: process.env.NODE_ENV === 'test', dropSchema: process.env.NODE_ENV === 'test', - cache: !config.db.disableCache && process.env.NODE_ENV !== 'test' ? { // dbをcloseしても何故かredisのコネクションが内部的に残り続けるようで、テストの際に支障が出るため無効にする(キャッシュも含めてテストしたいため本当は有効にしたいが...) + cache: config.db.disableCache === false && process.env.NODE_ENV !== 'test' ? { // dbをcloseしても何故かredisのコネクションが内部的に残り続けるようで、テストの際に支障が出るため無効にする(キャッシュも含めてテストしたいため本当は有効にしたいが...) type: 'ioredis', options: { ...config.redis, @@ -314,14 +334,13 @@ export function createPostgresDataSource(config: Config) { }, } : false, logging: log, - logger: log - ? new MyCustomLogger({ - disableQueryTruncation: config.logging?.sql?.disableQueryTruncation, - enableQueryParamLogging: config.logging?.sql?.enableQueryParamLogging, - printReplicationMode: !!config.dbReplications, - }) - : undefined, - maxQueryExecutionTime: 300, + logger: new MyCustomLogger({ + disableQueryTruncation: config.logging?.sql?.disableQueryTruncation, + enableQueryLogging: log, + enableQueryParamLogging: config.logging?.sql?.enableQueryParamLogging, + printReplicationMode: !!config.dbReplications, + }), + maxQueryExecutionTime: config.db.slowQueryThreshold, entities: entities, migrations: ['../../migration/*.js'], }); diff --git a/packages/backend/src/queue/QueueProcessorService.ts b/packages/backend/src/queue/QueueProcessorService.ts index 7f7ce2452c..4c1a6a1d9e 100644 --- a/packages/backend/src/queue/QueueProcessorService.ts +++ b/packages/backend/src/queue/QueueProcessorService.ts @@ -11,7 +11,7 @@ import { DI } from '@/di-symbols.js'; import type Logger from '@/logger.js'; import { bindThis } from '@/decorators.js'; import { CheckModeratorsActivityProcessorService } from '@/queue/processors/CheckModeratorsActivityProcessorService.js'; -import { StatusError } from '@/misc/status-error.js'; +import { renderFullError } from '@/misc/render-full-error.js'; import { UserWebhookDeliverProcessorService } from './processors/UserWebhookDeliverProcessorService.js'; import { SystemWebhookDeliverProcessorService } from './processors/SystemWebhookDeliverProcessorService.js'; import { EndedPollNotificationProcessorService } from './processors/EndedPollNotificationProcessorService.js'; @@ -73,7 +73,9 @@ function getJobInfo(job: Bull.Job | undefined, increment = false): string { const currentAttempts = job.attemptsMade + (increment ? 1 : 0); const maxAttempts = job.opts.attempts ?? 0; - return `id=${job.id} attempts=${currentAttempts}/${maxAttempts} age=${formated}`; + return job.name + ? `id=${job.id} attempts=${currentAttempts}/${maxAttempts} age=${formated} name=${job.name}` + : `id=${job.id} attempts=${currentAttempts}/${maxAttempts} age=${formated}`; } @Injectable() @@ -134,35 +136,6 @@ export class QueueProcessorService implements OnApplicationShutdown { ) { this.logger = this.queueLoggerService.logger; - function renderError(e?: Error) { - // 何故かeがundefinedで来ることがある - if (!e) return '?'; - - if (e instanceof Bull.UnrecoverableError || e.name === 'AbortError' || e instanceof StatusError) { - return `${e.name}: ${e.message}`; - } - - return { - stack: e.stack, - message: e.message, - name: e.name, - }; - } - - function renderJob(job?: Bull.Job) { - if (!job) return '?'; - - const info: Record = { - info: getJobInfo(job), - data: job.data, - }; - - if (job.name) info.name = job.name; - if (job.failedReason) info.failedReason = job.failedReason; - - return info; - } - //#region system { const processer = (job: Bull.Job) => { @@ -196,7 +169,7 @@ export class QueueProcessorService implements OnApplicationShutdown { .on('active', (job) => logger.debug(`active id=${job.id}`)) .on('completed', (job, result) => logger.debug(`completed(${result}) id=${job.id}`)) .on('failed', (job, err: Error) => { - logger.error(`failed(${err.name}: ${err.message}) id=${job?.id ?? '?'}`, { job: renderJob(job), e: renderError(err) }); + this.logError(logger, err, job); if (config.sentryForBackend) { Sentry.captureMessage(`Queue: System: ${job?.name ?? '?'}: ${err.name}: ${err.message}`, { level: 'error', @@ -204,7 +177,7 @@ export class QueueProcessorService implements OnApplicationShutdown { }); } }) - .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) })) + .on('error', (err: Error) => this.logError(logger, err)) .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); } //#endregion @@ -261,7 +234,7 @@ export class QueueProcessorService implements OnApplicationShutdown { .on('active', (job) => logger.debug(`active id=${job.id}`)) .on('completed', (job, result) => logger.debug(`completed(${result}) id=${job.id}`)) .on('failed', (job, err) => { - logger.error(`failed(${err.name}: ${err.message}) id=${job?.id ?? '?'}`, { job: renderJob(job), e: renderError(err) }); + this.logError(logger, err, job); if (config.sentryForBackend) { Sentry.captureMessage(`Queue: DB: ${job?.name ?? '?'}: ${err.name}: ${err.message}`, { level: 'error', @@ -269,7 +242,7 @@ export class QueueProcessorService implements OnApplicationShutdown { }); } }) - .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) })) + .on('error', (err: Error) => this.logError(logger, err)) .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); } //#endregion @@ -301,7 +274,7 @@ export class QueueProcessorService implements OnApplicationShutdown { .on('active', (job) => logger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`)) .on('completed', (job, result) => logger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`)) .on('failed', (job, err) => { - logger.error(`failed(${err.name}: ${err.message}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`); + this.logError(logger, err, job); if (config.sentryForBackend) { Sentry.captureMessage(`Queue: Deliver: ${err.name}: ${err.message}`, { level: 'error', @@ -309,7 +282,7 @@ export class QueueProcessorService implements OnApplicationShutdown { }); } }) - .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) })) + .on('error', (err: Error) => this.logError(logger, err)) .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); } //#endregion @@ -341,7 +314,7 @@ export class QueueProcessorService implements OnApplicationShutdown { .on('active', (job) => logger.debug(`active ${getJobInfo(job, true)}`)) .on('completed', (job, result) => logger.debug(`completed(${result}) ${getJobInfo(job, true)}`)) .on('failed', (job, err) => { - logger.error(`failed(${err.name}: ${err.message}) ${getJobInfo(job)} activity=${job ? (job.data.activity ? job.data.activity.id : 'none') : '-'}`, { job: renderJob(job), e: renderError(err) }); + this.logError(logger, err, job); if (config.sentryForBackend) { Sentry.captureMessage(`Queue: Inbox: ${err.name}: ${err.message}`, { level: 'error', @@ -349,7 +322,7 @@ export class QueueProcessorService implements OnApplicationShutdown { }); } }) - .on('error', (err: Error) => logger.error('inbox error:', renderError(err))) + .on('error', (err: Error) => this.logError(logger, err)) .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); } //#endregion @@ -381,7 +354,7 @@ export class QueueProcessorService implements OnApplicationShutdown { .on('active', (job) => logger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`)) .on('completed', (job, result) => logger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`)) .on('failed', (job, err) => { - logger.error(`failed(${err.name}: ${err.message}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`); + this.logError(logger, err, job); if (config.sentryForBackend) { Sentry.captureMessage(`Queue: UserWebhookDeliver: ${err.name}: ${err.message}`, { level: 'error', @@ -389,7 +362,7 @@ export class QueueProcessorService implements OnApplicationShutdown { }); } }) - .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) })) + .on('error', (err: Error) => this.logError(logger, err)) .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); } //#endregion @@ -421,7 +394,7 @@ export class QueueProcessorService implements OnApplicationShutdown { .on('active', (job) => logger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`)) .on('completed', (job, result) => logger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`)) .on('failed', (job, err) => { - logger.error(`failed(${err.name}: ${err.message}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`); + this.logError(logger, err, job); if (config.sentryForBackend) { Sentry.captureMessage(`Queue: SystemWebhookDeliver: ${err.name}: ${err.message}`, { level: 'error', @@ -429,7 +402,7 @@ export class QueueProcessorService implements OnApplicationShutdown { }); } }) - .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) })) + .on('error', (err: Error) => this.logError(logger, err)) .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); } //#endregion @@ -468,7 +441,7 @@ export class QueueProcessorService implements OnApplicationShutdown { .on('active', (job) => logger.debug(`active id=${job.id}`)) .on('completed', (job, result) => logger.debug(`completed(${result}) id=${job.id}`)) .on('failed', (job, err) => { - logger.error(`failed(${err.name}: ${err.message}) id=${job?.id ?? '?'}`, { job: renderJob(job), e: renderError(err) }); + this.logError(logger, err, job); if (config.sentryForBackend) { Sentry.captureMessage(`Queue: Relationship: ${job?.name ?? '?'}: ${err.name}: ${err.message}`, { level: 'error', @@ -476,7 +449,7 @@ export class QueueProcessorService implements OnApplicationShutdown { }); } }) - .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) })) + .on('error', (err: Error) => this.logError(logger, err)) .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); } //#endregion @@ -509,7 +482,7 @@ export class QueueProcessorService implements OnApplicationShutdown { .on('active', (job) => logger.debug(`active id=${job.id}`)) .on('completed', (job, result) => logger.debug(`completed(${result}) id=${job.id}`)) .on('failed', (job, err) => { - logger.error(`failed(${err.name}: ${err.message}) id=${job?.id ?? '?'}`, { job: renderJob(job), e: renderError(err) }); + this.logError(logger, err, job); if (config.sentryForBackend) { Sentry.captureMessage(`Queue: ObjectStorage: ${job?.name ?? '?'}: ${err.name}: ${err.message}`, { level: 'error', @@ -517,13 +490,15 @@ export class QueueProcessorService implements OnApplicationShutdown { }); } }) - .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) })) + .on('error', (err: Error) => this.logError(logger, err)) .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); } //#endregion //#region ended poll notification { + const logger = this.logger.createSubLogger('endedPollNotification'); + this.endedPollNotificationQueueWorker = new Bull.Worker(QUEUE.ENDED_POLL_NOTIFICATION, (job) => { if (this.config.sentryForBackend) { return Sentry.startSpan({ name: 'Queue: EndedPollNotification' }, () => this.endedPollNotificationProcessorService.process(job)); @@ -534,19 +509,75 @@ export class QueueProcessorService implements OnApplicationShutdown { ...baseWorkerOptions(this.config, QUEUE.ENDED_POLL_NOTIFICATION), autorun: false, }); + this.endedPollNotificationQueueWorker + .on('active', (job) => logger.debug(`active id=${job.id}`)) + .on('completed', (job, result) => logger.debug(`completed(${result}) id=${job.id}`)) + .on('failed', (job, err) => { + this.logError(logger, err, job); + if (config.sentryForBackend) { + Sentry.captureMessage(`Queue: EndedPollNotification: ${job?.name ?? '?'}: ${err.name}: ${err.message}`, { + level: 'error', + extra: { job, err }, + }); + } + }) + .on('error', (err: Error) => this.logError(logger, err)) + .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); } //#endregion //#region schedule note post { + const logger = this.logger.createSubLogger('scheduleNotePost'); + this.schedulerNotePostQueueWorker = new Bull.Worker(QUEUE.SCHEDULE_NOTE_POST, (job) => this.scheduleNotePostProcessorService.process(job), { ...baseWorkerOptions(this.config, QUEUE.SCHEDULE_NOTE_POST), autorun: false, }); + this.schedulerNotePostQueueWorker + .on('active', (job) => logger.debug(`active id=${job.id}`)) + .on('completed', (job, result) => logger.debug(`completed(${result}) id=${job.id}`)) + .on('failed', (job, err) => { + this.logError(logger, err, job); + if (config.sentryForBackend) { + Sentry.captureMessage(`Queue: ${QUEUE.SCHEDULE_NOTE_POST}: ${job?.name ?? '?'}: ${err.name}: ${err.message}`, { + level: 'error', + extra: { job, err }, + }); + } + }) + .on('error', (err: Error) => this.logError(logger, err)) + .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); } //#endregion } + private logError(logger: Logger, err: unknown, job?: Bull.Job | null): void { + const parts: string[] = []; + + // Render job + if (job) { + parts.push('job ['); + parts.push(getJobInfo(job)); + parts.push('] failed: '); + } else { + parts.push('job failed: '); + } + + // Render error + const fullError = renderFullError(err); + const errorText = typeof(fullError) === 'string' ? fullError : undefined; + if (errorText) { + parts.push(errorText); + } else if (job?.failedReason) { + parts.push(job.failedReason); + } + + const message = parts.join(''); + const data = typeof(fullError) !== 'string' ? { err: fullError } : undefined; + logger.error(message, data); + } + @bindThis public async start(): Promise { await Promise.all([ diff --git a/packages/backend/src/queue/processors/AggregateRetentionProcessorService.ts b/packages/backend/src/queue/processors/AggregateRetentionProcessorService.ts index 4769cccabf..30bdd6ccca 100644 --- a/packages/backend/src/queue/processors/AggregateRetentionProcessorService.ts +++ b/packages/backend/src/queue/processors/AggregateRetentionProcessorService.ts @@ -62,7 +62,7 @@ export class AggregateRetentionProcessorService { }); } catch (err) { if (isDuplicateKeyValueError(err)) { - this.logger.succ('Skip because it has already been processed by another worker.'); + this.logger.debug('Skip because it has already been processed by another worker.'); return; } throw err; @@ -87,6 +87,6 @@ export class AggregateRetentionProcessorService { }); } - this.logger.succ('Retention aggregated.'); + this.logger.info('Retention aggregated.'); } } diff --git a/packages/backend/src/queue/processors/BakeBufferedReactionsProcessorService.ts b/packages/backend/src/queue/processors/BakeBufferedReactionsProcessorService.ts index d49c99f694..83b375de3f 100644 --- a/packages/backend/src/queue/processors/BakeBufferedReactionsProcessorService.ts +++ b/packages/backend/src/queue/processors/BakeBufferedReactionsProcessorService.ts @@ -37,6 +37,6 @@ export class BakeBufferedReactionsProcessorService { await this.reactionsBufferingService.bake(); - this.logger.succ('All buffered reactions baked.'); + this.logger.info('All buffered reactions baked.'); } } diff --git a/packages/backend/src/queue/processors/CheckExpiredMutingsProcessorService.ts b/packages/backend/src/queue/processors/CheckExpiredMutingsProcessorService.ts index 448fc9c763..76d0cb4304 100644 --- a/packages/backend/src/queue/processors/CheckExpiredMutingsProcessorService.ts +++ b/packages/backend/src/queue/processors/CheckExpiredMutingsProcessorService.ts @@ -41,6 +41,6 @@ export class CheckExpiredMutingsProcessorService { await this.userMutingService.unmute(expired); } - this.logger.succ('All expired mutings checked.'); + this.logger.info('All expired mutings checked.'); } } diff --git a/packages/backend/src/queue/processors/CheckModeratorsActivityProcessorService.ts b/packages/backend/src/queue/processors/CheckModeratorsActivityProcessorService.ts index db8d2e789e..7821cd3d1d 100644 --- a/packages/backend/src/queue/processors/CheckModeratorsActivityProcessorService.ts +++ b/packages/backend/src/queue/processors/CheckModeratorsActivityProcessorService.ts @@ -98,16 +98,16 @@ export class CheckModeratorsActivityProcessorService { @bindThis public async process(): Promise { - this.logger.info('start.'); + this.logger.debug('start.'); const meta = await this.metaService.fetch(false); if (!meta.disableRegistration) { await this.processImpl(); } else { - this.logger.info('is already invitation only.'); + this.logger.debug('is already invitation only.'); } - this.logger.succ('finish.'); + this.logger.debug('finish.'); } @bindThis diff --git a/packages/backend/src/queue/processors/CleanChartsProcessorService.ts b/packages/backend/src/queue/processors/CleanChartsProcessorService.ts index 8c5faa8d07..c11682b0fe 100644 --- a/packages/backend/src/queue/processors/CleanChartsProcessorService.ts +++ b/packages/backend/src/queue/processors/CleanChartsProcessorService.ts @@ -62,6 +62,6 @@ export class CleanChartsProcessorService { await this.perUserDriveChart.clean(); await this.apRequestChart.clean(); - this.logger.succ('All charts successfully cleaned.'); + this.logger.info('All charts successfully cleaned.'); } } diff --git a/packages/backend/src/queue/processors/CleanProcessorService.ts b/packages/backend/src/queue/processors/CleanProcessorService.ts index a26b69cd2b..104d19103f 100644 --- a/packages/backend/src/queue/processors/CleanProcessorService.ts +++ b/packages/backend/src/queue/processors/CleanProcessorService.ts @@ -69,6 +69,6 @@ export class CleanProcessorService { this.reversiService.cleanOutdatedGames(); - this.logger.succ('Cleaned.'); + this.logger.info('Cleaned.'); } } diff --git a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts index 81842b221f..2eddae95c8 100644 --- a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts +++ b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts @@ -75,6 +75,6 @@ export class CleanRemoteFilesProcessorService { await job.updateProgress(100 / total * deletedCount); } - this.logger.succ(`All cached remote files processed. Total deleted: ${deletedCount}, Failed: ${errorCount}.`); + this.logger.info(`All cached remote files processed. Total deleted: ${deletedCount}, Failed: ${errorCount}.`); } } diff --git a/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts b/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts index 4e9779a41b..6a1a8bcc66 100644 --- a/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts +++ b/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts @@ -128,7 +128,7 @@ export class DeleteAccountProcessorService { userId: user.id, }); - this.logger.succ('All clips have been deleted.'); + this.logger.info('All clips have been deleted.'); } { // Delete favorites @@ -136,7 +136,7 @@ export class DeleteAccountProcessorService { userId: user.id, }); - this.logger.succ('All favorites have been deleted.'); + this.logger.info('All favorites have been deleted.'); } { // Delete user relations @@ -172,7 +172,7 @@ export class DeleteAccountProcessorService { muteeId: user.id, }); - this.logger.succ('All user relations have been deleted.'); + this.logger.info('All user relations have been deleted.'); } { // Delete reactions @@ -206,7 +206,7 @@ export class DeleteAccountProcessorService { } } - this.logger.succ('All reactions have been deleted'); + this.logger.info('All reactions have been deleted'); } { // Poll votes @@ -238,7 +238,7 @@ export class DeleteAccountProcessorService { }); } - this.logger.succ('All poll votes have been deleted'); + this.logger.info('All poll votes have been deleted'); } { // Delete scheduled notes @@ -254,7 +254,7 @@ export class DeleteAccountProcessorService { userId: user.id, }); - this.logger.succ('All scheduled notes deleted'); + this.logger.info('All scheduled notes deleted'); } { // Delete notes @@ -312,7 +312,7 @@ export class DeleteAccountProcessorService { } } - this.logger.succ('All of notes deleted'); + this.logger.info('All of notes deleted'); } { // Delete files @@ -341,7 +341,7 @@ export class DeleteAccountProcessorService { } } - this.logger.succ('All of files deleted'); + this.logger.info('All of files deleted'); } { // Delete actor logs @@ -353,7 +353,7 @@ export class DeleteAccountProcessorService { await this.apLogService.deleteInboxLogs(user.id) .catch(err => this.logger.error(err, `Failed to delete AP logs for user '${user.uri}'`)); - this.logger.succ('All AP logs deleted'); + this.logger.info('All AP logs deleted'); } // Do this BEFORE deleting the account! @@ -379,7 +379,7 @@ export class DeleteAccountProcessorService { await this.usersRepository.delete(user.id); } - this.logger.succ('Account data deleted'); + this.logger.info('Account data deleted'); } { // Send email notification diff --git a/packages/backend/src/queue/processors/DeleteDriveFilesProcessorService.ts b/packages/backend/src/queue/processors/DeleteDriveFilesProcessorService.ts index 291fa4a6d8..ac3cddbed0 100644 --- a/packages/backend/src/queue/processors/DeleteDriveFilesProcessorService.ts +++ b/packages/backend/src/queue/processors/DeleteDriveFilesProcessorService.ts @@ -74,6 +74,6 @@ export class DeleteDriveFilesProcessorService { job.updateProgress(deletedCount / total); } - this.logger.succ(`All drive files (${deletedCount}) of ${user.id} has been deleted.`); + this.logger.info(`All drive files (${deletedCount}) of ${user.id} has been deleted.`); } } diff --git a/packages/backend/src/queue/processors/DeliverProcessorService.ts b/packages/backend/src/queue/processors/DeliverProcessorService.ts index 391ccdac05..c54c27d661 100644 --- a/packages/backend/src/queue/processors/DeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/DeliverProcessorService.ts @@ -138,23 +138,18 @@ export class DeliverProcessorService { } }); - if (res instanceof StatusError) { + if (res instanceof StatusError && !res.isRetryable) { // 4xx - if (!res.isRetryable) { - // 相手が閉鎖していることを明示しているため、配送停止する - if (job.data.isSharedInbox && res.statusCode === 410) { - this.federatedInstanceService.fetchOrRegister(host).then(i => { - this.federatedInstanceService.update(i.id, { - suspensionState: 'goneSuspended', - }); + // 相手が閉鎖していることを明示しているため、配送停止する + if (job.data.isSharedInbox && res.statusCode === 410) { + this.federatedInstanceService.fetchOrRegister(host).then(i => { + this.federatedInstanceService.update(i.id, { + suspensionState: 'goneSuspended', }); - throw new Bull.UnrecoverableError(`${host} is gone`); - } - throw new Bull.UnrecoverableError(`${res.statusCode} ${res.statusMessage}`); + }); + throw new Bull.UnrecoverableError(`${host} is gone`); } - - // 5xx etc. - throw new Error(`${res.statusCode} ${res.statusMessage}`); + throw new Bull.UnrecoverableError(`${res.statusCode} ${res.statusMessage}`); } else { // DNS error, socket error, timeout ... throw res; diff --git a/packages/backend/src/queue/processors/ExportAccountDataProcessorService.ts b/packages/backend/src/queue/processors/ExportAccountDataProcessorService.ts index 33a2362c4a..58d542635f 100644 --- a/packages/backend/src/queue/processors/ExportAccountDataProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportAccountDataProcessorService.ts @@ -22,6 +22,7 @@ import { Packed } from '@/misc/json-schema.js'; import { UtilityService } from '@/core/UtilityService.js'; import { DownloadService } from '@/core/DownloadService.js'; import { EmailService } from '@/core/EmailService.js'; +import { renderInlineError } from '@/misc/render-inline-error.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; @@ -85,21 +86,23 @@ export class ExportAccountDataProcessorService { @bindThis public async process(job: Bull.Job): Promise { - this.logger.info('Exporting Account Data...'); - const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { + this.logger.debug(`Skip: user ${job.data.user.id} does not exist`); return; } const profile = await this.userProfilesRepository.findOneBy({ userId: job.data.user.id }); if (profile == null) { + this.logger.debug(`Skip: user ${job.data.user.id} has no profile`); return; } + this.logger.info(`Exporting account data for ${job.data.user.id} ...`); + const [path, cleanup] = await createTempDir(); - this.logger.info(`Temp dir is ${path}`); + this.logger.debug(`Temp dir is ${path}`); // User Export @@ -113,7 +116,7 @@ export class ExportAccountDataProcessorService { return new Promise((res, rej) => { userStream.write(text, err => { if (err) { - this.logger.error(err); + this.logger.error('Error writing user:', err); rej(err); } else { res(); @@ -145,7 +148,7 @@ export class ExportAccountDataProcessorService { return new Promise((res, rej) => { profileStream.write(text, err => { if (err) { - this.logger.error(err); + this.logger.error('Error writing profile:', err); rej(err); } else { res(); @@ -179,7 +182,7 @@ export class ExportAccountDataProcessorService { return new Promise((res, rej) => { ipStream.write(text, err => { if (err) { - this.logger.error(err); + this.logger.error('Error writing IPs:', err); rej(err); } else { res(); @@ -214,7 +217,7 @@ export class ExportAccountDataProcessorService { return new Promise((res, rej) => { notesStream.write(text, err => { if (err) { - this.logger.error(err); + this.logger.error('Error writing notes:', err); rej(err); } else { res(); @@ -275,7 +278,7 @@ export class ExportAccountDataProcessorService { return new Promise((res, rej) => { followingStream.write(text, err => { if (err) { - this.logger.error(err); + this.logger.error('Error writing following:', err); rej(err); } else { res(); @@ -345,7 +348,7 @@ export class ExportAccountDataProcessorService { return new Promise((res, rej) => { followerStream.write(text, err => { if (err) { - this.logger.error(err); + this.logger.error('Error writing followers:', err); rej(err); } else { res(); @@ -406,7 +409,7 @@ export class ExportAccountDataProcessorService { return new Promise((res, rej) => { filesStream.write(text, err => { if (err) { - this.logger.error(err); + this.logger.error('Error writing drive:', err); rej(err); } else { res(); @@ -432,7 +435,7 @@ export class ExportAccountDataProcessorService { await this.downloadService.downloadUrl(file.url, filePath); downloaded = true; } catch (e) { - this.logger.error(e instanceof Error ? e : new Error(e as string)); + this.logger.error(`Error writing drive file ${file.id} (${file.name}): ${renderInlineError(e)}`); } if (!downloaded) { @@ -464,7 +467,7 @@ export class ExportAccountDataProcessorService { return new Promise((res, rej) => { mutingStream.write(text, err => { if (err) { - this.logger.error(err); + this.logger.error('Error writing mutings:', err); rej(err); } else { res(); @@ -527,7 +530,7 @@ export class ExportAccountDataProcessorService { return new Promise((res, rej) => { blockingStream.write(text, err => { if (err) { - this.logger.error(err); + this.logger.error('Error writing blockings:', err); rej(err); } else { res(); @@ -589,7 +592,7 @@ export class ExportAccountDataProcessorService { return new Promise((res, rej) => { favoriteStream.write(text, err => { if (err) { - this.logger.error(err); + this.logger.error('Error writing favorites:', err); rej(err); } else { res(); @@ -650,7 +653,7 @@ export class ExportAccountDataProcessorService { return new Promise((res, rej) => { antennaStream.write(text, err => { if (err) { - this.logger.error(err); + this.logger.error('Error writing antennas:', err); rej(err); } else { res(); @@ -708,7 +711,7 @@ export class ExportAccountDataProcessorService { return new Promise((res, rej) => { listStream.write(text, err => { if (err) { - this.logger.error(err); + this.logger.error('Error writing lists:', err); rej(err); } else { res(); @@ -744,12 +747,12 @@ export class ExportAccountDataProcessorService { zlib: { level: 0 }, }); archiveStream.on('close', async () => { - this.logger.succ(`Exported to: ${archivePath}`); + this.logger.debug(`Exported to path: ${archivePath}`); const fileName = 'data-request-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.zip'; const driveFile = await this.driveService.addFile({ user, path: archivePath, name: fileName, force: true }); - this.logger.succ(`Exported to: ${driveFile.id}`); + this.logger.debug(`Exported to drive: ${driveFile.id}`); cleanup(); archiveCleanup(); if (profile.email) { diff --git a/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts b/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts index b3111865ad..61d76da5ac 100644 --- a/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts @@ -45,15 +45,19 @@ export class ExportAntennasProcessorService { public async process(job: Bull.Job): Promise { const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { + this.logger.debug(`Skip: user ${job.data.user.id} does not exist`); return; } + + this.logger.info(`Exporting antennas of ${job.data.user.id} ...`); + const [path, cleanup] = await createTemp(); const stream = fs.createWriteStream(path, { flags: 'a' }); const write = (input: string): Promise => { return new Promise((resolve, reject) => { stream.write(input, err => { if (err) { - this.logger.error(err); + this.logger.error('Error exporting antennas:', err); reject(); } else { resolve(); @@ -96,7 +100,7 @@ export class ExportAntennasProcessorService { const fileName = 'antennas-' + DateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.json'; const driveFile = await this.driveService.addFile({ user, path, name: fileName, force: true, ext: 'json' }); - this.logger.succ('Exported to: ' + driveFile.id); + this.logger.debug('Exported to: ' + driveFile.id); this.notificationService.createNotification(user.id, 'exportCompleted', { exportedEntity: 'antenna', diff --git a/packages/backend/src/queue/processors/ExportBlockingProcessorService.ts b/packages/backend/src/queue/processors/ExportBlockingProcessorService.ts index ecc439db69..4c17c3f718 100644 --- a/packages/backend/src/queue/processors/ExportBlockingProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportBlockingProcessorService.ts @@ -40,17 +40,18 @@ export class ExportBlockingProcessorService { @bindThis public async process(job: Bull.Job): Promise { - this.logger.info(`Exporting blocking of ${job.data.user.id} ...`); - const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { + this.logger.debug(`Skip: user ${job.data.user.id} does not exist`); return; } + this.logger.info(`Exporting blocking of ${job.data.user.id} ...`); + // Create temp file const [path, cleanup] = await createTemp(); - this.logger.info(`Temp file is ${path}`); + this.logger.debug(`Temp file is ${path}`); try { const stream = fs.createWriteStream(path, { flags: 'a' }); @@ -87,7 +88,7 @@ export class ExportBlockingProcessorService { await new Promise((res, rej) => { stream.write(content + '\n', err => { if (err) { - this.logger.error(err); + this.logger.error('Error exporting blocking:', err); rej(err); } else { res(); @@ -105,12 +106,12 @@ export class ExportBlockingProcessorService { } stream.end(); - this.logger.succ(`Exported to: ${path}`); + this.logger.debug(`Exported to: ${path}`); const fileName = 'blocking-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; const driveFile = await this.driveService.addFile({ user, path, name: fileName, force: true, ext: 'csv' }); - this.logger.succ(`Exported to: ${driveFile.id}`); + this.logger.debug(`Exported to: ${driveFile.id}`); this.notificationService.createNotification(user.id, 'exportCompleted', { exportedEntity: 'blocking', diff --git a/packages/backend/src/queue/processors/ExportClipsProcessorService.ts b/packages/backend/src/queue/processors/ExportClipsProcessorService.ts index 583ddbb745..1d34d2b4e6 100644 --- a/packages/backend/src/queue/processors/ExportClipsProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportClipsProcessorService.ts @@ -51,17 +51,18 @@ export class ExportClipsProcessorService { @bindThis public async process(job: Bull.Job): Promise { - this.logger.info(`Exporting clips of ${job.data.user.id} ...`); - const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { + this.logger.debug(`Skip: user ${job.data.user.id} does not exist`); return; } + this.logger.info(`Exporting clips of ${job.data.user.id} ...`); + // Create temp file const [path, cleanup] = await createTemp(); - this.logger.info(`Temp file is ${path}`); + this.logger.debug(`Temp file is ${path}`); try { const stream = Writable.toWeb(fs.createWriteStream(path, { flags: 'a' })); @@ -75,12 +76,12 @@ export class ExportClipsProcessorService { await writer.write(']'); await writer.close(); - this.logger.succ(`Exported to: ${path}`); + this.logger.debug(`Exported to: ${path}`); const fileName = 'clips-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.json'; const driveFile = await this.driveService.addFile({ user, path, name: fileName, force: true, ext: 'json' }); - this.logger.succ(`Exported to: ${driveFile.id}`); + this.logger.debug(`Exported to: ${driveFile.id}`); this.notificationService.createNotification(user.id, 'exportCompleted', { exportedEntity: 'clip', diff --git a/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts b/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts index 14d32e78b3..b8f208bbfc 100644 --- a/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts @@ -45,16 +45,17 @@ export class ExportCustomEmojisProcessorService { @bindThis public async process(job: Bull.Job): Promise { - this.logger.info('Exporting custom emojis ...'); - const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { + this.logger.debug(`Skip: user ${job.data.user.id} does not exist`); return; } + this.logger.info(`Exporting custom emojis of ${job.data.user.id} ...`); + const [path, cleanup] = await createTempDir(); - this.logger.info(`Temp dir is ${path}`); + this.logger.debug(`Temp dir is ${path}`); const metaPath = path + '/meta.json'; @@ -66,7 +67,7 @@ export class ExportCustomEmojisProcessorService { return new Promise((res, rej) => { metaStream.write(text, err => { if (err) { - this.logger.error(err); + this.logger.error('Error exporting custom emojis:', err); rej(err); } else { res(); @@ -101,7 +102,7 @@ export class ExportCustomEmojisProcessorService { await this.downloadService.downloadUrl(emoji.originalUrl, emojiPath); downloaded = true; } catch (e) { // TODO: 何度か再試行 - this.logger.error(e instanceof Error ? e : new Error(e as string)); + this.logger.error('Error exporting custom emojis:', e as Error); } if (!downloaded) { @@ -130,12 +131,12 @@ export class ExportCustomEmojisProcessorService { zlib: { level: 0 }, }); archiveStream.on('close', async () => { - this.logger.succ(`Exported to: ${archivePath}`); + this.logger.debug(`Exported to: ${archivePath}`); const fileName = 'custom-emojis-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.zip'; const driveFile = await this.driveService.addFile({ user, path: archivePath, name: fileName, force: true }); - this.logger.succ(`Exported to: ${driveFile.id}`); + this.logger.debug(`Exported to: ${driveFile.id}`); this.notificationService.createNotification(user.id, 'exportCompleted', { exportedEntity: 'customEmoji', diff --git a/packages/backend/src/queue/processors/ExportFavoritesProcessorService.ts b/packages/backend/src/queue/processors/ExportFavoritesProcessorService.ts index b81feece01..b5716f2d49 100644 --- a/packages/backend/src/queue/processors/ExportFavoritesProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportFavoritesProcessorService.ts @@ -45,17 +45,18 @@ export class ExportFavoritesProcessorService { @bindThis public async process(job: Bull.Job): Promise { - this.logger.info(`Exporting favorites of ${job.data.user.id} ...`); - const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { + this.logger.debug(`Skip: user ${job.data.user.id} does not exist`); return; } + this.logger.info(`Exporting favorites of ${job.data.user.id} ...`); + // Create temp file const [path, cleanup] = await createTemp(); - this.logger.info(`Temp file is ${path}`); + this.logger.debug(`Temp file is ${path}`); try { const stream = fs.createWriteStream(path, { flags: 'a' }); @@ -64,7 +65,7 @@ export class ExportFavoritesProcessorService { return new Promise((res, rej) => { stream.write(text, err => { if (err) { - this.logger.error(err); + this.logger.error('Error exporting favorites:', err); rej(err); } else { res(); @@ -119,12 +120,12 @@ export class ExportFavoritesProcessorService { await write(']'); stream.end(); - this.logger.succ(`Exported to: ${path}`); + this.logger.debug(`Exported to: ${path}`); const fileName = 'favorites-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.json'; const driveFile = await this.driveService.addFile({ user, path, name: fileName, force: true, ext: 'json' }); - this.logger.succ(`Exported to: ${driveFile.id}`); + this.logger.debug(`Exported to: ${driveFile.id}`); this.notificationService.createNotification(user.id, 'exportCompleted', { exportedEntity: 'favorite', diff --git a/packages/backend/src/queue/processors/ExportFollowingProcessorService.ts b/packages/backend/src/queue/processors/ExportFollowingProcessorService.ts index 903f962515..883f35e366 100644 --- a/packages/backend/src/queue/processors/ExportFollowingProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportFollowingProcessorService.ts @@ -44,17 +44,18 @@ export class ExportFollowingProcessorService { @bindThis public async process(job: Bull.Job): Promise { - this.logger.info(`Exporting following of ${job.data.user.id} ...`); - const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { + this.logger.debug(`Skip: user ${job.data.user.id} does not exist`); return; } + this.logger.info(`Exporting following of ${job.data.user.id} ...`); + // Create temp file const [path, cleanup] = await createTemp(); - this.logger.info(`Temp file is ${path}`); + this.logger.debug(`Temp file is ${path}`); try { const stream = fs.createWriteStream(path, { flags: 'a' }); @@ -98,7 +99,7 @@ export class ExportFollowingProcessorService { await new Promise((res, rej) => { stream.write(content + '\n', err => { if (err) { - this.logger.error(err); + this.logger.error('Error exporting following:', err); rej(err); } else { res(); @@ -109,12 +110,12 @@ export class ExportFollowingProcessorService { } stream.end(); - this.logger.succ(`Exported to: ${path}`); + this.logger.debug(`Exported to: ${path}`); const fileName = 'following-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; const driveFile = await this.driveService.addFile({ user, path, name: fileName, force: true, ext: 'csv' }); - this.logger.succ(`Exported to: ${driveFile.id}`); + this.logger.debug(`Exported to: ${driveFile.id}`); this.notificationService.createNotification(user.id, 'exportCompleted', { exportedEntity: 'following', diff --git a/packages/backend/src/queue/processors/ExportMutingProcessorService.ts b/packages/backend/src/queue/processors/ExportMutingProcessorService.ts index f9867ade29..9cdb94beaf 100644 --- a/packages/backend/src/queue/processors/ExportMutingProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportMutingProcessorService.ts @@ -40,17 +40,18 @@ export class ExportMutingProcessorService { @bindThis public async process(job: Bull.Job): Promise { - this.logger.info(`Exporting muting of ${job.data.user.id} ...`); - const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { + this.logger.debug(`Skip: user ${job.data.user.id} does not exist`); return; } + this.logger.debug(`Exporting muting of ${job.data.user.id} ...`); + // Create temp file const [path, cleanup] = await createTemp(); - this.logger.info(`Temp file is ${path}`); + this.logger.debug(`Temp file is ${path}`); try { const stream = fs.createWriteStream(path, { flags: 'a' }); @@ -88,7 +89,7 @@ export class ExportMutingProcessorService { await new Promise((res, rej) => { stream.write(content + '\n', err => { if (err) { - this.logger.error(err); + this.logger.error('Error exporting mutings:', err); rej(err); } else { res(); @@ -106,12 +107,12 @@ export class ExportMutingProcessorService { } stream.end(); - this.logger.succ(`Exported to: ${path}`); + this.logger.debug(`Exported to: ${path}`); const fileName = 'mute-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; const driveFile = await this.driveService.addFile({ user, path, name: fileName, force: true, ext: 'csv' }); - this.logger.succ(`Exported to: ${driveFile.id}`); + this.logger.debug(`Exported to: ${driveFile.id}`); this.notificationService.createNotification(user.id, 'exportCompleted', { exportedEntity: 'muting', diff --git a/packages/backend/src/queue/processors/ExportNotesProcessorService.ts b/packages/backend/src/queue/processors/ExportNotesProcessorService.ts index 9e2b678219..7d49a8dab2 100644 --- a/packages/backend/src/queue/processors/ExportNotesProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportNotesProcessorService.ts @@ -120,17 +120,18 @@ export class ExportNotesProcessorService { @bindThis public async process(job: Bull.Job): Promise { - this.logger.info(`Exporting notes of ${job.data.user.id} ...`); - const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { + this.logger.debug(`Skip: user ${job.data.user.id} does not exist`); return; } + this.logger.info(`Exporting notes of ${job.data.user.id} ...`); + // Create temp file const [path, cleanup] = await createTemp(); - this.logger.info(`Temp file is ${path}`); + this.logger.debug(`Temp file is ${path}`); try { // メモリが足りなくならないようにストリームで処理する @@ -146,12 +147,12 @@ export class ExportNotesProcessorService { .pipeThrough(new TextEncoderStream()) .pipeTo(new FileWriterStream(path)); - this.logger.succ(`Exported to: ${path}`); + this.logger.debug(`Exported to: ${path}`); const fileName = 'notes-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.json'; const driveFile = await this.driveService.addFile({ user, path, name: fileName, force: true, ext: 'json' }); - this.logger.succ(`Exported to: ${driveFile.id}`); + this.logger.debug(`Exported to: ${driveFile.id}`); this.notificationService.createNotification(user.id, 'exportCompleted', { exportedEntity: 'note', diff --git a/packages/backend/src/queue/processors/ExportUserListsProcessorService.ts b/packages/backend/src/queue/processors/ExportUserListsProcessorService.ts index c483d79854..43043e3a26 100644 --- a/packages/backend/src/queue/processors/ExportUserListsProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportUserListsProcessorService.ts @@ -43,13 +43,14 @@ export class ExportUserListsProcessorService { @bindThis public async process(job: Bull.Job): Promise { - this.logger.info(`Exporting user lists of ${job.data.user.id} ...`); - const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { + this.logger.debug(`Skip: user ${job.data.user.id} does not exist`); return; } + this.logger.info(`Exporting user lists of ${job.data.user.id} ...`); + const lists = await this.userListsRepository.findBy({ userId: user.id, }); @@ -57,7 +58,7 @@ export class ExportUserListsProcessorService { // Create temp file const [path, cleanup] = await createTemp(); - this.logger.info(`Temp file is ${path}`); + this.logger.debug(`Temp file is ${path}`); try { const stream = fs.createWriteStream(path, { flags: 'a' }); @@ -74,7 +75,7 @@ export class ExportUserListsProcessorService { await new Promise((res, rej) => { stream.write(content + '\n', err => { if (err) { - this.logger.error(err); + this.logger.error('Error exporting lists:', err); rej(err); } else { res(); @@ -85,12 +86,12 @@ export class ExportUserListsProcessorService { } stream.end(); - this.logger.succ(`Exported to: ${path}`); + this.logger.debug(`Exported to: ${path}`); const fileName = 'user-lists-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; const driveFile = await this.driveService.addFile({ user, path, name: fileName, force: true, ext: 'csv' }); - this.logger.succ(`Exported to: ${driveFile.id}`); + this.logger.debug(`Exported to: ${driveFile.id}`); this.notificationService.createNotification(user.id, 'exportCompleted', { exportedEntity: 'userList', diff --git a/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts b/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts index 9c033b73e2..f29a19ce66 100644 --- a/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts @@ -8,7 +8,7 @@ import _Ajv from 'ajv'; import { IdService } from '@/core/IdService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import Logger from '@/logger.js'; -import type { AntennasRepository } from '@/models/_.js'; +import type { AntennasRepository, UsersRepository } from '@/models/_.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; @@ -59,6 +59,9 @@ export class ImportAntennasProcessorService { @Inject(DI.antennasRepository) private antennasRepository: AntennasRepository, + @Inject(DI.usersRepository) + private usersRepository: UsersRepository, + private queueLoggerService: QueueLoggerService, private idService: IdService, private globalEventService: GlobalEventService, @@ -68,12 +71,20 @@ export class ImportAntennasProcessorService { @bindThis public async process(job: Bull.Job): Promise { + const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); + if (user == null) { + this.logger.debug(`Skip: user ${job.data.user.id} does not exist`); + return; + } + + this.logger.debug(`Importing blocking of ${job.data.user.id} ...`); + const now = new Date(); try { for (const antenna of job.data.antenna) { if (antenna.keywords.length === 0 || antenna.keywords[0].every(x => x === '')) continue; if (!validate(antenna)) { - this.logger.warn('Validation Failed'); + this.logger.warn('Antenna validation failed'); continue; } const result = await this.antennasRepository.insertOne({ @@ -92,11 +103,11 @@ export class ImportAntennasProcessorService { withReplies: antenna.withReplies, withFile: antenna.withFile, }); - this.logger.succ('Antenna created: ' + result.id); + this.logger.debug('Antenna created: ' + result.id); this.globalEventService.publishInternalEvent('antennaCreated', result); } } catch (err: any) { - this.logger.error(err); + this.logger.error('Error importing antennas:', err); } } } diff --git a/packages/backend/src/queue/processors/ImportBlockingProcessorService.ts b/packages/backend/src/queue/processors/ImportBlockingProcessorService.ts index b78229c648..e2de9532eb 100644 --- a/packages/backend/src/queue/processors/ImportBlockingProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportBlockingProcessorService.ts @@ -40,10 +40,9 @@ export class ImportBlockingProcessorService { @bindThis public async process(job: Bull.Job): Promise { - this.logger.info(`Importing blocking of ${job.data.user.id} ...`); - const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { + this.logger.debug(`Skip: user ${job.data.user.id} does not exist`); return; } @@ -51,14 +50,17 @@ export class ImportBlockingProcessorService { id: job.data.fileId, }); if (file == null) { + this.logger.debug(`Skip: file ${job.data.fileId} does not exist`); return; } + this.logger.debug(`Importing blocking of ${job.data.user.id} ...`); + const csv = await this.downloadService.downloadTextFile(file.url); const targets = csv.trim().split('\n'); this.queueService.createImportBlockingToDbJob({ id: user.id }, targets); - this.logger.succ('Import jobs created'); + this.logger.debug('Import jobs created'); } @bindThis @@ -93,11 +95,11 @@ export class ImportBlockingProcessorService { // skip myself if (target.id === job.data.user.id) return; - this.logger.info(`Block ${target.id} ...`); + this.logger.debug(`Block ${target.id} ...`); this.queueService.createBlockJob([{ from: { id: user.id }, to: { id: target.id }, silent: true }]); } catch (e) { - this.logger.warn(`Error: ${e}`); + this.logger.error('Error importing blockings:', e as Error); } } } diff --git a/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts b/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts index d08cadd378..4b909328cd 100644 --- a/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts @@ -16,6 +16,7 @@ import { DriveService } from '@/core/DriveService.js'; import { DownloadService } from '@/core/DownloadService.js'; import { bindThis } from '@/decorators.js'; import type { Config } from '@/config.js'; +import { renderInlineError } from '@/misc/render-inline-error.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; import type { DbUserImportJobData } from '../types.js'; @@ -45,18 +46,19 @@ export class ImportCustomEmojisProcessorService { @bindThis public async process(job: Bull.Job): Promise { - this.logger.info('Importing custom emojis ...'); - const file = await this.driveFilesRepository.findOneBy({ id: job.data.fileId, }); if (file == null) { + this.logger.debug(`Skip: file ${job.data.fileId} does not exist`); return; } + this.logger.info(`Importing custom emojis from ${file.id} (${file.name}) ...`); + const [path, cleanup] = await createTempDir(); - this.logger.info(`Temp dir is ${path}`); + this.logger.debug(`Temp dir is ${path}`); const destPath = path + '/emojis.zip'; @@ -65,14 +67,14 @@ export class ImportCustomEmojisProcessorService { await this.downloadService.downloadUrl(file.url, destPath, { operationTimeout: this.config.import?.downloadTimeout, maxSize: this.config.import?.maxFileSize }); } catch (e) { // TODO: 何度か再試行 if (e instanceof Error || typeof e === 'string') { - this.logger.error(e); + this.logger.error('Error importing custom emojis:', e as Error); } throw e; } const outputPath = path + '/emojis'; try { - this.logger.succ(`Unzipping to ${outputPath}`); + this.logger.debug(`Unzipping to ${outputPath}`); ZipReader.withDestinationPath(outputPath).viaBuffer(await fs.promises.readFile(destPath)); const metaRaw = fs.readFileSync(outputPath + '/meta.json', 'utf-8'); const meta = JSON.parse(metaRaw); @@ -117,7 +119,7 @@ export class ImportCustomEmojisProcessorService { }); } catch (e) { if (e instanceof Error || typeof e === 'string') { - this.logger.error(`couldn't import ${emojiPath} for ${emojiInfo.name}: ${e}`); + this.logger.error(`couldn't import ${emojiPath} for ${emojiInfo.name}: ${renderInlineError(e)}`); } continue; } @@ -125,11 +127,9 @@ export class ImportCustomEmojisProcessorService { cleanup(); - this.logger.succ('Imported'); + this.logger.debug('Imported'); } catch (e) { - if (e instanceof Error || typeof e === 'string') { - this.logger.error(e); - } + this.logger.error('Error importing custom emojis:', e as Error); cleanup(); throw e; } diff --git a/packages/backend/src/queue/processors/ImportFollowingProcessorService.ts b/packages/backend/src/queue/processors/ImportFollowingProcessorService.ts index 70c9f3a096..816d5cf65a 100644 --- a/packages/backend/src/queue/processors/ImportFollowingProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportFollowingProcessorService.ts @@ -40,10 +40,9 @@ export class ImportFollowingProcessorService { @bindThis public async process(job: Bull.Job): Promise { - this.logger.info(`Importing following of ${job.data.user.id} ...`); - const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { + this.logger.debug(`Skip: user ${job.data.user.id} does not exist`); return; } @@ -51,14 +50,17 @@ export class ImportFollowingProcessorService { id: job.data.fileId, }); if (file == null) { + this.logger.debug(`Skip: file ${job.data.fileId} does not exist`); return; } + this.logger.info(`Importing following of ${job.data.user.id} ...`); + const csv = await this.downloadService.downloadTextFile(file.url); const targets = csv.trim().split('\n'); this.queueService.createImportFollowingToDbJob({ id: user.id }, targets, job.data.withReplies); - this.logger.succ('Import jobs created'); + this.logger.debug('Import jobs created'); } @bindThis @@ -93,11 +95,11 @@ export class ImportFollowingProcessorService { // skip myself if (target.id === job.data.user.id) return; - this.logger.info(`Follow ${target.id} ${job.data.withReplies ? 'with replies' : 'without replies'} ...`); + this.logger.debug(`Follow ${target.id} ${job.data.withReplies ? 'with replies' : 'without replies'} ...`); this.queueService.createFollowJob([{ from: user, to: { id: target.id }, silent: true, withReplies: job.data.withReplies }]); } catch (e) { - this.logger.warn(`Error: ${e}`); + this.logger.error('Error importing followings:', e as Error); } } } diff --git a/packages/backend/src/queue/processors/ImportMutingProcessorService.ts b/packages/backend/src/queue/processors/ImportMutingProcessorService.ts index ec9d2b6c4c..d3827b12fd 100644 --- a/packages/backend/src/queue/processors/ImportMutingProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportMutingProcessorService.ts @@ -14,6 +14,7 @@ import { DownloadService } from '@/core/DownloadService.js'; import { UserMutingService } from '@/core/UserMutingService.js'; import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; +import { renderInlineError } from '@/misc/render-inline-error.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; import type { DbUserImportJobData } from '../types.js'; @@ -40,10 +41,9 @@ export class ImportMutingProcessorService { @bindThis public async process(job: Bull.Job): Promise { - this.logger.info(`Importing muting of ${job.data.user.id} ...`); - const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { + this.logger.debug(`Skip: user ${job.data.user.id} does not exist`); return; } @@ -51,9 +51,12 @@ export class ImportMutingProcessorService { id: job.data.fileId, }); if (file == null) { + this.logger.debug(`Skip: file ${job.data.fileId} does not exist`); return; } + this.logger.info(`Importing muting of ${job.data.user.id} ...`); + const csv = await this.downloadService.downloadTextFile(file.url); let linenum = 0; @@ -88,14 +91,14 @@ export class ImportMutingProcessorService { // skip myself if (target.id === job.data.user.id) continue; - this.logger.info(`Mute[${linenum}] ${target.id} ...`); + this.logger.debug(`Mute[${linenum}] ${target.id} ...`); await this.userMutingService.mute(user, target); } catch (e) { - this.logger.warn(`Error in line:${linenum} ${e}`); + this.logger.warn(`Error in line:${linenum} ${renderInlineError(e)}`); } } - this.logger.succ('Imported'); + this.logger.debug('Imported'); } } diff --git a/packages/backend/src/queue/processors/ImportNotesProcessorService.ts b/packages/backend/src/queue/processors/ImportNotesProcessorService.ts index 5e660e8081..e209855720 100644 --- a/packages/backend/src/queue/processors/ImportNotesProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportNotesProcessorService.ts @@ -159,10 +159,9 @@ export class ImportNotesProcessorService { @bindThis public async process(job: Bull.Job): Promise { - this.logger.info(`Starting note import of ${job.data.user.id} ...`); - const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { + this.logger.debug(`Skip: user ${job.data.user.id} does not exist`); return; } @@ -170,9 +169,12 @@ export class ImportNotesProcessorService { id: job.data.fileId, }); if (file == null) { + this.logger.debug(`Skip: file ${job.data.fileId} does not exist`); return; } + this.logger.info(`Starting note import of ${job.data.user.id} ...`); + let folder = await this.driveFoldersRepository.findOneBy({ name: 'Imports', userId: job.data.user.id }); if (folder == null) { await this.driveFoldersRepository.insert({ id: this.idService.gen(), name: 'Imports', userId: job.data.user.id }); @@ -184,7 +186,7 @@ export class ImportNotesProcessorService { if (type === 'Twitter' || file.name.startsWith('twitter') && file.name.endsWith('.zip')) { const [path, cleanup] = await createTempDir(); - this.logger.info(`Temp dir is ${path}`); + this.logger.debug(`Temp dir is ${path}`); const destPath = path + '/twitter.zip'; @@ -192,15 +194,13 @@ export class ImportNotesProcessorService { await fsp.writeFile(destPath, '', 'binary'); await this.downloadUrl(file.url, destPath); } catch (e) { // TODO: 何度か再試行 - if (e instanceof Error || typeof e === 'string') { - this.logger.error(e); - } + this.logger.error('Error importing notes:', e as Error); throw e; } const outputPath = path + '/twitter'; try { - this.logger.succ(`Unzipping to ${outputPath}`); + this.logger.debug(`Unzipping to ${outputPath}`); ZipReader.withDestinationPath(outputPath).viaBuffer(await fsp.readFile(destPath)); const unprocessedTweets = this.parseTwitterFile(await fsp.readFile(outputPath + '/data/tweets.js', 'utf-8')); @@ -214,7 +214,7 @@ export class ImportNotesProcessorService { } else if (type === 'Facebook' || file.name.startsWith('facebook-') && file.name.endsWith('.zip')) { const [path, cleanup] = await createTempDir(); - this.logger.info(`Temp dir is ${path}`); + this.logger.debug(`Temp dir is ${path}`); const destPath = path + '/facebook.zip'; @@ -222,15 +222,13 @@ export class ImportNotesProcessorService { await fsp.writeFile(destPath, '', 'binary'); await this.downloadUrl(file.url, destPath); } catch (e) { // TODO: 何度か再試行 - if (e instanceof Error || typeof e === 'string') { - this.logger.error(e); - } + this.logger.error('Error importing notes:', e as Error); throw e; } const outputPath = path + '/facebook'; try { - this.logger.succ(`Unzipping to ${outputPath}`); + this.logger.debug(`Unzipping to ${outputPath}`); ZipReader.withDestinationPath(outputPath).viaBuffer(await fsp.readFile(destPath)); const postsJson = await fsp.readFile(outputPath + '/your_activity_across_facebook/posts/your_posts__check_ins__photos_and_videos_1.json', 'utf-8'); const posts = JSON.parse(postsJson); @@ -247,7 +245,7 @@ export class ImportNotesProcessorService { } else if (file.name.endsWith('.zip')) { const [path, cleanup] = await createTempDir(); - this.logger.info(`Temp dir is ${path}`); + this.logger.debug(`Temp dir is ${path}`); const destPath = path + '/unknown.zip'; @@ -255,15 +253,13 @@ export class ImportNotesProcessorService { await fsp.writeFile(destPath, '', 'binary'); await this.downloadUrl(file.url, destPath); } catch (e) { // TODO: 何度か再試行 - if (e instanceof Error || typeof e === 'string') { - this.logger.error(e); - } + this.logger.error('Error importing notes:', e as Error); throw e; } const outputPath = path + '/unknown'; try { - this.logger.succ(`Unzipping to ${outputPath}`); + this.logger.debug(`Unzipping to ${outputPath}`); ZipReader.withDestinationPath(outputPath).viaBuffer(await fsp.readFile(destPath)); const isInstagram = type === 'Instagram' || fs.existsSync(outputPath + '/instagram_live') || fs.existsSync(outputPath + '/instagram_ads_and_businesses'); const isOutbox = type === 'Mastodon' || fs.existsSync(outputPath + '/outbox.json'); @@ -307,15 +303,13 @@ export class ImportNotesProcessorService { } else if (job.data.type === 'Misskey' || file.name.startsWith('notes-') && file.name.endsWith('.json')) { const [path, cleanup] = await createTemp(); - this.logger.info(`Temp dir is ${path}`); + this.logger.debug(`Temp dir is ${path}`); try { await fsp.writeFile(path, '', 'utf-8'); await this.downloadUrl(file.url, path); } catch (e) { // TODO: 何度か再試行 - if (e instanceof Error || typeof e === 'string') { - this.logger.error(e); - } + this.logger.error('Error importing notes:', e as Error); throw e; } @@ -326,7 +320,7 @@ export class ImportNotesProcessorService { cleanup(); } - this.logger.succ('Import jobs created'); + this.logger.debug('Import jobs created'); } @bindThis @@ -365,7 +359,7 @@ export class ImportNotesProcessorService { try { await this.downloadUrl(file.url, filePath); } catch (e) { // TODO: 何度か再試行 - this.logger.error(e instanceof Error ? e : new Error(e as string)); + this.logger.error('Error importing notes:', e as Error); } const driveFile = await this.driveService.addFile({ user: user, @@ -504,7 +498,7 @@ export class ImportNotesProcessorService { try { await this.downloadUrl(file.url, filePath); } catch (e) { // TODO: 何度か再試行 - this.logger.error(e instanceof Error ? e : new Error(e as string)); + this.logger.error('Error importing notes:', e as Error); } const driveFile = await this.driveService.addFile({ user: user, @@ -628,7 +622,7 @@ export class ImportNotesProcessorService { try { await this.downloadUrl(videos[0].url, filePath); } catch (e) { // TODO: 何度か再試行 - this.logger.error(e instanceof Error ? e : new Error(e as string)); + this.logger.error('Error importing notes:', e as Error); } const driveFile = await this.driveService.addFile({ user: user, @@ -653,7 +647,7 @@ export class ImportNotesProcessorService { try { await this.downloadUrl(file.media_url_https, filePath); } catch (e) { // TODO: 何度か再試行 - this.logger.error(e instanceof Error ? e : new Error(e as string)); + this.logger.error('Error importing notes:', e as Error); } const driveFile = await this.driveService.addFile({ @@ -673,7 +667,7 @@ export class ImportNotesProcessorService { const createdNote = await this.noteCreateService.import(user, { createdAt: date, reply: parentNote, text: text, files: files }); if (tweet.childNotes) this.queueService.createImportTweetsToDbJob(user, tweet.childNotes, createdNote.id); } catch (e) { - this.logger.warn(`Error: ${e}`); + this.logger.error('Error importing notes:', e as Error); } } diff --git a/packages/backend/src/queue/processors/ImportUserListsProcessorService.ts b/packages/backend/src/queue/processors/ImportUserListsProcessorService.ts index db9255b35d..482054e52f 100644 --- a/packages/backend/src/queue/processors/ImportUserListsProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportUserListsProcessorService.ts @@ -15,6 +15,7 @@ import { UserListService } from '@/core/UserListService.js'; import { IdService } from '@/core/IdService.js'; import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; +import { renderInlineError } from '@/misc/render-inline-error.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; import type { DbUserImportJobData } from '../types.js'; @@ -48,10 +49,9 @@ export class ImportUserListsProcessorService { @bindThis public async process(job: Bull.Job): Promise { - this.logger.info(`Importing user lists of ${job.data.user.id} ...`); - const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { + this.logger.debug(`Skip: user ${job.data.user.id} does not exist`); return; } @@ -59,9 +59,12 @@ export class ImportUserListsProcessorService { id: job.data.fileId, }); if (file == null) { + this.logger.debug(`Skip: file ${job.data.fileId} does not exist`); return; } + this.logger.info(`Importing user lists of ${job.data.user.id} ...`); + const csv = await this.downloadService.downloadTextFile(file.url); let linenum = 0; @@ -102,10 +105,10 @@ export class ImportUserListsProcessorService { this.userListService.addMember(target, list!, user); } catch (e) { - this.logger.warn(`Error in line:${linenum} ${e}`); + this.logger.warn(`Error in line:${linenum} ${renderInlineError(e)}`); } } - this.logger.succ('Imported'); + this.logger.debug('Imported'); } } diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index 9564724c62..612b16dfbf 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -31,6 +31,8 @@ import { SkApInboxLog } from '@/models/_.js'; import type { Config } from '@/config.js'; import { ApLogService, calculateDurationSince } from '@/core/ApLogService.js'; import { UpdateInstanceQueue } from '@/core/UpdateInstanceQueue.js'; +import { isRetryableError } from '@/misc/is-retryable-error.js'; +import { renderInlineError } from '@/misc/render-inline-error.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type { InboxJobData } from '../types.js'; @@ -125,6 +127,14 @@ export class InboxProcessorService implements OnApplicationShutdown { return `Old keyId is no longer supported. ${keyIdLower}`; } + if (activity.actor as unknown == null || (Array.isArray(activity.actor) && activity.actor.length < 1)) { + return 'skip: activity has no actor'; + } + if (typeof(activity.actor) !== 'string' && typeof(activity.actor) !== 'object') { + return `skip: activity actor has invalid type: ${typeof(activity.actor)}`; + } + const actorId = getApId(activity.actor); + // HTTP-Signature keyIdを元にDBから取得 let authUser: { user: MiRemoteUser; @@ -134,26 +144,25 @@ export class InboxProcessorService implements OnApplicationShutdown { // keyIdでわからなければ、activity.actorを元にDBから取得 || activity.actorを元にリモートから取得 if (authUser == null) { try { - authUser = await this.apDbResolverService.getAuthUserFromApId(getApId(activity.actor)); + authUser = await this.apDbResolverService.getAuthUserFromApId(actorId); } catch (err) { // 対象が4xxならスキップ - if (err instanceof StatusError) { - if (!err.isRetryable) { - throw new Bull.UnrecoverableError(`skip: Ignored deleted actors on both ends ${activity.actor} - ${err.statusCode}`); - } - throw new Error(`Error in actor ${activity.actor} - ${err.statusCode}`); + if (!isRetryableError(err)) { + throw new Bull.UnrecoverableError(`skip: Ignored deleted actors on both ends ${actorId}`); } + + throw err; } } // それでもわからなければ終了 if (authUser == null) { - throw new Bull.UnrecoverableError(`skip: failed to resolve user ${getApId(activity.actor)}`); + throw new Bull.UnrecoverableError(`skip: failed to resolve user ${actorId}`); } // publicKey がなくても終了 if (authUser.key == null) { - throw new Bull.UnrecoverableError(`skip: failed to resolve user publicKey ${getApId(activity.actor)}`); + throw new Bull.UnrecoverableError(`skip: failed to resolve user publicKey ${actorId}`); } // HTTP-Signatureの検証 @@ -168,7 +177,7 @@ export class InboxProcessorService implements OnApplicationShutdown { } // また、signatureのsignerは、activity.actorと一致する必要がある - if (!httpSignatureValidated || authUser.user.uri !== getApId(activity.actor)) { + if (!httpSignatureValidated || authUser.user.uri !== actorId) { // 一致しなくても、でもLD-Signatureがありそうならそっちも見る const ldSignature = activity.signature; if (ldSignature) { @@ -213,13 +222,13 @@ export class InboxProcessorService implements OnApplicationShutdown { activity.signature = ldSignature; // もう一度actorチェック - if (authUser.user.uri !== activity.actor) { - throw new Bull.UnrecoverableError(`skip: LD-Signature user(${authUser.user.uri}) !== activity.actor(${activity.actor})`); + if (authUser.user.uri !== actorId) { + throw new Bull.UnrecoverableError(`skip: LD-Signature user(${authUser.user.uri}) !== activity.actor(${actorId})`); } const ldHost = this.utilityService.extractDbHost(authUser.user.uri); if (!this.utilityService.isFederationAllowedHost(ldHost)) { - throw new Bull.UnrecoverableError(`Blocked request: ${ldHost}`); + throw new Bull.UnrecoverableError(`skip: request host is blocked: ${ldHost}`); } } else { throw new Bull.UnrecoverableError(`skip: http-signature verification failed and no LD-Signature. keyId=${signature.keyId}`); @@ -292,16 +301,8 @@ export class InboxProcessorService implements OnApplicationShutdown { } } - if (e instanceof StatusError && !e.isRetryable) { - return `skip: permanent error ${e.statusCode}`; - } - - if (e instanceof IdentifiableError && !e.isRetryable) { - if (e.message) { - return `skip: permanent error ${e.id}: ${e.message}`; - } else { - return `skip: permanent error ${e.id}`; - } + if (!isRetryableError(e)) { + return `skip: permanent error ${renderInlineError(e)}`; } throw e; diff --git a/packages/backend/src/queue/processors/ResyncChartsProcessorService.ts b/packages/backend/src/queue/processors/ResyncChartsProcessorService.ts index 0c47fdedb3..5b7a871af9 100644 --- a/packages/backend/src/queue/processors/ResyncChartsProcessorService.ts +++ b/packages/backend/src/queue/processors/ResyncChartsProcessorService.ts @@ -36,6 +36,6 @@ export class ResyncChartsProcessorService { await this.notesChart.resync(); await this.usersChart.resync(); - this.logger.succ('All charts successfully resynced.'); + this.logger.info('All charts successfully resynced.'); } } diff --git a/packages/backend/src/queue/processors/ScheduleNotePostProcessorService.ts b/packages/backend/src/queue/processors/ScheduleNotePostProcessorService.ts index d823d98ef1..73088f3312 100644 --- a/packages/backend/src/queue/processors/ScheduleNotePostProcessorService.ts +++ b/packages/backend/src/queue/processors/ScheduleNotePostProcessorService.ts @@ -12,6 +12,7 @@ import { DI } from '@/di-symbols.js'; import { NotificationService } from '@/core/NotificationService.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; import type { MiScheduleNoteType } from '@/models/NoteSchedule.js'; +import { renderInlineError } from '@/misc/render-inline-error.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; import type { ScheduleNotePostJobData } from '../types.js'; @@ -129,10 +130,11 @@ export class ScheduleNotePostProcessorService { channel, }).catch(async (err: IdentifiableError) => { this.notificationService.createNotification(me.id, 'scheduledNoteFailed', { - reason: err.message, + reason: renderInlineError(err), }); await this.noteScheduleRepository.remove(data); - throw this.logger.error(`Schedule Note Failed Reason: ${err.message}`); + this.logger.error(`Scheduled note failed: ${renderInlineError(err)}`); + throw err; }); await this.noteScheduleRepository.remove(data); this.notificationService.createNotification(me.id, 'scheduledNotePosted', { diff --git a/packages/backend/src/queue/processors/SystemWebhookDeliverProcessorService.ts b/packages/backend/src/queue/processors/SystemWebhookDeliverProcessorService.ts index f6bef52684..f9fcd1e928 100644 --- a/packages/backend/src/queue/processors/SystemWebhookDeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/SystemWebhookDeliverProcessorService.ts @@ -12,6 +12,7 @@ import type Logger from '@/logger.js'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { StatusError } from '@/misc/status-error.js'; import { bindThis } from '@/decorators.js'; +import { renderInlineError } from '@/misc/render-inline-error.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import { SystemWebhookDeliverJobData } from '../types.js'; @@ -63,21 +64,16 @@ export class SystemWebhookDeliverProcessorService { return 'Success'; } catch (res) { - this.logger.error(res as Error); + this.logger.error(`Failed to send webhook: ${renderInlineError(res)}`); this.systemWebhooksRepository.update({ id: job.data.webhookId }, { latestSentAt: new Date(), latestStatus: res instanceof StatusError ? res.statusCode : 1, }); - if (res instanceof StatusError) { + if (res instanceof StatusError && !res.isRetryable) { // 4xx - if (!res.isRetryable) { - throw new Bull.UnrecoverableError(`${res.statusCode} ${res.statusMessage}`); - } - - // 5xx etc. - throw new Error(`${res.statusCode} ${res.statusMessage}`); + throw new Bull.UnrecoverableError(`${res.statusCode} ${res.statusMessage}`); } else { // DNS error, socket error, timeout ... throw res; diff --git a/packages/backend/src/queue/processors/TickChartsProcessorService.ts b/packages/backend/src/queue/processors/TickChartsProcessorService.ts index fc8856a271..b4b8b1f205 100644 --- a/packages/backend/src/queue/processors/TickChartsProcessorService.ts +++ b/packages/backend/src/queue/processors/TickChartsProcessorService.ts @@ -62,6 +62,6 @@ export class TickChartsProcessorService { await this.perUserDriveChart.tick(false); await this.apRequestChart.tick(false); - this.logger.succ('All charts successfully ticked.'); + this.logger.info('All charts successfully ticked.'); } } diff --git a/packages/backend/src/queue/processors/UserWebhookDeliverProcessorService.ts b/packages/backend/src/queue/processors/UserWebhookDeliverProcessorService.ts index 9ec630ef70..0208ce6038 100644 --- a/packages/backend/src/queue/processors/UserWebhookDeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/UserWebhookDeliverProcessorService.ts @@ -69,14 +69,9 @@ export class UserWebhookDeliverProcessorService { latestStatus: res instanceof StatusError ? res.statusCode : 1, }); - if (res instanceof StatusError) { + if (res instanceof StatusError && !res.isRetryable) { // 4xx - if (!res.isRetryable) { - throw new Bull.UnrecoverableError(`${res.statusCode} ${res.statusMessage}`); - } - - // 5xx etc. - throw new Error(`${res.statusCode} ${res.statusMessage}`); + throw new Bull.UnrecoverableError(`${res.statusCode} ${res.statusMessage}`); } else { // DNS error, socket error, timeout ... throw res; diff --git a/packages/backend/src/server/FileServerService.ts b/packages/backend/src/server/FileServerService.ts index 4ef5539cff..1a372cb789 100644 --- a/packages/backend/src/server/FileServerService.ts +++ b/packages/backend/src/server/FileServerService.ts @@ -32,6 +32,7 @@ import { getIpHash } from '@/misc/get-ip-hash.js'; import { AuthenticateService } from '@/server/api/AuthenticateService.js'; import { SkRateLimiterService } from '@/server/SkRateLimiterService.js'; import { Keyed, RateLimit, sendRateLimitHeaders } from '@/misc/rate-limit-utils.js'; +import { renderInlineError } from '@/misc/render-inline-error.js'; import type { FastifyInstance, FastifyRequest, FastifyReply, FastifyPluginOptions } from 'fastify'; const _filename = fileURLToPath(import.meta.url); @@ -120,7 +121,7 @@ export class FileServerService { @bindThis private async errorHandler(request: FastifyRequest<{ Params?: { [x: string]: any }; Querystring?: { [x: string]: any }; }>, reply: FastifyReply, err?: any) { - this.logger.error(`${err}`); + this.logger.error(`Unhandled error in file server: ${renderInlineError(err)}`); reply.header('Cache-Control', 'max-age=300'); @@ -353,7 +354,7 @@ export class FileServerService { if (!request.headers['user-agent']) { throw new StatusError('User-Agent is required', 400, 'User-Agent is required'); } else if (request.headers['user-agent'].toLowerCase().indexOf('misskey/') !== -1) { - throw new StatusError('Refusing to proxy a request from another proxy', 403, 'Proxy is recursive'); + throw new StatusError(`Refusing to proxy recursive request to ${url} (from user-agent ${request.headers['user-agent']})`, 403, 'Proxy is recursive'); } // Create temp file @@ -383,7 +384,7 @@ export class FileServerService { ) { if (!isConvertibleImage) { // 画像でないなら404でお茶を濁す - throw new StatusError('Unexpected mime', 404); + throw new StatusError(`Unexpected non-convertible mime: ${file.mime}`, 404, 'Unexpected mime'); } } @@ -447,7 +448,7 @@ export class FileServerService { } else if (file.mime === 'image/svg+xml') { image = this.imageProcessingService.convertToWebpStream(file.path, 2048, 2048); } else if (!file.mime.startsWith('image/') || !FILE_TYPE_BROWSERSAFE.includes(file.mime)) { - throw new StatusError('Rejected type', 403, 'Rejected type'); + throw new StatusError(`Blocked mime type: ${file.mime}`, 403, 'Blocked mime type'); } if (!image) { @@ -521,7 +522,7 @@ export class FileServerService { > { if (url.startsWith(`${this.config.url}/files/`)) { const key = url.replace(`${this.config.url}/files/`, '').split('/').shift(); - if (!key) throw new StatusError('Invalid File Key', 400, 'Invalid File Key'); + if (!key) throw new StatusError(`Invalid file URL ${url}`, 400, 'Invalid file url'); return await this.getFileFromKey(key); } diff --git a/packages/backend/src/server/ServerService.ts b/packages/backend/src/server/ServerService.ts index 2d20aa1222..77b4519570 100644 --- a/packages/backend/src/server/ServerService.ts +++ b/packages/backend/src/server/ServerService.ts @@ -21,6 +21,7 @@ import { genIdenticon } from '@/misc/gen-identicon.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { LoggerService } from '@/core/LoggerService.js'; import { bindThis } from '@/decorators.js'; +import { renderInlineError } from '@/misc/render-inline-error.js'; import { ActivityPubServerService } from './ActivityPubServerService.js'; import { NodeinfoServerService } from './NodeinfoServerService.js'; import { ApiServerService } from './api/ApiServerService.js'; @@ -277,7 +278,7 @@ export class ServerService implements OnApplicationShutdown { this.logger.error(`Port ${this.config.port} is already in use by another process.`); break; default: - this.logger.error(err); + this.logger.error(`Unhandled error in server: ${renderInlineError(err)}`); break; } diff --git a/packages/backend/src/server/SkRateLimiterService.ts b/packages/backend/src/server/SkRateLimiterService.ts index 8978318045..35e87b0fe8 100644 --- a/packages/backend/src/server/SkRateLimiterService.ts +++ b/packages/backend/src/server/SkRateLimiterService.ts @@ -389,7 +389,7 @@ function createLimitKey(limit: ParsedLimit, actor: string, value: string): strin return `rl_${actor}_${limit.key}_${value}`; } -class ConflictError extends Error {} +export class ConflictError extends Error {} interface LimitCounter { timestamp: number; diff --git a/packages/backend/src/server/api/ApiCallService.ts b/packages/backend/src/server/api/ApiCallService.ts index 0d2dafd556..6d6c86bb82 100644 --- a/packages/backend/src/server/api/ApiCallService.ts +++ b/packages/backend/src/server/api/ApiCallService.ts @@ -20,12 +20,14 @@ import { RoleService } from '@/core/RoleService.js'; import type { Config } from '@/config.js'; import { sendRateLimitHeaders } from '@/misc/rate-limit-utils.js'; import { SkRateLimiterService } from '@/server/SkRateLimiterService.js'; +import { renderInlineError } from '@/misc/render-inline-error.js'; import { ApiError } from './error.js'; import { ApiLoggerService } from './ApiLoggerService.js'; import { AuthenticateService, AuthenticationError } from './AuthenticateService.js'; import type { FastifyRequest, FastifyReply } from 'fastify'; import type { OnApplicationShutdown } from '@nestjs/common'; import type { IEndpointMeta, IEndpoint } from './endpoints.js'; +import { renderFullError } from '@/misc/render-full-error.js'; const accessDenied = { message: 'Access denied.', @@ -100,26 +102,26 @@ export class ApiCallService implements OnApplicationShutdown { throw err; } else { const errId = randomUUID(); - this.logger.error(`Internal error occurred in ${ep.name}: ${err.message}`, { - ep: ep.name, - ps: data, - e: { - message: err.message, - code: err.name, - stack: err.stack, - id: errId, - }, + const fullError = renderFullError(err); + const message = typeof(fullError) === 'string' + ? `Internal error id=${errId} occurred in ${ep.name}: ${fullError}` + : `Internal error id=${errId} occurred in ${ep.name}:`; + const data = typeof(fullError) === 'object' + ? { e: fullError } + : {}; + this.logger.error(message, { + user: userId ?? '', + ...data, }); if (this.config.sentryForBackend) { - Sentry.captureMessage(`Internal error occurred in ${ep.name}: ${err.message}`, { + Sentry.captureMessage(`Internal error occurred in ${ep.name}: ${renderInlineError(err)}`, { level: 'error', user: { id: userId, }, extra: { ep: ep.name, - ps: data, e: { message: err.message, code: err.name, @@ -344,14 +346,14 @@ export class ApiCallService implements OnApplicationShutdown { } if (ep.meta.requireCredential || ep.meta.requireModerator || ep.meta.requireAdmin) { - if (user == null) { + if (user == null && ep.meta.requireCredential !== 'optional') { throw new ApiError({ message: 'Credential required.', code: 'CREDENTIAL_REQUIRED', id: '1384574d-a912-4b81-8601-c7b1c4085df1', httpStatusCode: 401, }); - } else if (user!.isSuspended) { + } else if (user?.isSuspended) { throw new ApiError({ message: 'Your account has been suspended.', code: 'YOUR_ACCOUNT_SUSPENDED', @@ -372,8 +374,8 @@ export class ApiCallService implements OnApplicationShutdown { } } - if ((ep.meta.requireModerator || ep.meta.requireAdmin) && (this.meta.rootUserId !== user!.id)) { - const myRoles = await this.roleService.getUserRoles(user!.id); + if ((ep.meta.requireModerator || ep.meta.requireAdmin) && (this.meta.rootUserId !== user?.id)) { + const myRoles = user ? await this.roleService.getUserRoles(user) : []; if (ep.meta.requireModerator && !myRoles.some(r => r.isModerator || r.isAdministrator)) { throw new ApiError({ message: 'You are not assigned to a moderator role.', @@ -392,9 +394,9 @@ export class ApiCallService implements OnApplicationShutdown { } } - if (ep.meta.requiredRolePolicy != null && (this.meta.rootUserId !== user!.id)) { - const myRoles = await this.roleService.getUserRoles(user!.id); - const policies = await this.roleService.getUserPolicies(user!.id); + if (ep.meta.requiredRolePolicy != null && (this.meta.rootUserId !== user?.id)) { + const myRoles = user ? await this.roleService.getUserRoles(user) : []; + const policies = await this.roleService.getUserPolicies(user ?? null); if (!policies[ep.meta.requiredRolePolicy] && !myRoles.some(r => r.isAdministrator)) { throw new ApiError({ message: 'You are not assigned to a required role.', @@ -418,7 +420,7 @@ export class ApiCallService implements OnApplicationShutdown { // Cast non JSON input if ((ep.meta.requireFile || request.method === 'GET') && ep.params.properties) { for (const k of Object.keys(ep.params.properties)) { - const param = ep.params.properties![k]; + const param = ep.params.properties[k]; if (['boolean', 'number', 'integer'].includes(param.type ?? '') && typeof data[k] === 'string') { try { data[k] = JSON.parse(data[k]); diff --git a/packages/backend/src/server/api/GetterService.ts b/packages/backend/src/server/api/GetterService.ts index 419017aaf4..f2850e6258 100644 --- a/packages/backend/src/server/api/GetterService.ts +++ b/packages/backend/src/server/api/GetterService.ts @@ -36,7 +36,7 @@ export class GetterService { const note = await this.notesRepository.findOneBy({ id: noteId }); if (note == null) { - throw new IdentifiableError('9725d0ce-ba28-4dde-95a7-2cbb2c15de24', 'No such note.'); + throw new IdentifiableError('9725d0ce-ba28-4dde-95a7-2cbb2c15de24', `Note ${noteId} does not exist`); } return note; @@ -47,7 +47,7 @@ export class GetterService { const note = await this.notesRepository.findOne({ where: { id: noteId }, relations: ['user'] }); if (note == null) { - throw new IdentifiableError('9725d0ce-ba28-4dde-95a7-2cbb2c15de24', 'No such note.'); + throw new IdentifiableError('9725d0ce-ba28-4dde-95a7-2cbb2c15de24', `Note ${noteId} does not exist`); } return note; @@ -59,7 +59,7 @@ export class GetterService { @bindThis public async getEdits(noteId: MiNote['id']) { const edits = await this.noteEditRepository.findBy({ noteId: noteId }).catch(() => { - throw new IdentifiableError('9725d0ce-ba28-4dde-95a7-2cbb2c15de24', 'No such note.'); + throw new IdentifiableError('9725d0ce-ba28-4dde-95a7-2cbb2c15de24', `Note ${noteId} does not exist`); }); return edits; @@ -73,7 +73,7 @@ export class GetterService { const user = await this.usersRepository.findOneBy({ id: userId }); if (user == null) { - throw new IdentifiableError('15348ddd-432d-49c2-8a5a-8069753becff', 'No such user.'); + throw new IdentifiableError('15348ddd-432d-49c2-8a5a-8069753becff', `User ${userId} does not exist`); } return user as MiLocalUser | MiRemoteUser; diff --git a/packages/backend/src/server/api/SigninApiService.ts b/packages/backend/src/server/api/SigninApiService.ts index 7f371ea309..a53fec88d0 100644 --- a/packages/backend/src/server/api/SigninApiService.ts +++ b/packages/backend/src/server/api/SigninApiService.ts @@ -205,37 +205,37 @@ export class SigninApiService { if (process.env.NODE_ENV !== 'test') { if (this.meta.enableHcaptcha && this.meta.hcaptchaSecretKey) { await this.captchaService.verifyHcaptcha(this.meta.hcaptchaSecretKey, body['hcaptcha-response']).catch(err => { - throw new FastifyReplyError(400, err); + throw new FastifyReplyError(400, String(err), err); }); } if (this.meta.enableMcaptcha && this.meta.mcaptchaSecretKey && this.meta.mcaptchaSitekey && this.meta.mcaptchaInstanceUrl) { await this.captchaService.verifyMcaptcha(this.meta.mcaptchaSecretKey, this.meta.mcaptchaSitekey, this.meta.mcaptchaInstanceUrl, body['m-captcha-response']).catch(err => { - throw new FastifyReplyError(400, err); + throw new FastifyReplyError(400, String(err), err); }); } if (this.meta.enableFC && this.meta.fcSecretKey) { await this.captchaService.verifyFriendlyCaptcha(this.meta.fcSecretKey, body['frc-captcha-solution']).catch(err => { - throw new FastifyReplyError(400, err); + throw new FastifyReplyError(400, String(err), err); }); } if (this.meta.enableRecaptcha && this.meta.recaptchaSecretKey) { await this.captchaService.verifyRecaptcha(this.meta.recaptchaSecretKey, body['g-recaptcha-response']).catch(err => { - throw new FastifyReplyError(400, err); + throw new FastifyReplyError(400, String(err), err); }); } if (this.meta.enableTurnstile && this.meta.turnstileSecretKey) { await this.captchaService.verifyTurnstile(this.meta.turnstileSecretKey, body['turnstile-response']).catch(err => { - throw new FastifyReplyError(400, err); + throw new FastifyReplyError(400, String(err), err); }); } if (this.meta.enableTestcaptcha) { await this.captchaService.verifyTestcaptcha(body['testcaptcha-response']).catch(err => { - throw new FastifyReplyError(400, err); + throw new FastifyReplyError(400, String(err), err); }); } } diff --git a/packages/backend/src/server/api/SigninWithPasskeyApiService.ts b/packages/backend/src/server/api/SigninWithPasskeyApiService.ts index f84f50523b..38886f8876 100644 --- a/packages/backend/src/server/api/SigninWithPasskeyApiService.ts +++ b/packages/backend/src/server/api/SigninWithPasskeyApiService.ts @@ -128,7 +128,7 @@ export class SigninWithPasskeyApiService { try { authorizedUserId = await this.webAuthnService.verifySignInWithPasskeyAuthentication(context, credential); } catch (err) { - this.logger.warn(`Passkey challenge Verify error! : ${err}`); + this.logger.warn('Passkey challenge verify error:', err as Error); const errorId = (err as IdentifiableError).id; return error(403, { id: errorId, diff --git a/packages/backend/src/server/api/SignupApiService.ts b/packages/backend/src/server/api/SignupApiService.ts index cb71047a24..81e3a5b706 100644 --- a/packages/backend/src/server/api/SignupApiService.ts +++ b/packages/backend/src/server/api/SignupApiService.ts @@ -83,37 +83,37 @@ export class SignupApiService { if (process.env.NODE_ENV !== 'test') { if (this.meta.enableHcaptcha && this.meta.hcaptchaSecretKey) { await this.captchaService.verifyHcaptcha(this.meta.hcaptchaSecretKey, body['hcaptcha-response']).catch(err => { - throw new FastifyReplyError(400, err); + throw new FastifyReplyError(400, String(err), err); }); } if (this.meta.enableMcaptcha && this.meta.mcaptchaSecretKey && this.meta.mcaptchaSitekey && this.meta.mcaptchaInstanceUrl) { await this.captchaService.verifyMcaptcha(this.meta.mcaptchaSecretKey, this.meta.mcaptchaSitekey, this.meta.mcaptchaInstanceUrl, body['m-captcha-response']).catch(err => { - throw new FastifyReplyError(400, err); + throw new FastifyReplyError(400, String(err), err); }); } if (this.meta.enableRecaptcha && this.meta.recaptchaSecretKey) { await this.captchaService.verifyRecaptcha(this.meta.recaptchaSecretKey, body['g-recaptcha-response']).catch(err => { - throw new FastifyReplyError(400, err); + throw new FastifyReplyError(400, String(err), err); }); } if (this.meta.enableTurnstile && this.meta.turnstileSecretKey) { await this.captchaService.verifyTurnstile(this.meta.turnstileSecretKey, body['turnstile-response']).catch(err => { - throw new FastifyReplyError(400, err); + throw new FastifyReplyError(400, String(err), err); }); } if (this.meta.enableFC && this.meta.fcSecretKey) { await this.captchaService.verifyFriendlyCaptcha(this.meta.fcSecretKey, body['frc-captcha-solution']).catch(err => { - throw new FastifyReplyError(400, err); + throw new FastifyReplyError(400, String(err), err); }); } if (this.meta.enableTestcaptcha) { await this.captchaService.verifyTestcaptcha(body['testcaptcha-response']).catch(err => { - throw new FastifyReplyError(400, err); + throw new FastifyReplyError(400, String(err), err); }); } } @@ -287,7 +287,7 @@ export class SignupApiService { token: secret, }; } catch (err) { - throw new FastifyReplyError(400, typeof err === 'string' ? err : (err as Error).toString()); + throw new FastifyReplyError(400, String(err), err); } } } @@ -356,7 +356,7 @@ export class SignupApiService { return this.signinService.signin(request, reply, account as MiLocalUser); } catch (err) { - throw new FastifyReplyError(400, typeof err === 'string' ? err : (err as Error).toString()); + throw new FastifyReplyError(400, String(err), err); } } } diff --git a/packages/backend/src/server/api/endpoints.ts b/packages/backend/src/server/api/endpoints.ts index 0ba041c536..c7d884cce1 100644 --- a/packages/backend/src/server/api/endpoints.ts +++ b/packages/backend/src/server/api/endpoints.ts @@ -92,7 +92,7 @@ export type IEndpointMeta = (Omit & { secure: true, }) | (Omit & { - requireCredential: true, + requireCredential: true | 'optional', kind: (typeof permissions)[number], }) | (Omit & { requireModerator: true, diff --git a/packages/backend/src/server/api/endpoints/admin/abuse-user-reports.ts b/packages/backend/src/server/api/endpoints/admin/abuse-user-reports.ts index 0dbfaae054..b8200c09aa 100644 --- a/packages/backend/src/server/api/endpoints/admin/abuse-user-reports.ts +++ b/packages/backend/src/server/api/endpoints/admin/abuse-user-reports.ts @@ -69,6 +69,11 @@ export const meta = { nullable: false, optional: false, ref: 'UserDetailedNotMe', }, + targetInstance: { + type: 'object', + nullable: true, optional: false, + ref: 'FederationInstance', + }, assignee: { type: 'object', nullable: true, optional: false, @@ -115,7 +120,15 @@ export default class extends Endpoint { // eslint- private queryService: QueryService, ) { super(meta, paramDef, async (ps, me) => { - const query = this.queryService.makePaginationQuery(this.abuseUserReportsRepository.createQueryBuilder('report'), ps.sinceId, ps.untilId); + const query = this.queryService.makePaginationQuery(this.abuseUserReportsRepository.createQueryBuilder('report'), ps.sinceId, ps.untilId) + .leftJoinAndSelect('report.targetUser', 'targetUser') + .leftJoinAndSelect('targetUser.userProfile', 'targetUserProfile') + .leftJoinAndSelect('report.targetUserInstance', 'targetUserInstance') + .leftJoinAndSelect('report.reporter', 'reporter') + .leftJoinAndSelect('reporter.userProfile', 'reporterProfile') + .leftJoinAndSelect('report.assignee', 'assignee') + .leftJoinAndSelect('assignee.userProfile', 'assigneeProfile') + ; switch (ps.state) { case 'resolved': query.andWhere('report.resolved = TRUE'); break; @@ -134,7 +147,7 @@ export default class extends Endpoint { // eslint- const reports = await query.limit(ps.limit).getMany(); - return await this.abuseUserReportEntityService.packMany(reports); + return await this.abuseUserReportEntityService.packMany(reports, me); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/relays/add.ts b/packages/backend/src/server/api/endpoints/admin/relays/add.ts index 129f69aca9..4644a069ee 100644 --- a/packages/backend/src/server/api/endpoints/admin/relays/add.ts +++ b/packages/backend/src/server/api/endpoints/admin/relays/add.ts @@ -68,11 +68,8 @@ export default class extends Endpoint { // eslint- private readonly moderationLogService: ModerationLogService, ) { super(meta, paramDef, async (ps, me) => { - try { - if (new URL(ps.inbox).protocol !== 'https:') throw new Error('https only'); - } catch { - throw new ApiError(meta.errors.invalidUrl); - } + if (!URL.canParse(ps.inbox)) throw new ApiError(meta.errors.invalidUrl); + if (new URL(ps.inbox).protocol !== 'https:') throw new ApiError(meta.errors.invalidUrl); await this.moderationLogService.log(me, 'addRelay', { inbox: ps.inbox, diff --git a/packages/backend/src/server/api/endpoints/admin/show-user.ts b/packages/backend/src/server/api/endpoints/admin/show-user.ts index 1579719246..6a77fc177f 100644 --- a/packages/backend/src/server/api/endpoints/admin/show-user.ts +++ b/packages/backend/src/server/api/endpoints/admin/show-user.ts @@ -122,6 +122,10 @@ export const meta = { type: 'boolean', optional: false, nullable: false, }, + isAdministrator: { + type: 'boolean', + optional: false, nullable: false, + }, isSystem: { type: 'boolean', optional: false, nullable: false, @@ -257,6 +261,7 @@ export default class extends Endpoint { // eslint- } const isModerator = await this.roleService.isModerator(user); + const isAdministrator = await this.roleService.isAdministrator(user); const isSilenced = user.isSilenced || !(await this.roleService.getUserPolicies(user.id)).canPublicNote; const _me = await this.usersRepository.findOneByOrFail({ id: me.id }); @@ -289,6 +294,7 @@ export default class extends Endpoint { // eslint- mutedInstances: profile.mutedInstances, notificationRecieveConfig: profile.notificationRecieveConfig, isModerator: isModerator, + isAdministrator: isAdministrator, isSystem: isSystemAccount(user), isSilenced: isSilenced, isSuspended: user.isSuspended, diff --git a/packages/backend/src/server/api/endpoints/antennas/notes.ts b/packages/backend/src/server/api/endpoints/antennas/notes.ts index cb5c25aaed..f8c12298cd 100644 --- a/packages/backend/src/server/api/endpoints/antennas/notes.ts +++ b/packages/backend/src/server/api/endpoints/antennas/notes.ts @@ -14,6 +14,7 @@ import { IdService } from '@/core/IdService.js'; import { FanoutTimelineService } from '@/core/FanoutTimelineService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import { trackPromise } from '@/misc/promise-tracker.js'; +import ActiveUsersChart from '@/core/chart/charts/active-users.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -75,6 +76,7 @@ export default class extends Endpoint { // eslint- private queryService: QueryService, private fanoutTimelineService: FanoutTimelineService, private globalEventService: GlobalEventService, + private readonly activeUsersChart: ActiveUsersChart, ) { super(meta, paramDef, async (ps, me) => { const untilId = ps.untilId ?? (ps.untilDate ? this.idService.gen(ps.untilDate!) : null); @@ -106,7 +108,8 @@ export default class extends Endpoint { // eslint- return []; } - const query = this.notesRepository.createQueryBuilder('note') + const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), + ps.sinceId, ps.untilId) .where('note.id IN (:...noteIds)', { noteIds: noteIds }) .innerJoinAndSelect('note.user', 'user') .leftJoinAndSelect('note.reply', 'reply') @@ -122,13 +125,13 @@ export default class extends Endpoint { // eslint- this.queryService.generateVisibilityQuery(query, me); this.queryService.generateMutedUserQueryForNotes(query, me); this.queryService.generateBlockedUserQueryForNotes(query, me); + this.queryService.generateMutedUserRenotesQueryForNotes(query, me); const notes = await query.getMany(); - if (sinceId != null && untilId == null) { - notes.sort((a, b) => a.id < b.id ? -1 : 1); - } else { - notes.sort((a, b) => a.id > b.id ? -1 : 1); - } + + process.nextTick(() => { + this.activeUsersChart.read(me); + }); return await this.noteEntityService.packMany(notes, me); }); diff --git a/packages/backend/src/server/api/endpoints/ap/get.ts b/packages/backend/src/server/api/endpoints/ap/get.ts index 14286bc23e..e3e68b50af 100644 --- a/packages/backend/src/server/api/endpoints/ap/get.ts +++ b/packages/backend/src/server/api/endpoints/ap/get.ts @@ -3,10 +3,17 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { Injectable } from '@nestjs/common'; +import { Inject, Injectable } from '@nestjs/common'; import ms from 'ms'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { ApResolverService } from '@/core/activitypub/ApResolverService.js'; +import { isCollectionOrOrderedCollection, isOrderedCollection, isOrderedCollectionPage } from '@/core/activitypub/type.js'; +import { ApiError } from '@/server/api/error.js'; +import { CacheService } from '@/core/CacheService.js'; +import { UserEntityService } from '@/core/entities/UserEntityService.js'; +import { DI } from '@/di-symbols.js'; +import type { NotesRepository } from '@/models/_.js'; +import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; export const meta = { tags: ['federation'], @@ -21,6 +28,16 @@ export const meta = { }, errors: { + noInputSpecified: { + message: 'uri, userId, or noteId must be specified.', + code: 'NO_INPUT_SPECIFIED', + id: 'b43ff2a7-e7a2-4237-ad7f-7b079563c09e', + }, + multipleInputsSpecified: { + message: 'Only one of uri, userId, or noteId can be specified', + code: 'MULTIPLE_INPUTS_SPECIFIED', + id: 'f1aa27ed-8f20-44f3-a92a-fe073c8ca52b', + }, }, res: { @@ -32,19 +49,57 @@ export const meta = { export const paramDef = { type: 'object', properties: { - uri: { type: 'string' }, + uri: { type: 'string', nullable: true }, + userId: { type: 'string', format: 'misskey:id', nullable: true }, + noteId: { type: 'string', format: 'misskey:id', nullable: true }, + expandCollectionItems: { type: 'boolean' }, + expandCollectionLimit: { type: 'integer', nullable: true }, + allowAnonymous: { type: 'boolean' }, }, - required: ['uri'], } as const; @Injectable() export default class extends Endpoint { // eslint-disable-line import/no-default-export constructor( + @Inject(DI.notesRepository) + private readonly notesRepository: NotesRepository, + + private readonly cacheService: CacheService, + private readonly userEntityService: UserEntityService, + private readonly noteEntityService: NoteEntityService, private apResolverService: ApResolverService, ) { - super(meta, paramDef, async (ps, me) => { + super(meta, paramDef, async (ps) => { + if (ps.uri && ps.userId && ps.noteId) { + throw new ApiError(meta.errors.multipleInputsSpecified); + } + + let uri: string; + if (ps.uri) { + uri = ps.uri; + } else if (ps.userId) { + const user = await this.cacheService.findUserById(ps.userId); + uri = user.uri ?? this.userEntityService.genLocalUserUri(ps.userId); + } else if (ps.noteId) { + const note = await this.notesRepository.findOneByOrFail({ id: ps.noteId }); + uri = note.uri ?? this.noteEntityService.genLocalNoteUri(ps.noteId); + } else { + throw new ApiError(meta.errors.noInputSpecified); + } + const resolver = this.apResolverService.createResolver(); - const object = await resolver.resolve(ps.uri); + const object = await resolver.resolve(uri, ps.allowAnonymous ?? false); + + if (ps.expandCollectionItems && isCollectionOrOrderedCollection(object)) { + const items = await resolver.resolveCollectionItems(object, ps.expandCollectionLimit, ps.allowAnonymous ?? false); + + if (isOrderedCollection(object) || isOrderedCollectionPage(object)) { + object.orderedItems = items; + } else { + object.items = items; + } + } + return object; }); } diff --git a/packages/backend/src/server/api/endpoints/ap/show.ts b/packages/backend/src/server/api/endpoints/ap/show.ts index d69850515c..d631b002cc 100644 --- a/packages/backend/src/server/api/endpoints/ap/show.ts +++ b/packages/backend/src/server/api/endpoints/ap/show.ts @@ -173,6 +173,10 @@ export default class extends Endpoint { // eslint- case '09d79f9e-64f1-4316-9cfa-e75c4d091574': throw new ApiError(meta.errors.federationNotAllowed); case '72180409-793c-4973-868e-5a118eb5519b': + case 'd09dc850-b76c-4f45-875a-7389339d78b8': + case 'dc110060-a5f2-461d-808b-39c62702ca64': + case '45793ab7-7648-4886-b503-429f8a0d0f73': + case '4bf8f36b-4d33-4ac9-ad76-63fa11f354e9': throw new ApiError(meta.errors.responseInvalid); // resolveLocal diff --git a/packages/backend/src/server/api/endpoints/channels/timeline.ts b/packages/backend/src/server/api/endpoints/channels/timeline.ts index dc42566b71..e675466196 100644 --- a/packages/backend/src/server/api/endpoints/channels/timeline.ts +++ b/packages/backend/src/server/api/endpoints/channels/timeline.ts @@ -96,7 +96,11 @@ export default class extends Endpoint { // eslint- throw new ApiError(meta.errors.noSuchChannel); } - if (me) this.activeUsersChart.read(me); + if (me) { + process.nextTick(() => { + this.activeUsersChart.read(me); + }); + } if (!this.serverSettings.enableFanoutTimeline) { return await this.noteEntityService.packMany(await this.getFromDb({ untilId, sinceId, limit: ps.limit, channelId: channel.id, withFiles: ps.withFiles, withRenotes: ps.withRenotes }, me), me); @@ -135,30 +139,29 @@ export default class extends Endpoint { // eslint- .leftJoinAndSelect('note.renote', 'renote') .leftJoinAndSelect('reply.user', 'replyUser') .leftJoinAndSelect('renote.user', 'renoteUser') - .leftJoinAndSelect('note.channel', 'channel'); + .leftJoinAndSelect('note.channel', 'channel') + .limit(ps.limit); + this.queryService.generateVisibilityQuery(query, me); this.queryService.generateBlockedHostQueryForNote(query); this.queryService.generateSuspendedUserQueryForNote(query); + this.queryService.generateSilencedUserQueryForNotes(query, me); if (me) { this.queryService.generateMutedUserQueryForNotes(query, me); this.queryService.generateBlockedUserQueryForNotes(query, me); } - if (ps.withRenotes === false) { - query.andWhere(new Brackets(qb => { - qb.orWhere('note.renoteId IS NULL'); - qb.orWhere(new Brackets(qb => { - qb.orWhere('note.text IS NOT NULL'); - qb.orWhere('note.fileIds != \'{}\''); - })); - })); - } - if (ps.withFiles) { query.andWhere('note.fileIds != \'{}\''); } + + if (!ps.withRenotes) { + this.queryService.generateExcludedRenotesQueryForNotes(query); + } else if (me) { + this.queryService.generateMutedUserRenotesQueryForNotes(query, me); + } //#endregion - return await query.limit(ps.limit).getMany(); + return await query.getMany(); } } diff --git a/packages/backend/src/server/api/endpoints/charts/active-users.ts b/packages/backend/src/server/api/endpoints/charts/active-users.ts index dcdcf46d0b..9f5064fe83 100644 --- a/packages/backend/src/server/api/endpoints/charts/active-users.ts +++ b/packages/backend/src/server/api/endpoints/charts/active-users.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/charts/ap-request.ts b/packages/backend/src/server/api/endpoints/charts/ap-request.ts index 28c64229e7..68dc87546e 100644 --- a/packages/backend/src/server/api/endpoints/charts/ap-request.ts +++ b/packages/backend/src/server/api/endpoints/charts/ap-request.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/charts/drive.ts b/packages/backend/src/server/api/endpoints/charts/drive.ts index 69ff3c5d7a..c0bfb00608 100644 --- a/packages/backend/src/server/api/endpoints/charts/drive.ts +++ b/packages/backend/src/server/api/endpoints/charts/drive.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/charts/federation.ts b/packages/backend/src/server/api/endpoints/charts/federation.ts index bd870cc3d9..bd15700670 100644 --- a/packages/backend/src/server/api/endpoints/charts/federation.ts +++ b/packages/backend/src/server/api/endpoints/charts/federation.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/charts/instance.ts b/packages/backend/src/server/api/endpoints/charts/instance.ts index 765bf024ee..e1053d05d8 100644 --- a/packages/backend/src/server/api/endpoints/charts/instance.ts +++ b/packages/backend/src/server/api/endpoints/charts/instance.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/charts/notes.ts b/packages/backend/src/server/api/endpoints/charts/notes.ts index ecac436311..4550e2f17e 100644 --- a/packages/backend/src/server/api/endpoints/charts/notes.ts +++ b/packages/backend/src/server/api/endpoints/charts/notes.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/charts/user/drive.ts b/packages/backend/src/server/api/endpoints/charts/user/drive.ts index 98ec40ade2..9475a8ab0a 100644 --- a/packages/backend/src/server/api/endpoints/charts/user/drive.ts +++ b/packages/backend/src/server/api/endpoints/charts/user/drive.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/charts/user/following.ts b/packages/backend/src/server/api/endpoints/charts/user/following.ts index cb3dd36bab..20d0ecb25d 100644 --- a/packages/backend/src/server/api/endpoints/charts/user/following.ts +++ b/packages/backend/src/server/api/endpoints/charts/user/following.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/charts/user/notes.ts b/packages/backend/src/server/api/endpoints/charts/user/notes.ts index 0742a21210..1d24dc2b77 100644 --- a/packages/backend/src/server/api/endpoints/charts/user/notes.ts +++ b/packages/backend/src/server/api/endpoints/charts/user/notes.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/charts/user/pv.ts b/packages/backend/src/server/api/endpoints/charts/user/pv.ts index a220381b00..e0026d5ff3 100644 --- a/packages/backend/src/server/api/endpoints/charts/user/pv.ts +++ b/packages/backend/src/server/api/endpoints/charts/user/pv.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/charts/user/reactions.ts b/packages/backend/src/server/api/endpoints/charts/user/reactions.ts index 3bb33622c2..c15056466f 100644 --- a/packages/backend/src/server/api/endpoints/charts/user/reactions.ts +++ b/packages/backend/src/server/api/endpoints/charts/user/reactions.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/charts/users.ts b/packages/backend/src/server/api/endpoints/charts/users.ts index b5452517ab..0f96fae202 100644 --- a/packages/backend/src/server/api/endpoints/charts/users.ts +++ b/packages/backend/src/server/api/endpoints/charts/users.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/clips/notes.ts b/packages/backend/src/server/api/endpoints/clips/notes.ts index 27fc98ba2e..a238d7d002 100644 --- a/packages/backend/src/server/api/endpoints/clips/notes.ts +++ b/packages/backend/src/server/api/endpoints/clips/notes.ts @@ -97,6 +97,7 @@ export default class extends Endpoint { // eslint- if (me) { this.queryService.generateMutedUserQueryForNotes(query, me); this.queryService.generateBlockedUserQueryForNotes(query, me); + this.queryService.generateMutedUserRenotesQueryForNotes(query, me); } const notes = await query diff --git a/packages/backend/src/server/api/endpoints/drive/files/attached-notes.ts b/packages/backend/src/server/api/endpoints/drive/files/attached-notes.ts index 32c2620915..9d70044db8 100644 --- a/packages/backend/src/server/api/endpoints/drive/files/attached-notes.ts +++ b/packages/backend/src/server/api/endpoints/drive/files/attached-notes.ts @@ -81,10 +81,22 @@ export default class extends Endpoint { // eslint- throw new ApiError(meta.errors.noSuchFile); } - const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId); - query.andWhere(':file <@ note.fileIds', { file: [file.id] }); + const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId) + .andWhere(':file <@ note.fileIds', { file: [file.id] }) + .innerJoinAndSelect('note.user', 'user') + .leftJoinAndSelect('note.reply', 'reply') + .leftJoinAndSelect('note.renote', 'renote') + .leftJoinAndSelect('reply.user', 'replyUser') + .leftJoinAndSelect('renote.user', 'renoteUser') + .limit(ps.limit); - const notes = await query.limit(ps.limit).getMany(); + this.queryService.generateVisibilityQuery(query, me); + this.queryService.generateBlockedHostQueryForNote(query); + this.queryService.generateSilencedUserQueryForNotes(query, me); + this.queryService.generateMutedUserQueryForNotes(query, me); + this.queryService.generateBlockedUserQueryForNotes(query, me); + + const notes = await query.getMany(); return await this.noteEntityService.packMany(notes, me, { detail: true, diff --git a/packages/backend/src/server/api/endpoints/drive/files/create.ts b/packages/backend/src/server/api/endpoints/drive/files/create.ts index f4c47d71bf..939eadad9b 100644 --- a/packages/backend/src/server/api/endpoints/drive/files/create.ts +++ b/packages/backend/src/server/api/endpoints/drive/files/create.ts @@ -10,6 +10,8 @@ import { Endpoint } from '@/server/api/endpoint-base.js'; import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js'; import { DriveService } from '@/core/DriveService.js'; import type { Config } from '@/config.js'; +import { ApiLoggerService } from '@/server/api/ApiLoggerService.js'; +import { renderInlineError } from '@/misc/render-inline-error.js'; import { ApiError } from '../../../error.js'; import { MiMeta } from '@/models/_.js'; import { DI } from '@/di-symbols.js'; @@ -95,6 +97,7 @@ export default class extends Endpoint { // eslint- private driveFileEntityService: DriveFileEntityService, private driveService: DriveService, + private readonly apiLoggerService: ApiLoggerService, ) { super(meta, paramDef, async (ps, me, _, file, cleanup, ip, headers) => { // Get 'name' parameter @@ -130,14 +133,14 @@ export default class extends Endpoint { // eslint- return await this.driveFileEntityService.pack(driveFile, { self: true }); } catch (err) { if (err instanceof Error || typeof err === 'string') { - console.error(err); + this.apiLoggerService.logger.error(`Error saving drive file: ${renderInlineError(err)}`); } if (err instanceof IdentifiableError) { if (err.id === '282f77bf-5816-4f72-9264-aa14d8261a21') throw new ApiError(meta.errors.inappropriate); if (err.id === 'c6244ed2-a39a-4e1c-bf93-f0fbd7764fa6') throw new ApiError(meta.errors.noFreeSpace); if (err.id === 'f9e4e5f3-4df4-40b5-b400-f236945f7073') throw new ApiError(meta.errors.maxFileSizeExceeded); } - throw new ApiError(); + throw err; } finally { cleanup!(); } diff --git a/packages/backend/src/server/api/endpoints/gallery/posts/create.ts b/packages/backend/src/server/api/endpoints/gallery/posts/create.ts index 504a9c789e..08abd7fed5 100644 --- a/packages/backend/src/server/api/endpoints/gallery/posts/create.ts +++ b/packages/backend/src/server/api/endpoints/gallery/posts/create.ts @@ -72,7 +72,7 @@ export default class extends Endpoint { // eslint- ))).filter(x => x != null); if (files.length === 0) { - throw new Error(); + throw new Error('no files specified'); } const post = await this.galleryPostsRepository.insertOne(new MiGalleryPost({ diff --git a/packages/backend/src/server/api/endpoints/gallery/posts/update.ts b/packages/backend/src/server/api/endpoints/gallery/posts/update.ts index 5243ee9603..d0f9b56863 100644 --- a/packages/backend/src/server/api/endpoints/gallery/posts/update.ts +++ b/packages/backend/src/server/api/endpoints/gallery/posts/update.ts @@ -73,7 +73,7 @@ export default class extends Endpoint { // eslint- ))).filter(x => x != null); if (files.length === 0) { - throw new Error(); + throw new Error('no files'); } } diff --git a/packages/backend/src/server/api/endpoints/i/2fa/remove-key.ts b/packages/backend/src/server/api/endpoints/i/2fa/remove-key.ts index d4098458d7..931c8d69b0 100644 --- a/packages/backend/src/server/api/endpoints/i/2fa/remove-key.ts +++ b/packages/backend/src/server/api/endpoints/i/2fa/remove-key.ts @@ -70,7 +70,7 @@ export default class extends Endpoint { // eslint- try { await this.userAuthService.twoFactorAuthenticate(profile, token); } catch (e) { - throw new Error('authentication failed'); + throw new Error('authentication failed', { cause: e }); } } diff --git a/packages/backend/src/server/api/endpoints/i/move.ts b/packages/backend/src/server/api/endpoints/i/move.ts index 7852b5a2e1..e2a14b61af 100644 --- a/packages/backend/src/server/api/endpoints/i/move.ts +++ b/packages/backend/src/server/api/endpoints/i/move.ts @@ -17,7 +17,7 @@ import { ApiLoggerService } from '@/server/api/ApiLoggerService.js'; import { GetterService } from '@/server/api/GetterService.js'; import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; - +import { renderInlineError } from '@/misc/render-inline-error.js'; import * as Acct from '@/misc/acct.js'; import { DI } from '@/di-symbols.js'; import { MiMeta } from '@/models/_.js'; @@ -105,7 +105,7 @@ export default class extends Endpoint { // eslint- const { username, host } = Acct.parse(ps.moveToAccount); // retrieve the destination account let moveTo = await this.remoteUserResolveService.resolveUser(username, host).catch((e) => { - this.apiLoggerService.logger.warn(`failed to resolve remote user: ${e}`); + this.apiLoggerService.logger.warn(`failed to resolve remote user: ${renderInlineError(e)}`); throw new ApiError(meta.errors.noSuchUser); }); const destination = await this.getterService.getUser(moveTo.id) as MiLocalUser | MiRemoteUser; diff --git a/packages/backend/src/server/api/endpoints/i/registry/get.ts b/packages/backend/src/server/api/endpoints/i/registry/get.ts index dff33016e0..d284334834 100644 --- a/packages/backend/src/server/api/endpoints/i/registry/get.ts +++ b/packages/backend/src/server/api/endpoints/i/registry/get.ts @@ -20,9 +20,7 @@ export const meta = { }, }, - res: { - type: 'object', - }, + res: {}, // 10 calls per 5 seconds limit: { diff --git a/packages/backend/src/server/api/endpoints/i/update.ts b/packages/backend/src/server/api/endpoints/i/update.ts index f35e395841..dda42ce0e4 100644 --- a/packages/backend/src/server/api/endpoints/i/update.ts +++ b/packages/backend/src/server/api/endpoints/i/update.ts @@ -34,6 +34,7 @@ import { verifyFieldLinks } from '@/misc/verify-field-link.js'; import { AvatarDecorationService } from '@/core/AvatarDecorationService.js'; import { notificationRecieveConfig } from '@/models/json-schema/user.js'; import { userUnsignedFetchOptions } from '@/const.js'; +import { renderInlineError } from '@/misc/render-inline-error.js'; import { ApiLoggerService } from '../../ApiLoggerService.js'; import { ApiError } from '../../error.js'; @@ -263,6 +264,15 @@ export const paramDef = { enum: userUnsignedFetchOptions, nullable: false, }, + attributionDomains: { + type: 'array', + items: { + type: 'string', + minLength: 1, + maxLength: 128, + }, + maxItems: 32, + }, }, } as const; @@ -373,6 +383,7 @@ export default class extends Endpoint { // eslint- } if (ps.mutedInstances !== undefined) profileUpdates.mutedInstances = ps.mutedInstances; if (ps.notificationRecieveConfig !== undefined) profileUpdates.notificationRecieveConfig = ps.notificationRecieveConfig; + if (ps.attributionDomains !== undefined) updates.attributionDomains = ps.attributionDomains; if (typeof ps.isLocked === 'boolean') updates.isLocked = ps.isLocked; if (typeof ps.isExplorable === 'boolean') updates.isExplorable = ps.isExplorable; if (typeof ps.hideOnlineStatus === 'boolean') updates.hideOnlineStatus = ps.hideOnlineStatus; @@ -506,7 +517,7 @@ export default class extends Endpoint { // eslint- // Retrieve the old account const knownAs = await this.remoteUserResolveService.resolveUser(username, host).catch((e) => { - this.apiLoggerService.logger.warn(`failed to resolve dstination user: ${e}`); + this.apiLoggerService.logger.warn(`failed to resolve destination user: ${renderInlineError(e)}`); throw new ApiError(meta.errors.noSuchUser); }); if (knownAs.id === _user.id) throw new ApiError(meta.errors.forbiddenToSetYourself); @@ -663,7 +674,7 @@ export default class extends Endpoint { // eslint- // these two methods need to be kept in sync with // `ApRendererService.renderPerson` private userNeedsPublishing(oldUser: MiLocalUser, newUser: Partial): boolean { - const basicFields: (keyof MiUser)[] = ['avatarId', 'bannerId', 'backgroundId', 'isBot', 'username', 'name', 'isLocked', 'isExplorable', 'isCat', 'noindex', 'speakAsCat', 'movedToUri', 'alsoKnownAs', 'hideOnlineStatus', 'enableRss', 'requireSigninToViewContents', 'makeNotesFollowersOnlyBefore', 'makeNotesHiddenBefore']; + const basicFields: (keyof MiUser)[] = ['avatarId', 'bannerId', 'backgroundId', 'isBot', 'username', 'name', 'isLocked', 'isExplorable', 'isCat', 'noindex', 'speakAsCat', 'movedToUri', 'alsoKnownAs', 'hideOnlineStatus', 'enableRss', 'requireSigninToViewContents', 'makeNotesFollowersOnlyBefore', 'makeNotesHiddenBefore', 'attributionDomains']; for (const field of basicFields) { if ((field in newUser) && oldUser[field] !== newUser[field]) { return true; diff --git a/packages/backend/src/server/api/endpoints/notes.ts b/packages/backend/src/server/api/endpoints/notes.ts index f6c37023e1..00a88521fd 100644 --- a/packages/backend/src/server/api/endpoints/notes.ts +++ b/packages/backend/src/server/api/endpoints/notes.ts @@ -64,7 +64,16 @@ export default class extends Endpoint { // eslint- .leftJoinAndSelect('note.reply', 'reply') .leftJoinAndSelect('note.renote', 'renote') .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser'); + .leftJoinAndSelect('renote.user', 'renoteUser') + .limit(ps.limit); + + this.queryService.generateVisibilityQuery(query, me); + this.queryService.generateBlockedHostQueryForNote(query); + if (me) { + this.queryService.generateSilencedUserQueryForNotes(query, me); + this.queryService.generateMutedUserQueryForNotes(query, me); + this.queryService.generateBlockedUserQueryForNotes(query, me); + } if (ps.local) { query.andWhere('note.userHost IS NULL'); @@ -75,7 +84,15 @@ export default class extends Endpoint { // eslint- } if (ps.renote !== undefined) { - query.andWhere(ps.renote ? 'note.renoteId IS NOT NULL' : 'note.renoteId IS NULL'); + if (ps.renote) { + this.queryService.andIsRenote(query, 'note'); + + if (me) { + this.queryService.generateMutedUserRenotesQueryForNotes(query, me); + } + } else { + this.queryService.andIsNotRenote(query, 'note'); + } } if (ps.withFiles !== undefined) { @@ -91,7 +108,7 @@ export default class extends Endpoint { // eslint- // query.isBot = bot; //} - const notes = await query.limit(ps.limit).getMany(); + const notes = await query.getMany(); return await this.noteEntityService.packMany(notes); }); diff --git a/packages/backend/src/server/api/endpoints/notes/bubble-timeline.ts b/packages/backend/src/server/api/endpoints/notes/bubble-timeline.ts index df030d90aa..84d6aa0dc7 100644 --- a/packages/backend/src/server/api/endpoints/notes/bubble-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/bubble-timeline.ts @@ -1,13 +1,16 @@ +/* + * SPDX-FileCopyrightText: Marie and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + import { Inject, Injectable } from '@nestjs/common'; -import { Brackets } from 'typeorm'; -import type { NotesRepository, MiMeta } from '@/models/_.js'; +import type { NotesRepository } from '@/models/_.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { QueryService } from '@/core/QueryService.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import ActiveUsersChart from '@/core/chart/charts/active-users.js'; import { DI } from '@/di-symbols.js'; import { RoleService } from '@/core/RoleService.js'; -import { CacheService } from '@/core/CacheService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -56,9 +59,6 @@ export const paramDef = { @Injectable() export default class extends Endpoint { // eslint-disable-line import/no-default-export constructor( - @Inject(DI.meta) - private serverSettings: MiMeta, - @Inject(DI.notesRepository) private notesRepository: NotesRepository, @@ -66,7 +66,6 @@ export default class extends Endpoint { // eslint- private queryService: QueryService, private roleService: RoleService, private activeUsersChart: ActiveUsersChart, - private cacheService: CacheService, ) { super(meta, paramDef, async (ps, me) => { const policies = await this.roleService.getUserPolicies(me ? me.id : null); @@ -74,29 +73,34 @@ export default class extends Endpoint { // eslint- throw new ApiError(meta.errors.btlDisabled); } - const [ - followings, - ] = me ? await Promise.all([ - this.cacheService.userFollowingsCache.fetch(me.id), - ]) : [undefined]; - //#region Construct query const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate) .andWhere('note.visibility = \'public\'') .andWhere('note.channelId IS NULL') - .andWhere('note.userHost IN (:...hosts)', { hosts: this.serverSettings.bubbleInstances }) + .andWhere('note.userHost IS NOT NULL') .innerJoinAndSelect('note.user', 'user') .leftJoinAndSelect('note.reply', 'reply') .leftJoinAndSelect('note.renote', 'renote') .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser'); + .leftJoinAndSelect('renote.user', 'renoteUser') + .limit(ps.limit); + + // This subquery mess teaches postgres how to use the right indexes. + // Using WHERE or ON conditions causes a fallback to full sequence scan, which times out. + // Important: don't use a query builder here or TypeORM will get confused and stop quoting column names! (known, unfixed bug apparently) + query + .leftJoin('(select "host" from "instance" where "isBubbled" = true)', 'bubbleInstance', '"bubbleInstance"."host" = "note"."userHost"') + .andWhere('"bubbleInstance" IS NOT NULL'); + this.queryService + .leftJoinInstance(query, 'note.userInstance', 'userInstance', '"userInstance"."host" = "bubbleInstance"."host"'); - this.queryService.generateVisibilityQuery(query, me); this.queryService.generateBlockedHostQueryForNote(query); - if (me) this.queryService.generateMutedUserQueryForNotes(query, me); - if (me) this.queryService.generateBlockedUserQueryForNotes(query, me); - if (me) this.queryService.generateMutedUserRenotesQueryForNotes(query, me); + this.queryService.generateSilencedUserQueryForNotes(query, me); + if (me) { + this.queryService.generateMutedUserQueryForNotes(query, me); + this.queryService.generateBlockedUserQueryForNotes(query, me); + } if (ps.withFiles) { query.andWhere('note.fileIds != \'{}\''); @@ -104,29 +108,20 @@ export default class extends Endpoint { // eslint- if (!ps.withBots) query.andWhere('user.isBot = FALSE'); - if (ps.withRenotes === false) { - query.andWhere(new Brackets(qb => { - qb.where('note.renoteId IS NULL'); - qb.orWhere(new Brackets(qb => { - qb.where('note.text IS NOT NULL'); - qb.orWhere('note.fileIds != \'{}\''); - })); - })); + if (!ps.withRenotes) { + this.queryService.generateExcludedRenotesQueryForNotes(query); + } else if (me) { + this.queryService.generateMutedUserRenotesQueryForNotes(query, me); } //#endregion - let timeline = await query.limit(ps.limit).getMany(); + const timeline = await query.getMany(); - timeline = timeline.filter(note => { - if (note.user?.isSilenced && me && followings && note.userId !== me.id && !followings[note.userId]) return false; - return true; - }); - - process.nextTick(() => { - if (me) { + if (me) { + process.nextTick(() => { this.activeUsersChart.read(me); - } - }); + }); + } return await this.noteEntityService.packMany(timeline, me); }); diff --git a/packages/backend/src/server/api/endpoints/notes/children.ts b/packages/backend/src/server/api/endpoints/notes/children.ts index 97fc454b34..22b064d2a0 100644 --- a/packages/backend/src/server/api/endpoints/notes/children.ts +++ b/packages/backend/src/server/api/endpoints/notes/children.ts @@ -57,26 +57,22 @@ export default class extends Endpoint { // eslint- super(meta, paramDef, async (ps, me) => { const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId) .andWhere(new Brackets(qb => { - qb - .where('note.replyId = :noteId', { noteId: ps.noteId }); - if (ps.showQuotes) { - qb.orWhere(new Brackets(qb => { - qb - .where('note.renoteId = :noteId', { noteId: ps.noteId }) - .andWhere(new Brackets(qb => { - qb - .where('note.text IS NOT NULL') - .orWhere('note.fileIds != \'{}\'') - .orWhere('note.hasPoll = TRUE'); - })); - })); - } + qb.orWhere('note.replyId = :noteId'); + + if (ps.showQuotes) { + qb.orWhere(new Brackets(qbb => this.queryService + .andIsQuote(qbb, 'note') + .andWhere('note.renoteId = :noteId'), + )); + } })) .innerJoinAndSelect('note.user', 'user') .leftJoinAndSelect('note.reply', 'reply') .leftJoinAndSelect('note.renote', 'renote') .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser'); + .leftJoinAndSelect('renote.user', 'renoteUser') + .setParameters({ noteId: ps.noteId }) + .limit(ps.limit); this.queryService.generateVisibilityQuery(query, me); this.queryService.generateBlockedHostQueryForNote(query); @@ -86,7 +82,7 @@ export default class extends Endpoint { // eslint- this.queryService.generateBlockedUserQueryForNotes(query, me); } - const notes = await query.limit(ps.limit).getMany(); + const notes = await query.getMany(); return await this.noteEntityService.packMany(notes, me); }); diff --git a/packages/backend/src/server/api/endpoints/notes/following.ts b/packages/backend/src/server/api/endpoints/notes/following.ts index 5f6ee9f903..0f8c61ab3e 100644 --- a/packages/backend/src/server/api/endpoints/notes/following.ts +++ b/packages/backend/src/server/api/endpoints/notes/following.ts @@ -4,7 +4,7 @@ */ import { Inject, Injectable } from '@nestjs/common'; -import { ObjectLiteral, SelectQueryBuilder } from 'typeorm'; +import { IsNull, ObjectLiteral, SelectQueryBuilder } from 'typeorm'; import { SkLatestNote, MiFollowing } from '@/models/_.js'; import type { NotesRepository } from '@/models/_.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; @@ -12,6 +12,7 @@ import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import { DI } from '@/di-symbols.js'; import { QueryService } from '@/core/QueryService.js'; import { ApiError } from '@/server/api/error.js'; +import ActiveUsersChart from '@/core/chart/charts/active-users.js'; export const meta = { tags: ['notes'], @@ -76,8 +77,9 @@ export default class extends Endpoint { // eslint- @Inject(DI.notesRepository) private notesRepository: NotesRepository, - private noteEntityService: NoteEntityService, - private queryService: QueryService, + private readonly noteEntityService: NoteEntityService, + private readonly queryService: QueryService, + private readonly activeUsersChart: ActiveUsersChart, ) { super(meta, paramDef, async (ps, me) => { if (ps.includeReplies && ps.filesOnly) throw new ApiError(meta.errors.bothWithRepliesAndWithFiles); @@ -85,7 +87,7 @@ export default class extends Endpoint { // eslint- const query = this.notesRepository .createQueryBuilder('note') - .setParameter('me', me.id) + .setParameters({ meId: me.id }) // Limit to latest notes .innerJoin( @@ -130,7 +132,9 @@ export default class extends Endpoint { // eslint- .leftJoinAndSelect('note.renote', 'renote') .leftJoinAndSelect('reply.user', 'replyUser') .leftJoinAndSelect('renote.user', 'renoteUser') - .leftJoinAndSelect('note.channel', 'channel') + + // Exclude channel notes + .andWhere({ channelId: IsNull() }) ; // Limit to files, if requested @@ -145,23 +149,26 @@ export default class extends Endpoint { // eslint- // Hide blocked users / instances query.andWhere('"user"."isSuspended" = false'); - query.andWhere('("replyUser" IS NULL OR "replyUser"."isSuspended" = false)'); - query.andWhere('("renoteUser" IS NULL OR "renoteUser"."isSuspended" = false)'); this.queryService.generateBlockedHostQueryForNote(query); - // Respect blocks and mutes + // Respect blocks, mutes, and privacy + this.queryService.generateVisibilityQuery(query, me); this.queryService.generateBlockedUserQueryForNotes(query, me); this.queryService.generateMutedUserQueryForNotes(query, me); // Support pagination this.queryService .makePaginationQuery(query, ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate) - .orderBy('note.id', 'DESC') .take(ps.limit); // Query and return the next page const notes = await query.getMany(); - return await this.noteEntityService.packMany(notes, me); + + process.nextTick(() => { + this.activeUsersChart.read(me); + }); + + return await this.noteEntityService.packMany(notes, me, { skipHide: true }); }); } } @@ -170,14 +177,14 @@ export default class extends Endpoint { // eslint- * Limit to followers (they follow us) */ function addFollower>(query: T): T { - return query.innerJoin(MiFollowing, 'follower', 'follower."followerId" = latest.user_id AND follower."followeeId" = :me'); + return query.innerJoin(MiFollowing, 'follower', 'follower."followerId" = latest.user_id AND follower."followeeId" = :meId'); } /** * Limit to followees (we follow them) */ function addFollowee>(query: T): T { - return query.innerJoin(MiFollowing, 'followee', 'followee."followerId" = :me AND followee."followeeId" = latest.user_id'); + return query.innerJoin(MiFollowing, 'followee', 'followee."followerId" = :meId AND followee."followeeId" = latest.user_id'); } /** diff --git a/packages/backend/src/server/api/endpoints/notes/global-timeline.ts b/packages/backend/src/server/api/endpoints/notes/global-timeline.ts index e82d9ca7af..506ea6fcda 100644 --- a/packages/backend/src/server/api/endpoints/notes/global-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/global-timeline.ts @@ -12,7 +12,6 @@ import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import ActiveUsersChart from '@/core/chart/charts/active-users.js'; import { DI } from '@/di-symbols.js'; import { RoleService } from '@/core/RoleService.js'; -import { CacheService } from '@/core/CacheService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -68,7 +67,6 @@ export default class extends Endpoint { // eslint- private queryService: QueryService, private roleService: RoleService, private activeUsersChart: ActiveUsersChart, - private cacheService: CacheService, ) { super(meta, paramDef, async (ps, me) => { const policies = await this.roleService.getUserPolicies(me ? me.id : null); @@ -76,8 +74,6 @@ export default class extends Endpoint { // eslint- throw new ApiError(meta.errors.gtlDisabled); } - const followings = me ? await this.cacheService.userFollowingsCache.fetch(me.id) : {}; - //#region Construct query const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate) @@ -90,11 +86,10 @@ export default class extends Endpoint { // eslint- .leftJoinAndSelect('renote.user', 'renoteUser'); this.queryService.generateBlockedHostQueryForNote(query); - + this.queryService.generateSilencedUserQueryForNotes(query, me); if (me) { this.queryService.generateMutedUserQueryForNotes(query, me); this.queryService.generateBlockedUserQueryForNotes(query, me); - this.queryService.generateMutedUserRenotesQueryForNotes(query, me); } if (ps.withFiles) { @@ -103,29 +98,20 @@ export default class extends Endpoint { // eslint- if (!ps.withBots) query.andWhere('user.isBot = FALSE'); - if (ps.withRenotes === false) { - query.andWhere(new Brackets(qb => { - qb.where('note.renoteId IS NULL'); - qb.orWhere(new Brackets(qb => { - qb.where('note.text IS NOT NULL'); - qb.orWhere('note.fileIds != \'{}\''); - })); - })); + if (!ps.withRenotes) { + this.queryService.generateExcludedRenotesQueryForNotes(query); + } else if (me) { + this.queryService.generateMutedUserRenotesQueryForNotes(query, me); } //#endregion - let timeline = await query.limit(ps.limit).getMany(); + const timeline = await query.limit(ps.limit).getMany(); - timeline = timeline.filter(note => { - if (note.user?.isSilenced && me && followings && note.userId !== me.id && !followings[note.userId]) return false; - return true; - }); - - process.nextTick(() => { - if (me) { + if (me) { + process.nextTick(() => { this.activeUsersChart.read(me); - } - }); + }); + } return await this.noteEntityService.packMany(timeline, me); }); diff --git a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts index 184d1807d5..613cf7b9b4 100644 --- a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts @@ -66,9 +66,6 @@ export const paramDef = { sinceDate: { type: 'integer' }, untilDate: { type: 'integer' }, allowPartial: { type: 'boolean', default: false }, // true is recommended but for compatibility false by default - includeMyRenotes: { type: 'boolean', default: true }, - includeRenotedMyNotes: { type: 'boolean', default: true }, - includeLocalRenotes: { type: 'boolean', default: true }, withFiles: { type: 'boolean', default: false }, withRenotes: { type: 'boolean', default: true }, withReplies: { type: 'boolean', default: false }, @@ -114,12 +111,10 @@ export default class extends Endpoint { // eslint- untilId, sinceId, limit: ps.limit, - includeMyRenotes: ps.includeMyRenotes, - includeRenotedMyNotes: ps.includeRenotedMyNotes, - includeLocalRenotes: ps.includeLocalRenotes, withFiles: ps.withFiles, withReplies: ps.withReplies, withBots: ps.withBots, + withRenotes: ps.withRenotes, }, me); process.nextTick(() => { @@ -178,12 +173,10 @@ export default class extends Endpoint { // eslint- untilId, sinceId, limit, - includeMyRenotes: ps.includeMyRenotes, - includeRenotedMyNotes: ps.includeRenotedMyNotes, - includeLocalRenotes: ps.includeLocalRenotes, withFiles: ps.withFiles, withReplies: ps.withReplies, withBots: ps.withBots, + withRenotes: ps.withRenotes, }, me), }); @@ -199,104 +192,59 @@ export default class extends Endpoint { // eslint- untilId: string | null, sinceId: string | null, limit: number, - includeMyRenotes: boolean, - includeRenotedMyNotes: boolean, - includeLocalRenotes: boolean, withFiles: boolean, withReplies: boolean, withBots: boolean, + withRenotes: boolean, }, me: MiLocalUser) { - const followees = await this.userFollowingService.getFollowees(me.id); - const followingChannels = await this.channelFollowingsRepository.find({ - where: { - followerId: me.id, - }, - }); - const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId) - .andWhere(new Brackets(qb => { - if (followees.length > 0) { - const meOrFolloweeIds = [me.id, ...followees.map(f => f.followeeId)]; - qb.where('note.userId IN (:...meOrFolloweeIds)', { meOrFolloweeIds: meOrFolloweeIds }); - qb.orWhere('(note.visibility = \'public\') AND (note.userHost IS NULL)'); - } else { - qb.where('note.userId = :meId', { meId: me.id }); - qb.orWhere('(note.visibility = \'public\') AND (note.userHost IS NULL)'); - } - })) + // 1. by a user I follow, 2. a public local post, 3. my own post + .andWhere(new Brackets(qb => this.queryService + .orFollowingUser(qb, ':meId', 'note.userId') + .orWhere(new Brackets(qbb => qbb + .andWhere('note.visibility = \'public\'') + .andWhere('note.userHost IS NULL'))) + .orWhere(':meId = note.userId'))) + // 1. in a channel I follow, 2. not in a channel + .andWhere(new Brackets(qb => this.queryService + .orFollowingChannel(qb, ':meId', 'note.channelId') + .orWhere('note.channelId IS NULL'))) + .setParameters({ meId: me.id }) .innerJoinAndSelect('note.user', 'user') .leftJoinAndSelect('note.reply', 'reply') .leftJoinAndSelect('note.renote', 'renote') .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser'); - - if (followingChannels.length > 0) { - const followingChannelIds = followingChannels.map(x => x.followeeId); - - query.andWhere(new Brackets(qb => { - qb.where('note.channelId IN (:...followingChannelIds)', { followingChannelIds }); - qb.orWhere('note.channelId IS NULL'); - })); - } else { - query.andWhere('note.channelId IS NULL'); - } + .leftJoinAndSelect('renote.user', 'renoteUser') + .limit(ps.limit); if (!ps.withReplies) { - query.andWhere(new Brackets(qb => { - qb - .where('note.replyId IS NULL') // 返信ではない - .orWhere(new Brackets(qb => { - qb // 返信だけど投稿者自身への返信 - .where('note.replyId IS NOT NULL') - .andWhere('note.replyUserId = note.userId'); - })); - })); + query + // 1. Not a reply, 2. a self-reply + .andWhere(new Brackets(qb => qb + .orWhere('note.replyId IS NULL') // 返信ではない + .orWhere('note.replyUserId = note.userId'))); } this.queryService.generateVisibilityQuery(query, me); this.queryService.generateBlockedHostQueryForNote(query); this.queryService.generateSuspendedUserQueryForNote(query); + this.queryService.generateSilencedUserQueryForNotes(query, me); this.queryService.generateMutedUserQueryForNotes(query, me); this.queryService.generateBlockedUserQueryForNotes(query, me); - this.queryService.generateMutedUserRenotesQueryForNotes(query, me); - - if (ps.includeMyRenotes === false) { - query.andWhere(new Brackets(qb => { - qb.orWhere('note.userId != :meId', { meId: me.id }); - qb.orWhere('note.renoteId IS NULL'); - qb.orWhere('note.text IS NOT NULL'); - qb.orWhere('note.fileIds != \'{}\''); - qb.orWhere('0 < (SELECT COUNT(*) FROM poll WHERE poll."noteId" = note.id)'); - })); - } - - if (ps.includeRenotedMyNotes === false) { - query.andWhere(new Brackets(qb => { - qb.orWhere('note.renoteUserId != :meId', { meId: me.id }); - qb.orWhere('note.renoteId IS NULL'); - qb.orWhere('note.text IS NOT NULL'); - qb.orWhere('note.fileIds != \'{}\''); - qb.orWhere('0 < (SELECT COUNT(*) FROM poll WHERE poll."noteId" = note.id)'); - })); - } - - if (ps.includeLocalRenotes === false) { - query.andWhere(new Brackets(qb => { - qb.orWhere('note.renoteUserHost IS NOT NULL'); - qb.orWhere('note.renoteId IS NULL'); - qb.orWhere('note.text IS NOT NULL'); - qb.orWhere('note.fileIds != \'{}\''); - qb.orWhere('0 < (SELECT COUNT(*) FROM poll WHERE poll."noteId" = note.id)'); - })); - } if (ps.withFiles) { query.andWhere('note.fileIds != \'{}\''); } if (!ps.withBots) query.andWhere('user.isBot = FALSE'); + + if (!ps.withRenotes) { + this.queryService.generateExcludedRenotesQueryForNotes(query); + } else { + this.queryService.generateMutedUserRenotesQueryForNotes(query, me); + } //#endregion - return await query.limit(ps.limit).getMany(); + return await query.getMany(); } } diff --git a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts index 7caf657b4c..a0fa72ce09 100644 --- a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts @@ -103,13 +103,14 @@ export default class extends Endpoint { // eslint- withFiles: ps.withFiles, withReplies: ps.withReplies, withBots: ps.withBots, + withRenotes: ps.withRenotes, }, me); - process.nextTick(() => { - if (me) { + if (me) { + process.nextTick(() => { this.activeUsersChart.read(me); - } - }); + }); + } return await this.noteEntityService.packMany(timeline, me); } @@ -136,14 +137,15 @@ export default class extends Endpoint { // eslint- withFiles: ps.withFiles, withReplies: ps.withReplies, withBots: ps.withBots, + withRenotes: ps.withRenotes, }, me), }); - process.nextTick(() => { - if (me) { + if (me) { + process.nextTick(() => { this.activeUsersChart.read(me); - } - }); + }); + } return timeline; }); @@ -156,41 +158,48 @@ export default class extends Endpoint { // eslint- withFiles: boolean, withReplies: boolean, withBots: boolean, + withRenotes: boolean, }, me: MiLocalUser | null) { const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId) - .andWhere('(note.visibility = \'public\') AND (note.userHost IS NULL) AND (note.channelId IS NULL)') + .andWhere('note.visibility = \'public\'') + .andWhere('note.channelId IS NULL') + .andWhere('note.userHost IS NULL') .innerJoinAndSelect('note.user', 'user') .leftJoinAndSelect('note.reply', 'reply') .leftJoinAndSelect('note.renote', 'renote') .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser'); + .leftJoinAndSelect('renote.user', 'renoteUser') + .limit(ps.limit); + + if (!ps.withReplies) { + query + // 1. Not a reply, 2. a self-reply + .andWhere(new Brackets(qb => qb + .orWhere('note.replyId IS NULL') // 返信ではない + .orWhere('note.replyUserId = note.userId'))); + } - this.queryService.generateVisibilityQuery(query, me); this.queryService.generateBlockedHostQueryForNote(query); this.queryService.generateSuspendedUserQueryForNote(query); - if (me) this.queryService.generateMutedUserQueryForNotes(query, me); - if (me) this.queryService.generateBlockedUserQueryForNotes(query, me); - if (me) this.queryService.generateMutedUserRenotesQueryForNotes(query, me); + this.queryService.generateSilencedUserQueryForNotes(query, me); + if (me) { + this.queryService.generateMutedUserQueryForNotes(query, me); + this.queryService.generateBlockedUserQueryForNotes(query, me); + } if (ps.withFiles) { query.andWhere('note.fileIds != \'{}\''); } - if (!ps.withReplies) { - query.andWhere(new Brackets(qb => { - qb - .where('note.replyId IS NULL') // 返信ではない - .orWhere(new Brackets(qb => { - qb // 返信だけど投稿者自身への返信 - .where('note.replyId IS NOT NULL') - .andWhere('note.replyUserId = note.userId'); - })); - })); - } - if (!ps.withBots) query.andWhere('user.isBot = FALSE'); - return await query.limit(ps.limit).getMany(); + if (!ps.withRenotes) { + this.queryService.generateExcludedRenotesQueryForNotes(query); + } else if (me) { + this.queryService.generateMutedUserRenotesQueryForNotes(query, me); + } + + return await query.getMany(); } } diff --git a/packages/backend/src/server/api/endpoints/notes/mentions.ts b/packages/backend/src/server/api/endpoints/notes/mentions.ts index 44812635af..889e85b4a8 100644 --- a/packages/backend/src/server/api/endpoints/notes/mentions.ts +++ b/packages/backend/src/server/api/endpoints/notes/mentions.ts @@ -6,10 +6,12 @@ import { Brackets } from 'typeorm'; import { Inject, Injectable } from '@nestjs/common'; import type { NotesRepository, FollowingsRepository } from '@/models/_.js'; +import { MiNote } from '@/models/_.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { QueryService } from '@/core/QueryService.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import { DI } from '@/di-symbols.js'; +import ActiveUsersChart from '@/core/chart/charts/active-users.js'; export const meta = { tags: ['notes'], @@ -57,43 +59,60 @@ export default class extends Endpoint { // eslint- private noteEntityService: NoteEntityService, private queryService: QueryService, + private readonly activeUsersChart: ActiveUsersChart, ) { super(meta, paramDef, async (ps, me) => { - const followingQuery = this.followingsRepository.createQueryBuilder('following') - .select('following.followeeId') - .where('following.followerId = :followerId', { followerId: me.id }); - const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId) - .andWhere(new Brackets(qb => { - qb // このmeIdAsListパラメータはqueryServiceのgenerateVisibilityQueryでセットされる - .where(':meIdAsList <@ note.mentions') - .orWhere(':meIdAsList <@ note.visibleUserIds'); - })) - // Avoid scanning primary key index - .orderBy('CONCAT(note.id)', 'DESC') + .innerJoin(qb => { + qb + .select('note.id', 'id') + .from(qbb => qbb + .select('note.id', 'id') + .from(MiNote, 'note') + .where(new Brackets(qbbb => qbbb + // DM to me + .orWhere(':meIdAsList <@ note.visibleUserIds') + // Mentions me + .orWhere(':meIdAsList <@ note.mentions'), + )) + .setParameters({ meIdAsList: [me.id] }) + , 'source') + .innerJoin(MiNote, 'note', 'note.id = source.id'); + + // Mentioned or visible users can always access + //this.queryService.generateVisibilityQuery(query, me); + this.queryService.generateBlockedHostQueryForNote(qb); + this.queryService.generateSuspendedUserQueryForNote(query); + this.queryService.generateMutedUserQueryForNotes(qb, me); + this.queryService.generateMutedNoteThreadQuery(qb, me); + this.queryService.generateBlockedUserQueryForNotes(qb, me); + // A renote can't mention a user, so it will never appear here anyway. + //this.queryService.generateMutedUserRenotesQueryForNotes(query, me); + + if (ps.visibility) { + qb.andWhere('note.visibility = :visibility', { visibility: ps.visibility }); + } + + if (ps.following) { + this.queryService + .andFollowingUser(qb, ':meId', 'note.userId') + .setParameters({ meId: me.id }); + } + + return qb; + }, 'source', 'source.id = note.id') .innerJoinAndSelect('note.user', 'user') .leftJoinAndSelect('note.reply', 'reply') .leftJoinAndSelect('note.renote', 'renote') .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser'); + .leftJoinAndSelect('renote.user', 'renoteUser') + .limit(ps.limit); - this.queryService.generateVisibilityQuery(query, me); - this.queryService.generateBlockedHostQueryForNote(query); - this.queryService.generateSuspendedUserQueryForNote(query); - this.queryService.generateMutedUserQueryForNotes(query, me); - this.queryService.generateMutedNoteThreadQuery(query, me); - this.queryService.generateBlockedUserQueryForNotes(query, me); + const mentions = await query.getMany(); - if (ps.visibility) { - query.andWhere('note.visibility = :visibility', { visibility: ps.visibility }); - } - - if (ps.following) { - query.andWhere(`((note.userId IN (${ followingQuery.getQuery() })) OR (note.userId = :meId))`, { meId: me.id }); - query.setParameters(followingQuery.getParameters()); - } - - const mentions = await query.limit(ps.limit).getMany(); + process.nextTick(() => { + this.activeUsersChart.read(me); + }); return await this.noteEntityService.packMany(mentions, me); }); diff --git a/packages/backend/src/server/api/endpoints/notes/polls/recommendation.ts b/packages/backend/src/server/api/endpoints/notes/polls/recommendation.ts index 33a9c281b3..6f96821a63 100644 --- a/packages/backend/src/server/api/endpoints/notes/polls/recommendation.ts +++ b/packages/backend/src/server/api/endpoints/notes/polls/recommendation.ts @@ -9,13 +9,13 @@ import type { NotesRepository, MutingsRepository, PollsRepository, PollVotesRepo import { Endpoint } from '@/server/api/endpoint-base.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import { DI } from '@/di-symbols.js'; +import { QueryService } from '@/core/QueryService.js'; +import { RoleService } from '@/core/RoleService.js'; +import { ApiError } from '@/server/api/error.js'; export const meta = { tags: ['notes'], - requireCredential: true, - kind: 'read:account', - res: { type: 'array', optional: false, nullable: false, @@ -26,10 +26,24 @@ export const meta = { }, }, - // 2 calls per second + errors: { + ltlDisabled: { + message: 'Local timeline has been disabled.', + code: 'LTL_DISABLED', + id: '45a6eb02-7695-4393-b023-dd3be9aaaefd', + }, + gtlDisabled: { + message: 'Global timeline has been disabled.', + code: 'GTL_DISABLED', + id: '0332fc13-6ab2-4427-ae80-a9fadffd1a6b', + }, + }, + + // Up to 10 calls, then 2 per second limit: { - duration: 1000, - max: 2, + type: 'bucket', + size: 10, + dripRate: 500, }, } as const; @@ -39,6 +53,8 @@ export const paramDef = { limit: { type: 'integer', minimum: 1, maximum: 100, default: 10 }, offset: { type: 'integer', default: 0 }, excludeChannels: { type: 'boolean', default: false }, + local: { type: 'boolean', nullable: true, default: null }, + expired: { type: 'boolean', default: false }, }, required: [], } as const; @@ -59,18 +75,54 @@ export default class extends Endpoint { // eslint- private mutingsRepository: MutingsRepository, private noteEntityService: NoteEntityService, + private readonly queryService: QueryService, + private readonly roleService: RoleService, ) { super(meta, paramDef, async (ps, me) => { const query = this.pollsRepository.createQueryBuilder('poll') - .where('poll.userHost IS NULL') - .andWhere('poll.userId != :meId', { meId: me.id }) - .andWhere('poll.noteVisibility = \'public\'') - .andWhere(new Brackets(qb => { + .innerJoinAndSelect('poll.note', 'note') + .innerJoinAndSelect('note.user', 'user') + .leftJoinAndSelect('note.renote', 'renote') + .leftJoinAndSelect('note.reply', 'reply') + .leftJoinAndSelect('renote.user', 'renoteUser') + .leftJoinAndSelect('reply.user', 'replyUser') + .andWhere('user.isExplorable = TRUE') + ; + + if (me) { + query.andWhere('poll.userId != :meId', { meId: me.id }); + } + + if (ps.expired) { + query.andWhere('poll.expiresAt IS NOT NULL'); + query.andWhere('poll.expiresAt <= :expiresMax', { + expiresMax: new Date(), + }); + query.andWhere('poll.expiresAt >= :expiresMin', { + expiresMin: new Date(Date.now() - (1000 * 60 * 60 * 24 * 7)), + }); + } else { + query.andWhere(new Brackets(qb => { qb .where('poll.expiresAt IS NULL') .orWhere('poll.expiresAt > :now', { now: new Date() }); })); + } + const policies = await this.roleService.getUserPolicies(me?.id ?? null); + if (ps.local != null) { + if (ps.local) { + if (!policies.ltlAvailable) throw new ApiError(meta.errors.ltlDisabled); + query.andWhere('poll.userHost IS NULL'); + } else { + if (!policies.gtlAvailable) throw new ApiError(meta.errors.gtlDisabled); + query.andWhere('poll.userHost IS NOT NULL'); + } + } else { + if (!policies.gtlAvailable) throw new ApiError(meta.errors.gtlDisabled); + } + + /* //#region exclude arleady voted polls const votedQuery = this.pollVotesRepository.createQueryBuilder('vote') .select('vote.noteId') @@ -81,16 +133,15 @@ export default class extends Endpoint { // eslint- query.setParameters(votedQuery.getParameters()); //#endregion + */ - //#region mute - const mutingQuery = this.mutingsRepository.createQueryBuilder('muting') - .select('muting.muteeId') - .where('muting.muterId = :muterId', { muterId: me.id }); - - query - .andWhere(`poll.userId NOT IN (${ mutingQuery.getQuery() })`); - - query.setParameters(mutingQuery.getParameters()); + //#region block/mute/vis + this.queryService.generateVisibilityQuery(query, me); + this.queryService.generateBlockedHostQueryForNote(query); + if (me) { + this.queryService.generateBlockedUserQueryForNotes(query, me); + this.queryService.generateMutedUserQueryForNotes(query, me); + } //#endregion //#region exclude channels @@ -107,6 +158,7 @@ export default class extends Endpoint { // eslint- if (polls.length === 0) return []; + /* const notes = await this.notesRepository.find({ where: { id: In(polls.map(poll => poll.noteId)), @@ -115,6 +167,10 @@ export default class extends Endpoint { // eslint- id: 'DESC', }, }); + */ + + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const notes = polls.map(poll => poll.note!); return await this.noteEntityService.packMany(notes, me, { detail: true, diff --git a/packages/backend/src/server/api/endpoints/notes/renotes.ts b/packages/backend/src/server/api/endpoints/notes/renotes.ts index 4ccbec3e2f..4a4ef0b77e 100644 --- a/packages/backend/src/server/api/endpoints/notes/renotes.ts +++ b/packages/backend/src/server/api/endpoints/notes/renotes.ts @@ -47,7 +47,7 @@ export const paramDef = { type: 'object', properties: { noteId: { type: 'string', format: 'misskey:id' }, - userId: { type: "string", format: "misskey:id" }, + userId: { type: 'string', format: 'misskey:id' }, limit: { type: 'integer', minimum: 1, maximum: 100, default: 10 }, sinceId: { type: 'string', format: 'misskey:id' }, untilId: { type: 'string', format: 'misskey:id' }, @@ -81,20 +81,22 @@ export default class extends Endpoint { // eslint- .leftJoinAndSelect('renote.user', 'renoteUser'); if (ps.userId) { - query.andWhere("user.id = :userId", { userId: ps.userId }); + query.andWhere('user.id = :userId', { userId: ps.userId }); } if (ps.quote) { - query.andWhere("note.text IS NOT NULL"); + this.queryService.andIsQuote(query, 'note'); } else { - query.andWhere("note.text IS NULL"); + this.queryService.andIsRenote(query, 'note'); } this.queryService.generateVisibilityQuery(query, me); this.queryService.generateBlockedHostQueryForNote(query); this.queryService.generateSuspendedUserQueryForNote(query); - if (me) this.queryService.generateMutedUserQueryForNotes(query, me); - if (me) this.queryService.generateBlockedUserQueryForNotes(query, me); + if (me) { + this.queryService.generateMutedUserQueryForNotes(query, me); + this.queryService.generateBlockedUserQueryForNotes(query, me); + } const renotes = await query.limit(ps.limit).getMany(); diff --git a/packages/backend/src/server/api/endpoints/notes/replies.ts b/packages/backend/src/server/api/endpoints/notes/replies.ts index fd6fb58130..d6142b581a 100644 --- a/packages/backend/src/server/api/endpoints/notes/replies.ts +++ b/packages/backend/src/server/api/endpoints/notes/replies.ts @@ -59,15 +59,18 @@ export default class extends Endpoint { // eslint- .leftJoinAndSelect('note.reply', 'reply') .leftJoinAndSelect('note.renote', 'renote') .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser'); + .leftJoinAndSelect('renote.user', 'renoteUser') + .limit(ps.limit); this.queryService.generateVisibilityQuery(query, me); this.queryService.generateBlockedHostQueryForNote(query); this.queryService.generateSuspendedUserQueryForNote(query); - if (me) this.queryService.generateMutedUserQueryForNotes(query, me); - if (me) this.queryService.generateBlockedUserQueryForNotes(query, me); + if (me) { + this.queryService.generateMutedUserQueryForNotes(query, me); + this.queryService.generateBlockedUserQueryForNotes(query, me); + } - const timeline = await query.limit(ps.limit).getMany(); + const timeline = await query.getMany(); return await this.noteEntityService.packMany(timeline, me); }); diff --git a/packages/backend/src/server/api/endpoints/notes/search-by-tag.ts b/packages/backend/src/server/api/endpoints/notes/search-by-tag.ts index fea05a80a4..5ef54856da 100644 --- a/packages/backend/src/server/api/endpoints/notes/search-by-tag.ts +++ b/packages/backend/src/server/api/endpoints/notes/search-by-tag.ts @@ -12,8 +12,6 @@ import { Endpoint } from '@/server/api/endpoint-base.js'; import { QueryService } from '@/core/QueryService.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import { DI } from '@/di-symbols.js'; -import { CacheService } from '@/core/CacheService.js'; -import { UtilityService } from '@/core/UtilityService.js'; export const meta = { tags: ['notes', 'hashtags'], @@ -82,27 +80,27 @@ export default class extends Endpoint { // eslint- private noteEntityService: NoteEntityService, private queryService: QueryService, - private cacheService: CacheService, - private utilityService: UtilityService, ) { super(meta, paramDef, async (ps, me) => { const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId) - .andWhere("note.visibility IN ('public', 'home')") // keep in sync with NoteCreateService call to `hashtagService.updateHashtags()` + .andWhere(new Brackets(qb => qb + .orWhere('note.visibility = \'public\'') + .orWhere('note.visibility = \'home\''))) // keep in sync with NoteCreateService call to `hashtagService.updateHashtags()` .innerJoinAndSelect('note.user', 'user') .leftJoinAndSelect('note.reply', 'reply') .leftJoinAndSelect('note.renote', 'renote') .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser'); + .leftJoinAndSelect('renote.user', 'renoteUser') + .limit(ps.limit); - if (!this.serverSettings.enableBotTrending) query.andWhere('user.isBot = FALSE'); - - this.queryService.generateVisibilityQuery(query, me); this.queryService.generateBlockedHostQueryForNote(query); this.queryService.generateSuspendedUserQueryForNote(query); + this.queryService.generateSilencedUserQueryForNotes(query, me); if (me) this.queryService.generateMutedUserQueryForNotes(query, me); if (me) this.queryService.generateBlockedUserQueryForNotes(query, me); + if (me) this.queryService.generateMutedUserRenotesQueryForNotes(query, me); - const followings = me ? await this.cacheService.userFollowingsCache.fetch(me.id) : {}; + if (!this.serverSettings.enableBotTrending) query.andWhere('user.isBot = FALSE'); try { if (ps.tag) { @@ -135,9 +133,9 @@ export default class extends Endpoint { // eslint- if (ps.renote != null) { if (ps.renote) { - query.andWhere('note.renoteId IS NOT NULL'); + this.queryService.andIsRenote(query, 'note'); } else { - query.andWhere('note.renoteId IS NULL'); + this.queryService.andIsNotRenote(query, 'note'); } } @@ -154,17 +152,7 @@ export default class extends Endpoint { // eslint- } // Search notes - let notes = await query.limit(ps.limit).getMany(); - - notes = notes.filter(note => { - if (note.user?.isSilenced && me && followings && note.userId !== me.id && !followings[note.userId]) return false; - if (note.user?.isSuspended) return false; - if (note.userHost) { - if (!this.utilityService.isFederationAllowedHost(note.userHost)) return false; - if (this.utilityService.isSilencedHost(this.serverSettings.silencedHosts, note.userHost)) return false; - } - return true; - }); + const notes = await query.getMany(); return await this.noteEntityService.packMany(notes, me); }); diff --git a/packages/backend/src/server/api/endpoints/notes/timeline.ts b/packages/backend/src/server/api/endpoints/notes/timeline.ts index 2fea7dcc35..17065bdd9a 100644 --- a/packages/backend/src/server/api/endpoints/notes/timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/timeline.ts @@ -49,9 +49,6 @@ export const paramDef = { sinceDate: { type: 'integer' }, untilDate: { type: 'integer' }, allowPartial: { type: 'boolean', default: false }, // true is recommended but for compatibility false by default - includeMyRenotes: { type: 'boolean', default: true }, - includeRenotedMyNotes: { type: 'boolean', default: true }, - includeLocalRenotes: { type: 'boolean', default: true }, withFiles: { type: 'boolean', default: false }, withRenotes: { type: 'boolean', default: true }, withBots: { type: 'boolean', default: true }, @@ -88,9 +85,6 @@ export default class extends Endpoint { // eslint- untilId, sinceId, limit: ps.limit, - includeMyRenotes: ps.includeMyRenotes, - includeRenotedMyNotes: ps.includeRenotedMyNotes, - includeLocalRenotes: ps.includeLocalRenotes, withFiles: ps.withFiles, withRenotes: ps.withRenotes, withBots: ps.withBots, @@ -131,9 +125,6 @@ export default class extends Endpoint { // eslint- untilId, sinceId, limit, - includeMyRenotes: ps.includeMyRenotes, - includeRenotedMyNotes: ps.includeRenotedMyNotes, - includeLocalRenotes: ps.includeLocalRenotes, withFiles: ps.withFiles, withRenotes: ps.withRenotes, withBots: ps.withBots, @@ -148,114 +139,49 @@ export default class extends Endpoint { // eslint- }); } - private async getFromDb(ps: { untilId: string | null; sinceId: string | null; limit: number; includeMyRenotes: boolean; includeRenotedMyNotes: boolean; includeLocalRenotes: boolean; withFiles: boolean; withRenotes: boolean; withBots: boolean; }, me: MiLocalUser) { - const followees = await this.userFollowingService.getFollowees(me.id); - const followingChannels = await this.channelFollowingsRepository.find({ - where: { - followerId: me.id, - }, - }); - + private async getFromDb(ps: { untilId: string | null; sinceId: string | null; limit: number; withFiles: boolean; withRenotes: boolean; withBots: boolean; }, me: MiLocalUser) { //#region Construct query const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId) + // 1. in a channel I follow, 2. my own post, 3. by a user I follow + .andWhere(new Brackets(qb => this.queryService + .orFollowingChannel(qb, ':meId', 'note.channelId') + .orWhere(':meId = note.userId') + .orWhere(new Brackets(qb2 => this.queryService + .andFollowingUser(qb2, ':meId', 'note.userId') + .andWhere('note.channelId IS NULL'))), + )) + // 1. Not a reply, 2. a self-reply + .andWhere(new Brackets(qb => qb + .orWhere('note.replyId IS NULL') // 返信ではない + .orWhere('note.replyUserId = note.userId'))) + .setParameters({ meId: me.id }) .innerJoinAndSelect('note.user', 'user') .leftJoinAndSelect('note.reply', 'reply') .leftJoinAndSelect('note.renote', 'renote') .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser'); - - if (followees.length > 0 && followingChannels.length > 0) { - // ユーザー・チャンネルともにフォローあり - const meOrFolloweeIds = [me.id, ...followees.map(f => f.followeeId)]; - const followingChannelIds = followingChannels.map(x => x.followeeId); - query.andWhere(new Brackets(qb => { - qb - .where(new Brackets(qb2 => { - qb2 - .where('note.userId IN (:...meOrFolloweeIds)', { meOrFolloweeIds: meOrFolloweeIds }) - .andWhere('note.channelId IS NULL'); - })) - .orWhere('note.channelId IN (:...followingChannelIds)', { followingChannelIds }); - })); - } else if (followees.length > 0) { - // ユーザーフォローのみ（チャンネルフォローなし） - const meOrFolloweeIds = [me.id, ...followees.map(f => f.followeeId)]; - query - .andWhere('note.channelId IS NULL') - .andWhere('note.userId IN (:...meOrFolloweeIds)', { meOrFolloweeIds: meOrFolloweeIds }); - } else if (followingChannels.length > 0) { - // チャンネルフォローのみ（ユーザーフォローなし） - const followingChannelIds = followingChannels.map(x => x.followeeId); - query.andWhere(new Brackets(qb => { - qb - .where('note.channelId IN (:...followingChannelIds)', { followingChannelIds }) - .orWhere('note.userId = :meId', { meId: me.id }); - })); - } else { - // フォローなし - query - .andWhere('note.channelId IS NULL') - .andWhere('note.userId = :meId', { meId: me.id }); - } - - query.andWhere(new Brackets(qb => { - qb - .where('note.replyId IS NULL') // 返信ではない - .orWhere(new Brackets(qb => { - qb // 返信だけど投稿者自身への返信 - .where('note.replyId IS NOT NULL') - .andWhere('note.replyUserId = note.userId'); - })); - })); + .leftJoinAndSelect('renote.user', 'renoteUser') + .limit(ps.limit); this.queryService.generateVisibilityQuery(query, me); this.queryService.generateBlockedHostQueryForNote(query); this.queryService.generateSuspendedUserQueryForNote(query); + this.queryService.generateSilencedUserQueryForNotes(query, me); this.queryService.generateMutedUserQueryForNotes(query, me); this.queryService.generateBlockedUserQueryForNotes(query, me); - this.queryService.generateMutedUserRenotesQueryForNotes(query, me); - - if (ps.includeMyRenotes === false) { - query.andWhere(new Brackets(qb => { - qb.orWhere('note.userId != :meId', { meId: me.id }); - qb.orWhere('note.renoteId IS NULL'); - qb.orWhere('note.text IS NOT NULL'); - qb.orWhere('note.fileIds != \'{}\''); - qb.orWhere('0 < (SELECT COUNT(*) FROM poll WHERE poll."noteId" = note.id)'); - })); - } - - if (ps.includeRenotedMyNotes === false) { - query.andWhere(new Brackets(qb => { - qb.orWhere('note.renoteUserId != :meId', { meId: me.id }); - qb.orWhere('note.renoteId IS NULL'); - qb.orWhere('note.text IS NOT NULL'); - qb.orWhere('note.fileIds != \'{}\''); - qb.orWhere('0 < (SELECT COUNT(*) FROM poll WHERE poll."noteId" = note.id)'); - })); - } - - if (ps.includeLocalRenotes === false) { - query.andWhere(new Brackets(qb => { - qb.orWhere('note.renoteUserHost IS NOT NULL'); - qb.orWhere('note.renoteId IS NULL'); - qb.orWhere('note.text IS NOT NULL'); - qb.orWhere('note.fileIds != \'{}\''); - qb.orWhere('0 < (SELECT COUNT(*) FROM poll WHERE poll."noteId" = note.id)'); - })); - } if (ps.withFiles) { query.andWhere('note.fileIds != \'{}\''); } - if (ps.withRenotes === false) { - query.andWhere('note.renoteId IS NULL'); - } - if (!ps.withBots) query.andWhere('user.isBot = FALSE'); + + if (!ps.withRenotes) { + this.queryService.generateExcludedRenotesQueryForNotes(query); + } else { + this.queryService.generateMutedUserRenotesQueryForNotes(query, me); + } //#endregion - return await query.limit(ps.limit).getMany(); + return await query.getMany(); } } diff --git a/packages/backend/src/server/api/endpoints/notes/translate.ts b/packages/backend/src/server/api/endpoints/notes/translate.ts index a97542c063..e55168e296 100644 --- a/packages/backend/src/server/api/endpoints/notes/translate.ts +++ b/packages/backend/src/server/api/endpoints/notes/translate.ts @@ -20,11 +20,9 @@ import { ApiError } from '../../error.js'; export const meta = { tags: ['notes'], - // TODO allow unauthenticated if default template allows? - // Maybe a value 'optional' that allows unauthenticated OR a token w/ appropriate role. - // This will allow unauthenticated requests without leaking post data to restricted clients. - requireCredential: true, + requireCredential: 'optional', kind: 'read:account', + requiredRolePolicy: 'canUseTranslator', res: { type: 'object', @@ -88,17 +86,12 @@ export default class extends Endpoint { // eslint- private readonly loggerService: ApiLoggerService, ) { super(meta, paramDef, async (ps, me) => { - const policies = await this.roleService.getUserPolicies(me.id); - if (!policies.canUseTranslator) { - throw new ApiError(meta.errors.unavailable); - } - const note = await this.getterService.getNote(ps.noteId).catch(err => { if (err.id === '9725d0ce-ba28-4dde-95a7-2cbb2c15de24') throw new ApiError(meta.errors.noSuchNote); throw err; }); - if (!(await this.noteEntityService.isVisibleForMe(note, me.id))) { + if (!(await this.noteEntityService.isVisibleForMe(note, me?.id ?? null))) { throw new ApiError(meta.errors.cannotTranslateInvisibleNote); } @@ -140,7 +133,7 @@ export default class extends Endpoint { // eslint- if (this.serverSettings.deeplAuthKey) params.append('auth_key', this.serverSettings.deeplAuthKey); params.append('text', note.text); params.append('target_lang', targetLang); - const endpoint = deeplFreeInstance ?? this.serverSettings.deeplIsPro ? 'https://api.deepl.com/v2/translate' : 'https://api-free.deepl.com/v2/translate'; + const endpoint = deeplFreeInstance ?? ( this.serverSettings.deeplIsPro ? 'https://api.deepl.com/v2/translate' : 'https://api-free.deepl.com/v2/translate' ); const res = await this.httpRequestService.send(endpoint, { method: 'POST', diff --git a/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts b/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts index 0e46752452..d54a24f0a1 100644 --- a/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts @@ -57,9 +57,6 @@ export const paramDef = { sinceDate: { type: 'integer' }, untilDate: { type: 'integer' }, allowPartial: { type: 'boolean', default: false }, // true is recommended but for compatibility false by default - includeMyRenotes: { type: 'boolean', default: true }, - includeRenotedMyNotes: { type: 'boolean', default: true }, - includeLocalRenotes: { type: 'boolean', default: true }, withRenotes: { type: 'boolean', default: true }, withFiles: { type: 'boolean', @@ -109,14 +106,13 @@ export default class extends Endpoint { // eslint- untilId, sinceId, limit: ps.limit, - includeMyRenotes: ps.includeMyRenotes, - includeRenotedMyNotes: ps.includeRenotedMyNotes, - includeLocalRenotes: ps.includeLocalRenotes, withFiles: ps.withFiles, withRenotes: ps.withRenotes, }, me); - this.activeUsersChart.read(me); + process.nextTick(() => { + this.activeUsersChart.read(me); + }); return await this.noteEntityService.packMany(timeline, me); } @@ -135,15 +131,14 @@ export default class extends Endpoint { // eslint- untilId, sinceId, limit, - includeMyRenotes: ps.includeMyRenotes, - includeRenotedMyNotes: ps.includeRenotedMyNotes, - includeLocalRenotes: ps.includeLocalRenotes, withFiles: ps.withFiles, withRenotes: ps.withRenotes, }, me), }); - this.activeUsersChart.read(me); + process.nextTick(() => { + this.activeUsersChart.read(me); + }); return timeline; }); @@ -153,94 +148,50 @@ export default class extends Endpoint { // eslint- untilId: string | null, sinceId: string | null, limit: number, - includeMyRenotes: boolean, - includeRenotedMyNotes: boolean, - includeLocalRenotes: boolean, withFiles: boolean, withRenotes: boolean, }, me: MiLocalUser) { //#region Construct query const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId) .innerJoin(this.userListMembershipsRepository.metadata.targetName, 'userListMemberships', 'userListMemberships.userId = note.userId') + .andWhere('userListMemberships.userListId = :userListId', { userListId: list.id }) + .andWhere('note.channelId IS NULL') // チャンネルノートではない + .andWhere(new Brackets(qb => qb + // 返信ではない + .orWhere('note.replyId IS NULL') + // 返信だけど投稿者自身への返信 + .orWhere('note.replyUserId = note.userId') + // 返信だけど自分宛ての返信 + .orWhere('note.replyUserId = :meId') + // 返信だけどwithRepliesがtrueの場合 + .orWhere('userListMemberships.withReplies = true'), + )) + .setParameters({ meId: me.id }) .innerJoinAndSelect('note.user', 'user') .leftJoinAndSelect('note.reply', 'reply') .leftJoinAndSelect('note.renote', 'renote') .leftJoinAndSelect('reply.user', 'replyUser') .leftJoinAndSelect('renote.user', 'renoteUser') - .andWhere('userListMemberships.userListId = :userListId', { userListId: list.id }) - .andWhere('note.channelId IS NULL') // チャンネルノートではない - .andWhere(new Brackets(qb => { - qb - .where('note.replyId IS NULL') // 返信ではない - .orWhere(new Brackets(qb => { - qb // 返信だけど投稿者自身への返信 - .where('note.replyId IS NOT NULL') - .andWhere('note.replyUserId = note.userId'); - })) - .orWhere(new Brackets(qb => { - qb // 返信だけど自分宛ての返信 - .where('note.replyId IS NOT NULL') - .andWhere('note.replyUserId = :meId', { meId: me.id }); - })) - .orWhere(new Brackets(qb => { - qb // 返信だけどwithRepliesがtrueの場合 - .where('note.replyId IS NOT NULL') - .andWhere('userListMemberships.withReplies = true'); - })); - })); + .limit(ps.limit); this.queryService.generateVisibilityQuery(query, me); this.queryService.generateBlockedHostQueryForNote(query); this.queryService.generateSuspendedUserQueryForNote(query); this.queryService.generateMutedUserQueryForNotes(query, me); this.queryService.generateBlockedUserQueryForNotes(query, me); - this.queryService.generateMutedUserRenotesQueryForNotes(query, me); - - if (ps.includeMyRenotes === false) { - query.andWhere(new Brackets(qb => { - qb.orWhere('note.userId != :meId', { meId: me.id }); - qb.orWhere('note.renoteId IS NULL'); - qb.orWhere('note.text IS NOT NULL'); - qb.orWhere('note.fileIds != \'{}\''); - qb.orWhere('0 < (SELECT COUNT(*) FROM poll WHERE poll."noteId" = note.id)'); - })); - } - - if (ps.includeRenotedMyNotes === false) { - query.andWhere(new Brackets(qb => { - qb.orWhere('note.renoteUserId != :meId', { meId: me.id }); - qb.orWhere('note.renoteId IS NULL'); - qb.orWhere('note.text IS NOT NULL'); - qb.orWhere('note.fileIds != \'{}\''); - qb.orWhere('0 < (SELECT COUNT(*) FROM poll WHERE poll."noteId" = note.id)'); - })); - } - - if (ps.includeLocalRenotes === false) { - query.andWhere(new Brackets(qb => { - qb.orWhere('note.renoteUserHost IS NOT NULL'); - qb.orWhere('note.renoteId IS NULL'); - qb.orWhere('note.text IS NOT NULL'); - qb.orWhere('note.fileIds != \'{}\''); - qb.orWhere('0 < (SELECT COUNT(*) FROM poll WHERE poll."noteId" = note.id)'); - })); - } - - if (ps.withRenotes === false) { - query.andWhere(new Brackets(qb => { - qb.orWhere('note.renoteId IS NULL'); - qb.orWhere(new Brackets(qb => { - qb.orWhere('note.text IS NOT NULL'); - qb.orWhere('note.fileIds != \'{}\''); - })); - })); - } if (ps.withFiles) { query.andWhere('note.fileIds != \'{}\''); } + + if (!ps.withRenotes) { + this.queryService.generateExcludedRenotesQueryForNotes(query); + } else { + this.queryService.generateMutedUserRenotesQueryForNotes(query, me); + } + //#endregion - return await query.limit(ps.limit).getMany(); + return await query.getMany(); } } diff --git a/packages/backend/src/server/api/endpoints/roles/notes.ts b/packages/backend/src/server/api/endpoints/roles/notes.ts index df577f2964..33cb3f94a0 100644 --- a/packages/backend/src/server/api/endpoints/roles/notes.ts +++ b/packages/backend/src/server/api/endpoints/roles/notes.ts @@ -12,6 +12,7 @@ import { DI } from '@/di-symbols.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import { IdService } from '@/core/IdService.js'; import { FanoutTimelineService } from '@/core/FanoutTimelineService.js'; +import ActiveUsersChart from '@/core/chart/charts/active-users.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -74,6 +75,7 @@ export default class extends Endpoint { // eslint- private noteEntityService: NoteEntityService, private queryService: QueryService, private fanoutTimelineService: FanoutTimelineService, + private readonly activeUsersChart: ActiveUsersChart, ) { super(meta, paramDef, async (ps, me) => { const untilId = ps.untilId ?? (ps.untilDate ? this.idService.gen(ps.untilDate!) : null); @@ -101,20 +103,25 @@ export default class extends Endpoint { // eslint- const query = this.notesRepository.createQueryBuilder('note') .where('note.id IN (:...noteIds)', { noteIds: noteIds }) .andWhere('(note.visibility = \'public\')') + .orderBy('note.id', 'DESC') .innerJoinAndSelect('note.user', 'user') .leftJoinAndSelect('note.reply', 'reply') .leftJoinAndSelect('note.renote', 'renote') .leftJoinAndSelect('reply.user', 'replyUser') .leftJoinAndSelect('renote.user', 'renoteUser'); - this.queryService.generateVisibilityQuery(query, me); this.queryService.generateBlockedHostQueryForNote(query); this.queryService.generateSuspendedUserQueryForNote(query); + this.queryService.generateSilencedUserQueryForNotes(query, me); this.queryService.generateMutedUserQueryForNotes(query, me); this.queryService.generateBlockedUserQueryForNotes(query, me); + this.queryService.generateMutedUserRenotesQueryForNotes(query, me); const notes = await query.getMany(); - notes.sort((a, b) => a.id > b.id ? -1 : 1); + + process.nextTick(() => { + this.activeUsersChart.read(me); + }); return await this.noteEntityService.packMany(notes, me); }); diff --git a/packages/backend/src/server/api/endpoints/users/notes.ts b/packages/backend/src/server/api/endpoints/users/notes.ts index 8aa7e9ab62..cd5ef53915 100644 --- a/packages/backend/src/server/api/endpoints/users/notes.ts +++ b/packages/backend/src/server/api/endpoints/users/notes.ts @@ -206,7 +206,8 @@ export default class extends Endpoint { // eslint- .leftJoinAndSelect('note.renote', 'renote') .leftJoinAndSelect('note.channel', 'channel') .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser'); + .leftJoinAndSelect('renote.user', 'renoteUser') + .limit(ps.limit); if (ps.withChannelNotes) { if (!isSelf) query.andWhere(new Brackets(qb => { @@ -232,26 +233,9 @@ export default class extends Endpoint { // eslint- if (!ps.withRenotes && !ps.withQuotes) { query.andWhere('note.renoteId IS NULL'); } else if (!ps.withRenotes) { - query.andWhere(new Brackets(qb => { - qb.orWhere('note.userId != :userId', { userId: ps.userId }); - qb.orWhere('note.renoteId IS NULL'); - qb.orWhere('note.text IS NOT NULL'); - qb.orWhere('note.fileIds != \'{}\''); - qb.orWhere('0 < (SELECT COUNT(*) FROM poll WHERE poll."noteId" = note.id)'); - })); + this.queryService.andIsNotRenote(query, 'note'); } else if (!ps.withQuotes) { - query.andWhere(` - ( - note."renoteId" IS NULL - OR ( - note.text IS NULL - AND note.cw IS NULL - AND note."replyId" IS NULL - AND note."hasPoll" IS FALSE - AND note."fileIds" = '{}' - ) - ) - `); + this.queryService.andIsNotQuote(query, 'note'); } if (!ps.withRepliesToOthers && !ps.withRepliesToSelf) { @@ -270,6 +254,6 @@ export default class extends Endpoint { // eslint- query.andWhere('"user"."isBot" = false'); } - return await query.limit(ps.limit).getMany(); + return await query.getMany(); } } diff --git a/packages/backend/src/server/api/endpoints/users/reactions.ts b/packages/backend/src/server/api/endpoints/users/reactions.ts index f751266910..f7a40a2b9f 100644 --- a/packages/backend/src/server/api/endpoints/users/reactions.ts +++ b/packages/backend/src/server/api/endpoints/users/reactions.ts @@ -105,7 +105,7 @@ export default class extends Endpoint { // eslint- const query = this.queryService.makePaginationQuery(this.noteReactionsRepository.createQueryBuilder('reaction'), ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate) .andWhere('reaction.userId = :userId', { userId: ps.userId }) - .leftJoinAndSelect('reaction.note', 'note') + .innerJoinAndSelect('reaction.note', 'note'); .leftJoinAndSelect('note.user', 'user') .leftJoinAndSelect('note.reply', 'reply') .leftJoinAndSelect('note.renote', 'renote') @@ -115,6 +115,11 @@ export default class extends Endpoint { // eslint- this.queryService.generateVisibilityQuery(query, me); this.queryService.generateBlockedHostQueryForNote(query); this.queryService.generateSuspendedUserQueryForNote(query); + if (me) { + this.queryService.generateMutedUserQueryForNotes(query, me); + this.queryService.generateBlockedUserQueryForNotes(query, me); + this.queryService.generateMutedUserRenotesQueryForNotes(query, me); + } const reactions = (await query .limit(ps.limit) diff --git a/packages/backend/src/server/api/endpoints/users/show.ts b/packages/backend/src/server/api/endpoints/users/show.ts index 7b1c8adfb8..84eb661742 100644 --- a/packages/backend/src/server/api/endpoints/users/show.ts +++ b/packages/backend/src/server/api/endpoints/users/show.ts @@ -13,6 +13,7 @@ import { RemoteUserResolveService } from '@/core/RemoteUserResolveService.js'; import { DI } from '@/di-symbols.js'; import PerUserPvChart from '@/core/chart/charts/per-user-pv.js'; import { RoleService } from '@/core/RoleService.js'; +import { renderInlineError } from '@/misc/render-inline-error.js'; import { ApiError } from '../../error.js'; import { ApiLoggerService } from '../../ApiLoggerService.js'; import type { FindOptionsWhere } from 'typeorm'; @@ -131,7 +132,7 @@ export default class extends Endpoint { // eslint- // Lookup user if (typeof ps.host === 'string' && typeof ps.username === 'string') { user = await this.remoteUserResolveService.resolveUser(ps.username, ps.host).catch(err => { - this.apiLoggerService.logger.warn(`failed to resolve remote user: ${err}`); + this.apiLoggerService.logger.warn(`failed to resolve remote user: ${renderInlineError(err)}`); throw new ApiError(meta.errors.failedToResolveRemoteUser); }); } else { diff --git a/packages/backend/src/server/api/mastodon/MastodonConverters.ts b/packages/backend/src/server/api/mastodon/MastodonConverters.ts index 375ea1ef08..02ce31c4f8 100644 --- a/packages/backend/src/server/api/mastodon/MastodonConverters.ts +++ b/packages/backend/src/server/api/mastodon/MastodonConverters.ts @@ -252,10 +252,10 @@ export class MastodonConverters { return await this.convertStatus(status, me); } - public async convertStatus(status: Entity.Status, me: MiLocalUser | null): Promise { + public async convertStatus(status: Entity.Status, me: MiLocalUser | null, hints?: { note?: MiNote, user?: MiUser }): Promise { const convertedAccount = this.convertAccount(status.account); - const note = await this.mastodonDataService.requireNote(status.id, me); - const noteUser = await this.getUser(status.account.id); + const note = hints?.note ?? await this.mastodonDataService.requireNote(status.id, me); + const noteUser = hints?.user ?? note.user ?? await this.getUser(status.account.id); const mentionedRemoteUsers = JSON.parse(note.mentionedRemoteUsers); const emojis = await this.customEmojiService.populateEmojis(note.emojis, noteUser.host ? noteUser.host : this.config.host); diff --git a/packages/backend/src/server/api/mastodon/MastodonDataService.ts b/packages/backend/src/server/api/mastodon/MastodonDataService.ts index db257756de..73cd553b9a 100644 --- a/packages/backend/src/server/api/mastodon/MastodonDataService.ts +++ b/packages/backend/src/server/api/mastodon/MastodonDataService.ts @@ -7,8 +7,8 @@ import { Inject, Injectable } from '@nestjs/common'; import { IsNull } from 'typeorm'; import { DI } from '@/di-symbols.js'; import { QueryService } from '@/core/QueryService.js'; -import type { MiNote, NotesRepository } from '@/models/_.js'; -import type { MiLocalUser } from '@/models/User.js'; +import type { MiChannel, MiNote, NotesRepository } from '@/models/_.js'; +import type { MiLocalUser, MiUser } from '@/models/User.js'; import { ApiError } from '../error.js'; /** @@ -27,8 +27,8 @@ export class MastodonDataService { /** * Fetches a note in the context of the current user, and throws an exception if not found. */ - public async requireNote(noteId: string, me?: MiLocalUser | null): Promise { - const note = await this.getNote(noteId, me); + public async requireNote(noteId: string, me: MiLocalUser | null | undefined, relations?: Rel): Promise> { + const note = await this.getNote(noteId, me, relations); if (!note) { throw new ApiError({ @@ -46,12 +46,39 @@ export class MastodonDataService { /** * Fetches a note in the context of the current user. */ - public async getNote(noteId: string, me?: MiLocalUser | null): Promise { + public async getNote(noteId: string, me: MiLocalUser | null | undefined, relations?: Rel): Promise | null> { // Root query: note + required dependencies const query = this.notesRepository .createQueryBuilder('note') - .where('note.id = :noteId', { noteId }) - .innerJoinAndSelect('note.user', 'user'); + .where('note.id = :noteId', { noteId }); + + // Load relations + if (relations) { + if (relations.reply) { + query.leftJoinAndSelect('note.reply', 'reply'); + if (typeof(relations.reply) === 'object') { + if (relations.reply.reply) query.leftJoinAndSelect('note.reply.reply', 'replyReply'); + if (relations.reply.renote) query.leftJoinAndSelect('note.reply.renote', 'replyRenote'); + if (relations.reply.user) query.innerJoinAndSelect('note.reply.user', 'replyUser'); + if (relations.reply.channel) query.leftJoinAndSelect('note.reply.channel', 'replyChannel'); + } + } + if (relations.renote) { + query.leftJoinAndSelect('note.renote', 'renote'); + if (typeof(relations.renote) === 'object') { + if (relations.renote.reply) query.leftJoinAndSelect('note.renote.reply', 'renoteReply'); + if (relations.renote.renote) query.leftJoinAndSelect('note.renote.renote', 'renoteRenote'); + if (relations.renote.user) query.innerJoinAndSelect('note.renote.user', 'renoteUser'); + if (relations.renote.channel) query.leftJoinAndSelect('note.renote.channel', 'renoteChannel'); + } + } + if (relations.user) { + query.innerJoinAndSelect('note.user', 'user'); + } + if (relations.channel) { + query.leftJoinAndSelect('note.channel', 'channel'); + } + } // Restrict visibility this.queryService.generateVisibilityQuery(query, me); @@ -59,7 +86,7 @@ export class MastodonDataService { this.queryService.generateBlockedUserQueryForNotes(query, me); } - return await query.getOne(); + return await query.getOne() as NoteWithRelations | null; } /** @@ -82,3 +109,41 @@ export class MastodonDataService { }); } } + +interface NoteRelations { + reply?: boolean | { + reply?: boolean; + renote?: boolean; + user?: boolean; + channel?: boolean; + }; + renote?: boolean | { + reply?: boolean; + renote?: boolean; + user?: boolean; + channel?: boolean; + }; + user?: boolean; + channel?: boolean; +} + +type NoteWithRelations = MiNote & { + reply: Rel extends { reply: false } + ? null + : null | (MiNote & { + reply: Rel['reply'] extends { reply: true } ? MiNote | null : null; + renote: Rel['reply'] extends { renote: true } ? MiNote | null : null; + user: Rel['reply'] extends { user: true } ? MiUser : null; + channel: Rel['reply'] extends { channel: true } ? MiChannel | null : null; + }); + renote: Rel extends { renote: false } + ? null + : null | (MiNote & { + reply: Rel['renote'] extends { reply: true } ? MiNote | null : null; + renote: Rel['renote'] extends { renote: true } ? MiNote | null : null; + user: Rel['renote'] extends { user: true } ? MiUser : null; + channel: Rel['renote'] extends { channel: true } ? MiChannel | null : null; + }); + user: Rel extends { user: true } ? MiUser : null; + channel: Rel extends { channel: true } ? MiChannel | null : null; +}; diff --git a/packages/backend/src/server/api/mastodon/endpoints/status.ts b/packages/backend/src/server/api/mastodon/endpoints/status.ts index 22b8a911ca..7a058a0ed9 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/status.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/status.ts @@ -8,6 +8,10 @@ import { Injectable } from '@nestjs/common'; import { emojiRegexAtStartToEnd } from '@/misc/emoji-regex.js'; import { parseTimelineArgs, TimelineArgs, toBoolean, toInt } from '@/server/api/mastodon/argsUtils.js'; import { MastodonClientService } from '@/server/api/mastodon/MastodonClientService.js'; +import { MastodonDataService } from '@/server/api/mastodon/MastodonDataService.js'; +import { getNoteSummary } from '@/misc/get-note-summary.js'; +import type { Packed } from '@/misc/json-schema.js'; +import { isPureRenote } from '@/misc/is-renote.js'; import { convertAttachment, convertPoll, MastodonConverters } from '../MastodonConverters.js'; import type { Entity } from 'megalodon'; import type { FastifyInstance } from 'fastify'; @@ -22,6 +26,7 @@ export class ApiStatusMastodon { constructor( private readonly mastoConverters: MastodonConverters, private readonly clientService: MastodonClientService, + private readonly mastodonDataService: MastodonDataService, ) {} public register(fastify: FastifyInstance): void { @@ -29,13 +34,24 @@ export class ApiStatusMastodon { if (!_request.params.id) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required parameter "id"' }); const { client, me } = await this.clientService.getAuthClient(_request); - const data = await client.getStatus(_request.params.id); - const response = await this.mastoConverters.convertStatus(data.data, me); + const note = await this.mastodonDataService.requireNote(_request.params.id, me, { user: true, renote: { user: true } }); + + // Unpack renote for Discord, otherwise the preview breaks + const appearNote = (isPureRenote(note) && _request.headers['user-agent']?.match(/\bDiscordbot\//)) + ? note.renote as NonNullable + : note; + + const data = await client.getStatus(appearNote.id); + const response = await this.mastoConverters.convertStatus(data.data, me, { note: appearNote, user: appearNote.user }); // Fixup - Discord ignores CWs and renders the entire post. if (response.sensitive && _request.headers['user-agent']?.match(/\bDiscordbot\//)) { - response.content = '(preview disabled for sensitive content)'; + response.content = getNoteSummary(data.data satisfies Packed<'Note'>); response.media_attachments = []; + response.in_reply_to_id = null; + response.in_reply_to_account_id = null; + response.reblog = null; + response.quote = null; } return reply.send(response); @@ -170,7 +186,7 @@ export class ApiStatusMastodon { const data = await client.deleteEmojiReaction(id, react); return reply.send(data.data); } - if (!body.media_ids) body.media_ids = undefined; + body.media_ids ??= undefined; if (body.media_ids && !body.media_ids.length) body.media_ids = undefined; if (body.poll && !body.poll.options) { diff --git a/packages/backend/src/server/api/stream/channel.ts b/packages/backend/src/server/api/stream/channel.ts index 9af816dfbb..3a82865577 100644 --- a/packages/backend/src/server/api/stream/channel.ts +++ b/packages/backend/src/server/api/stream/channel.ts @@ -61,12 +61,30 @@ export default abstract class Channel { return this.connection.subscriber; } + /** + * Checks if a note is visible to the current user *excluding* blocks and mutes. + */ + protected isNoteVisibleToMe(note: Packed<'Note'>): boolean { + if (note.visibility === 'public') return true; + if (note.visibility === 'home') return true; + if (!this.user) return false; + if (this.user.id === note.userId) return true; + if (note.visibility === 'followers') { + return this.following[note.userId] != null; + } + if (!note.visibleUserIds) return false; + return note.visibleUserIds.includes(this.user.id); + } + /* * ミュートとブロックされてるを処理する */ protected isNoteMutedOrBlocked(note: Packed<'Note'>): boolean { + // Ignore notes that require sign-in + if (note.user.requireSigninToViewContents && !this.user) return true; + // 流れてきたNoteがインスタンスミュートしたインスタンスが関わる - if (isInstanceMuted(note, new Set(this.userProfile?.mutedInstances ?? [])) && !this.following[note.userId]) return true; + if (isInstanceMuted(note, this.userMutedInstances) && !this.following[note.userId]) return true; // 流れてきたNoteがミュートしているユーザーが関わる if (isUserRelated(note, this.userIdsWhoMeMuting)) return true; @@ -79,6 +97,15 @@ export default abstract class Channel { // If it's a boost (pure renote) then we need to check the target as well if (isPackedPureRenote(note) && note.renote && this.isNoteMutedOrBlocked(note.renote)) return true; + // Hide silenced notes + if (note.user.isSilenced || note.user.instance?.isSilenced) { + if (this.user == null) return true; + if (this.user.id === note.userId) return false; + if (this.following[note.userId] == null) return true; + } + + // TODO muted threads + return false; } diff --git a/packages/backend/src/server/api/stream/channels/bubble-timeline.ts b/packages/backend/src/server/api/stream/channels/bubble-timeline.ts index d29101cbc5..393fe3883c 100644 --- a/packages/backend/src/server/api/stream/channels/bubble-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/bubble-timeline.ts @@ -5,13 +5,12 @@ import { Injectable } from '@nestjs/common'; import type { Packed } from '@/misc/json-schema.js'; -import { MetaService } from '@/core/MetaService.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import { bindThis } from '@/decorators.js'; import { RoleService } from '@/core/RoleService.js'; -import type { MiMeta } from '@/models/Meta.js'; import { isRenotePacked, isQuotePacked } from '@/misc/is-renote.js'; import type { JsonObject } from '@/misc/json-value.js'; +import { UtilityService } from '@/core/UtilityService.js'; import Channel, { MiChannelService } from '../channel.js'; class BubbleTimelineChannel extends Channel { @@ -21,11 +20,10 @@ class BubbleTimelineChannel extends Channel { private withRenotes: boolean; private withFiles: boolean; private withBots: boolean; - private instance: MiMeta; constructor( - private metaService: MetaService, private roleService: RoleService, + private readonly utilityService: UtilityService, noteEntityService: NoteEntityService, id: string, @@ -42,7 +40,6 @@ class BubbleTimelineChannel extends Channel { this.withRenotes = !!(params.withRenotes ?? true); this.withFiles = !!(params.withFiles ?? false); this.withBots = !!(params.withBots ?? true); - this.instance = await this.metaService.fetch(); // Subscribe events this.subscriber.on('notesStream', this.onNote); @@ -50,21 +47,37 @@ class BubbleTimelineChannel extends Channel { @bindThis private async onNote(note: Packed<'Note'>) { + const isMe = this.user?.id === note.userId; + if (this.withFiles && (note.fileIds == null || note.fileIds.length === 0)) return; if (!this.withBots && note.user.isBot) return; if (note.visibility !== 'public') return; if (note.channelId != null) return; - if (note.user.host == null) return; - if (!this.instance.bubbleInstances.includes(note.user.host)) return; - if (note.user.requireSigninToViewContents && this.user == null) return; - - if (isRenotePacked(note) && !isQuotePacked(note) && !this.withRenotes) return; - - if (note.user.isSilenced && !this.following[note.userId] && note.userId !== this.user!.id) return; + if (!this.utilityService.isBubbledHost(note.user.host)) return; if (this.isNoteMutedOrBlocked(note)) return; + if (note.reply) { + const reply = note.reply; + // 自分のフォローしていないユーザーの visibility: followers な投稿への返信は弾く + if (!this.isNoteVisibleToMe(reply)) return; + if (!this.following[note.userId]?.withReplies) { + // 「チャンネル接続主への返信」でもなければ、「チャンネル接続主が行った返信」でもなければ、「投稿者の投稿者自身への返信」でもない場合 + if (reply.userId !== this.user?.id && !isMe && reply.userId !== note.userId) return; + } + } + + // 純粋なリノート（引用リノートでないリノート）の場合 + if (isRenotePacked(note) && !isQuotePacked(note) && note.renote) { + if (!this.withRenotes) return; + if (note.renote.reply) { + const reply = note.renote.reply; + // 自分のフォローしていないユーザーの visibility: followers な投稿への返信のリノートは弾く + if (!this.isNoteVisibleToMe(reply)) return; + } + } + const clonedNote = await this.assignMyReaction(note); await this.hideNote(clonedNote); @@ -85,17 +98,17 @@ export class BubbleTimelineChannelService implements MiChannelService { public readonly kind = BubbleTimelineChannel.kind; constructor( - private metaService: MetaService, private roleService: RoleService, private noteEntityService: NoteEntityService, + private readonly utilityService: UtilityService, ) { } @bindThis public create(id: string, connection: Channel['connection']): BubbleTimelineChannel { return new BubbleTimelineChannel( - this.metaService, this.roleService, + this.utilityService, this.noteEntityService, id, connection, diff --git a/packages/backend/src/server/api/stream/channels/global-timeline.ts b/packages/backend/src/server/api/stream/channels/global-timeline.ts index c899ad9490..bac0277538 100644 --- a/packages/backend/src/server/api/stream/channels/global-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/global-timeline.ts @@ -48,20 +48,36 @@ class GlobalTimelineChannel extends Channel { @bindThis private async onNote(note: Packed<'Note'>) { + const isMe = this.user?.id === note.userId; + if (this.withFiles && (note.fileIds == null || note.fileIds.length === 0)) return; if (!this.withBots && note.user.isBot) return; if (note.visibility !== 'public') return; if (note.channelId != null) return; - if (note.user.requireSigninToViewContents && this.user == null) return; - if (note.renote && note.renote.user.requireSigninToViewContents && this.user == null) return; - if (note.reply && note.reply.user.requireSigninToViewContents && this.user == null) return; - - if (isRenotePacked(note) && !isQuotePacked(note) && !this.withRenotes) return; - - if (note.user.isSilenced && !this.following[note.userId] && note.userId !== this.user!.id) return; if (this.isNoteMutedOrBlocked(note)) return; + if (!this.isNoteVisibleToMe(note)) return; + + if (note.reply) { + const reply = note.reply; + // 自分のフォローしていないユーザーの visibility: followers な投稿への返信は弾く + if (!this.isNoteVisibleToMe(reply)) return; + if (!this.following[note.userId]?.withReplies) { + // 「チャンネル接続主への返信」でもなければ、「チャンネル接続主が行った返信」でもなければ、「投稿者の投稿者自身への返信」でもない場合 + if (reply.userId !== this.user?.id && !isMe && reply.userId !== note.userId) return; + } + } + + // 純粋なリノート（引用リノートでないリノート）の場合 + if (isRenotePacked(note) && !isQuotePacked(note) && note.renote) { + if (!this.withRenotes) return; + if (note.renote.reply) { + const reply = note.renote.reply; + // 自分のフォローしていないユーザーの visibility: followers な投稿への返信のリノートは弾く + if (!this.isNoteVisibleToMe(reply)) return; + } + } const clonedNote = await this.assignMyReaction(note); await this.hideNote(clonedNote); diff --git a/packages/backend/src/server/api/stream/channels/home-timeline.ts b/packages/backend/src/server/api/stream/channels/home-timeline.ts index dfdb491113..d1dcbd07e5 100644 --- a/packages/backend/src/server/api/stream/channels/home-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/home-timeline.ts @@ -50,37 +50,29 @@ class HomeTimelineChannel extends Channel { if (!isMe && !Object.hasOwn(this.following, note.userId)) return; } - if (note.visibility === 'followers') { - if (!isMe && !Object.hasOwn(this.following, note.userId)) return; - } else if (note.visibility === 'specified') { - if (!isMe && !note.visibleUserIds!.includes(this.user!.id)) return; - } + if (this.isNoteMutedOrBlocked(note)) return; + if (!this.isNoteVisibleToMe(note)) return; if (note.reply) { const reply = note.reply; - if (this.following[note.userId]?.withReplies) { - // 自分のフォローしていないユーザーの visibility: followers な投稿への返信は弾く - if (reply.visibility === 'followers' && !Object.hasOwn(this.following, reply.userId) && reply.userId !== this.user!.id) return; - } else { + // 自分のフォローしていないユーザーの visibility: followers な投稿への返信は弾く + if (!this.isNoteVisibleToMe(reply)) return; + if (!this.following[note.userId]?.withReplies) { // 「チャンネル接続主への返信」でもなければ、「チャンネル接続主が行った返信」でもなければ、「投稿者の投稿者自身への返信」でもない場合 if (reply.userId !== this.user!.id && !isMe && reply.userId !== note.userId) return; } } - if (note.user.isSilenced && !this.following[note.userId] && note.userId !== this.user!.id) return; - // 純粋なリノート（引用リノートでないリノート）の場合 if (isRenotePacked(note) && !isQuotePacked(note) && note.renote) { if (!this.withRenotes) return; if (note.renote.reply) { const reply = note.renote.reply; // 自分のフォローしていないユーザーの visibility: followers な投稿への返信のリノートは弾く - if (reply.visibility === 'followers' && !Object.hasOwn(this.following, reply.userId) && reply.userId !== this.user!.id) return; + if (!this.isNoteVisibleToMe(reply)) return; } } - if (this.isNoteMutedOrBlocked(note)) return; - const clonedNote = await this.assignMyReaction(note); await this.hideNote(clonedNote); diff --git a/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts b/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts index 6cb425ff81..d923167e04 100644 --- a/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts @@ -67,34 +67,26 @@ class HybridTimelineChannel extends Channel { (note.channelId != null && this.followingChannels.has(note.channelId)) )) return; - if (note.visibility === 'followers') { - if (!isMe && !Object.hasOwn(this.following, note.userId)) return; - } else if (note.visibility === 'specified') { - if (!isMe && !note.visibleUserIds!.includes(this.user!.id)) return; - } - if (this.isNoteMutedOrBlocked(note)) return; + if (!this.isNoteVisibleToMe(note)) return; if (note.reply) { const reply = note.reply; - if ((this.following[note.userId]?.withReplies ?? false) || this.withReplies) { - // 自分のフォローしていないユーザーの visibility: followers な投稿への返信は弾く - if (reply.visibility === 'followers' && !Object.hasOwn(this.following, reply.userId) && reply.userId !== this.user!.id) return; - } else { + // 自分のフォローしていないユーザーの visibility: followers な投稿への返信は弾く + if (!this.isNoteVisibleToMe(reply)) return; + if (!this.following[note.userId]?.withReplies && !this.withReplies) { // 「チャンネル接続主への返信」でもなければ、「チャンネル接続主が行った返信」でもなければ、「投稿者の投稿者自身への返信」でもない場合 if (reply.userId !== this.user!.id && !isMe && reply.userId !== note.userId) return; } } - if (note.user.isSilenced && !this.following[note.userId] && note.userId !== this.user!.id) return; - // 純粋なリノート（引用リノートでないリノート）の場合 if (isRenotePacked(note) && !isQuotePacked(note) && note.renote) { if (!this.withRenotes) return; if (note.renote.reply) { const reply = note.renote.reply; // 自分のフォローしていないユーザーの visibility: followers な投稿への返信のリノートは弾く - if (reply.visibility === 'followers' && !Object.hasOwn(this.following, reply.userId) && reply.userId !== this.user!.id) return; + if (!this.isNoteVisibleToMe(reply)) return; } } diff --git a/packages/backend/src/server/api/stream/channels/local-timeline.ts b/packages/backend/src/server/api/stream/channels/local-timeline.ts index 82b128eae0..2eb3460efa 100644 --- a/packages/backend/src/server/api/stream/channels/local-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/local-timeline.ts @@ -50,28 +50,37 @@ class LocalTimelineChannel extends Channel { @bindThis private async onNote(note: Packed<'Note'>) { + const isMe = this.user?.id === note.userId; + if (this.withFiles && (note.fileIds == null || note.fileIds.length === 0)) return; if (!this.withBots && note.user.isBot) return; if (note.user.host !== null) return; if (note.visibility !== 'public') return; if (note.channelId != null) return; - if (note.user.requireSigninToViewContents && this.user == null) return; - if (note.renote && note.renote.user.requireSigninToViewContents && this.user == null) return; - if (note.reply && note.reply.user.requireSigninToViewContents && this.user == null) return; - - // 関係ない返信は除外 - if (note.reply && this.user && !this.following[note.userId]?.withReplies && !this.withReplies) { - const reply = note.reply; - // 「チャンネル接続主への返信」でもなければ、「チャンネル接続主が行った返信」でもなければ、「投稿者の投稿者自身への返信」でもない場合 - if (reply.userId !== this.user.id && note.userId !== this.user.id && reply.userId !== note.userId) return; - } - - if (note.user.isSilenced && !this.following[note.userId] && note.userId !== this.user!.id) return; - - if (isRenotePacked(note) && !isQuotePacked(note) && !this.withRenotes) return; if (this.isNoteMutedOrBlocked(note)) return; + if (!this.isNoteVisibleToMe(note)) return; + + // 関係ない返信は除外 + if (note.reply) { + const reply = note.reply; + // 自分のフォローしていないユーザーの visibility: followers な投稿への返信は弾く + if (!this.isNoteVisibleToMe(reply)) return; + if (!this.following[note.userId]?.withReplies) { + // 「チャンネル接続主への返信」でもなければ、「チャンネル接続主が行った返信」でもなければ、「投稿者の投稿者自身への返信」でもない場合 + if (reply.userId !== this.user?.id && !isMe && reply.userId !== note.userId) return; + } + } + + if (isRenotePacked(note) && !isQuotePacked(note) && note.renote) { + if (!this.withRenotes) return; + if (note.renote.reply) { + const reply = note.renote.reply; + // 自分のフォローしていないユーザーの visibility: followers な投稿への返信のリノートは弾く + if (!this.isNoteVisibleToMe(reply)) return; + } + } const clonedNote = await this.assignMyReaction(note); await this.hideNote(clonedNote); diff --git a/packages/backend/src/server/api/stream/channels/main.ts b/packages/backend/src/server/api/stream/channels/main.ts index 6194bb78dd..193907504a 100644 --- a/packages/backend/src/server/api/stream/channels/main.ts +++ b/packages/backend/src/server/api/stream/channels/main.ts @@ -32,10 +32,12 @@ class MainChannel extends Channel { switch (data.type) { case 'notification': { // Ignore notifications from instances the user has muted - if (isUserFromMutedInstance(data.body, new Set(this.userProfile?.mutedInstances ?? []))) return; + if (isUserFromMutedInstance(data.body, this.userMutedInstances)) return; if (data.body.userId && this.userIdsWhoMeMuting.has(data.body.userId)) return; if (data.body.note && data.body.note.isHidden) { + if (this.isNoteMutedOrBlocked(data.body.note)) return; + if (!this.isNoteVisibleToMe(data.body.id)) return; const note = await this.noteEntityService.pack(data.body.note.id, this.user, { detail: true, }); @@ -44,9 +46,7 @@ class MainChannel extends Channel { break; } case 'mention': { - if (isInstanceMuted(data.body, new Set(this.userProfile?.mutedInstances ?? []))) return; - - if (this.userIdsWhoMeMuting.has(data.body.userId)) return; + if (this.isNoteMutedOrBlocked(data.body)) return; if (data.body.isHidden) { const note = await this.noteEntityService.pack(data.body.id, this.user, { detail: true, diff --git a/packages/backend/src/server/api/stream/channels/role-timeline.ts b/packages/backend/src/server/api/stream/channels/role-timeline.ts index 78cd9bf868..f5984b5ae9 100644 --- a/packages/backend/src/server/api/stream/channels/role-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/role-timeline.ts @@ -9,6 +9,7 @@ import { bindThis } from '@/decorators.js'; import { RoleService } from '@/core/RoleService.js'; import type { GlobalEvents } from '@/core/GlobalEventService.js'; import type { JsonObject } from '@/misc/json-value.js'; +import { isQuotePacked, isRenotePacked } from '@/misc/is-renote.js'; import Channel, { type MiChannelService } from '../channel.js'; class RoleTimelineChannel extends Channel { @@ -40,7 +41,9 @@ class RoleTimelineChannel extends Channel { private async onEvent(data: GlobalEvents['roleTimeline']['payload']) { if (data.type === 'note') { const note = data.body; + const isMe = this.user?.id === note.userId; + // TODO this should be cached if (!(await this.roleservice.isExplorable({ id: this.roleId }))) { return; } @@ -48,6 +51,25 @@ class RoleTimelineChannel extends Channel { if (this.isNoteMutedOrBlocked(note)) return; + if (note.reply) { + const reply = note.reply; + // 自分のフォローしていないユーザーの visibility: followers な投稿への返信は弾く + if (!this.isNoteVisibleToMe(reply)) return; + if (!this.following[note.userId]?.withReplies) { + // 「チャンネル接続主への返信」でもなければ、「チャンネル接続主が行った返信」でもなければ、「投稿者の投稿者自身への返信」でもない場合 + if (reply.userId !== this.user?.id && !isMe && reply.userId !== note.userId) return; + } + } + + // 純粋なリノート（引用リノートでないリノート）の場合 + if (isRenotePacked(note) && !isQuotePacked(note) && note.renote) { + if (note.renote.reply) { + const reply = note.renote.reply; + // 自分のフォローしていないユーザーの visibility: followers な投稿への返信のリノートは弾く + if (!this.isNoteVisibleToMe(reply)) return; + } + } + const clonedNote = await this.assignMyReaction(note); await this.hideNote(clonedNote); diff --git a/packages/backend/src/server/api/stream/channels/user-list.ts b/packages/backend/src/server/api/stream/channels/user-list.ts index 8a7c2b2633..3f1a5a8f8f 100644 --- a/packages/backend/src/server/api/stream/channels/user-list.ts +++ b/packages/backend/src/server/api/stream/channels/user-list.ts @@ -16,7 +16,8 @@ import Channel, { type MiChannelService } from '../channel.js'; class UserListChannel extends Channel { public readonly chName = 'userList'; public static shouldShare = false; - public static requireCredential = false as const; + public static requireCredential = true as const; + public static kind = 'read:account'; private listId: string; private membershipsMap: Record | undefined> = {}; private listUsersClock: NodeJS.Timeout; @@ -81,7 +82,7 @@ class UserListChannel extends Channel { @bindThis private async onNote(note: Packed<'Note'>) { - const isMe = this.user!.id === note.userId; + const isMe = this.user?.id === note.userId; // チャンネル投稿は無視する if (note.channelId) return; @@ -90,26 +91,28 @@ class UserListChannel extends Channel { if (!Object.hasOwn(this.membershipsMap, note.userId)) return; - if (note.visibility === 'followers') { - if (!isMe && !Object.hasOwn(this.following, note.userId)) return; - } else if (note.visibility === 'specified') { - if (!note.visibleUserIds!.includes(this.user!.id)) return; - } + if (this.isNoteMutedOrBlocked(note)) return; + if (!this.isNoteVisibleToMe(note)) return; if (note.reply) { const reply = note.reply; - if (this.membershipsMap[note.userId]?.withReplies) { - // 自分のフォローしていないユーザーの visibility: followers な投稿への返信は弾く - if (reply.visibility === 'followers' && !Object.hasOwn(this.following, reply.userId)) return; - } else { + // 自分のフォローしていないユーザーの visibility: followers な投稿への返信は弾く + if (!this.isNoteVisibleToMe(reply)) return; + if (!this.following[note.userId]?.withReplies) { // 「チャンネル接続主への返信」でもなければ、「チャンネル接続主が行った返信」でもなければ、「投稿者の投稿者自身への返信」でもない場合 if (reply.userId !== this.user!.id && !isMe && reply.userId !== note.userId) return; } } - if (isRenotePacked(note) && !isQuotePacked(note) && !this.withRenotes) return; - - if (this.isNoteMutedOrBlocked(note)) return; + // 純粋なリノート（引用リノートでないリノート）の場合 + if (isRenotePacked(note) && !isQuotePacked(note) && note.renote) { + if (!this.withRenotes) return; + if (note.renote.reply) { + const reply = note.renote.reply; + // 自分のフォローしていないユーザーの visibility: followers な投稿への返信のリノートは弾く + if (!this.isNoteVisibleToMe(reply)) return; + } + } const clonedNote = await this.assignMyReaction(note); await this.hideNote(clonedNote); @@ -128,7 +131,7 @@ class UserListChannel extends Channel { } @Injectable() -export class UserListChannelService implements MiChannelService { +export class UserListChannelService implements MiChannelService { public readonly shouldShare = UserListChannel.shouldShare; public readonly requireCredential = UserListChannel.requireCredential; public readonly kind = UserListChannel.kind; diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index 203bc908a8..ed5d87d15d 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -20,6 +20,7 @@ import { RedisKVCache } from '@/misc/cache.js'; import { UtilityService } from '@/core/UtilityService.js'; import { ApDbResolverService } from '@/core/activitypub/ApDbResolverService.js'; import type { MiAccessToken, NotesRepository } from '@/models/_.js'; +import { RemoteUserResolveService } from '@/core/RemoteUserResolveService.js'; import { ApUtilityService } from '@/core/activitypub/ApUtilityService.js'; import { ApRequestService } from '@/core/activitypub/ApRequestService.js'; import { SystemAccountService } from '@/core/SystemAccountService.js'; @@ -30,14 +31,19 @@ import { BucketRateLimit, Keyed, sendRateLimitHeaders } from '@/misc/rate-limit- import type { MiLocalUser } from '@/models/User.js'; import { getIpHash } from '@/misc/get-ip-hash.js'; import { isRetryableError } from '@/misc/is-retryable-error.js'; +import * as Acct from '@/misc/acct.js'; +import { isNote } from '@/core/activitypub/type.js'; import type { FastifyRequest, FastifyReply } from 'fastify'; export type LocalSummalyResult = SummalyResult & { haveNoteLocally?: boolean; + linkAttribution?: { + userId: string, + } }; // Increment this to invalidate cached previews after a major change. -const cacheFormatVersion = 3; +const cacheFormatVersion = 4; type PreviewRoute = { Querystring: { @@ -82,6 +88,7 @@ export class UrlPreviewService { private readonly utilityService: UtilityService, private readonly apUtilityService: ApUtilityService, private readonly apDbResolverService: ApDbResolverService, + private readonly remoteUserResolveService: RemoteUserResolveService, private readonly apRequestService: ApRequestService, private readonly systemAccountService: SystemAccountService, private readonly apNoteService: ApNoteService, @@ -117,18 +124,6 @@ export class UrlPreviewService { request: FastifyRequest, reply: FastifyReply, ): Promise { - const url = request.query.url; - if (typeof url !== 'string' || !URL.canParse(url)) { - reply.code(400); - return; - } - - const lang = request.query.lang; - if (Array.isArray(lang)) { - reply.code(400); - return; - } - if (!this.meta.urlPreviewEnabled) { return reply.code(403).send({ error: { @@ -139,13 +134,44 @@ export class UrlPreviewService { }); } + const url = request.query.url; + if (typeof url !== 'string' || !URL.canParse(url)) { + reply.code(400); + return; + } + + // Enforce HTTP(S) for input URLs + const urlScheme = this.utilityService.getUrlScheme(url); + if (urlScheme !== 'http:' && urlScheme !== 'https:') { + reply.code(400); + return; + } + + const lang = request.query.lang; + if (Array.isArray(lang)) { + reply.code(400); + return; + } + + // Strip out hash (anchor) + const urlObj = new URL(url); + if (urlObj.hash) { + urlObj.hash = ''; + const params = new URLSearchParams({ url: urlObj.href }); + if (lang) params.set('lang', lang); + const newUrl = `/url?${params.toString()}`; + + reply.redirect(newUrl, 301); + return; + } + // Check rate limit const auth = await this.authenticate(request); if (!await this.checkRateLimit(auth, reply)) { return; } - if (this.utilityService.isBlockedHost(this.meta.blockedHosts, new URL(url).host)) { + if (this.utilityService.isBlockedHost(this.meta.blockedHosts, urlObj.host)) { return reply.code(403).send({ error: { message: 'URL is blocked', @@ -160,7 +186,7 @@ export class UrlPreviewService { return; } - const cacheKey = `${url}@${lang}@${cacheFormatVersion}`; + const cacheKey = getCacheKey(url, lang); if (await this.sendCachedPreview(cacheKey, reply, fetch)) { return; } @@ -206,9 +232,23 @@ export class UrlPreviewService { } } + await this.validateLinkAttribution(summary); + // Await this to avoid hammering redis when a bunch of URLs are fetched at once await this.previewCache.set(cacheKey, summary); + // Also cache the response URL in case of redirects + if (summary.url !== url) { + const responseCacheKey = getCacheKey(summary.url, lang); + await this.previewCache.set(responseCacheKey, summary); + } + + // Also cache the ActivityPub URL, if different from the others + if (summary.activityPub && summary.activityPub !== summary.url) { + const apCacheKey = getCacheKey(summary.activityPub, lang); + await this.previewCache.set(apCacheKey, summary); + } + // Cache 1 day (matching redis), but only once we finalize the result if (!summary.activityPub || summary.haveNoteLocally) { reply.header('Cache-Control', 'public, max-age=86400'); @@ -370,7 +410,7 @@ export class UrlPreviewService { // Finally, attempt a signed GET in case it's a direct link to an instance with authorized fetch. const instanceActor = await this.systemAccountService.getInstanceActor(); const remoteObject = await this.apRequestService.signedGet(summary.url, instanceActor).catch(() => null); - if (remoteObject && this.apUtilityService.haveSameAuthority(remoteObject.id, summary.url)) { + if (remoteObject && isNote(remoteObject) && this.apUtilityService.haveSameAuthority(remoteObject.id, summary.url)) { summary.activityPub = remoteObject.id; return; } @@ -426,6 +466,30 @@ export class UrlPreviewService { } } + private async validateLinkAttribution(summary: LocalSummalyResult) { + if (!summary.fediverseCreator) return; + if (!URL.canParse(summary.url)) return; + + const url = URL.parse(summary.url); + + const acct = Acct.parse(summary.fediverseCreator); + if (acct.host?.toLowerCase() === this.config.host) { + acct.host = null; + } + try { + const user = await this.remoteUserResolveService.resolveUser(acct.username, acct.host); + + const attributionDomains = user.attributionDomains; + if (attributionDomains.some(x => `.${url?.host.toLowerCase()}`.endsWith(`.${x}`))) { + summary.linkAttribution = { + userId: user.id, + }; + } + } catch { + this.logger.debug('User not found: ' + summary.fediverseCreator); + } + } + // Adapted from ApiCallService private async checkFetchPermissions(auth: AuthArray, reply: FastifyReply): Promise { const [user, app] = auth; @@ -501,3 +565,7 @@ export class UrlPreviewService { return true; } } + +function getCacheKey(url: string, lang = 'none') { + return `${url}@${lang}@${cacheFormatVersion}`; +} diff --git a/packages/backend/test-federation/tsconfig.json b/packages/backend/test-federation/tsconfig.json index 3a1cb3b9f3..16b333f877 100644 --- a/packages/backend/test-federation/tsconfig.json +++ b/packages/backend/test-federation/tsconfig.json @@ -3,7 +3,7 @@ /* Visit https://aka.ms/tsconfig to read more about this file */ /* Projects */ - // "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */ + "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */ // "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */ // "tsBuildInfoFile": "./.tsbuildinfo", /* Specify the path to .tsbuildinfo incremental compilation file. */ // "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects. */ diff --git a/packages/backend/test-server/tsconfig.json b/packages/backend/test-server/tsconfig.json index 10313699c2..cb394ecccd 100644 --- a/packages/backend/test-server/tsconfig.json +++ b/packages/backend/test-server/tsconfig.json @@ -23,6 +23,7 @@ "emitDecoratorMetadata": true, "resolveJsonModule": true, "isolatedModules": true, + "incremental": true, "rootDir": "../src", "baseUrl": "./", "paths": { diff --git a/packages/backend/test/tsconfig.json b/packages/backend/test/tsconfig.json index 2b562acda8..f3b6a5108d 100644 --- a/packages/backend/test/tsconfig.json +++ b/packages/backend/test/tsconfig.json @@ -23,6 +23,7 @@ "emitDecoratorMetadata": true, "resolveJsonModule": true, "isolatedModules": true, + "incremental": true, "baseUrl": "./", "paths": { "@/*": ["../src/*"] diff --git a/packages/backend/test/unit/AbuseReportNotificationService.ts b/packages/backend/test/unit/AbuseReportNotificationService.ts index 6d555326fb..ee68b10f1b 100644 --- a/packages/backend/test/unit/AbuseReportNotificationService.ts +++ b/packages/backend/test/unit/AbuseReportNotificationService.ts @@ -367,8 +367,10 @@ describe('AbuseReportNotificationService', () => { id: idService.gen(), targetUserId: alice.id, targetUser: alice, + targetUserInstance: null, reporterId: bob.id, reporter: bob, + reporterInstance: null, assigneeId: null, assignee: null, resolved: false, diff --git a/packages/backend/test/unit/MetaService.ts b/packages/backend/test/unit/MetaService.ts index 19c98eab3d..056838e180 100644 --- a/packages/backend/test/unit/MetaService.ts +++ b/packages/backend/test/unit/MetaService.ts @@ -11,6 +11,7 @@ import { GlobalModule } from '@/GlobalModule.js'; import { DI } from '@/di-symbols.js'; import { MetaService } from '@/core/MetaService.js'; import { CoreModule } from '@/core/CoreModule.js'; +import { MetasRepository } from '@/models/_.js'; import type { TestingModule } from '@nestjs/testing'; import type { DataSource } from 'typeorm'; @@ -39,8 +40,8 @@ describe('MetaService', () => { }); test('fetch (cache)', async () => { - const db = app.get(DI.db); - const spy = jest.spyOn(db, 'transaction'); + const metasRepository = app.get(DI.metasRepository); + const spy = jest.spyOn(metasRepository, 'createQueryBuilder'); const result = await metaService.fetch(); @@ -49,12 +50,12 @@ describe('MetaService', () => { }); test('fetch (force)', async () => { - const db = app.get(DI.db); - const spy = jest.spyOn(db, 'transaction'); + const metasRepository = app.get(DI.metasRepository); + const spy = jest.spyOn(metasRepository, 'createQueryBuilder'); const result = await metaService.fetch(true); expect(result.id).toBe('x'); - expect(spy).toHaveBeenCalledTimes(1); + expect(spy).toHaveBeenCalled(); }); }); diff --git a/packages/backend/test/unit/NoteCreateService.ts b/packages/backend/test/unit/NoteCreateService.ts index f4ecfef34d..63e3795a84 100644 --- a/packages/backend/test/unit/NoteCreateService.ts +++ b/packages/backend/test/unit/NoteCreateService.ts @@ -57,10 +57,13 @@ describe('NoteCreateService', () => { channelId: null, channel: null, userHost: null, + userInstance: null, replyUserId: null, replyUserHost: null, + replyUserInstance: null, renoteUserId: null, renoteUserHost: null, + renoteUserInstance: null, processErrors: [], }; diff --git a/packages/backend/test/unit/RoleService.ts b/packages/backend/test/unit/RoleService.ts index 553ff0982a..839402418e 100644 --- a/packages/backend/test/unit/RoleService.ts +++ b/packages/backend/test/unit/RoleService.ts @@ -15,6 +15,7 @@ import type { MockFunctionMetadata } from 'jest-mock'; import { GlobalModule } from '@/GlobalModule.js'; import { RoleService } from '@/core/RoleService.js'; import { + InstancesRepository, MiMeta, MiRole, MiRoleAssignment, @@ -39,6 +40,7 @@ const moduleMocker = new ModuleMocker(global); describe('RoleService', () => { let app: TestingModule; let roleService: RoleService; + let instancesRepository: InstancesRepository; let usersRepository: UsersRepository; let rolesRepository: RolesRepository; let roleAssignmentsRepository: RoleAssignmentsRepository; @@ -47,6 +49,19 @@ describe('RoleService', () => { let clock: lolex.InstalledClock; async function createUser(data: Partial = {}) { + if (data.host != null) { + await instancesRepository + .createQueryBuilder('instance') + .insert() + .values({ + id: genAidx(Date.now()), + firstRetrievedAt: new Date(), + host: data.host, + }) + .orIgnore() + .execute(); + } + const un = secureRndstr(16); const x = await usersRepository.insert({ id: genAidx(Date.now()), @@ -145,6 +160,7 @@ describe('RoleService', () => { app.enableShutdownHooks(); roleService = app.get(RoleService); + instancesRepository = app.get(DI.instancesRepository); usersRepository = app.get(DI.usersRepository); rolesRepository = app.get(DI.rolesRepository); roleAssignmentsRepository = app.get(DI.roleAssignmentsRepository); diff --git a/packages/backend/test/unit/UserSearchService.ts b/packages/backend/test/unit/UserSearchService.ts index 697425beb8..a6b331d1cb 100644 --- a/packages/backend/test/unit/UserSearchService.ts +++ b/packages/backend/test/unit/UserSearchService.ts @@ -7,16 +7,18 @@ import { Test, TestingModule } from '@nestjs/testing'; import { describe, jest, test } from '@jest/globals'; import { In } from 'typeorm'; import { UserSearchService } from '@/core/UserSearchService.js'; -import { FollowingsRepository, MiUser, UserProfilesRepository, UsersRepository } from '@/models/_.js'; +import { FollowingsRepository, InstancesRepository, MiUser, UserProfilesRepository, UsersRepository } from '@/models/_.js'; import { IdService } from '@/core/IdService.js'; import { GlobalModule } from '@/GlobalModule.js'; import { DI } from '@/di-symbols.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; +import { genAidx } from '@/misc/id/aidx.js'; describe('UserSearchService', () => { let app: TestingModule; let service: UserSearchService; + let instancesRepository: InstancesRepository; let usersRepository: UsersRepository; let followingsRepository: FollowingsRepository; let idService: IdService; @@ -35,6 +37,19 @@ describe('UserSearchService', () => { let bobby: MiUser; async function createUser(data: Partial = {}) { + if (data.host != null) { + await instancesRepository + .createQueryBuilder('instance') + .insert() + .values({ + id: genAidx(Date.now()), + firstRetrievedAt: new Date(), + host: data.host, + }) + .orIgnore() + .execute(); + } + const user = await usersRepository .insert({ id: idService.gen(), @@ -104,6 +119,7 @@ describe('UserSearchService', () => { await app.init(); + instancesRepository = app.get(DI.instancesRepository); usersRepository = app.get(DI.usersRepository); userProfilesRepository = app.get(DI.userProfilesRepository); followingsRepository = app.get(DI.followingsRepository); diff --git a/packages/backend/test/unit/activitypub.ts b/packages/backend/test/unit/activitypub.ts index 6f6d4c4121..94dec16401 100644 --- a/packages/backend/test/unit/activitypub.ts +++ b/packages/backend/test/unit/activitypub.ts @@ -103,6 +103,25 @@ describe('ActivityPub', () => { let config: Config; const metaInitial = { + id: 'x', + name: 'Test Instance', + shortName: 'Test Instance', + description: 'Test Instance', + langs: [] as string[], + pinnedUsers: [] as string[], + hiddenTags: [] as string[], + prohibitedWordsForNameOfUser: [] as string[], + silencedHosts: [] as string[], + mediaSilencedHosts: [] as string[], + policies: {}, + serverRules: [] as string[], + bannedEmailDomains: [] as string[], + preservedUsernames: [] as string[], + bubbleInstances: [] as string[], + trustedLinkUrlPatterns: [] as string[], + federation: 'all', + federationHosts: [] as string[], + allowUnsignedFetch: 'always', cacheRemoteFiles: true, cacheRemoteSensitiveFiles: true, enableFanoutTimeline: true, diff --git a/packages/backend/test/unit/misc/diff-arrays.ts b/packages/backend/test/unit/misc/diff-arrays.ts new file mode 100644 index 0000000000..b6db5e2eca --- /dev/null +++ b/packages/backend/test/unit/misc/diff-arrays.ts @@ -0,0 +1,91 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { diffArrays, diffArraysSimple } from '@/misc/diff-arrays.js'; + +describe(diffArrays, () => { + it('should return empty result when both inputs are null', () => { + const result = diffArrays(null, null); + expect(result.added).toHaveLength(0); + expect(result.removed).toHaveLength(0); + }); + + it('should return empty result when both inputs are empty', () => { + const result = diffArrays([], []); + expect(result.added).toHaveLength(0); + expect(result.removed).toHaveLength(0); + }); + + it('should remove before when after is empty', () => { + const result = diffArrays([1, 2, 3], []); + expect(result.added).toHaveLength(0); + expect(result.removed).toEqual([1, 2, 3]); + }); + + it('should deduplicate before when after is empty', () => { + const result = diffArrays([1, 1, 2, 2, 3], []); + expect(result.added).toHaveLength(0); + expect(result.removed).toEqual([1, 2, 3]); + }); + + it('should add after when before is empty', () => { + const result = diffArrays([], [1, 2, 3]); + expect(result.added).toEqual([1, 2, 3]); + expect(result.removed).toHaveLength(0); + }); + + it('should deduplicate after when before is empty', () => { + const result = diffArrays([], [1, 1, 2, 2, 3]); + expect(result.added).toEqual([1, 2, 3]); + expect(result.removed).toHaveLength(0); + }); + + it('should return diff when both have values', () => { + const result = diffArrays( + ['a', 'b', 'c', 'd'], + ['a', 'c', 'e', 'f'], + ); + expect(result.added).toEqual(['e', 'f']); + expect(result.removed).toEqual(['b', 'd']); + }); +}); + +describe(diffArraysSimple, () => { + it('should return false when both inputs are null', () => { + const result = diffArraysSimple(null, null); + expect(result).toBe(false); + }); + + it('should return false when both inputs are empty', () => { + const result = diffArraysSimple([], []); + expect(result).toBe(false); + }); + + it('should return true when before is populated and after is empty', () => { + const result = diffArraysSimple([1, 2, 3], []); + expect(result).toBe(true); + }); + + it('should return true when before is empty and after is populated', () => { + const result = diffArraysSimple([], [1, 2, 3]); + expect(result).toBe(true); + }); + + it('should return true when values have changed', () => { + const result = diffArraysSimple( + ['a', 'a', 'b', 'c'], + ['a', 'b', 'c', 'd'], + ); + expect(result).toBe(true); + }); + + it('should return false when values have not changed', () => { + const result = diffArraysSimple( + ['a', 'a', 'b', 'c'], + ['a', 'b', 'c', 'c'], + ); + expect(result).toBe(false); + }); +}); diff --git a/packages/backend/test/unit/misc/is-renote.ts b/packages/backend/test/unit/misc/is-renote.ts index 24cd2236bb..b6cfa53466 100644 --- a/packages/backend/test/unit/misc/is-renote.ts +++ b/packages/backend/test/unit/misc/is-renote.ts @@ -40,10 +40,13 @@ const base: MiNote = { channelId: null, channel: null, userHost: null, + userInstance: null, replyUserId: null, replyUserHost: null, + replyUserInstance: null, renoteUserId: null, renoteUserHost: null, + renoteUserInstance: null, processErrors: [], }; diff --git a/packages/backend/test/unit/misc/is-retryable-error.ts b/packages/backend/test/unit/misc/is-retryable-error.ts index 096bf64d4f..6d241066f7 100644 --- a/packages/backend/test/unit/misc/is-retryable-error.ts +++ b/packages/backend/test/unit/misc/is-retryable-error.ts @@ -8,6 +8,9 @@ import { AbortError } from 'node-fetch'; import { isRetryableError } from '@/misc/is-retryable-error.js'; import { StatusError } from '@/misc/status-error.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; +import { CaptchaError, captchaErrorCodes } from '@/core/CaptchaService.js'; +import { FastifyReplyError } from '@/misc/fastify-reply-error.js'; +import { ConflictError } from '@/server/SkRateLimiterService.js'; describe(isRetryableError, () => { it('should return true for retryable StatusError', () => { @@ -55,6 +58,78 @@ describe(isRetryableError, () => { expect(result).toBeTruthy(); }); + it('should return false for CaptchaError with verificationFailed', () => { + const error = new CaptchaError(captchaErrorCodes.verificationFailed, 'verificationFailed'); + const result = isRetryableError(error); + expect(result).toBeFalsy(); + }); + + it('should return false for CaptchaError with invalidProvider', () => { + const error = new CaptchaError(captchaErrorCodes.invalidProvider, 'invalidProvider'); + const result = isRetryableError(error); + expect(result).toBeFalsy(); + }); + + it('should return false for CaptchaError with invalidParameters', () => { + const error = new CaptchaError(captchaErrorCodes.invalidParameters, 'invalidParameters'); + const result = isRetryableError(error); + expect(result).toBeFalsy(); + }); + + it('should return true for CaptchaError with noResponseProvided', () => { + const error = new CaptchaError(captchaErrorCodes.noResponseProvided, 'noResponseProvided'); + const result = isRetryableError(error); + expect(result).toBeTruthy(); + }); + + it('should return true for CaptchaError with requestFailed', () => { + const error = new CaptchaError(captchaErrorCodes.requestFailed, 'requestFailed'); + const result = isRetryableError(error); + expect(result).toBeTruthy(); + }); + + it('should return true for CaptchaError with unknown', () => { + const error = new CaptchaError(captchaErrorCodes.unknown, 'unknown'); + const result = isRetryableError(error); + expect(result).toBeTruthy(); + }); + + it('should return true for CaptchaError with any other', () => { + const error = new CaptchaError(Symbol('temp'), 'unknown'); + const result = isRetryableError(error); + expect(result).toBeTruthy(); + }); + + it('should return false for FastifyReplyError', () => { + const error = new FastifyReplyError(400, 'test error'); + const result = isRetryableError(error); + expect(result).toBeFalsy(); + }); + + it('should return true for ConflictError', () => { + const error = new ConflictError('test error'); + const result = isRetryableError(error); + expect(result).toBeTruthy(); + }); + + it('should return true for AggregateError when all inners are retryable', () => { + const error = new AggregateError([ + new ConflictError(), + new ConflictError(), + ]); + const result = isRetryableError(error); + expect(result).toBeTruthy(); + }); + + it('should return true for AggregateError when any error is not retryable', () => { + const error = new AggregateError([ + new ConflictError(), + new StatusError('test err', 400), + ]); + const result = isRetryableError(error); + expect(result).toBeFalsy(); + }); + const nonErrorInputs = [ [null, 'null'], [undefined, 'undefined'], diff --git a/packages/backend/tsconfig.json b/packages/backend/tsconfig.json index 392da169ad..afed1f186c 100644 --- a/packages/backend/tsconfig.json +++ b/packages/backend/tsconfig.json @@ -23,6 +23,7 @@ "emitDecoratorMetadata": true, "resolveJsonModule": true, "isolatedModules": true, + "incremental": true, "rootDir": "./src", "baseUrl": "./", "paths": { diff --git a/packages/frontend-embed/src/boot.ts b/packages/frontend-embed/src/boot.ts index 7c8336ce3f..f11cfef8fd 100644 --- a/packages/frontend-embed/src/boot.ts +++ b/packages/frontend-embed/src/boot.ts @@ -28,7 +28,7 @@ console.log('Sharkey Embed'); //#region Embedパラメータの取得・パース const params = new URLSearchParams(location.search); const embedParams = parseEmbedParams(params); -if (_DEV_) console.log(embedParams); +if (_DEV_) console.debug(embedParams); //#endregion //#region テーマ diff --git a/packages/frontend-embed/src/post-message.ts b/packages/frontend-embed/src/post-message.ts index 93b57c380b..f44b58acd3 100644 --- a/packages/frontend-embed/src/post-message.ts +++ b/packages/frontend-embed/src/post-message.ts @@ -28,7 +28,7 @@ let defaultIframeId: string | null = null; export function setIframeId(id: string): void { if (defaultIframeId != null) return; - if (_DEV_) console.log('setIframeId', id); + if (_DEV_) console.debug('setIframeId', id); defaultIframeId = id; } @@ -40,7 +40,7 @@ export function postMessageToParentWindow { + return new Promise((resolve) => { + window.setTimeout(() => { + resolve(); + }, ms); + }); +} + +export async function retryOnThrottled(f: () => Promise, retryCount = 5): Promise { + let lastError; + for (let i = 0; i < Math.min(retryCount, 1); i++) { + try { + return await f(); + } catch (err: any) { + // RATE_LIMIT_EXCEEDED + if (typeof err === 'object' && err?.id === 'd5826d14-3982-4d2e-8011-b9e9f02499ef') { + lastError = err; + await sleep(err?.info?.fullResetMs ?? 1000); + } else { + throw err; + } + } + } + + throw lastError; +} diff --git a/packages/frontend-shared/js/worker-multi-dispatch.ts b/packages/frontend-shared/js/worker-multi-dispatch.ts index 5d393ed1ed..808b568d66 100644 --- a/packages/frontend-shared/js/worker-multi-dispatch.ts +++ b/packages/frontend-shared/js/worker-multi-dispatch.ts @@ -28,13 +28,13 @@ export class WorkerMultiDispatch { }); this.finalizationRegistry.register(this, this.symbol); - if (_DEV_) console.log('WorkerMultiDispatch: Created', this); + if (_DEV_) console.debug('WorkerMultiDispatch: Created', this); } public postMessage(message: POST, options?: Transferable[] | StructuredSerializeOptions, useWorkerNumber: WorkerNumberGetter = this.getUseWorkerNumber) { let workerNumber = useWorkerNumber(this.prevWorkerNumber, this.workers.length); workerNumber = Math.abs(Math.round(workerNumber)) % this.workers.length; - if (_DEV_) console.log('WorkerMultiDispatch: Posting message to worker', workerNumber, useWorkerNumber); + if (_DEV_) console.debug('WorkerMultiDispatch: Posting message to worker', workerNumber, useWorkerNumber); this.prevWorkerNumber = workerNumber; // 不毛だがunionをoverloadに突っ込めない @@ -64,7 +64,7 @@ export class WorkerMultiDispatch { public terminate() { this.terminated = true; - if (_DEV_) console.log('WorkerMultiDispatch: Terminating', this); + if (_DEV_) console.debug('WorkerMultiDispatch: Terminating', this); this.workers.forEach(worker => { worker.terminate(); }); diff --git a/packages/frontend-shared/tsconfig.json b/packages/frontend-shared/tsconfig.json index 8f76763e10..0512b50caf 100644 --- a/packages/frontend-shared/tsconfig.json +++ b/packages/frontend-shared/tsconfig.json @@ -18,6 +18,7 @@ "esModuleInterop": true, "verbatimModuleSyntax": true, "skipLibCheck": true, + "incremental": true, "baseUrl": ".", "paths": { "@/*": ["./*"], diff --git a/packages/frontend/.storybook/tsconfig.json b/packages/frontend/.storybook/tsconfig.json index f325114522..18baf516ba 100644 --- a/packages/frontend/.storybook/tsconfig.json +++ b/packages/frontend/.storybook/tsconfig.json @@ -18,6 +18,7 @@ "esModuleInterop": true, "skipLibCheck": true, "forceConsistentCasingInFileNames": true, + "incremental": true, "jsx": "react", "jsxFactory": "h" }, diff --git a/packages/frontend/package.json b/packages/frontend/package.json index 275a3321cb..cdd67de8b2 100644 --- a/packages/frontend/package.json +++ b/packages/frontend/package.json @@ -60,6 +60,7 @@ "misskey-reversi": "workspace:*", "moment": "^2.30.1", "photoswipe": "5.4.4", + "promise-limit": "2.7.0", "punycode.js": "2.3.1", "rollup": "4.40.0", "sanitize-html": "2.16.0", diff --git a/packages/frontend/src/components/MkAbuseReport.vue b/packages/frontend/src/components/MkAbuseReport.vue index c52fdb898e..7ad0ad647c 100644 --- a/packages/frontend/src/components/MkAbuseReport.vue +++ b/packages/frontend/src/components/MkAbuseReport.vue @@ -33,10 +33,28 @@ SPDX-License-Identifier: AGPL-3.0-only - + -
- +
+ +
+ + + + + + + +
+
@@ -44,23 +62,24 @@ SPDX-License-Identifier: AGPL-3.0-only
- + +
- + -
- +
+
- +
@@ -78,8 +97,9 @@ SPDX-License-Identifier: AGPL-3.0-only diff --git a/packages/frontend/src/components/MkCaptcha.vue b/packages/frontend/src/components/MkCaptcha.vue index 21f604aa43..e19c6435ef 100644 --- a/packages/frontend/src/components/MkCaptcha.vue +++ b/packages/frontend/src/components/MkCaptcha.vue @@ -142,7 +142,7 @@ function reset() { function remove() { if (captcha.value.remove && captchaWidgetId.value) { try { - if (_DEV_) console.log('remove', props.provider, captchaWidgetId.value); + if (_DEV_) console.debug('remove', props.provider, captchaWidgetId.value); captcha.value.remove(captchaWidgetId.value); } catch (error: unknown) { // ignore diff --git a/packages/frontend/src/components/MkCode.core.vue b/packages/frontend/src/components/MkCode.core.vue index 40f41f5d0f..36c08a8c64 100644 --- a/packages/frontend/src/components/MkCode.core.vue +++ b/packages/frontend/src/components/MkCode.core.vue @@ -52,7 +52,7 @@ async function fetchLanguage(to: string): Promise { return bundle.id === language || bundle.aliases?.includes(language); }); if (bundles.length > 0) { - if (_DEV_) console.log(`Loading language: ${language}`); + if (_DEV_) console.debug(`Loading language: ${language}`); await highlighter.loadLanguage(bundles[0].import); codeLang.value = language; } else { diff --git a/packages/frontend/src/components/MkDateSeparatedList.vue b/packages/frontend/src/components/MkDateSeparatedList.vue index 63e6b74154..8cf4e5fa2d 100644 --- a/packages/frontend/src/components/MkDateSeparatedList.vue +++ b/packages/frontend/src/components/MkDateSeparatedList.vue @@ -16,6 +16,7 @@ import { instance } from '@/instance.js'; import { prefer } from '@/preferences.js'; import { getDateText } from '@/utility/timeline-date-separate.js'; import { $i } from '@/i.js'; +import SkTransitionGroup from '@/components/SkTransitionGroup.vue'; export default defineComponent({ props: { @@ -146,14 +147,12 @@ export default defineComponent({ [$style['direction-up']]: props.direction === 'up', }; - return () => prefer.s.animation ? h(TransitionGroup, { + return () => h(SkTransitionGroup, { class: classes, name: 'list', tag: 'div', onBeforeLeave, onLeaveCancelled, - }, { default: renderChildren }) : h('div', { - class: classes, }, { default: renderChildren }); }, }); diff --git a/packages/frontend/src/components/MkDonation.vue b/packages/frontend/src/components/MkDonation.vue index 43d2002204..dfdfc0a871 100644 --- a/packages/frontend/src/components/MkDonation.vue +++ b/packages/frontend/src/components/MkDonation.vue @@ -53,6 +53,7 @@ import { i18n } from '@/i18n.js'; import * as os from '@/os.js'; import { miLocalStorage } from '@/local-storage.js'; import { instance } from '@/instance.js'; +import { prefer } from '@/preferences.js'; const emit = defineEmits<{ (ev: 'closed'): void; @@ -66,6 +67,7 @@ function close() { } function neverShow() { + prefer.commit('neverShowDonationInfo', 'true'); miLocalStorage.setItem('neverShowDonationInfo', 'true'); close(); } diff --git a/packages/frontend/src/components/MkFolder.vue b/packages/frontend/src/components/MkFolder.vue index e735a08f6f..adeedaefc3 100644 --- a/packages/frontend/src/components/MkFolder.vue +++ b/packages/frontend/src/components/MkFolder.vue @@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only