diff --git a/.config/ci.yml b/.config/ci.yml index fefa45643c..5657b8beae 100644 --- a/.config/ci.yml +++ b/.config/ci.yml @@ -297,6 +297,10 @@ proxyBypassHosts: #proxySmtp: socks4://127.0.0.1:1080 # use SOCKS4 #proxySmtp: socks5://127.0.0.1:1080 # use SOCKS5 +# Path to the directory that uploaded media will be saved to +# Defaults to a folder called "files" in the Sharkey directory +#mediaDirectory: /var/lib/sharkey + # Media Proxy #mediaProxy: https://example.com/proxy @@ -321,9 +325,24 @@ attachLdSignatureForRelays: true # For security reasons, uploading attachments from the intranet is prohibited, # but exceptions can be made from the following settings. Default value is "undefined". # Read changelog to learn more (Improvements of 12.90.0 (2021/09/04)). -#allowedPrivateNetworks: [ -# '127.0.0.1/32' -#] +# Some example configurations: +#allowedPrivateNetworks: +# # Allow connections to 127.0.0.1 on any port +# - '127.0.0.1/32' +# # Allow connections to 127.0.0.* on any port +# - '127.0.0.1/24' +# # Allow connections to 127.0.0.1 on any port +# - '127.0.0.1' +# # Allow connections to 127.0.0.1 on any port +# - network: '127.0.0.1' +# # Allow connections to 127.0.0.1 on port 80 +# - network: '127.0.0.1' +# ports: [80] +# # Allow connections to 127.0.0.1 on port 80 or 443 +# - network: '127.0.0.1' +# ports: +# - 80 +# - 443 #customMOTD: ['Hello World', 'The sharks rule all', 'Shonks'] diff --git a/.config/cypress-devcontainer.yml b/.config/cypress-devcontainer.yml index e4eb8cc805..ca62616462 100644 --- a/.config/cypress-devcontainer.yml +++ b/.config/cypress-devcontainer.yml @@ -260,6 +260,10 @@ proxyBypassHosts: #proxySmtp: socks4://127.0.0.1:1080 # use SOCKS4 #proxySmtp: socks5://127.0.0.1:1080 # use SOCKS5 +# Path to the directory that uploaded media will be saved to +# Defaults to a folder called "files" in the Sharkey directory +#mediaDirectory: /var/lib/sharkey + # Media Proxy #mediaProxy: https://example.com/proxy @@ -269,9 +273,27 @@ proxyRemoteFiles: true # Sign to ActivityPub GET request (default: true) signToActivityPubGet: true -allowedPrivateNetworks: [ - '127.0.0.1/32' -] +# For security reasons, uploading attachments from the intranet is prohibited, +# but exceptions can be made from the following settings. Default value is "undefined". +# Read changelog to learn more (Improvements of 12.90.0 (2021/09/04)). +# Some example configurations: +allowedPrivateNetworks: + # Allow connections to 127.0.0.1 on any port + - '127.0.0.1/32' +# # Allow connections to 127.0.0.* on any port +# - '127.0.0.1/24' +# # Allow connections to 127.0.0.1 on any port +# - '127.0.0.1' +# # Allow connections to 127.0.0.1 on any port +# - network: '127.0.0.1' +# # Allow connections to 127.0.0.1 on port 80 +# - network: '127.0.0.1' +# ports: [80] +# # Allow connections to 127.0.0.1 on port 80 or 443 +# - network: '127.0.0.1' +# ports: +# - 80 +# - 443 # Disable automatic redirect for ActivityPub object lookup. (default: false) # This is a strong defense against potential impersonation attacks if the viewer instance has inadequate validation. diff --git a/.config/docker_example.yml b/.config/docker_example.yml index 7968a7d1f4..df5d77a97f 100644 --- a/.config/docker_example.yml +++ b/.config/docker_example.yml @@ -351,6 +351,10 @@ proxyBypassHosts: #proxySmtp: socks4://127.0.0.1:1080 # use SOCKS4 #proxySmtp: socks5://127.0.0.1:1080 # use SOCKS5 +# Path to the directory that uploaded media will be saved to +# Defaults to a folder called "files" in the Sharkey directory +#mediaDirectory: /var/lib/sharkey + # Media Proxy # Reference Implementation: https://github.com/misskey-dev/media-proxy # * Deliver a common cache between instances @@ -378,9 +382,24 @@ attachLdSignatureForRelays: true # For security reasons, uploading attachments from the intranet is prohibited, # but exceptions can be made from the following settings. Default value is "undefined". # Read changelog to learn more (Improvements of 12.90.0 (2021/09/04)). -#allowedPrivateNetworks: [ -# '127.0.0.1/32' -#] +# Some example configurations: +#allowedPrivateNetworks: +# # Allow connections to 127.0.0.1 on any port +# - '127.0.0.1/32' +# # Allow connections to 127.0.0.* on any port +# - '127.0.0.1/24' +# # Allow connections to 127.0.0.1 on any port +# - '127.0.0.1' +# # Allow connections to 127.0.0.1 on any port +# - network: '127.0.0.1' +# # Allow connections to 127.0.0.1 on port 80 +# - network: '127.0.0.1' +# ports: [80] +# # Allow connections to 127.0.0.1 on port 80 or 443 +# - network: '127.0.0.1' +# ports: +# - 80 +# - 443 #customMOTD: ['Hello World', 'The sharks rule all', 'Shonks'] diff --git a/.config/example.yml b/.config/example.yml index d0ed4defaa..9e2b6d6da5 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -354,6 +354,10 @@ proxyBypassHosts: #proxySmtp: socks4://127.0.0.1:1080 # use SOCKS4 #proxySmtp: socks5://127.0.0.1:1080 # use SOCKS5 +# Path to the directory that uploaded media will be saved to +# Defaults to a folder called "files" in the Sharkey directory +#mediaDirectory: /var/lib/sharkey + # Media Proxy # Reference Implementation: https://github.com/misskey-dev/media-proxy # * Deliver a common cache between instances @@ -381,9 +385,24 @@ attachLdSignatureForRelays: true # For security reasons, uploading attachments from the intranet is prohibited, # but exceptions can be made from the following settings. Default value is "undefined". # Read changelog to learn more (Improvements of 12.90.0 (2021/09/04)). -#allowedPrivateNetworks: [ -# '127.0.0.1/32' -#] +# Some example configurations: +#allowedPrivateNetworks: +# # Allow connections to 127.0.0.1 on any port +# - '127.0.0.1/32' +# # Allow connections to 127.0.0.* on any port +# - '127.0.0.1/24' +# # Allow connections to 127.0.0.1 on any port +# - '127.0.0.1' +# # Allow connections to 127.0.0.1 on any port +# - network: '127.0.0.1' +# # Allow connections to 127.0.0.1 on port 80 +# - network: '127.0.0.1' +# ports: [80] +# # Allow connections to 127.0.0.1 on port 80 or 443 +# - network: '127.0.0.1' +# ports: +# - 80 +# - 443 #customMOTD: ['Hello World', 'The sharks rule all', 'Shonks'] diff --git a/.devcontainer/devcontainer.yml b/.devcontainer/devcontainer.yml index 6d904e87b9..1f8192101b 100644 --- a/.devcontainer/devcontainer.yml +++ b/.devcontainer/devcontainer.yml @@ -199,6 +199,10 @@ proxyBypassHosts: #proxySmtp: socks4://127.0.0.1:1080 # use SOCKS4 #proxySmtp: socks5://127.0.0.1:1080 # use SOCKS5 +# Path to the directory that uploaded media will be saved to +# Defaults to a folder called "files" in the Sharkey directory +#mediaDirectory: /var/lib/sharkey + # Media Proxy #mediaProxy: https://example.com/proxy diff --git a/locales/index.d.ts b/locales/index.d.ts index 5b3b69bdcf..b446fbb8f6 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -2451,7 +2451,7 @@ export interface Locale extends ILocale { */ "disablePagesScript": string; /** - * リモートユーザー情報の更新 + * Refresh remote data */ "updateRemoteUser": string; /** @@ -7631,6 +7631,10 @@ export interface Locale extends ILocale { * Maximum number of scheduled notes */ "scheduleNoteMax": string; + /** + * Can appear in trending notes / users + */ + "canTrend": string; }; "_condition": { /** @@ -12961,6 +12965,14 @@ export interface Locale extends ILocale { * Fetch linked note */ "fetchLinkedNote": string; + /** + * Add "Translate" to note action menu + */ + "showTranslationButtonInNoteFooter": string; + /** + * Failed to translate note. Please try again later or contact an administrator for assistance. + */ + "translationFailed": string; "_processErrors": { /** * Unable to process quote. This post may be missing context. @@ -13065,6 +13077,10 @@ export interface Locale extends ILocale { * Text does not match any patterns. */ "wordMuteTestNoMatch": string; + /** + * All word mutes are *case-sensitive* and match on any substring, including part of a longer word or name. You can use regular expressions for more precise control. + */ + "wordMuteWarning": string; /** * Bubble timeline */ @@ -13077,6 +13093,74 @@ export interface Locale extends ILocale { * Note: the bubble timeline is hidden by default, and must be enabled via roles. */ "bubbleTimelineMustBeEnabled": string; + /** + * Users popular on the global network + */ + "popularUsersGlobal": string; + /** + * Users popular on {name} + */ + "popularUsersLocal": ParameterizedString<"name">; + /** + * Silenced + */ + "silenced": string; + /** + * Total followers + */ + "totalFollowers": string; + /** + * Total following + */ + "totalFollowing": string; + /** + * Local followers + */ + "localFollowers": string; + /** + * Local following + */ + "localFollowing": string; + /** + * Remote followers + */ + "remoteFollowers": string; + /** + * Remote following + */ + "remoteFollowing": string; + /** + * Activity Pub + */ + "activityPub": string; + /** + * IP + */ + "ip": string; + /** + * The date is when IP address was first used. + */ + "ipTip": string; + /** + * Period + */ + "rolePeriod": string; + /** + * Assigned + */ + "roleAssigned": string; + /** + * automatic + */ + "roleAutomatic": string; + /** + * Translation timeout + */ + "translationTimeoutLabel": string; + /** + * Timeout in milliseconds for translation API requests. + */ + "translationTimeoutCaption": string; } declare const locales: { [lang: string]: Locale; diff --git a/package.json b/package.json index dae0574ba0..1db16bcd55 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sharkey", - "version": "2025.5.1-rc", + "version": "2025.5.2-dev", "codename": "shonk", "repository": { "type": "git", diff --git a/packages/backend/migration/1747023091463-add_meta_translationTimeout.js b/packages/backend/migration/1747023091463-add_meta_translationTimeout.js new file mode 100644 index 0000000000..cf291f8d74 --- /dev/null +++ b/packages/backend/migration/1747023091463-add_meta_translationTimeout.js @@ -0,0 +1,18 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class AddMetaTranslationTimeout1747023091463 { + name = 'AddMetaTranslationTimeout1747023091463' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" ADD "translationTimeout" integer NOT NULL DEFAULT '5000'`); + await queryRunner.query(`COMMENT ON COLUMN "meta"."translationTimeout" IS 'Timeout in milliseconds for translation API requests'`); + } + + async down(queryRunner) { + await queryRunner.query(`COMMENT ON COLUMN "meta"."translationTimeout" IS 'Timeout in milliseconds for translation API requests'`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "translationTimeout"`); + } +} diff --git a/packages/backend/migration/1747934911491-rename_followingVisibility.js b/packages/backend/migration/1747934911491-rename_followingVisibility.js new file mode 100644 index 0000000000..75f6db49ec --- /dev/null +++ b/packages/backend/migration/1747934911491-rename_followingVisibility.js @@ -0,0 +1,16 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class RenameFollowingVisibility1747934911491 { + name = 'RenameFollowingVisibility1747934911491' + + async up(queryRunner) { + await queryRunner.query(`ALTER TYPE "public"."user_profile_followingvisibility_enum" RENAME TO "user_profile_followingVisibility_enum"`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TYPE "public"."user_profile_followingVisibility_enum" RENAME TO "user_profile_followingvisibility_enum"`); + } +} diff --git a/packages/backend/migration/1747935197708-add_entity_comments.js b/packages/backend/migration/1747935197708-add_entity_comments.js new file mode 100644 index 0000000000..687c957425 --- /dev/null +++ b/packages/backend/migration/1747935197708-add_entity_comments.js @@ -0,0 +1,30 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class AddEntityComments1747935197708 { + name = 'AddEntityComments1747935197708' + + async up(queryRunner) { + await queryRunner.query(`COMMENT ON COLUMN "user"."backgroundId" IS 'The ID of background DriveFile.'`); + await queryRunner.query(`COMMENT ON COLUMN "user"."isSilenced" IS 'Whether the User is silenced.'`); + await queryRunner.query(`COMMENT ON COLUMN "user"."noindex" IS 'Whether the User''s notes dont get indexed.'`); + await queryRunner.query(`COMMENT ON COLUMN "user"."speakAsCat" IS 'Whether the User speaks in nya.'`); + await queryRunner.query(`COMMENT ON COLUMN "user_profile"."listenbrainz" IS 'The ListenBrainz username of the User.'`); + await queryRunner.query(`COMMENT ON COLUMN "note"."updatedAt" IS 'The update time of the Note.'`); + await queryRunner.query(`COMMENT ON COLUMN "meta"."trustedLinkUrlPatterns" IS 'An array of URL strings or regex that can be used to omit warnings about redirects to external sites. Separate them with spaces to specify AND, and enclose them with slashes to specify regular expressions. Each item is regarded as an OR.'`); + await queryRunner.query(`COMMENT ON COLUMN "note_edit"."oldDate" IS 'The old date from before the edit'`); + } + + async down(queryRunner) { + await queryRunner.query(`COMMENT ON COLUMN "note_edit"."oldDate" IS NULL`); + await queryRunner.query(`COMMENT ON COLUMN "meta"."trustedLinkUrlPatterns" IS NULL`); + await queryRunner.query(`COMMENT ON COLUMN "note"."updatedAt" IS 'The updated date of the Note.'`); + await queryRunner.query(`COMMENT ON COLUMN "user_profile"."listenbrainz" IS 'listenbrainz username to fetch currently playing.'`); + await queryRunner.query(`COMMENT ON COLUMN "user"."speakAsCat" IS 'Whether to speak as a cat if chosen.'`); + await queryRunner.query(`COMMENT ON COLUMN "user"."noindex" IS NULL`); + await queryRunner.query(`COMMENT ON COLUMN "user"."isSilenced" IS NULL`); + await queryRunner.query(`COMMENT ON COLUMN "user"."backgroundId" IS NULL`); + } +} diff --git a/packages/backend/migration/1747937504140-fix-system_webhook-updatedAt-default.js b/packages/backend/migration/1747937504140-fix-system_webhook-updatedAt-default.js new file mode 100644 index 0000000000..d30c6c4872 --- /dev/null +++ b/packages/backend/migration/1747937504140-fix-system_webhook-updatedAt-default.js @@ -0,0 +1,16 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class FixSystemWebhookUpdatedAtDefault1747937504140 { + name = 'FixSystemWebhookUpdatedAtDefault1747937504140' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "system_webhook" ALTER COLUMN "updatedAt" SET DEFAULT now()`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "system_webhook" ALTER COLUMN "updatedAt" SET DEFAULT CURRENT_TIMESTAMP`); + } +} diff --git a/packages/backend/migration/1747937670341-fix-abuse_report_notification_recipient-updatedAt-default.js b/packages/backend/migration/1747937670341-fix-abuse_report_notification_recipient-updatedAt-default.js new file mode 100644 index 0000000000..4364bbb56d --- /dev/null +++ b/packages/backend/migration/1747937670341-fix-abuse_report_notification_recipient-updatedAt-default.js @@ -0,0 +1,16 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class FixAbuseReportNotificationRecipientUpdatedAtDefault1747937670341 { + name = 'FixAbuseReportNotificationRecipientUpdatedAtDefault1747937670341' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "abuse_report_notification_recipient" ALTER COLUMN "updatedAt" SET DEFAULT now()`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "abuse_report_notification_recipient" ALTER COLUMN "updatedAt" SET DEFAULT CURRENT_TIMESTAMP`); + } +} diff --git a/packages/backend/migration/1747937796573-fix-flash-visibility-nullable.js b/packages/backend/migration/1747937796573-fix-flash-visibility-nullable.js new file mode 100644 index 0000000000..7076d05e19 --- /dev/null +++ b/packages/backend/migration/1747937796573-fix-flash-visibility-nullable.js @@ -0,0 +1,16 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class FixFlashVisibilityNullable1747937796573 { + name = 'FixFlashVisibilityNullable1747937796573' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "flash" ALTER COLUMN "visibility" SET NOT NULL`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "flash" ALTER COLUMN "visibility" DROP NOT NULL`); + } +} diff --git a/packages/backend/migration/1747938136399-fix-abuse_report_notification_recipient-defaults.js b/packages/backend/migration/1747938136399-fix-abuse_report_notification_recipient-defaults.js new file mode 100644 index 0000000000..6cf48bcbe9 --- /dev/null +++ b/packages/backend/migration/1747938136399-fix-abuse_report_notification_recipient-defaults.js @@ -0,0 +1,18 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class FixAbuseReportNotificationRecipientDefaults1747938136399 { + name = 'FixAbuseReportNotificationRecipientDefaults1747938136399' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "abuse_report_notification_recipient" ALTER COLUMN "userId" DROP DEFAULT`); + await queryRunner.query(`ALTER TABLE "abuse_report_notification_recipient" ALTER COLUMN "systemWebhookId" DROP DEFAULT`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "abuse_report_notification_recipient" ALTER COLUMN "systemWebhookId" SET DEFAULT NULL`); + await queryRunner.query(`ALTER TABLE "abuse_report_notification_recipient" ALTER COLUMN "userId" SET DEFAULT NULL`); + } +} diff --git a/packages/backend/migration/1747938263980-fix-meta-urlPreviewUserAgent-default.js b/packages/backend/migration/1747938263980-fix-meta-urlPreviewUserAgent-default.js new file mode 100644 index 0000000000..d04c1ac377 --- /dev/null +++ b/packages/backend/migration/1747938263980-fix-meta-urlPreviewUserAgent-default.js @@ -0,0 +1,16 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class FixMetaUrlPreviewUserAgentDefault1747938263980 { + name = 'FixMetaUrlPreviewUserAgentDefault1747938263980' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" ALTER COLUMN "urlPreviewUserAgent" DROP DEFAULT`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" ALTER COLUMN "urlPreviewUserAgent" SET DEFAULT NULL`); + } +} diff --git a/packages/backend/migration/1747938628395-add-missing-indexes.js b/packages/backend/migration/1747938628395-add-missing-indexes.js new file mode 100644 index 0000000000..745b39c855 --- /dev/null +++ b/packages/backend/migration/1747938628395-add-missing-indexes.js @@ -0,0 +1,22 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class AddMissingIndexes1747938628395 { + name = 'AddMissingIndexes1747938628395' + + async up(queryRunner) { + await queryRunner.query(`CREATE INDEX "IDX_58699f75b9cf904f5f007909cb" ON "user_profile" ("birthday") `); + await queryRunner.query(`CREATE INDEX "IDX_021015e6683570ae9f6b0c62be" ON "user_list_membership" ("userId") `); + await queryRunner.query(`CREATE INDEX "IDX_cddcaf418dc4d392ecfcca842a" ON "user_list_membership" ("userListId") `); + await queryRunner.query(`CREATE UNIQUE INDEX "IDX_e4f3094c43f2d665e6030b0337" ON "user_list_membership" ("userId", "userListId") `); + } + + async down(queryRunner) { + await queryRunner.query(`DROP INDEX "public"."IDX_e4f3094c43f2d665e6030b0337"`); + await queryRunner.query(`DROP INDEX "public"."IDX_cddcaf418dc4d392ecfcca842a"`); + await queryRunner.query(`DROP INDEX "public"."IDX_021015e6683570ae9f6b0c62be"`); + await queryRunner.query(`DROP INDEX "public"."IDX_58699f75b9cf904f5f007909cb"`); + } +} diff --git a/packages/backend/migration/1747944466178-alter-meta-defaultLike-not-null.js b/packages/backend/migration/1747944466178-alter-meta-defaultLike-not-null.js new file mode 100644 index 0000000000..b206a15ee2 --- /dev/null +++ b/packages/backend/migration/1747944466178-alter-meta-defaultLike-not-null.js @@ -0,0 +1,16 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class AlterMetaDefaultLikeNotNull1747944466178 { + name = 'AlterMetaDefaultLikeNotNull1747944466178' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" ALTER COLUMN "defaultLike" SET NOT NULL`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" ALTER COLUMN "defaultLike" DROP NOT NULL`); + } +} diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 92fc2b8a13..8507420839 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -8,9 +8,11 @@ import { fileURLToPath } from 'node:url'; import { dirname, resolve } from 'node:path'; import * as yaml from 'js-yaml'; import { globSync } from 'glob'; +import ipaddr from 'ipaddr.js'; import type * as Sentry from '@sentry/node'; import type * as SentryVue from '@sentry/vue'; import type { RedisOptions } from 'ioredis'; +import type { IPv4, IPv6 } from 'ipaddr.js'; type RedisOptionsSource = Partial & { host?: string; @@ -82,7 +84,7 @@ type Source = { proxySmtp?: string; proxyBypassHosts?: string[]; - allowedPrivateNetworks?: string[]; + allowedPrivateNetworks?: PrivateNetworkSource[]; disallowExternalApRedirect?: boolean; maxFileSize?: number; @@ -109,6 +111,7 @@ type Source = { deliverJobMaxAttempts?: number; inboxJobMaxAttempts?: number; + mediaDirectory?: string; mediaProxy?: string; proxyRemoteFiles?: boolean; videoThumbnailGenerator?: string; @@ -152,6 +155,60 @@ type Source = { } }; +export type PrivateNetworkSource = string | { network?: string, ports?: number[] }; + +export type PrivateNetwork = { + /** + * CIDR IP/netmask definition of the IP range to match. + */ + cidr: CIDR; + + /** + * List of ports to match. + * If undefined, then all ports match. + * If empty, then NO ports match. + */ + ports?: number[]; +}; + +export type CIDR = [ip: IPv4 | IPv6, prefixLength: number]; + +export function parsePrivateNetworks(patterns: PrivateNetworkSource[]): PrivateNetwork[]; +export function parsePrivateNetworks(patterns: undefined): undefined; +export function parsePrivateNetworks(patterns: PrivateNetworkSource[] | undefined): PrivateNetwork[] | undefined; +export function parsePrivateNetworks(patterns: PrivateNetworkSource[] | undefined): PrivateNetwork[] | undefined { + if (!patterns) return undefined; + return patterns + .map(e => { + if (typeof(e) === 'string') { + const cidr = parseIpOrMask(e); + if (cidr) { + return { cidr } satisfies PrivateNetwork; + } + } else if (e.network) { + const cidr = parseIpOrMask(e.network); + if (cidr) { + return { cidr, ports: e.ports } satisfies PrivateNetwork; + } + } + + console.warn('[config] Skipping invalid entry in allowedPrivateNetworks: ', e); + return null; + }) + .filter(p => p != null); +} + +function parseIpOrMask(ipOrMask: string): CIDR | null { + if (ipaddr.isValidCIDR(ipOrMask)) { + return ipaddr.parseCIDR(ipOrMask); + } + if (ipaddr.isValid(ipOrMask)) { + const ip = ipaddr.parse(ipOrMask); + return [ip, 32]; + } + return null; +} + export type Config = { url: string; port: number; @@ -190,7 +247,7 @@ export type Config = { proxy: string | undefined; proxySmtp: string | undefined; proxyBypassHosts: string[] | undefined; - allowedPrivateNetworks: string[] | undefined; + allowedPrivateNetworks: PrivateNetwork[] | undefined; disallowExternalApRedirect: boolean; maxFileSize: number; maxNoteLength: number; @@ -241,6 +298,7 @@ export type Config = { frontendManifestExists: boolean; frontendEmbedEntry: string; frontendEmbedManifestExists: boolean; + mediaDirectory: string; mediaProxy: string; externalMediaProxyEnabled: boolean; videoThumbnailGenerator: string | null; @@ -290,7 +348,7 @@ const _dirname = dirname(_filename); /** * Path of configuration directory */ -const dir = `${_dirname}/../../../.config`; +const dir = process.env.MISSKEY_CONFIG_DIR ?? `${_dirname}/../../../.config`; /** * Path of configuration file @@ -382,7 +440,7 @@ export function loadConfig(): Config { proxy: config.proxy, proxySmtp: config.proxySmtp, proxyBypassHosts: config.proxyBypassHosts, - allowedPrivateNetworks: config.allowedPrivateNetworks, + allowedPrivateNetworks: parsePrivateNetworks(config.allowedPrivateNetworks), disallowExternalApRedirect: config.disallowExternalApRedirect ?? false, maxFileSize: config.maxFileSize ?? 262144000, maxNoteLength: config.maxNoteLength ?? 3000, @@ -407,6 +465,7 @@ export function loadConfig(): Config { signToActivityPubGet: config.signToActivityPubGet ?? true, attachLdSignatureForRelays: config.attachLdSignatureForRelays ?? true, checkActivityPubGetSignature: config.checkActivityPubGetSignature, + mediaDirectory: config.mediaDirectory ?? resolve(_dirname, '../../../files'), mediaProxy: externalMediaProxy ?? internalMediaProxy, externalMediaProxyEnabled: externalMediaProxy !== null && externalMediaProxy !== internalMediaProxy, videoThumbnailGenerator: config.videoThumbnailGenerator ? @@ -575,14 +634,14 @@ function applyEnvOverrides(config: Source) { ['host', 'port', 'username', 'pass', 'db', 'prefix'], ]); _apply_top(['fulltextSearch', 'provider']); - _apply_top(['meilisearch', ['host', 'port', 'apikey', 'ssl', 'index', 'scope']]); + _apply_top(['meilisearch', ['host', 'port', 'apiKey', 'ssl', 'index', 'scope']]); _apply_top([['sentryForFrontend', 'sentryForBackend'], 'options', ['dsn', 'profileSampleRate', 'serverName', 'includeLocalVariables', 'proxy', 'keepAlive', 'caCerts']]); _apply_top(['sentryForBackend', 'enableNodeProfiling']); _apply_top(['sentryForFrontend', 'vueIntegration', ['attachProps', 'attachErrorHandler']]); _apply_top(['sentryForFrontend', 'vueIntegration', 'tracingOptions', 'timeout']); _apply_top(['sentryForFrontend', 'browserTracingIntegration', 'routeLabel']); _apply_top([['clusterLimit', 'deliverJobConcurrency', 'inboxJobConcurrency', 'relashionshipJobConcurrency', 'deliverJobPerSec', 'inboxJobPerSec', 'relashionshipJobPerSec', 'deliverJobMaxAttempts', 'inboxJobMaxAttempts']]); - _apply_top([['outgoingAddress', 'outgoingAddressFamily', 'proxy', 'proxySmtp', 'mediaProxy', 'proxyRemoteFiles', 'videoThumbnailGenerator']]); + _apply_top([['outgoingAddress', 'outgoingAddressFamily', 'proxy', 'proxySmtp', 'mediaDirectory', 'mediaProxy', 'proxyRemoteFiles', 'videoThumbnailGenerator']]); _apply_top([['maxFileSize', 'maxNoteLength', 'maxRemoteNoteLength', 'maxAltTextLength', 'maxRemoteAltTextLength', 'pidFile', 'filePermissionBits']]); _apply_top(['import', ['downloadTimeout', 'maxFileSize']]); _apply_top([['signToActivityPubGet', 'checkActivityPubGetSignature', 'setupPassword', 'disallowExternalApRedirect']]); diff --git a/packages/backend/src/core/CacheService.ts b/packages/backend/src/core/CacheService.ts index 822bb9d42c..1cf63221f9 100644 --- a/packages/backend/src/core/CacheService.ts +++ b/packages/backend/src/core/CacheService.ts @@ -6,7 +6,7 @@ import { Inject, Injectable } from '@nestjs/common'; import * as Redis from 'ioredis'; import { IsNull } from 'typeorm'; -import type { BlockingsRepository, FollowingsRepository, MutingsRepository, RenoteMutingsRepository, MiUserProfile, UserProfilesRepository, UsersRepository, MiFollowing } from '@/models/_.js'; +import type { BlockingsRepository, FollowingsRepository, MutingsRepository, RenoteMutingsRepository, MiUserProfile, UserProfilesRepository, UsersRepository, MiFollowing, MiNote } from '@/models/_.js'; import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js'; import type { MiLocalUser, MiUser } from '@/models/User.js'; import { DI } from '@/di-symbols.js'; @@ -22,6 +22,17 @@ export interface FollowStats { remoteFollowers: number; } +export interface CachedTranslation { + sourceLang: string | undefined; + text: string | undefined; +} + +interface CachedTranslationEntity { + l?: string; + t?: string; + u?: number; +} + @Injectable() export class CacheService implements OnApplicationShutdown { public userByIdCache: MemoryKVCache; @@ -35,6 +46,7 @@ export class CacheService implements OnApplicationShutdown { public renoteMutingsCache: RedisKVCache>; public userFollowingsCache: RedisKVCache | undefined>>; private readonly userFollowStatsCache = new MemoryKVCache(1000 * 60 * 10); // 10 minutes + private readonly translationsCache: RedisKVCache; constructor( @Inject(DI.redis) @@ -124,6 +136,11 @@ export class CacheService implements OnApplicationShutdown { fromRedisConverter: (value) => JSON.parse(value), }); + this.translationsCache = new RedisKVCache(this.redisClient, 'translations', { + lifetime: 1000 * 60 * 60 * 24 * 7, // 1 week, + memoryCacheLifetime: 1000 * 60, // 1 minute + }); + // NOTE: チャンネルのフォロー状況キャッシュはChannelFollowingServiceで行っている this.redisForSub.on('message', this.onMessage); @@ -253,6 +270,34 @@ export class CacheService implements OnApplicationShutdown { }); } + @bindThis + public async getCachedTranslation(note: MiNote, targetLang: string): Promise { + const cacheKey = `${note.id}@${targetLang}`; + + // Use cached translation, if present and up-to-date + const cached = await this.translationsCache.get(cacheKey); + if (cached && cached.u === note.updatedAt?.valueOf()) { + return { + sourceLang: cached.l, + text: cached.t, + }; + } + + // No cache entry :( + return null; + } + + @bindThis + public async setCachedTranslation(note: MiNote, targetLang: string, translation: CachedTranslation): Promise { + const cacheKey = `${note.id}@${targetLang}`; + + await this.translationsCache.set(cacheKey, { + l: translation.sourceLang, + t: translation.text, + u: note.updatedAt?.valueOf(), + }); + } + @bindThis public dispose(): void { this.redisForSub.off('message', this.onMessage); diff --git a/packages/backend/src/core/DeleteAccountService.ts b/packages/backend/src/core/DeleteAccountService.ts index 48f27d558e..efbe6a2d59 100644 --- a/packages/backend/src/core/DeleteAccountService.ts +++ b/packages/backend/src/core/DeleteAccountService.ts @@ -63,8 +63,6 @@ export class DeleteAccountService { // 知り得る全SharedInboxにDelete配信 const content = this.apRendererService.addContext(this.apRendererService.renderDelete(this.userEntityService.genLocalUserUri(user.id), user)); - const queue: string[] = []; - const followings = await this.followingsRepository.find({ where: [ { followerSharedInbox: Not(IsNull()) }, @@ -73,22 +71,17 @@ export class DeleteAccountService { select: ['followerSharedInbox', 'followeeSharedInbox'], }); - const inboxes = followings.map(x => x.followerSharedInbox ?? x.followeeSharedInbox); + const inboxes = followings.map(x => [x.followerSharedInbox ?? x.followeeSharedInbox as string, true] as const); + const queue = new Map(inboxes); - for (const inbox of inboxes) { - if (inbox != null && !queue.includes(inbox)) queue.push(inbox); - } + await this.queueService.deliverMany(user, content, queue); - for (const inbox of queue) { - this.queueService.deliver(user, content, inbox, true); - } - - this.queueService.createDeleteAccountJob(user, { + await this.queueService.createDeleteAccountJob(user, { soft: false, }); } else { // リモートユーザーの削除は、完全にDBから物理削除してしまうと再度連合してきてアカウントが復活する可能性があるため、soft指定する - this.queueService.createDeleteAccountJob(user, { + await this.queueService.createDeleteAccountJob(user, { soft: true, }); } diff --git a/packages/backend/src/core/FeaturedService.ts b/packages/backend/src/core/FeaturedService.ts index b3335e38da..cabbb46504 100644 --- a/packages/backend/src/core/FeaturedService.ts +++ b/packages/backend/src/core/FeaturedService.ts @@ -8,6 +8,7 @@ import * as Redis from 'ioredis'; import type { MiGalleryPost, MiNote, MiUser } from '@/models/_.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; +import { RoleService } from '@/core/RoleService.js'; const GLOBAL_NOTES_RANKING_WINDOW = 1000 * 60 * 60 * 24 * 3; // 3日ごと export const GALLERY_POSTS_RANKING_WINDOW = 1000 * 60 * 60 * 24 * 3; // 3日ごと @@ -21,6 +22,8 @@ export class FeaturedService { constructor( @Inject(DI.redis) private redisClient: Redis.Redis, // TODO: 専用のRedisサーバーを設定できるようにする + + private readonly roleService: RoleService, ) { } @@ -31,7 +34,14 @@ export class FeaturedService { } @bindThis - private async updateRankingOf(name: string, windowRange: number, element: string, score = 1): Promise { + private async updateRankingOf(name: string, windowRange: number, element: string, score: number, userId: string | null): Promise { + if (userId) { + const policies = await this.roleService.getUserPolicies(userId); + if (!policies.canTrend) { + return; + } + } + const currentWindow = this.getCurrentWindow(windowRange); const redisTransaction = this.redisClient.multi(); redisTransaction.zincrby( @@ -89,28 +99,28 @@ export class FeaturedService { } @bindThis - public updateGlobalNotesRanking(noteId: MiNote['id'], score = 1): Promise { - return this.updateRankingOf('featuredGlobalNotesRanking', GLOBAL_NOTES_RANKING_WINDOW, noteId, score); + public updateGlobalNotesRanking(note: Pick, score = 1): Promise { + return this.updateRankingOf('featuredGlobalNotesRanking', GLOBAL_NOTES_RANKING_WINDOW, note.id, score, note.userId); } @bindThis - public updateGalleryPostsRanking(galleryPostId: MiGalleryPost['id'], score = 1): Promise { - return this.updateRankingOf('featuredGalleryPostsRanking', GALLERY_POSTS_RANKING_WINDOW, galleryPostId, score); + public updateGalleryPostsRanking(galleryPost: Pick, score = 1): Promise { + return this.updateRankingOf('featuredGalleryPostsRanking', GALLERY_POSTS_RANKING_WINDOW, galleryPost.id, score, galleryPost.userId); } @bindThis - public updateInChannelNotesRanking(channelId: MiNote['channelId'], noteId: MiNote['id'], score = 1): Promise { - return this.updateRankingOf(`featuredInChannelNotesRanking:${channelId}`, GLOBAL_NOTES_RANKING_WINDOW, noteId, score); + public updateInChannelNotesRanking(channelId: MiNote['channelId'], note: Pick, score = 1): Promise { + return this.updateRankingOf(`featuredInChannelNotesRanking:${channelId}`, GLOBAL_NOTES_RANKING_WINDOW, note.id, score, note.userId); } @bindThis - public updatePerUserNotesRanking(userId: MiUser['id'], noteId: MiNote['id'], score = 1): Promise { - return this.updateRankingOf(`featuredPerUserNotesRanking:${userId}`, PER_USER_NOTES_RANKING_WINDOW, noteId, score); + public updatePerUserNotesRanking(userId: MiUser['id'], note: Pick, score = 1): Promise { + return this.updateRankingOf(`featuredPerUserNotesRanking:${userId}`, PER_USER_NOTES_RANKING_WINDOW, note.id, score, userId); } @bindThis public updateHashtagsRanking(hashtag: string, score = 1): Promise { - return this.updateRankingOf('featuredHashtagsRanking', HASHTAG_RANKING_WINDOW, hashtag, score); + return this.updateRankingOf('featuredHashtagsRanking', HASHTAG_RANKING_WINDOW, hashtag, score, null); } @bindThis diff --git a/packages/backend/src/core/HttpRequestService.ts b/packages/backend/src/core/HttpRequestService.ts index 12047346fb..5c271b81e3 100644 --- a/packages/backend/src/core/HttpRequestService.ts +++ b/packages/backend/src/core/HttpRequestService.ts @@ -12,7 +12,7 @@ import fetch from 'node-fetch'; import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent'; import { Inject, Injectable } from '@nestjs/common'; import { DI } from '@/di-symbols.js'; -import type { Config } from '@/config.js'; +import type { Config, PrivateNetwork } from '@/config.js'; import { StatusError } from '@/misc/status-error.js'; import { bindThis } from '@/decorators.js'; import { validateContentTypeSetAsActivityPub } from '@/core/activitypub/misc/validator.js'; @@ -20,12 +20,36 @@ import type { IObject, IObjectWithId } from '@/core/activitypub/type.js'; import { ApUtilityService } from './activitypub/ApUtilityService.js'; import type { Response } from 'node-fetch'; import type { URL } from 'node:url'; +import type { Socket } from 'node:net'; export type HttpRequestSendOptions = { throwErrorWhenResponseNotOk: boolean; validators?: ((res: Response) => void)[]; }; +export function isPrivateIp(allowedPrivateNetworks: PrivateNetwork[] | undefined, ip: string, port?: number): boolean { + const parsedIp = ipaddr.parse(ip); + + for (const { cidr, ports } of allowedPrivateNetworks ?? []) { + if (cidr[0].kind() === parsedIp.kind() && parsedIp.match(cidr)) { + if (ports == null || (port != null && ports.includes(port))) { + return false; + } + } + } + + return parsedIp.range() !== 'unicast'; +} + +export function validateSocketConnect(allowedPrivateNetworks: PrivateNetwork[] | undefined, socket: Socket): void { + const address = socket.remoteAddress; + if (address && ipaddr.isValid(address)) { + if (isPrivateIp(allowedPrivateNetworks, address, socket.remotePort)) { + socket.destroy(new Error(`Blocked address: ${address}`)); + } + } +} + declare module 'node:http' { interface Agent { createConnection(options: net.NetConnectOpts, callback?: (err: unknown, stream: net.Socket) => void): net.Socket; @@ -44,31 +68,12 @@ class HttpRequestServiceAgent extends http.Agent { public createConnection(options: net.NetConnectOpts, callback?: (err: unknown, stream: net.Socket) => void): net.Socket { const socket = super.createConnection(options, callback) .on('connect', () => { - const address = socket.remoteAddress; if (process.env.NODE_ENV === 'production') { - if (address && ipaddr.isValid(address)) { - if (this.isPrivateIp(address)) { - socket.destroy(new Error(`Blocked address: ${address}`)); - } - } + validateSocketConnect(this.config.allowedPrivateNetworks, socket); } }); return socket; } - - @bindThis - private isPrivateIp(ip: string): boolean { - const parsedIp = ipaddr.parse(ip); - - for (const net of this.config.allowedPrivateNetworks ?? []) { - const cidr = ipaddr.parseCIDR(net); - if (cidr[0].kind() === parsedIp.kind() && parsedIp.match(ipaddr.parseCIDR(net))) { - return false; - } - } - - return parsedIp.range() !== 'unicast'; - } } class HttpsRequestServiceAgent extends https.Agent { @@ -83,31 +88,12 @@ class HttpsRequestServiceAgent extends https.Agent { public createConnection(options: net.NetConnectOpts, callback?: (err: unknown, stream: net.Socket) => void): net.Socket { const socket = super.createConnection(options, callback) .on('connect', () => { - const address = socket.remoteAddress; if (process.env.NODE_ENV === 'production') { - if (address && ipaddr.isValid(address)) { - if (this.isPrivateIp(address)) { - socket.destroy(new Error(`Blocked address: ${address}`)); - } - } + validateSocketConnect(this.config.allowedPrivateNetworks, socket); } }); return socket; } - - @bindThis - private isPrivateIp(ip: string): boolean { - const parsedIp = ipaddr.parse(ip); - - for (const net of this.config.allowedPrivateNetworks ?? []) { - const cidr = ipaddr.parseCIDR(net); - if (cidr[0].kind() === parsedIp.kind() && parsedIp.match(ipaddr.parseCIDR(net))) { - return false; - } - } - - return parsedIp.range() !== 'unicast'; - } } @Injectable() @@ -250,6 +236,8 @@ export class HttpRequestService { @bindThis public async getActivityJson(url: string, isLocalAddressAllowed = false): Promise { + this.apUtilityService.assertApUrl(url); + const res = await this.send(url, { method: 'GET', headers: { diff --git a/packages/backend/src/core/InternalStorageService.ts b/packages/backend/src/core/InternalStorageService.ts index b00c5796d2..abdbbc61d3 100644 --- a/packages/backend/src/core/InternalStorageService.ts +++ b/packages/backend/src/core/InternalStorageService.ts @@ -6,18 +6,11 @@ import * as fs from 'node:fs'; import { copyFile, unlink, writeFile, chmod } from 'node:fs/promises'; import * as Path from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { dirname } from 'node:path'; import { Inject, Injectable } from '@nestjs/common'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import { bindThis } from '@/decorators.js'; -const _filename = fileURLToPath(import.meta.url); -const _dirname = dirname(_filename); - -const path = Path.resolve(_dirname, '../../../../files'); - @Injectable() export class InternalStorageService { constructor( @@ -25,12 +18,12 @@ export class InternalStorageService { private config: Config, ) { // No one should erase the working directory *while the server is running*. - fs.mkdirSync(path, { recursive: true }); + fs.mkdirSync(this.config.mediaDirectory, { recursive: true }); } @bindThis public resolvePath(key: string) { - return Path.resolve(path, key); + return Path.resolve(this.config.mediaDirectory, key); } @bindThis diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index e961d4236c..097d657ba3 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -592,6 +592,8 @@ export class NoteCreateService implements OnApplicationShutdown { if (!this.isRenote(note) || this.isQuote(note)) { // Increment notes count (user) this.incNotesCountOfUser(user); + } else { + this.usersRepository.update({ id: user.id }, { updatedAt: new Date() }); } this.pushToTl(note, user); @@ -631,7 +633,7 @@ export class NoteCreateService implements OnApplicationShutdown { } if (this.isRenote(data) && !this.isQuote(data) && data.renote.userId !== user.id && !user.isBot) { - this.incRenoteCount(data.renote); + this.incRenoteCount(data.renote, user); } if (data.poll && data.poll.expiresAt) { @@ -814,8 +816,8 @@ export class NoteCreateService implements OnApplicationShutdown { } @bindThis - private incRenoteCount(renote: MiNote) { - this.notesRepository.createQueryBuilder().update() + private async incRenoteCount(renote: MiNote, user: MiUser) { + await this.notesRepository.createQueryBuilder().update() .set({ renoteCount: () => '"renoteCount" + 1', }) @@ -823,15 +825,18 @@ export class NoteCreateService implements OnApplicationShutdown { .execute(); // 30%の確率、3日以内に投稿されたノートの場合ハイライト用ランキング更新 - if (Math.random() < 0.3 && (Date.now() - this.idService.parse(renote.id).date.getTime()) < 1000 * 60 * 60 * 24 * 3) { - if (renote.channelId != null) { - if (renote.replyId == null) { - this.featuredService.updateInChannelNotesRanking(renote.channelId, renote.id, 5); - } - } else { - if (renote.visibility === 'public' && renote.userHost == null && renote.replyId == null) { - this.featuredService.updateGlobalNotesRanking(renote.id, 5); - this.featuredService.updatePerUserNotesRanking(renote.userId, renote.id, 5); + if (user.isExplorable && Math.random() < 0.3 && (Date.now() - this.idService.parse(renote.id).date.getTime()) < 1000 * 60 * 60 * 24 * 3) { + const policies = await this.roleService.getUserPolicies(user); + if (policies.canTrend) { + if (renote.channelId != null) { + if (renote.replyId == null) { + this.featuredService.updateInChannelNotesRanking(renote.channelId, renote, 5); + } + } else { + if (renote.visibility === 'public' && renote.userHost == null && renote.replyId == null) { + this.featuredService.updateGlobalNotesRanking(renote, 5); + this.featuredService.updatePerUserNotesRanking(renote.userId, renote, 5); + } } } } diff --git a/packages/backend/src/core/NoteDeleteService.ts b/packages/backend/src/core/NoteDeleteService.ts index 9b6c4754d1..9ce8cb6731 100644 --- a/packages/backend/src/core/NoteDeleteService.ts +++ b/packages/backend/src/core/NoteDeleteService.ts @@ -124,9 +124,11 @@ export class NoteDeleteService { this.perUserNotesChart.update(user, note, false); } - if (note.renoteId && note.text || !note.renoteId) { + if (!isRenote(note) || isQuote(note)) { // Decrement notes count (user) this.decNotesCountOfUser(user); + } else { + this.usersRepository.update({ id: user.id }, { updatedAt: new Date() }); } if (this.meta.enableStatsForFederatedInstances) { @@ -165,8 +167,11 @@ export class NoteDeleteService { }); } - if (note.uri) { - this.apLogService.deleteObjectLogs(note.uri) + const deletedUris = [note, ...cascadingNotes] + .map(n => n.uri) + .filter((u): u is string => u != null); + if (deletedUris.length > 0) { + this.apLogService.deleteObjectLogs(deletedUris) .catch(err => this.logger.error(err, `Failed to delete AP logs for note '${note.uri}'`)); } } diff --git a/packages/backend/src/core/NoteEditService.ts b/packages/backend/src/core/NoteEditService.ts index e9637c56c7..58233b90ee 100644 --- a/packages/backend/src/core/NoteEditService.ts +++ b/packages/backend/src/core/NoteEditService.ts @@ -610,6 +610,8 @@ export class NoteEditService implements OnApplicationShutdown { } } + this.usersRepository.update({ id: user.id }, { updatedAt: new Date() }); + // ハッシュタグ更新 this.pushToTl(note, user); diff --git a/packages/backend/src/core/ReactionService.ts b/packages/backend/src/core/ReactionService.ts index 0179b0680f..f05ee2ee73 100644 --- a/packages/backend/src/core/ReactionService.ts +++ b/packages/backend/src/core/ReactionService.ts @@ -30,6 +30,7 @@ import { trackPromise } from '@/misc/promise-tracker.js'; import { isQuote, isRenote } from '@/misc/is-renote.js'; import { ReactionsBufferingService } from '@/core/ReactionsBufferingService.js'; import { PER_NOTE_REACTION_USER_PAIR_CACHE_MAX } from '@/const.js'; +import { CacheService } from '@/core/CacheService.js'; const FALLBACK = '\u2764'; @@ -102,6 +103,7 @@ export class ReactionService { private apDeliverManagerService: ApDeliverManagerService, private notificationService: NotificationService, private perUserReactionsChart: PerUserReactionsChart, + private readonly cacheService: CacheService, ) { } @@ -212,20 +214,28 @@ export class ReactionService { .execute(); } + this.usersRepository.update({ id: user.id }, { updatedAt: new Date() }); + // 30%の確率、セルフではない、3日以内に投稿されたノートの場合ハイライト用ランキング更新 if ( Math.random() < 0.3 && note.userId !== user.id && (Date.now() - this.idService.parse(note.id).date.getTime()) < 1000 * 60 * 60 * 24 * 3 ) { - if (note.channelId != null) { - if (note.replyId == null) { - this.featuredService.updateInChannelNotesRanking(note.channelId, note.id, 1); - } - } else { - if (note.visibility === 'public' && note.userHost == null && note.replyId == null) { - this.featuredService.updateGlobalNotesRanking(note.id, 1); - this.featuredService.updatePerUserNotesRanking(note.userId, note.id, 1); + const author = await this.cacheService.findUserById(note.userId); + if (author.isExplorable) { + const policies = await this.roleService.getUserPolicies(author); + if (policies.canTrend) { + if (note.channelId != null) { + if (note.replyId == null) { + this.featuredService.updateInChannelNotesRanking(note.channelId, note, 1); + } + } else { + if (note.visibility === 'public' && note.userHost == null && note.replyId == null) { + this.featuredService.updateGlobalNotesRanking(note, 1); + this.featuredService.updatePerUserNotesRanking(note.userId, note, 1); + } + } } } } @@ -298,9 +308,9 @@ export class ReactionService { } @bindThis - public async delete(user: { id: MiUser['id']; host: MiUser['host']; isBot: MiUser['isBot']; }, note: MiNote) { + public async delete(user: { id: MiUser['id']; host: MiUser['host']; isBot: MiUser['isBot']; }, note: MiNote, exist?: MiNoteReaction | null) { // if already unreacted - const exist = await this.noteReactionsRepository.findOneBy({ + exist ??= await this.noteReactionsRepository.findOneBy({ noteId: note.id, userId: user.id, }); @@ -330,6 +340,8 @@ export class ReactionService { .execute(); } + this.usersRepository.update({ id: user.id }, { updatedAt: new Date() }); + this.globalEventService.publishNoteStream(note.id, 'unreacted', { reaction: this.decodeReaction(exist.reaction).reaction, userId: user.id, diff --git a/packages/backend/src/core/RoleService.ts b/packages/backend/src/core/RoleService.ts index 039932b76d..d3c458eec7 100644 --- a/packages/backend/src/core/RoleService.ts +++ b/packages/backend/src/core/RoleService.ts @@ -69,6 +69,7 @@ export type RolePolicies = { canImportMuting: boolean; canImportUserLists: boolean; chatAvailability: 'available' | 'readonly' | 'unavailable'; + canTrend: boolean; }; export const DEFAULT_POLICIES: RolePolicies = { @@ -108,6 +109,7 @@ export const DEFAULT_POLICIES: RolePolicies = { canImportMuting: true, canImportUserLists: true, chatAvailability: 'available', + canTrend: true, }; @Injectable() @@ -149,6 +151,7 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { ) { this.rolesCache = new MemorySingleCache(1000 * 60 * 60); // 1h this.roleAssignmentByUserIdCache = new MemoryKVCache(1000 * 60 * 5); // 5m + // TODO additional cache for final calculation? this.redisForSub.on('message', this.onMessage); } @@ -358,8 +361,9 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { } @bindThis - public async getUserAssigns(userId: MiUser['id']) { + public async getUserAssigns(userOrId: MiUser | MiUser['id']) { const now = Date.now(); + const userId = typeof(userOrId) === 'object' ? userOrId.id : userOrId; let assigns = await this.roleAssignmentByUserIdCache.fetch(userId, () => this.roleAssignmentsRepository.findBy({ userId })); // 期限切れのロールを除外 assigns = assigns.filter(a => a.expiresAt == null || (a.expiresAt.getTime() > now)); @@ -367,12 +371,13 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { } @bindThis - public async getUserRoles(userId: MiUser['id']) { + public async getUserRoles(userOrId: MiUser | MiUser['id']) { const roles = await this.rolesCache.fetch(() => this.rolesRepository.findBy({})); + const userId = typeof(userOrId) === 'object' ? userOrId.id : userOrId; const followStats = await this.cacheService.getFollowStats(userId); - const assigns = await this.getUserAssigns(userId); + const assigns = await this.getUserAssigns(userOrId); const assignedRoles = roles.filter(r => assigns.map(x => x.roleId).includes(r.id)); - const user = roles.some(r => r.target === 'conditional') ? await this.cacheService.findUserById(userId) : null; + const user = typeof(userOrId) === 'object' ? userOrId : roles.some(r => r.target === 'conditional') ? await this.cacheService.findUserById(userOrId) : null; const matchedCondRoles = roles.filter(r => r.target === 'conditional' && this.evalCond(user!, assignedRoles, r.condFormula, followStats)); return [...assignedRoles, ...matchedCondRoles]; } @@ -381,8 +386,9 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { * 指定ユーザーのバッジロール一覧取得 */ @bindThis - public async getUserBadgeRoles(userId: MiUser['id']) { + public async getUserBadgeRoles(userOrId: MiUser | MiUser['id']) { const now = Date.now(); + const userId = typeof(userOrId) === 'object' ? userOrId.id : userOrId; let assigns = await this.roleAssignmentByUserIdCache.fetch(userId, () => this.roleAssignmentsRepository.findBy({ userId })); // 期限切れのロールを除外 assigns = assigns.filter(a => a.expiresAt == null || (a.expiresAt.getTime() > now)); @@ -392,7 +398,7 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { const assignedBadgeRoles = assignedRoles.filter(r => r.asBadge); const badgeCondRoles = roles.filter(r => r.asBadge && (r.target === 'conditional')); if (badgeCondRoles.length > 0) { - const user = roles.some(r => r.target === 'conditional') ? await this.cacheService.findUserById(userId) : null; + const user = typeof(userOrId) === 'object' ? userOrId : roles.some(r => r.target === 'conditional') ? await this.cacheService.findUserById(userOrId) : null; const matchedBadgeCondRoles = badgeCondRoles.filter(r => this.evalCond(user!, assignedRoles, r.condFormula, followStats)); return [...assignedBadgeRoles, ...matchedBadgeCondRoles]; } else { @@ -401,12 +407,12 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { } @bindThis - public async getUserPolicies(userId: MiUser['id'] | null): Promise { + public async getUserPolicies(userOrId: MiUser | MiUser['id'] | null): Promise { const basePolicies = { ...DEFAULT_POLICIES, ...this.meta.policies }; - if (userId == null) return basePolicies; + if (userOrId == null) return basePolicies; - const roles = await this.getUserRoles(userId); + const roles = await this.getUserRoles(userOrId); function calc(name: T, aggregate: (values: RolePolicies[T][]) => RolePolicies[T]) { if (roles.length === 0) return basePolicies[name]; @@ -465,6 +471,7 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { canImportMuting: calc('canImportMuting', vs => vs.some(v => v === true)), canImportUserLists: calc('canImportUserLists', vs => vs.some(v => v === true)), chatAvailability: calc('chatAvailability', aggregateChatAvailability), + canTrend: calc('canTrend', vs => vs.some(v => v === true)), }; } diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index 7118ce1e02..b665b51700 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -155,6 +155,8 @@ export class ApRequestService { @bindThis public async signedPost(user: { id: MiUser['id'] }, url: string, object: unknown, digest?: string): Promise { + this.apUtilityService.assertApUrl(url); + const body = typeof object === 'string' ? object : JSON.stringify(object); const keypair = await this.userKeypairService.getUserKeypair(user.id); @@ -186,6 +188,8 @@ export class ApRequestService { */ @bindThis public async signedGet(url: string, user: { id: MiUser['id'] }, followAlternate?: boolean): Promise { + this.apUtilityService.assertApUrl(url); + const _followAlternate = followAlternate ?? true; const keypair = await this.userKeypairService.getUserKeypair(user.id); diff --git a/packages/backend/src/core/activitypub/ApUtilityService.ts b/packages/backend/src/core/activitypub/ApUtilityService.ts index ae6e4997e4..c3958cdf42 100644 --- a/packages/backend/src/core/activitypub/ApUtilityService.ts +++ b/packages/backend/src/core/activitypub/ApUtilityService.ts @@ -77,16 +77,48 @@ export class ApUtilityService { return acceptableUrls[0]?.url ?? null; } + /** + * Verifies that a provided URL is in a format acceptable for federation. + * @throws {IdentifiableError} If URL cannot be parsed + * @throws {IdentifiableError} If URL contains a fragment + * @throws {IdentifiableError} If URL is not HTTPS + */ + public assertApUrl(url: string | URL): void { + // If string, parse and validate + if (typeof(url) === 'string') { + try { + url = new URL(url); + } catch { + throw new IdentifiableError('0bedd29b-e3bf-4604-af51-d3352e2518af', `invalid AP url ${url}: not a valid URL`); + } + } + + // Hash component breaks federation + if (url.hash) { + throw new IdentifiableError('0bedd29b-e3bf-4604-af51-d3352e2518af', `invalid AP url ${url}: contains a fragment (#)`); + } + + // Must be HTTPS + if (!this.checkHttps(url)) { + throw new IdentifiableError('0bedd29b-e3bf-4604-af51-d3352e2518af', `invalid AP url ${url}: unsupported protocol ${url.protocol}`); + } + } + /** * Checks if the URL contains HTTPS. * Additionally, allows HTTP in non-production environments. * Based on check-https.ts. */ - private checkHttps(url: string): boolean { + private checkHttps(url: string | URL): boolean { const isNonProd = this.envService.env.NODE_ENV !== 'production'; - // noinspection HttpUrlsUsage - return url.startsWith('https://') || (url.startsWith('http://') && isNonProd); + try { + const proto = new URL(url).protocol; + return proto === 'https:' || (proto === 'http:' && isNonProd); + } catch { + // Invalid URLs don't "count" as HTTPS + return false; + } } } diff --git a/packages/backend/src/core/activitypub/models/ApNoteService.ts b/packages/backend/src/core/activitypub/models/ApNoteService.ts index f6152e3888..7811b81795 100644 --- a/packages/backend/src/core/activitypub/models/ApNoteService.ts +++ b/packages/backend/src/core/activitypub/models/ApNoteService.ts @@ -95,6 +95,7 @@ export class ApNoteService { actor?: MiRemoteUser, user?: MiRemoteUser, ): Error | null { + this.apUtilityService.assertApUrl(uri); const expectHost = this.utilityService.extractDbHost(uri); const apType = getApType(object); diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index 4b685f7e1b..5c6716a0b8 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -153,6 +153,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { */ @bindThis private validateActor(x: IObject, uri: string): IActor { + this.apUtilityService.assertApUrl(uri); const expectHost = this.utilityService.punyHostPSLDomain(uri); if (!isActor(x)) { @@ -167,6 +168,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { throw new UnrecoverableError(`invalid Actor ${uri} - wrong inbox type`); } + this.apUtilityService.assertApUrl(x.inbox); const inboxHost = this.utilityService.punyHostPSLDomain(x.inbox); if (inboxHost !== expectHost) { throw new UnrecoverableError(`invalid Actor ${uri} - wrong inbox ${inboxHost}`); @@ -175,6 +177,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { const sharedInboxObject = x.sharedInbox ?? (x.endpoints ? x.endpoints.sharedInbox : undefined); if (sharedInboxObject != null) { const sharedInbox = getApId(sharedInboxObject); + this.apUtilityService.assertApUrl(sharedInbox); if (!(typeof sharedInbox === 'string' && sharedInbox.length > 0 && this.utilityService.punyHostPSLDomain(sharedInbox) === expectHost)) { throw new UnrecoverableError(`invalid Actor ${uri} - wrong shared inbox ${sharedInbox}`); } @@ -185,6 +188,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { if (xCollection != null) { const collectionUri = getApId(xCollection); if (typeof collectionUri === 'string' && collectionUri.length > 0) { + this.apUtilityService.assertApUrl(collectionUri); if (this.utilityService.punyHostPSLDomain(collectionUri) !== expectHost) { throw new UnrecoverableError(`invalid Actor ${uri} - wrong ${collection} ${collectionUri}`); } diff --git a/packages/backend/src/core/entities/NotificationEntityService.ts b/packages/backend/src/core/entities/NotificationEntityService.ts index 77e6a1c7e7..cc8edfc666 100644 --- a/packages/backend/src/core/entities/NotificationEntityService.ts +++ b/packages/backend/src/core/entities/NotificationEntityService.ts @@ -23,6 +23,13 @@ import type { NoteEntityService } from './NoteEntityService.js'; const NOTE_REQUIRED_NOTIFICATION_TYPES = new Set(['note', 'mention', 'reply', 'renote', 'renote:grouped', 'quote', 'reaction', 'reaction:grouped', 'pollEnded', 'edited', 'scheduledNotePosted'] as (typeof groupedNotificationTypes[number])[]); +function undefOnMissing(packPromise: Promise): Promise { + return packPromise.catch(err => { + if (err instanceof EntityNotFoundError) return undefined; + throw err; + }); +} + @Injectable() export class NotificationEntityService implements OnModuleInit { private userEntityService: UserEntityService; @@ -75,9 +82,9 @@ export class NotificationEntityService implements OnModuleInit { const noteIfNeed = needsNote ? ( hint?.packedNotes != null ? hint.packedNotes.get(notification.noteId) - : this.noteEntityService.pack(notification.noteId, { id: meId }, { + : undefOnMissing(this.noteEntityService.pack(notification.noteId, { id: meId }, { detail: true, - }) + })) ) : undefined; // if the note has been deleted, don't show this notification if (needsNote && !noteIfNeed) return null; @@ -86,7 +93,7 @@ export class NotificationEntityService implements OnModuleInit { const userIfNeed = needsUser ? ( hint?.packedUsers != null ? hint.packedUsers.get(notification.notifierId) - : this.userEntityService.pack(notification.notifierId, { id: meId }) + : undefOnMissing(this.userEntityService.pack(notification.notifierId, { id: meId })) ) : undefined; // if the user has been deleted, don't show this notification if (needsUser && !userIfNeed) return null; @@ -96,7 +103,7 @@ export class NotificationEntityService implements OnModuleInit { const reactions = (await Promise.all(notification.reactions.map(async reaction => { const user = hint?.packedUsers != null ? hint.packedUsers.get(reaction.userId)! - : await this.userEntityService.pack(reaction.userId, { id: meId }); + : await undefOnMissing(this.userEntityService.pack(reaction.userId, { id: meId })); return { user, reaction: reaction.reaction, @@ -121,7 +128,7 @@ export class NotificationEntityService implements OnModuleInit { return packedUser; } - return this.userEntityService.pack(userId, { id: meId }); + return undefOnMissing(this.userEntityService.pack(userId, { id: meId })); }))).filter(x => x != null); // if all users have been deleted, don't show this notification if (users.length === 0) { @@ -140,10 +147,7 @@ export class NotificationEntityService implements OnModuleInit { const needsRole = notification.type === 'roleAssigned'; const role = needsRole - ? await this.roleEntityService.pack(notification.roleId).catch(err => { - if (err instanceof EntityNotFoundError) return undefined; - throw err; - }) + ? await undefOnMissing(this.roleEntityService.pack(notification.roleId)) : undefined; // if the role has been deleted, don't show this notification if (needsRole && !role) { diff --git a/packages/backend/src/misc/cache.ts b/packages/backend/src/misc/cache.ts index f9692ce5d5..48b8f43678 100644 --- a/packages/backend/src/misc/cache.ts +++ b/packages/backend/src/misc/cache.ts @@ -19,16 +19,16 @@ export class RedisKVCache { opts: { lifetime: RedisKVCache['lifetime']; memoryCacheLifetime: number; - fetcher: RedisKVCache['fetcher']; - toRedisConverter: RedisKVCache['toRedisConverter']; - fromRedisConverter: RedisKVCache['fromRedisConverter']; + fetcher?: RedisKVCache['fetcher']; + toRedisConverter?: RedisKVCache['toRedisConverter']; + fromRedisConverter?: RedisKVCache['fromRedisConverter']; }, ) { this.lifetime = opts.lifetime; this.memoryCache = new MemoryKVCache(opts.memoryCacheLifetime); - this.fetcher = opts.fetcher; - this.toRedisConverter = opts.toRedisConverter; - this.fromRedisConverter = opts.fromRedisConverter; + this.fetcher = opts.fetcher ?? (() => { throw new Error('fetch not supported - use get/set directly'); }); + this.toRedisConverter = opts.toRedisConverter ?? ((value) => JSON.stringify(value)); + this.fromRedisConverter = opts.fromRedisConverter ?? ((value) => JSON.parse(value)); } @bindThis diff --git a/packages/backend/src/models/AbuseReportNotificationRecipient.ts b/packages/backend/src/models/AbuseReportNotificationRecipient.ts index fbff880afc..fd31354a80 100644 --- a/packages/backend/src/models/AbuseReportNotificationRecipient.ts +++ b/packages/backend/src/models/AbuseReportNotificationRecipient.ts @@ -22,7 +22,7 @@ export class MiAbuseReportNotificationRecipient { /** * 有効かどうか. */ - @Index() + @Index('IDX_abuse_report_notification_recipient_isActive') @Column('boolean', { default: true, }) @@ -47,7 +47,7 @@ export class MiAbuseReportNotificationRecipient { /** * 通知方法. */ - @Index() + @Index('IDX_abuse_report_notification_recipient_method') @Column('varchar', { length: 64, }) @@ -56,7 +56,7 @@ export class MiAbuseReportNotificationRecipient { /** * 通知先のユーザID. */ - @Index() + @Index('IDX_abuse_report_notification_recipient_userId') @Column({ ...id(), nullable: true, @@ -75,14 +75,16 @@ export class MiAbuseReportNotificationRecipient { /** * 通知先のユーザプロフィール. */ - @ManyToOne(type => MiUserProfile, {}) + @ManyToOne(type => MiUserProfile, { + onDelete: 'CASCADE', + }) @JoinColumn({ name: 'userId', referencedColumnName: 'userId', foreignKeyConstraintName: 'FK_abuse_report_notification_recipient_userId2' }) public userProfile: MiUserProfile | null; /** * 通知先のシステムWebhookId. */ - @Index() + @Index('IDX_abuse_report_notification_recipient_systemWebhookId') @Column({ ...id(), nullable: true, @@ -95,6 +97,8 @@ export class MiAbuseReportNotificationRecipient { @ManyToOne(type => MiSystemWebhook, { onDelete: 'CASCADE', }) - @JoinColumn() + @JoinColumn({ + foreignKeyConstraintName: 'FK_abuse_report_notification_recipient_systemWebhookId', + }) public systemWebhook: MiSystemWebhook | null; } diff --git a/packages/backend/src/models/Emoji.ts b/packages/backend/src/models/Emoji.ts index d62b6e9f6f..9f31455b83 100644 --- a/packages/backend/src/models/Emoji.ts +++ b/packages/backend/src/models/Emoji.ts @@ -29,6 +29,7 @@ export class MiEmoji { }) public host: string | null; + @Index('IDX_EMOJI_CATEGORY') @Column('varchar', { length: 128, nullable: true, }) @@ -77,6 +78,8 @@ export class MiEmoji { public isSensitive: boolean; // TODO: 定期ジョブで存在しなくなったロールIDを除去するようにする + // Synchronize: false is needed because TypeORM doesn't understand GIN indexes + @Index('IDX_EMOJI_ROLE_IDS', { synchronize: false }) @Column('varchar', { array: true, length: 128, default: '{}', }) diff --git a/packages/backend/src/models/LatestNote.ts b/packages/backend/src/models/LatestNote.ts index 064fcccc0a..37efb0d4b6 100644 --- a/packages/backend/src/models/LatestNote.ts +++ b/packages/backend/src/models/LatestNote.ts @@ -45,6 +45,7 @@ export class SkLatestNote { }) @JoinColumn({ name: 'user_id', + foreignKeyConstraintName: 'FK_20e346fffe4a2174585005d6d80', }) public user: MiUser | null; @@ -60,6 +61,7 @@ export class SkLatestNote { }) @JoinColumn({ name: 'note_id', + foreignKeyConstraintName: 'FK_47a38b1c13de6ce4e5090fb1acd', }) public note: MiNote | null; diff --git a/packages/backend/src/models/Meta.ts b/packages/backend/src/models/Meta.ts index 20b6554632..182495e95f 100644 --- a/packages/backend/src/models/Meta.ts +++ b/packages/backend/src/models/Meta.ts @@ -60,7 +60,7 @@ export class MiMeta { public maintainerEmail: string | null; @Column('boolean', { - default: false, + default: true, }) public disableRegistration: boolean; @@ -382,6 +382,12 @@ export class MiMeta { }) public swPrivateKey: string | null; + @Column('integer', { + default: 5000, + comment: 'Timeout in milliseconds for translation API requests', + }) + public translationTimeout: number; + @Column('varchar', { length: 1024, nullable: true, @@ -425,7 +431,7 @@ export class MiMeta { @Column('varchar', { length: 1024, default: 'https://activitypub.software/TransFem-org/Sharkey/', - nullable: false, + nullable: true, }) public repositoryUrl: string | null; @@ -612,8 +618,8 @@ export class MiMeta { }) public enableAchievements: boolean; - @Column('varchar', { - length: 2048, nullable: true, + @Column('text', { + nullable: true, }) public robotsTxt: string | null; @@ -643,7 +649,7 @@ export class MiMeta { public bannedEmailDomains: string[]; @Column('varchar', { - length: 1024, array: true, default: '{ "admin", "administrator", "root", "system", "maintainer", "host", "mod", "moderator", "owner", "superuser", "staff", "auth", "i", "me", "everyone", "all", "mention", "mentions", "example", "user", "users", "account", "accounts", "official", "help", "helps", "support", "supports", "info", "information", "informations", "announce", "announces", "announcement", "announcements", "notice", "notification", "notifications", "dev", "developer", "developers", "tech", "misskey" }', + length: 1024, array: true, default: '{admin,administrator,root,system,maintainer,host,mod,moderator,owner,superuser,staff,auth,i,me,everyone,all,mention,mentions,example,user,users,account,accounts,official,help,helps,support,supports,info,information,informations,announce,announces,announcement,announcements,notice,notification,notifications,dev,developer,developers,tech,misskey}', }) public preservedUsernames: string[]; @@ -658,22 +664,22 @@ export class MiMeta { public enableFanoutTimelineDbFallback: boolean; @Column('integer', { - default: 300, + default: 800, }) public perLocalUserUserTimelineCacheMax: number; @Column('integer', { - default: 100, + default: 800, }) public perRemoteUserUserTimelineCacheMax: number; @Column('integer', { - default: 300, + default: 800, }) public perUserHomeTimelineCacheMax: number; @Column('integer', { - default: 300, + default: 800, }) public perUserListTimelineCacheMax: number; @@ -689,9 +695,9 @@ export class MiMeta { @Column('varchar', { length: 500, - nullable: true, + default: '❤️', }) - public defaultLike: string | null; + public defaultLike: string; @Column('varchar', { length: 256, array: true, default: '{}', @@ -714,7 +720,7 @@ export class MiMeta { public urlPreviewMaximumContentLength: number; @Column('boolean', { - default: true, + default: false, }) public urlPreviewRequireContentLength: boolean; diff --git a/packages/backend/src/models/Note.ts b/packages/backend/src/models/Note.ts index e143d6f22a..c4d5e794fa 100644 --- a/packages/backend/src/models/Note.ts +++ b/packages/backend/src/models/Note.ts @@ -20,7 +20,7 @@ import type { MiDriveFile } from './DriveFile.js'; // You should not use `@Index({ concurrent: true })` decorator because database initialization for test will fail // because it will always run CREATE INDEX in transaction based on decorators. // Not appending `{ concurrent: true }` to `@Index` will not cause any problem in production, -@Index(['userId', 'id']) +@Index('IDX_724b311e6f883751f261ebe378', ['userId', 'id']) @Entity('note') export class MiNote { @PrimaryColumn(id()) @@ -273,3 +273,7 @@ export type IMentionedRemoteUsers = { username: string; host: string; }[]; + +export function hasText(note: MiNote): note is MiNote & { text: string } { + return note.text != null; +} diff --git a/packages/backend/src/models/User.ts b/packages/backend/src/models/User.ts index 760ef52d2b..46f8e84a94 100644 --- a/packages/backend/src/models/User.ts +++ b/packages/backend/src/models/User.ts @@ -129,7 +129,9 @@ export class MiUser { @OneToOne(() => MiDriveFile, { onDelete: 'SET NULL', }) - @JoinColumn() + @JoinColumn({ + foreignKeyConstraintName: 'FK_q5lm0tbgejtfskzg0rc4wd7t1n', + }) public background: MiDriveFile | null; // avatarId が null になったとしてもこれが null でない可能性があるため、このフィールドを使うときは avatarId の non-null チェックをすること @@ -345,7 +347,7 @@ export class MiUser { */ @Column('boolean', { name: 'enable_rss', - default: true, + default: false, }) public enableRss: boolean; diff --git a/packages/backend/src/models/UserListMembership.ts b/packages/backend/src/models/UserListMembership.ts index af659d071d..99ec8bdc26 100644 --- a/packages/backend/src/models/UserListMembership.ts +++ b/packages/backend/src/models/UserListMembership.ts @@ -24,7 +24,9 @@ export class MiUserListMembership { @ManyToOne(type => MiUser, { onDelete: 'CASCADE', }) - @JoinColumn() + @JoinColumn({ + foreignKeyConstraintName: 'FK_d844bfc6f3f523a05189076efaa', + }) public user: MiUser | null; @Index() @@ -37,7 +39,9 @@ export class MiUserListMembership { @ManyToOne(type => MiUserList, { onDelete: 'CASCADE', }) - @JoinColumn() + @JoinColumn({ + foreignKeyConstraintName: 'FK_605472305f26818cc93d1baaa74', + }) public userList: MiUserList | null; // タイムラインにその人のリプライまで含めるかどうか diff --git a/packages/backend/src/models/UserPending.ts b/packages/backend/src/models/UserPending.ts index 961ae344f1..972c862a1a 100644 --- a/packages/backend/src/models/UserPending.ts +++ b/packages/backend/src/models/UserPending.ts @@ -34,6 +34,7 @@ export class MiUserPending { @Column('varchar', { length: 1000, + nullable: true, }) public reason: string; } diff --git a/packages/backend/src/models/UserProfile.ts b/packages/backend/src/models/UserProfile.ts index 42b955041e..6db555b060 100644 --- a/packages/backend/src/models/UserProfile.ts +++ b/packages/backend/src/models/UserProfile.ts @@ -110,12 +110,14 @@ export class MiUserProfile { @Column('enum', { enum: followingVisibilities, + enumName: 'user_profile_followingVisibility_enum', default: 'public', }) public followingVisibility: typeof followingVisibilities[number]; @Column('enum', { enum: followersVisibilities, + enumName: 'user_profile_followersVisibility_enum', default: 'public', }) public followersVisibility: typeof followersVisibilities[number]; diff --git a/packages/backend/src/models/json-schema/role.ts b/packages/backend/src/models/json-schema/role.ts index 307c114c96..363be921ed 100644 --- a/packages/backend/src/models/json-schema/role.ts +++ b/packages/backend/src/models/json-schema/role.ts @@ -309,6 +309,10 @@ export const packedRolePoliciesSchema = { optional: false, nullable: false, enum: ['available', 'readonly', 'unavailable'], }, + canTrend: { + type: 'boolean', + optional: false, nullable: false, + }, }, } as const; diff --git a/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts b/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts index 46cee096cf..4e9779a41b 100644 --- a/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts +++ b/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts @@ -4,9 +4,9 @@ */ import { Inject, Injectable } from '@nestjs/common'; -import { MoreThan } from 'typeorm'; +import { In, MoreThan } from 'typeorm'; import { DI } from '@/di-symbols.js'; -import type { DriveFilesRepository, NoteReactionsRepository, NotesRepository, UserProfilesRepository, UsersRepository, NoteScheduleRepository, MiNoteSchedule } from '@/models/_.js'; +import type { DriveFilesRepository, NoteReactionsRepository, NotesRepository, UserProfilesRepository, UsersRepository, NoteScheduleRepository, MiNoteSchedule, FollowingsRepository, FollowRequestsRepository, BlockingsRepository, MutingsRepository, ClipsRepository, ClipNotesRepository, LatestNotesRepository, NoteEditRepository, NoteFavoritesRepository, PollVotesRepository, PollsRepository, SigninsRepository, UserIpsRepository, RegistryItemsRepository } from '@/models/_.js'; import type Logger from '@/logger.js'; import { DriveService } from '@/core/DriveService.js'; import type { MiDriveFile } from '@/models/DriveFile.js'; @@ -17,10 +17,10 @@ import { bindThis } from '@/decorators.js'; import { SearchService } from '@/core/SearchService.js'; import { ApLogService } from '@/core/ApLogService.js'; import { ReactionService } from '@/core/ReactionService.js'; +import { QueueService } from '@/core/QueueService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; import type { DbUserDeleteJobData } from '../types.js'; -import { QueueService } from '@/core/QueueService.js'; @Injectable() export class DeleteAccountProcessorService { @@ -45,6 +45,48 @@ export class DeleteAccountProcessorService { @Inject(DI.noteScheduleRepository) private noteScheduleRepository: NoteScheduleRepository, + @Inject(DI.followingsRepository) + private readonly followingsRepository: FollowingsRepository, + + @Inject(DI.followRequestsRepository) + private readonly followRequestsRepository: FollowRequestsRepository, + + @Inject(DI.blockingsRepository) + private readonly blockingsRepository: BlockingsRepository, + + @Inject(DI.mutingsRepository) + private readonly mutingsRepository: MutingsRepository, + + @Inject(DI.clipsRepository) + private readonly clipsRepository: ClipsRepository, + + @Inject(DI.clipNotesRepository) + private readonly clipNotesRepository: ClipNotesRepository, + + @Inject(DI.latestNotesRepository) + private readonly latestNotesRepository: LatestNotesRepository, + + @Inject(DI.noteEditRepository) + private readonly noteEditRepository: NoteEditRepository, + + @Inject(DI.noteFavoritesRepository) + private readonly noteFavoritesRepository: NoteFavoritesRepository, + + @Inject(DI.pollVotesRepository) + private readonly pollVotesRepository: PollVotesRepository, + + @Inject(DI.pollsRepository) + private readonly pollsRepository: PollsRepository, + + @Inject(DI.signinsRepository) + private readonly signinsRepository: SigninsRepository, + + @Inject(DI.userIpsRepository) + private readonly userIpsRepository: UserIpsRepository, + + @Inject(DI.registryItemsRepository) + private readonly registryItemsRepository: RegistryItemsRepository, + private queueService: QueueService, private driveService: DriveService, private emailService: EmailService, @@ -65,6 +107,140 @@ export class DeleteAccountProcessorService { return; } + { // Delete user clips + const userClips = await this.clipsRepository.find({ + select: { + id: true, + }, + where: { + userId: user.id, + }, + }) as { id: string }[]; + + // Delete one-at-a-time because there can be a lot + for (const clip of userClips) { + await this.clipNotesRepository.delete({ + id: clip.id, + }); + } + + await this.clipsRepository.delete({ + userId: user.id, + }); + + this.logger.succ('All clips have been deleted.'); + } + + { // Delete favorites + await this.noteFavoritesRepository.delete({ + userId: user.id, + }); + + this.logger.succ('All favorites have been deleted.'); + } + + { // Delete user relations + await this.followingsRepository.delete({ + followerId: user.id, + }); + + await this.followingsRepository.delete({ + followeeId: user.id, + }); + + await this.followRequestsRepository.delete({ + followerId: user.id, + }); + + await this.followRequestsRepository.delete({ + followeeId: user.id, + }); + + await this.blockingsRepository.delete({ + blockerId: user.id, + }); + + await this.blockingsRepository.delete({ + blockeeId: user.id, + }); + + await this.mutingsRepository.delete({ + muterId: user.id, + }); + + await this.mutingsRepository.delete({ + muteeId: user.id, + }); + + this.logger.succ('All user relations have been deleted.'); + } + + { // Delete reactions + let cursor: MiNoteReaction['id'] | null = null; + + while (true) { + const reactions = await this.noteReactionsRepository.find({ + where: { + userId: user.id, + ...(cursor ? { id: MoreThan(cursor) } : {}), + }, + take: 100, + order: { + id: 1, + }, + relations: { + note: true, + }, + }) as MiNoteReaction[]; + + if (reactions.length === 0) { + break; + } + + cursor = reactions.at(-1)?.id ?? null; + + for (const reaction of reactions) { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const note = reaction.note!; + await this.reactionService.delete(user, note, reaction); + } + } + + this.logger.succ('All reactions have been deleted'); + } + + { // Poll votes + let cursor: MiNoteReaction['id'] | null = null; + + while (true) { + const votes = await this.pollVotesRepository.find({ + where: { + userId: user.id, + ...(cursor ? { id: MoreThan(cursor) } : {}), + }, + select: { + id: true, + }, + take: 100, + order: { + id: 1, + }, + }) as { id: string }[]; + + if (votes.length === 0) { + break; + } + + cursor = votes.at(-1)?.id ?? null; + + await this.pollVotesRepository.delete({ + id: In(votes.map(v => v.id)), + }); + } + + this.logger.succ('All poll votes have been deleted'); + } + { // Delete scheduled notes const scheduledNotes = await this.noteScheduleRepository.findBy({ userId: user.id, @@ -82,6 +258,10 @@ export class DeleteAccountProcessorService { } { // Delete notes + await this.latestNotesRepository.delete({ + userId: user.id, + }); + let cursor: MiNote['id'] | null = null; while (true) { @@ -102,7 +282,23 @@ export class DeleteAccountProcessorService { cursor = notes.at(-1)?.id ?? null; - await this.notesRepository.delete(notes.map(note => note.id)); + // Delete associated polls one-at-a-time, since it can cascade to a LOT of vote entries + for (const note of notes) { + if (note.hasPoll) { + await this.pollsRepository.delete({ + noteId: note.id, + }); + } + } + + const ids = notes.map(note => note.id); + + await this.noteEditRepository.delete({ + noteId: In(ids), + }); + await this.notesRepository.delete({ + id: In(ids), + }); for (const note of notes) { await this.searchService.unindexNote(note); @@ -119,37 +315,6 @@ export class DeleteAccountProcessorService { this.logger.succ('All of notes deleted'); } - { // Delete reactions - let cursor: MiNoteReaction['id'] | null = null; - - while (true) { - const reactions = await this.noteReactionsRepository.find({ - where: { - userId: user.id, - ...(cursor ? { id: MoreThan(cursor) } : {}), - }, - take: 100, - order: { - id: 1, - }, - }) as MiNoteReaction[]; - - if (reactions.length === 0) { - break; - } - - cursor = reactions.at(-1)?.id ?? null; - - for (const reaction of reactions) { - const note = await this.notesRepository.findOneBy({ id: reaction.noteId }) as MiNote; - - await this.reactionService.delete(user, note); - } - } - - this.logger.succ('All reactions have been deleted'); - } - { // Delete files let cursor: MiDriveFile['id'] | null = null; @@ -191,20 +356,42 @@ export class DeleteAccountProcessorService { this.logger.succ('All AP logs deleted'); } - { // Send email notification - const profile = await this.userProfilesRepository.findOneByOrFail({ userId: user.id }); - if (profile.email && profile.emailVerified) { - this.emailService.sendEmail(profile.email, 'Account deleted', - 'Your account has been deleted.', - 'Your account has been deleted.'); + // Do this BEFORE deleting the account! + const profile = await this.userProfilesRepository.findOneBy({ userId: user.id }); + + { // Delete the actual account + await this.userIpsRepository.delete({ + userId: user.id, + }); + + await this.signinsRepository.delete({ + userId: user.id, + }); + + await this.registryItemsRepository.delete({ + userId: user.id, + }); + + // soft指定されている場合は物理削除しない + if (job.data.soft) { + // nop + } else { + await this.usersRepository.delete(user.id); } + + this.logger.succ('Account data deleted'); } - // soft指定されている場合は物理削除しない - if (job.data.soft) { - // nop - } else { - await this.usersRepository.delete(job.data.user.id); + { // Send email notification + if (profile && profile.email && profile.emailVerified) { + try { + await this.emailService.sendEmail(profile.email, 'Account deleted', + 'Your account has been deleted.', + 'Your account has been deleted.'); + } catch (e) { + this.logger.warn('Failed to send account deletion message:', { e }); + } + } } return 'Account deleted'; diff --git a/packages/backend/src/server/FileServerService.ts b/packages/backend/src/server/FileServerService.ts index 34ef683983..4ef5539cff 100644 --- a/packages/backend/src/server/FileServerService.ts +++ b/packages/backend/src/server/FileServerService.ts @@ -675,9 +675,11 @@ export class FileServerService { if (info.blocked) { reply.code(429); reply.send({ - message: 'Rate limit exceeded. Please try again later.', - code: 'RATE_LIMIT_EXCEEDED', - id: 'd5826d14-3982-4d2e-8011-b9e9f02499ef', + error: { + message: 'Rate limit exceeded. Please try again later.', + code: 'RATE_LIMIT_EXCEEDED', + id: 'd5826d14-3982-4d2e-8011-b9e9f02499ef', + }, }); return false; diff --git a/packages/backend/src/server/api/endpoints/admin/meta.ts b/packages/backend/src/server/api/endpoints/admin/meta.ts index a6b7401efe..4bcdc06aa3 100644 --- a/packages/backend/src/server/api/endpoints/admin/meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/meta.ts @@ -445,6 +445,10 @@ export const meta = { type: 'string', optional: false, nullable: true, }, + translationTimeout: { + type: 'number', + optional: false, nullable: false, + }, deeplAuthKey: { type: 'string', optional: false, nullable: true, @@ -477,6 +481,10 @@ export const meta = { type: 'string', optional: false, nullable: true, }, + defaultLike: { + type: 'string', + optional: false, nullable: false, + }, description: { type: 'string', optional: false, nullable: true, @@ -741,6 +749,7 @@ export default class extends Endpoint { // eslint- objectStorageUseProxy: instance.objectStorageUseProxy, objectStorageSetPublicRead: instance.objectStorageSetPublicRead, objectStorageS3ForcePathStyle: instance.objectStorageS3ForcePathStyle, + translationTimeout: instance.translationTimeout, deeplAuthKey: instance.deeplAuthKey, deeplIsPro: instance.deeplIsPro, deeplFreeMode: instance.deeplFreeMode, diff --git a/packages/backend/src/server/api/endpoints/admin/show-user.ts b/packages/backend/src/server/api/endpoints/admin/show-user.ts index 6dbfbf9d9a..1579719246 100644 --- a/packages/backend/src/server/api/endpoints/admin/show-user.ts +++ b/packages/backend/src/server/api/endpoints/admin/show-user.ts @@ -12,6 +12,7 @@ import { RoleEntityService } from '@/core/entities/RoleEntityService.js'; import { IdService } from '@/core/IdService.js'; import { notificationRecieveConfig } from '@/models/json-schema/user.js'; import { isSystemAccount } from '@/misc/is-system-account.js'; +import { CacheService } from '@/core/CacheService.js'; export const meta = { tags: ['admin'], @@ -186,6 +187,36 @@ export const meta = { }, }, }, + followStats: { + type: 'object', + optional: false, nullable: false, + properties: { + totalFollowing: { + type: 'number', + optional: false, nullable: false, + }, + totalFollowers: { + type: 'number', + optional: false, nullable: false, + }, + localFollowing: { + type: 'number', + optional: false, nullable: false, + }, + localFollowers: { + type: 'number', + optional: false, nullable: false, + }, + remoteFollowing: { + type: 'number', + optional: false, nullable: false, + }, + remoteFollowers: { + type: 'number', + optional: false, nullable: false, + }, + }, + }, }, }, } as const; @@ -213,6 +244,7 @@ export default class extends Endpoint { // eslint- private roleService: RoleService, private roleEntityService: RoleEntityService, private idService: IdService, + private readonly cacheService: CacheService, ) { super(meta, paramDef, async (ps, me) => { const [user, profile] = await Promise.all([ @@ -237,6 +269,8 @@ export default class extends Endpoint { // eslint- const roleAssigns = await this.roleService.getUserAssigns(user.id); const roles = await this.roleService.getUserRoles(user.id); + const followStats = await this.cacheService.getFollowStats(user.id); + return { email: profile.email, emailVerified: profile.emailVerified, @@ -269,6 +303,11 @@ export default class extends Endpoint { // eslint- expiresAt: a.expiresAt ? a.expiresAt.toISOString() : null, roleId: a.roleId, })), + followStats: { + ...followStats, + totalFollowers: Math.max(user.followersCount, followStats.localFollowers + followStats.remoteFollowers), + totalFollowing: Math.max(user.followingCount, followStats.localFollowing + followStats.remoteFollowing), + }, }; }); } diff --git a/packages/backend/src/server/api/endpoints/admin/update-meta.ts b/packages/backend/src/server/api/endpoints/admin/update-meta.ts index 5a0feb1a31..2de8b11f7d 100644 --- a/packages/backend/src/server/api/endpoints/admin/update-meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/update-meta.ts @@ -69,7 +69,7 @@ export const paramDef = { description: { type: 'string', nullable: true }, defaultLightTheme: { type: 'string', nullable: true }, defaultDarkTheme: { type: 'string', nullable: true }, - defaultLike: { type: 'string', nullable: true }, + defaultLike: { type: 'string' }, cacheRemoteFiles: { type: 'boolean' }, cacheRemoteSensitiveFiles: { type: 'boolean' }, emailRequiredForSignup: { type: 'boolean' }, @@ -103,6 +103,7 @@ export const paramDef = { type: 'string', }, }, + translationTimeout: { type: 'number' }, deeplAuthKey: { type: 'string', nullable: true }, deeplIsPro: { type: 'boolean' }, deeplFreeMode: { type: 'boolean' }, @@ -571,6 +572,10 @@ export default class extends Endpoint { // eslint- set.objectStorageS3ForcePathStyle = ps.objectStorageS3ForcePathStyle; } + if (ps.translationTimeout !== undefined) { + set.translationTimeout = ps.translationTimeout; + } + if (ps.deeplAuthKey !== undefined) { if (ps.deeplAuthKey === '') { set.deeplAuthKey = null; diff --git a/packages/backend/src/server/api/endpoints/gallery/posts/like.ts b/packages/backend/src/server/api/endpoints/gallery/posts/like.ts index e73110648c..ae8ad6c044 100644 --- a/packages/backend/src/server/api/endpoints/gallery/posts/like.ts +++ b/packages/backend/src/server/api/endpoints/gallery/posts/like.ts @@ -98,7 +98,7 @@ export default class extends Endpoint { // eslint- // ランキング更新 if (Date.now() - this.idService.parse(post.id).date.getTime() < GALLERY_POSTS_RANKING_WINDOW) { - await this.featuredService.updateGalleryPostsRanking(post.id, 1); + await this.featuredService.updateGalleryPostsRanking(post, 1); } this.galleryPostsRepository.increment({ id: post.id }, 'likedCount', 1); diff --git a/packages/backend/src/server/api/endpoints/gallery/posts/unlike.ts b/packages/backend/src/server/api/endpoints/gallery/posts/unlike.ts index b0fad1eff2..be0a5a5584 100644 --- a/packages/backend/src/server/api/endpoints/gallery/posts/unlike.ts +++ b/packages/backend/src/server/api/endpoints/gallery/posts/unlike.ts @@ -81,7 +81,7 @@ export default class extends Endpoint { // eslint- // ランキング更新 if (Date.now() - this.idService.parse(post.id).date.getTime() < GALLERY_POSTS_RANKING_WINDOW) { - await this.featuredService.updateGalleryPostsRanking(post.id, -1); + await this.featuredService.updateGalleryPostsRanking(post, -1); } this.galleryPostsRepository.decrement({ id: post.id }, 'likedCount', 1); diff --git a/packages/backend/src/server/api/endpoints/hashtags/users.ts b/packages/backend/src/server/api/endpoints/hashtags/users.ts index eb2289960a..68c795de73 100644 --- a/packages/backend/src/server/api/endpoints/hashtags/users.ts +++ b/packages/backend/src/server/api/endpoints/hashtags/users.ts @@ -10,6 +10,7 @@ import { safeForSql } from "@/misc/safe-for-sql.js"; import { normalizeForSearch } from '@/misc/normalize-for-search.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { DI } from '@/di-symbols.js'; +import { RoleService } from '@/core/RoleService.js'; export const meta = { requireCredential: false, @@ -41,6 +42,7 @@ export const paramDef = { sort: { type: 'string', enum: ['+follower', '-follower', '+createdAt', '-createdAt', '+updatedAt', '-updatedAt'] }, state: { type: 'string', enum: ['all', 'alive'], default: 'all' }, origin: { type: 'string', enum: ['combined', 'local', 'remote'], default: 'local' }, + trending: { type: 'boolean', default: false }, }, required: ['tag', 'sort'], } as const; @@ -52,6 +54,7 @@ export default class extends Endpoint { // eslint- private usersRepository: UsersRepository, private userEntityService: UserEntityService, + private readonly roleService: RoleService, ) { super(meta, paramDef, async (ps, me) => { if (!safeForSql(normalizeForSearch(ps.tag))) throw new Error('Injection'); @@ -80,7 +83,18 @@ export default class extends Endpoint { // eslint- case '-updatedAt': query.orderBy('user.updatedAt', 'ASC'); break; } - const users = await query.limit(ps.limit).getMany(); + let users = await query.limit(ps.limit).getMany(); + + // This is not ideal, for a couple of reasons: + // 1. It may return less than "limit" results. + // 2. A span of more than "limit" consecutive non-trendable users may cause the pagination to stop early. + // Unfortunately, there's no better solution unless we refactor role policies to be persisted to the DB. + if (ps.trending) { + const usersWithRoles = await Promise.all(users.map(async u => [u, await this.roleService.getUserPolicies(u)] as const)); + users = usersWithRoles + .filter(([,p]) => p.canTrend) + .map(([u]) => u); + } return await this.userEntityService.packMany(users, me, { schema: 'UserDetailed' }); }); diff --git a/packages/backend/src/server/api/endpoints/notes/featured.ts b/packages/backend/src/server/api/endpoints/notes/featured.ts index ae4fbbef50..12d928c9a7 100644 --- a/packages/backend/src/server/api/endpoints/notes/featured.ts +++ b/packages/backend/src/server/api/endpoints/notes/featured.ts @@ -117,7 +117,8 @@ export default class extends Endpoint { // eslint- .leftJoinAndSelect('note.renote', 'renote') .leftJoinAndSelect('reply.user', 'replyUser') .leftJoinAndSelect('renote.user', 'renoteUser') - .leftJoinAndSelect('note.channel', 'channel'); + .leftJoinAndSelect('note.channel', 'channel') + .andWhere('user.isExplorable = TRUE'); this.queryService.generateBlockedHostQueryForNote(query); this.queryService.generateSuspendedUserQueryForNote(query); diff --git a/packages/backend/src/server/api/endpoints/notes/translate.ts b/packages/backend/src/server/api/endpoints/notes/translate.ts index 39119bc206..a97542c063 100644 --- a/packages/backend/src/server/api/endpoints/notes/translate.ts +++ b/packages/backend/src/server/api/endpoints/notes/translate.ts @@ -10,22 +10,28 @@ import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { GetterService } from '@/server/api/GetterService.js'; import { RoleService } from '@/core/RoleService.js'; -import { ApiError } from '../../error.js'; -import { MiMeta } from '@/models/_.js'; +import type { MiMeta, MiNote } from '@/models/_.js'; import { DI } from '@/di-symbols.js'; +import { CacheService } from '@/core/CacheService.js'; +import { hasText } from '@/models/Note.js'; +import { ApiLoggerService } from '@/server/api/ApiLoggerService.js'; +import { ApiError } from '../../error.js'; export const meta = { tags: ['notes'], + // TODO allow unauthenticated if default template allows? + // Maybe a value 'optional' that allows unauthenticated OR a token w/ appropriate role. + // This will allow unauthenticated requests without leaking post data to restricted clients. requireCredential: true, kind: 'read:account', res: { type: 'object', - optional: true, nullable: false, + optional: false, nullable: false, properties: { - sourceLang: { type: 'string' }, - text: { type: 'string' }, + sourceLang: { type: 'string', optional: true, nullable: false }, + text: { type: 'string', optional: true, nullable: false }, }, }, @@ -45,6 +51,11 @@ export const meta = { code: 'CANNOT_TRANSLATE_INVISIBLE_NOTE', id: 'ea29f2ca-c368-43b3-aaf1-5ac3e74bbe5d', }, + translationFailed: { + message: 'Failed to translate note. Please try again later or contact an administrator for assistance.', + code: 'TRANSLATION_FAILED', + id: '4e7a1a4f-521c-4ba2-b10a-69e5e2987b2f', + }, }, // 10 calls per 5 seconds @@ -73,6 +84,8 @@ export default class extends Endpoint { // eslint- private getterService: GetterService, private httpRequestService: HttpRequestService, private roleService: RoleService, + private readonly cacheService: CacheService, + private readonly loggerService: ApiLoggerService, ) { super(meta, paramDef, async (ps, me) => { const policies = await this.roleService.getUserPolicies(me.id); @@ -89,8 +102,8 @@ export default class extends Endpoint { // eslint- throw new ApiError(meta.errors.cannotTranslateInvisibleNote); } - if (note.text == null) { - return; + if (!hasText(note)) { + return {}; } const canDeeplFree = this.serverSettings.deeplFreeMode && !!this.serverSettings.deeplFreeInstance; @@ -101,13 +114,33 @@ export default class extends Endpoint { // eslint- let targetLang = ps.targetLang; if (targetLang.includes('-')) targetLang = targetLang.split('-')[0]; + let response = await this.cacheService.getCachedTranslation(note, targetLang); + if (!response) { + this.loggerService.logger.debug(`Fetching new translation for note=${note.id} lang=${targetLang}`); + response = await this.fetchTranslation(note, targetLang); + if (!response) { + throw new ApiError(meta.errors.translationFailed); + } + + await this.cacheService.setCachedTranslation(note, targetLang, response); + } + return response; + }); + } + + private async fetchTranslation(note: MiNote & { text: string }, targetLang: string) { + // Load-bearing try/catch - removing this will shift indentation and cause ~80 lines of upstream merge conflicts + try { + // Ignore deeplFreeInstance unless deeplFreeMode is set + const deeplFreeInstance = this.serverSettings.deeplFreeMode ? this.serverSettings.deeplFreeInstance : null; + // DeepL/DeepLX handling - if (canDeepl) { + if (this.serverSettings.deeplAuthKey || deeplFreeInstance) { const params = new URLSearchParams(); if (this.serverSettings.deeplAuthKey) params.append('auth_key', this.serverSettings.deeplAuthKey); params.append('text', note.text); params.append('target_lang', targetLang); - const endpoint = canDeeplFree ? this.serverSettings.deeplFreeInstance as string : this.serverSettings.deeplIsPro ? 'https://api.deepl.com/v2/translate' : 'https://api-free.deepl.com/v2/translate'; + const endpoint = deeplFreeInstance ?? this.serverSettings.deeplIsPro ? 'https://api.deepl.com/v2/translate' : 'https://api-free.deepl.com/v2/translate'; const res = await this.httpRequestService.send(endpoint, { method: 'POST', @@ -116,6 +149,7 @@ export default class extends Endpoint { // eslint- Accept: 'application/json, */*', }, body: params.toString(), + timeout: this.serverSettings.translationTimeout, }); if (this.serverSettings.deeplAuthKey) { const json = (await res.json()) as { @@ -151,8 +185,8 @@ export default class extends Endpoint { // eslint- } // LibreTranslate handling - if (canLibre) { - const res = await this.httpRequestService.send(this.serverSettings.libreTranslateURL as string, { + if (this.serverSettings.libreTranslateURL) { + const res = await this.httpRequestService.send(this.serverSettings.libreTranslateURL, { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -165,6 +199,7 @@ export default class extends Endpoint { // eslint- format: 'text', api_key: this.serverSettings.libreTranslateKey ?? '', }), + timeout: this.serverSettings.translationTimeout, }); const json = (await res.json()) as { @@ -182,8 +217,10 @@ export default class extends Endpoint { // eslint- text: json.translatedText, }; } + } catch (e) { + this.loggerService.logger.error('Unhandled error from translation API: ', { e }); + } - return; - }); + return null; } } diff --git a/packages/backend/src/server/api/endpoints/users.ts b/packages/backend/src/server/api/endpoints/users.ts index ee5c44cedd..defd38fe96 100644 --- a/packages/backend/src/server/api/endpoints/users.ts +++ b/packages/backend/src/server/api/endpoints/users.ts @@ -4,11 +4,14 @@ */ import { Inject, Injectable } from '@nestjs/common'; -import type { UsersRepository } from '@/models/_.js'; +import { MiFollowing } from '@/models/_.js'; +import type { MiUser, UsersRepository } from '@/models/_.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { QueryService } from '@/core/QueryService.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { DI } from '@/di-symbols.js'; +import { RoleService } from '@/core/RoleService.js'; +import type { SelectQueryBuilder } from 'typeorm'; export const meta = { tags: ['users'], @@ -38,7 +41,7 @@ export const paramDef = { properties: { limit: { type: 'integer', minimum: 1, maximum: 100, default: 10 }, offset: { type: 'integer', default: 0 }, - sort: { type: 'string', enum: ['+follower', '-follower', '+createdAt', '-createdAt', '+updatedAt', '-updatedAt'] }, + sort: { type: 'string', enum: ['+follower', '-follower', '+localFollower', '-localFollower', '+createdAt', '-createdAt', '+updatedAt', '-updatedAt'] }, state: { type: 'string', enum: ['all', 'alive'], default: 'all' }, origin: { type: 'string', enum: ['combined', 'local', 'remote'], default: 'local' }, hostname: { @@ -59,6 +62,7 @@ export default class extends Endpoint { // eslint- private userEntityService: UserEntityService, private queryService: QueryService, + private readonly roleService: RoleService, ) { super(meta, paramDef, async (ps, me) => { const query = this.usersRepository.createQueryBuilder('user') @@ -81,6 +85,8 @@ export default class extends Endpoint { // eslint- switch (ps.sort) { case '+follower': query.orderBy('user.followersCount', 'DESC'); break; case '-follower': query.orderBy('user.followersCount', 'ASC'); break; + case '+localFollower': this.addLocalFollowers(query); query.orderBy('f."localFollowers"', 'DESC'); break; + case '-localFollower': this.addLocalFollowers(query); query.orderBy('f."localFollowers"', 'ASC'); break; case '+createdAt': query.orderBy('user.id', 'DESC'); break; case '-createdAt': query.orderBy('user.id', 'ASC'); break; case '+updatedAt': query.andWhere('user.updatedAt IS NOT NULL').orderBy('user.updatedAt', 'DESC'); break; @@ -94,9 +100,29 @@ export default class extends Endpoint { // eslint- query.limit(ps.limit); query.offset(ps.offset); - const users = await query.getMany(); + const allUsers = await query.getMany(); + + // This is not ideal, for a couple of reasons: + // 1. It may return less than "limit" results. + // 2. A span of more than "limit" consecutive non-trendable users may cause the pagination to stop early. + // Unfortunately, there's no better solution unless we refactor role policies to be persisted to the DB. + const usersWithRoles = await Promise.all(allUsers.map(async u => [u, await this.roleService.getUserPolicies(u)] as const)); + const users = usersWithRoles + .filter(([,p]) => p.canTrend) + .map(([u]) => u); return await this.userEntityService.packMany(users, me, { schema: 'UserDetailed' }); }); } + + private addLocalFollowers(query: SelectQueryBuilder) { + query.innerJoin(qb => { + return qb + .from(MiFollowing, 'f') + .addSelect('f."followeeId"') + .addSelect('COUNT(*) FILTER (where f."followerHost" IS NULL)', 'localFollowers') + .addSelect('COUNT(*) FILTER (where f."followeeHost" IS NOT NULL)', 'remoteFollowers') + .groupBy('"followeeId"'); + }, 'f', 'user.id = f."followeeId"'); + } } diff --git a/packages/backend/src/server/web/ClientServerService.ts b/packages/backend/src/server/web/ClientServerService.ts index 1321cf6338..c40d042fa4 100644 --- a/packages/backend/src/server/web/ClientServerService.ts +++ b/packages/backend/src/server/web/ClientServerService.ts @@ -890,6 +890,7 @@ export class ClientServerService { return await reply.view('info-card', { version: this.config.version, host: this.config.host, + url: this.config.url, meta: this.meta, originalUsersCount: await this.usersRepository.countBy({ host: IsNull() }), originalNotesCount: await this.notesRepository.countBy({ userHost: IsNull() }), diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index 0cab657c23..203bc908a8 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -15,15 +15,21 @@ import type Logger from '@/logger.js'; import { query } from '@/misc/prelude/url.js'; import { LoggerService } from '@/core/LoggerService.js'; import { bindThis } from '@/decorators.js'; -import { ApiError } from '@/server/api/error.js'; import { MiMeta } from '@/models/Meta.js'; import { RedisKVCache } from '@/misc/cache.js'; import { UtilityService } from '@/core/UtilityService.js'; import { ApDbResolverService } from '@/core/activitypub/ApDbResolverService.js'; -import type { NotesRepository } from '@/models/_.js'; +import type { MiAccessToken, NotesRepository } from '@/models/_.js'; import { ApUtilityService } from '@/core/activitypub/ApUtilityService.js'; import { ApRequestService } from '@/core/activitypub/ApRequestService.js'; import { SystemAccountService } from '@/core/SystemAccountService.js'; +import { ApNoteService } from '@/core/activitypub/models/ApNoteService.js'; +import { AuthenticateService, AuthenticationError } from '@/server/api/AuthenticateService.js'; +import { SkRateLimiterService } from '@/server/SkRateLimiterService.js'; +import { BucketRateLimit, Keyed, sendRateLimitHeaders } from '@/misc/rate-limit-utils.js'; +import type { MiLocalUser } from '@/models/User.js'; +import { getIpHash } from '@/misc/get-ip-hash.js'; +import { isRetryableError } from '@/misc/is-retryable-error.js'; import type { FastifyRequest, FastifyReply } from 'fastify'; export type LocalSummalyResult = SummalyResult & { @@ -31,7 +37,27 @@ export type LocalSummalyResult = SummalyResult & { }; // Increment this to invalidate cached previews after a major change. -const cacheFormatVersion = 2; +const cacheFormatVersion = 3; + +type PreviewRoute = { + Querystring: { + url?: string + lang?: string, + fetch?: string, + i?: string, + }, +}; + +type AuthArray = [user: MiLocalUser | null | undefined, app: MiAccessToken | null | undefined, actor: MiLocalUser | string]; + +// Up to 50 requests, then 10 / second (at 2 / 200ms rate) +const previewLimit: Keyed = { + key: '/url', + type: 'bucket', + size: 50, + dripSize: 2, + dripRate: 200, +}; @Injectable() export class UrlPreviewService { @@ -58,6 +84,9 @@ export class UrlPreviewService { private readonly apDbResolverService: ApDbResolverService, private readonly apRequestService: ApRequestService, private readonly systemAccountService: SystemAccountService, + private readonly apNoteService: ApNoteService, + private readonly authenticateService: AuthenticateService, + private readonly rateLimiterService: SkRateLimiterService, ) { this.logger = this.loggerService.getLogger('url-preview'); this.previewCache = new RedisKVCache(this.redisClient, 'summaly', { @@ -85,9 +114,9 @@ export class UrlPreviewService { @bindThis public async handle( - request: FastifyRequest<{ Querystring: { url?: string; lang?: string; } }>, + request: FastifyRequest, reply: FastifyReply, - ): Promise { + ): Promise { const url = request.query.url; if (typeof url !== 'string' || !URL.canParse(url)) { reply.code(400); @@ -101,38 +130,39 @@ export class UrlPreviewService { } if (!this.meta.urlPreviewEnabled) { - reply.code(403); - return { - error: new ApiError({ + return reply.code(403).send({ + error: { message: 'URL preview is disabled', code: 'URL_PREVIEW_DISABLED', id: '58b36e13-d2f5-0323-b0c6-76aa9dabefb8', - }), - }; + }, + }); + } + + // Check rate limit + const auth = await this.authenticate(request); + if (!await this.checkRateLimit(auth, reply)) { + return; } if (this.utilityService.isBlockedHost(this.meta.blockedHosts, new URL(url).host)) { - reply.code(403); - return { - error: new ApiError({ + return reply.code(403).send({ + error: { message: 'URL is blocked', code: 'URL_PREVIEW_BLOCKED', id: '50294652-857b-4b13-9700-8e5c7a8deae8', - }), - }; + }, + }); + } + + const fetch = !!request.query.fetch; + if (fetch && !await this.checkFetchPermissions(auth, reply)) { + return; } const cacheKey = `${url}@${lang}@${cacheFormatVersion}`; - const cached = await this.previewCache.get(cacheKey); - if (cached !== undefined) { - // Cache 1 day (matching redis) - reply.header('Cache-Control', 'public, max-age=86400'); - - if (cached.activityPub) { - cached.haveNoteLocally = !! await this.apDbResolverService.getNoteFromApId(cached.activityPub); - } - - return cached; + if (await this.sendCachedPreview(cacheKey, reply, fetch)) { + return; } try { @@ -144,14 +174,13 @@ export class UrlPreviewService { // Repeat check, since redirects are allowed. if (this.utilityService.isBlockedHost(this.meta.blockedHosts, new URL(summary.url).host)) { - reply.code(403); - return { - error: new ApiError({ + return reply.code(403).send({ + error: { message: 'URL is blocked', code: 'URL_PREVIEW_BLOCKED', id: '50294652-857b-4b13-9700-8e5c7a8deae8', - }), - }; + }, + }); } this.logger.info(`Got preview of ${url} in ${lang}: ${summary.title}`); @@ -164,33 +193,76 @@ export class UrlPreviewService { await this.inferActivityPubLink(summary); } - if (summary.activityPub) { + if (summary.activityPub && !summary.haveNoteLocally) { // Avoid duplicate checks in case inferActivityPubLink already set this. - summary.haveNoteLocally ||= !!await this.apDbResolverService.getNoteFromApId(summary.activityPub); + const exists = await this.noteExists(summary.activityPub, fetch); + + // Remove the AP flag if we encounter a permanent error fetching the note. + if (exists === false) { + summary.activityPub = null; + summary.haveNoteLocally = undefined; + } else { + summary.haveNoteLocally = exists ?? false; + } } // Await this to avoid hammering redis when a bunch of URLs are fetched at once await this.previewCache.set(cacheKey, summary); - // Cache 1 day (matching redis) - reply.header('Cache-Control', 'public, max-age=86400'); + // Cache 1 day (matching redis), but only once we finalize the result + if (!summary.activityPub || summary.haveNoteLocally) { + reply.header('Cache-Control', 'public, max-age=86400'); + } - return summary; + return reply.code(200).send(summary); } catch (err) { this.logger.warn(`Failed to get preview of ${url} for ${lang}: ${err}`); - reply.code(422); reply.header('Cache-Control', 'max-age=3600'); - return { - error: new ApiError({ + return reply.code(422).send({ + error: { message: 'Failed to get preview', code: 'URL_PREVIEW_FAILED', id: '09d01cb5-53b9-4856-82e5-38a50c290a3b', - }), - }; + }, + }); } } + private async sendCachedPreview(cacheKey: string, reply: FastifyReply, fetch: boolean): Promise { + const summary = await this.previewCache.get(cacheKey); + if (summary === undefined) { + return false; + } + + // Check if note has loaded since we last cached the preview + if (summary.activityPub && !summary.haveNoteLocally) { + // Avoid duplicate checks in case inferActivityPubLink already set this. + const exists = await this.noteExists(summary.activityPub, fetch); + + // Remove the AP flag if we encounter a permanent error fetching the note. + if (exists === false) { + summary.activityPub = null; + summary.haveNoteLocally = undefined; + } else { + summary.haveNoteLocally = exists ?? false; + } + + // Persist the result once we finalize the result + if (!summary.activityPub || summary.haveNoteLocally) { + await this.previewCache.set(cacheKey, summary); + } + } + + // Cache 1 day (matching redis), but only once we finalize the result + if (!summary.activityPub || summary.haveNoteLocally) { + reply.header('Cache-Control', 'public, max-age=86400'); + } + + reply.code(200).send(summary); + return true; + } + private fetchSummary(url: string, meta: MiMeta, lang?: string): Promise { const agent = this.config.proxy ? { @@ -211,6 +283,7 @@ export class UrlPreviewService { } private fetchSummaryFromProxy(url: string, meta: MiMeta, lang?: string): Promise { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const proxy = meta.urlPreviewSummaryProxyUrl!; const queryStr = query({ followRedirects: true, @@ -302,4 +375,129 @@ export class UrlPreviewService { return; } } + + // true = exists, false = does not exist (permanently), null = does not exist (temporarily) + private async noteExists(uri: string, fetch = false): Promise { + try { + // Local note or cached remote note + if (await this.apDbResolverService.getNoteFromApId(uri)) { + return true; + } + + // Un-cached remote note + if (!fetch) { + return null; + } + + // Newly cached remote note + if (await this.apNoteService.resolveNote(uri)) { + return true; + } + + // Non-existent or deleted note + return false; + } catch (err) { + // Errors, including invalid notes and network errors + return isRetryableError(err) ? null : false; + } + } + + // Adapted from ApiCallService + private async authenticate(request: FastifyRequest<{ Querystring?: { i?: string | string[] }, Body?: { i?: string | string[] } }>): Promise { + const body = request.method === 'GET' ? request.query : request.body; + + // https://datatracker.ietf.org/doc/html/rfc6750.html#section-2.1 (case sensitive) + const token = request.headers.authorization?.startsWith('Bearer ') + ? request.headers.authorization.slice(7) + : body?.['i']; + if (token != null && typeof token !== 'string') { + return [undefined, undefined, getIpHash(request.ip)]; + } + + try { + const auth = await this.authenticateService.authenticate(token); + return [auth[0], auth[1], auth[0] ?? getIpHash(request.ip)]; + } catch (err) { + if (err instanceof AuthenticationError) { + return [undefined, undefined, getIpHash(request.ip)]; + } else { + throw err; + } + } + } + + // Adapted from ApiCallService + private async checkFetchPermissions(auth: AuthArray, reply: FastifyReply): Promise { + const [user, app] = auth; + + // Authentication + if (user === undefined) { + reply.code(401).send({ + error: { + message: 'Authentication failed. Please ensure your token is correct.', + code: 'AUTHENTICATION_FAILED', + id: 'b0a7f5f8-dc2f-4171-b91f-de88ad238e14', + }, + }); + return false; + } + if (user === null) { + reply.code(401).send({ + error: { + message: 'Credential required.', + code: 'CREDENTIAL_REQUIRED', + id: '1384574d-a912-4b81-8601-c7b1c4085df1', + }, + }); + return false; + } + + // Authorization + if (user.isSuspended || user.isDeleted) { + reply.code(403).send({ + error: { + message: 'Your account has been suspended.', + code: 'YOUR_ACCOUNT_SUSPENDED', + kind: 'permission', + + id: 'a8c724b3-6e9c-4b46-b1a8-bc3ed6258370', + }, + }); + return false; + } + if (app && !app.permission.includes('read:account')) { + reply.code(403).send({ + error: { + message: 'Your app does not have the necessary permissions to use this endpoint.', + code: 'PERMISSION_DENIED', + kind: 'permission', + id: '1370e5b7-d4eb-4566-bb1d-7748ee6a1838', + }, + }); + return false; + } + + return true; + } + + private async checkRateLimit(auth: AuthArray, reply: FastifyReply): Promise { + const info = await this.rateLimiterService.limit(previewLimit, auth[2]); + + // Always send headers, even if not blocked + sendRateLimitHeaders(reply, info); + + if (info.blocked) { + reply.code(429).send({ + error: { + message: 'Rate limit exceeded. Please try again later.', + code: 'RATE_LIMIT_EXCEEDED', + id: 'd5826d14-3982-4d2e-8011-b9e9f02499ef', + }, + }); + + return false; + } + + return true; + } } diff --git a/packages/backend/src/server/web/views/info-card.pug b/packages/backend/src/server/web/views/info-card.pug index 4a9d00a596..0a95ea7b17 100644 --- a/packages/backend/src/server/web/views/info-card.pug +++ b/packages/backend/src/server/web/views/info-card.pug @@ -43,7 +43,7 @@ html } body - a#a(href=`https://${host}` target="_blank") + a#a(href=url target="_blank") header#banner(style=`background-image: url(${meta.bannerUrl})`) div#title= meta.name || host div#content diff --git a/packages/backend/test/unit/core/HttpRequestService.ts b/packages/backend/test/unit/core/HttpRequestService.ts new file mode 100644 index 0000000000..a2f4604e7b --- /dev/null +++ b/packages/backend/test/unit/core/HttpRequestService.ts @@ -0,0 +1,113 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { jest } from '@jest/globals'; +import type { Mock } from 'jest-mock'; +import type { PrivateNetwork } from '@/config.js'; +import type { Socket } from 'net'; +import { HttpRequestService, isPrivateIp, validateSocketConnect } from '@/core/HttpRequestService.js'; +import { parsePrivateNetworks } from '@/config.js'; + +describe(HttpRequestService, () => { + let allowedPrivateNetworks: PrivateNetwork[] | undefined; + + beforeEach(() => { + allowedPrivateNetworks = parsePrivateNetworks([ + '10.0.0.1/32', + { network: '127.0.0.1/32', ports: [1] }, + { network: '127.0.0.1/32', ports: [3, 4, 5] }, + ]); + }); + + describe('isPrivateIp', () => { + it('should return false when ip public', () => { + const result = isPrivateIp(allowedPrivateNetworks, '74.125.127.100', 80); + expect(result).toBeFalsy(); + }); + + it('should return false when ip private and port matches', () => { + const result = isPrivateIp(allowedPrivateNetworks, '127.0.0.1', 1); + expect(result).toBeFalsy(); + }); + + it('should return false when ip private and all ports undefined', () => { + const result = isPrivateIp(allowedPrivateNetworks, '10.0.0.1', undefined); + expect(result).toBeFalsy(); + }); + + it('should return true when ip private and no ports specified', () => { + const result = isPrivateIp(allowedPrivateNetworks, '10.0.0.2', 80); + expect(result).toBeTruthy(); + }); + + it('should return true when ip private and port does not match', () => { + const result = isPrivateIp(allowedPrivateNetworks, '127.0.0.1', 80); + expect(result).toBeTruthy(); + }); + + it('should return true when ip private and port is null but ports are specified', () => { + const result = isPrivateIp(allowedPrivateNetworks, '127.0.0.1', undefined); + expect(result).toBeTruthy(); + }); + }); + + describe('validateSocketConnect', () => { + let fakeSocket: Socket; + let fakeSocketMutable: { + remoteAddress: string | undefined; + remotePort: number | undefined; + destroy: Mock<(error?: Error) => void>; + }; + + beforeEach(() => { + fakeSocketMutable = { + remoteAddress: '74.125.127.100', + remotePort: 80, + destroy: jest.fn<(error?: Error) => void>(), + }; + fakeSocket = fakeSocketMutable as unknown as Socket; + }); + + it('should accept when IP is empty', () => { + fakeSocketMutable.remoteAddress = undefined; + + validateSocketConnect(allowedPrivateNetworks, fakeSocket); + + expect(fakeSocket.destroy).not.toHaveBeenCalled(); + }); + + it('should accept when IP is invalid', () => { + fakeSocketMutable.remoteAddress = 'AB939ajd9jdajsdja8jj'; + + validateSocketConnect(allowedPrivateNetworks, fakeSocket); + + expect(fakeSocket.destroy).not.toHaveBeenCalled(); + }); + + it('should accept when IP is valid', () => { + validateSocketConnect(allowedPrivateNetworks, fakeSocket); + + expect(fakeSocket.destroy).not.toHaveBeenCalled(); + }); + + it('should accept when IP is private and port match', () => { + fakeSocketMutable.remoteAddress = '127.0.0.1'; + fakeSocketMutable.remotePort = 1; + + validateSocketConnect(allowedPrivateNetworks, fakeSocket); + + expect(fakeSocket.destroy).not.toHaveBeenCalled(); + }); + + it('should reject when IP is private and port no match', () => { + fakeSocketMutable.remoteAddress = '127.0.0.1'; + fakeSocketMutable.remotePort = 2; + + validateSocketConnect(allowedPrivateNetworks, fakeSocket); + + expect(fakeSocket.destroy).toHaveBeenCalled(); + }); + }); +}); diff --git a/packages/frontend-embed/package.json b/packages/frontend-embed/package.json index eda5428136..fa54842914 100644 --- a/packages/frontend-embed/package.json +++ b/packages/frontend-embed/package.json @@ -18,7 +18,7 @@ "@transfem-org/sfm-js": "0.24.5", "@twemoji/parser": "15.1.1", "@vitejs/plugin-vue": "5.2.3", - "@vue/compiler-sfc": "3.5.13", + "@vue/compiler-sfc": "3.5.14", "astring": "1.9.0", "buraha": "0.0.1", "estree-walker": "3.0.3", @@ -35,7 +35,7 @@ "typescript": "5.8.3", "uuid": "11.1.0", "vite": "6.3.4", - "vue": "3.5.13" + "vue": "3.5.14" }, "devDependencies": { "@misskey-dev/summaly": "5.2.1", @@ -49,7 +49,7 @@ "@typescript-eslint/eslint-plugin": "8.31.0", "@typescript-eslint/parser": "8.31.0", "@vitest/coverage-v8": "3.1.2", - "@vue/runtime-core": "3.5.13", + "@vue/runtime-core": "3.5.14", "acorn": "8.14.1", "cross-env": "7.0.3", "eslint-plugin-import": "2.31.0", diff --git a/packages/frontend-shared/js/const.ts b/packages/frontend-shared/js/const.ts index 2f0f126b90..bded34ea41 100644 --- a/packages/frontend-shared/js/const.ts +++ b/packages/frontend-shared/js/const.ts @@ -176,6 +176,7 @@ export const ROLE_POLICIES = [ 'canImportMuting', 'canImportUserLists', 'chatAvailability', + 'canTrend', ] as const; export const MFM_TAGS = ['tada', 'jelly', 'twitch', 'shake', 'spin', 'jump', 'bounce', 'flip', 'x2', 'x3', 'x4', 'scale', 'position', 'fg', 'bg', 'border', 'font', 'blur', 'rainbow', 'sparkle', 'rotate', 'ruby', 'unixtime', 'crop', 'fade', 'followmouse']; diff --git a/packages/frontend-shared/js/i18n.ts b/packages/frontend-shared/js/i18n.ts index 480cfcd642..d38bad45d9 100644 --- a/packages/frontend-shared/js/i18n.ts +++ b/packages/frontend-shared/js/i18n.ts @@ -59,6 +59,7 @@ export class I18n { if (typeof value === 'string') { const parameters = Array.from(value.matchAll(/\{(\w+)\}/g), ([, parameter]) => parameter); + // TODO add a flag to suppress this warning from uses of component if (parameters.length) { console.error(`Missing locale parameters: ${parameters.join(', ')} at ${String(p)}`); } diff --git a/packages/frontend/package.json b/packages/frontend/package.json index a3553e7b3a..275a3321cb 100644 --- a/packages/frontend/package.json +++ b/packages/frontend/package.json @@ -30,7 +30,7 @@ "@transfem-org/sfm-js": "0.24.6", "@twemoji/parser": "15.1.1", "@vitejs/plugin-vue": "5.2.3", - "@vue/compiler-sfc": "3.5.13", + "@vue/compiler-sfc": "3.5.14", "aiscript-vscode": "github:aiscript-dev/aiscript-vscode#v0.1.15", "astring": "1.9.0", "broadcast-channel": "7.1.0", @@ -76,7 +76,7 @@ "uuid": "11.1.0", "v-code-diff": "1.13.1", "vite": "6.3.4", - "vue": "3.5.13", + "vue": "3.5.14", "vuedraggable": "next", "wanakana": "5.3.1" }, @@ -119,8 +119,8 @@ "@typescript-eslint/eslint-plugin": "8.31.0", "@typescript-eslint/parser": "8.31.0", "@vitest/coverage-v8": "3.1.2", - "@vue/compiler-core": "3.5.13", - "@vue/runtime-core": "3.5.13", + "@vue/compiler-core": "3.5.14", + "@vue/runtime-core": "3.5.14", "acorn": "8.14.1", "cross-env": "7.0.3", "eslint-plugin-import": "2.31.0", diff --git a/packages/frontend/src/components/DynamicNote.vue b/packages/frontend/src/components/DynamicNote.vue index a5008e9ddf..67707bfda9 100644 --- a/packages/frontend/src/components/DynamicNote.vue +++ b/packages/frontend/src/components/DynamicNote.vue @@ -23,10 +23,10 @@ import type MkNote from '@/components/MkNote.vue'; import type SkNote from '@/components/SkNote.vue'; import { prefer } from '@/preferences'; -const XNote = computed(() => - prefer.r.noteDesign.value === 'misskey' - ? defineAsyncComponent(() => import('@/components/MkNote.vue')) - : defineAsyncComponent(() => import('@/components/SkNote.vue')), +const XNote = defineAsyncComponent(() => + prefer.s.noteDesign === 'misskey' + ? import('@/components/MkNote.vue') + : import('@/components/SkNote.vue') ); const rootEl = useTemplateRef>('rootEl'); diff --git a/packages/frontend/src/components/DynamicNoteDetailed.vue b/packages/frontend/src/components/DynamicNoteDetailed.vue index 21bf00ccbe..8594db2328 100644 --- a/packages/frontend/src/components/DynamicNoteDetailed.vue +++ b/packages/frontend/src/components/DynamicNoteDetailed.vue @@ -20,10 +20,10 @@ import type MkNoteDetailed from '@/components/MkNoteDetailed.vue'; import type SkNoteDetailed from '@/components/SkNoteDetailed.vue'; import { prefer } from '@/preferences'; -const XNoteDetailed = computed(() => - prefer.r.noteDesign.value === 'misskey' - ? defineAsyncComponent(() => import('@/components/MkNoteDetailed.vue')) - : defineAsyncComponent(() => import('@/components/SkNoteDetailed.vue')), +const XNoteDetailed = defineAsyncComponent(() => + prefer.s.noteDesign === 'misskey' + ? import('@/components/MkNoteDetailed.vue') + : import('@/components/SkNoteDetailed.vue'), ); const rootEl = useTemplateRef>('rootEl'); diff --git a/packages/frontend/src/components/DynamicNoteSimple.vue b/packages/frontend/src/components/DynamicNoteSimple.vue index 7ca345ccfa..5eaeaf6c23 100644 --- a/packages/frontend/src/components/DynamicNoteSimple.vue +++ b/packages/frontend/src/components/DynamicNoteSimple.vue @@ -21,10 +21,10 @@ import type MkNoteSimple from '@/components/MkNoteSimple.vue'; import type SkNoteSimple from '@/components/SkNoteSimple.vue'; import { prefer } from '@/preferences'; -const XNoteSimple = computed(() => - prefer.r.noteDesign.value === 'misskey' - ? defineAsyncComponent(() => import('@/components/MkNoteSimple.vue')) - : defineAsyncComponent(() => import('@/components/SkNoteSimple.vue')), +const XNoteSimple = defineAsyncComponent(() => + prefer.s.noteDesign === 'misskey' + ? import('@/components/MkNoteSimple.vue') + : import('@/components/SkNoteSimple.vue'), ); const rootEl = useTemplateRef>('rootEl'); diff --git a/packages/frontend/src/components/MkButton.vue b/packages/frontend/src/components/MkButton.vue index d37f7f39f8..53453be2c1 100644 --- a/packages/frontend/src/components/MkButton.vue +++ b/packages/frontend/src/components/MkButton.vue @@ -7,7 +7,7 @@ SPDX-License-Identifier: AGPL-3.0-only - - + @@ -171,24 +168,7 @@ SPDX-License-Identifier: AGPL-3.0-only
- - - - - - - - - - +
diff --git a/packages/frontend/src/components/SkNote.vue b/packages/frontend/src/components/SkNote.vue index 8c89cdd5a3..5184cbd801 100644 --- a/packages/frontend/src/components/SkNote.vue +++ b/packages/frontend/src/components/SkNote.vue @@ -88,13 +88,7 @@ SPDX-License-Identifier: AGPL-3.0-only :isAnim="allowAnim" :isBlock="true" /> -
- -
- {{ i18n.tsx.translatedFrom({ x: translation.sourceLang }) }}: - -
-
+ {{ i18n.ts._animatedMFM.play }} {{ i18n.ts._animatedMFM.stop }}
@@ -103,7 +97,7 @@ SPDX-License-Identifier: AGPL-3.0-only
- +
- - + @@ -172,24 +169,7 @@ SPDX-License-Identifier: AGPL-3.0-only
- - - - - - - - - - +
+ + + + + + diff --git a/packages/frontend/src/components/SkOldNoteWindow.vue b/packages/frontend/src/components/SkOldNoteWindow.vue index bd5368351b..b6dbec81c5 100644 --- a/packages/frontend/src/components/SkOldNoteWindow.vue +++ b/packages/frontend/src/components/SkOldNoteWindow.vue @@ -29,7 +29,7 @@ SPDX-License-Identifier: AGPL-3.0-only
- +
@@ -42,18 +42,12 @@ SPDX-License-Identifier: AGPL-3.0-only RN: -
- -
- {{ i18n.t('translatedFrom', { x: translation.sourceLang }) }}: - -
-
+
- +
{{ appearNote.channel.name }} @@ -92,12 +86,14 @@ import MkPoll from '@/components/MkPoll.vue'; import MkUrlPreview from '@/components/MkUrlPreview.vue'; import MkInstanceTicker from '@/components/MkInstanceTicker.vue'; import { userPage } from '@/filters/user.js'; -import { extractUrlFromMfm } from '@/utility/extract-url-from-mfm.js'; import { i18n } from '@/i18n.js'; import { deepClone } from '@/utility/clone.js'; import { dateTimeFormat } from '@/utility/intl-const.js'; import { prefer } from '@/preferences'; import { getPluginHandlers } from '@/plugin.js'; +import SkNoteTranslation from '@/components/SkNoteTranslation.vue'; +import { getSelfNoteIds } from '@/utility/get-self-note-ids'; +import { extractPreviewUrls } from '@/utility/extract-preview-urls.js'; const props = defineProps<{ note: Misskey.entities.Note; @@ -146,14 +142,14 @@ const isRenote = ( ); const el = shallowRef(); -let appearNote = computed(() => isRenote ? note.value.renote as Misskey.entities.Note : note.value); -const renoteUrl = appearNote.value.renote ? appearNote.value.renote.url : null; -const renoteUri = appearNote.value.renote ? appearNote.value.renote.uri : null; +const appearNote = computed(() => isRenote ? note.value.renote as Misskey.entities.Note : note.value); +const parsed = computed(() => appearNote.value.text ? mfm.parse(appearNote.value.text) : null); const showContent = ref(false); -const translation = ref(null); +const translation = ref(null); const translating = ref(false); -const urls = appearNote.value.text ? extractUrlFromMfm(mfm.parse(appearNote.value.text)).filter(u => u !== renoteUrl && u !== renoteUri) : null; +const urls = computed(() => parsed.value ? extractPreviewUrls(props.note, parsed.value) : null); +const selfNoteIds = computed(() => getSelfNoteIds(props.note)); const showTicker = (prefer.s.instanceTicker === 'always') || (prefer.s.instanceTicker === 'remote' && appearNote.value.user.instance); @@ -259,13 +255,6 @@ const showTicker = (prefer.s.instanceTicker === 'always') || (prefer.s.instanceT color: var(--MI_THEME-renote); } -.translation { - border: solid 0.5px var(--MI_THEME-divider); - border-radius: var(--MI-radius); - padding: 12px; - margin-top: 8px; -} - .poll { font-size: 80%; } diff --git a/packages/frontend/src/components/SkPatternTest.vue b/packages/frontend/src/components/SkPatternTest.vue index 2ed2b3fdc3..fe82c1df2f 100644 --- a/packages/frontend/src/components/SkPatternTest.vue +++ b/packages/frontend/src/components/SkPatternTest.vue @@ -12,7 +12,7 @@ SPDX-License-Identifier: AGPL-3.0-only
{{ i18n.ts.wordMuteTestTest }}
-
{{ i18n.ts.wordMuteTestNoResults}}
+
{{ i18n.ts.wordMuteTestNoResults }}
{{ i18n.ts.wordMuteTestNoMatch }}
{{ i18n.tsx.wordMuteTestMatch({ words: testMatches }) }}
@@ -44,7 +44,7 @@ function testWordMutes() { try { const mutes = parseMutes(props.mutedWords); const matches = checkWordMute(testWords.value, null, mutes); - testMatches.value = matches ? matches.flat(2).join(', ') : ''; + testMatches.value = matches ? matches.join(', ') : ''; } catch { // Error is displayed by above function testMatches.value = null; diff --git a/packages/frontend/src/pages/admin-user.vue b/packages/frontend/src/pages/admin-user.vue index efe547ff21..d3c0de3040 100644 --- a/packages/frontend/src/pages/admin-user.vue +++ b/packages/frontend/src/pages/admin-user.vue @@ -7,38 +7,49 @@ SPDX-License-Identifier: AGPL-3.0-only
-
-
+
+
- @{{ acct(user) }} + + @{{ acct(user) }} + + + + {{ user.id }} + + {{ i18n.ts.notApproved }} {{ i18n.ts.approved }} - Suspended - Silenced - Moderator + {{ i18n.ts.suspended }} + {{ i18n.ts.silenced }} + {{ i18n.ts.moderator }}
{{ i18n.ts.isSystemAccount }} - {{ i18n.ts.instanceInfo }} - -
- - - - - - -
+ + + + + + + + + + + + + + + + + + + + + + + + +
+ - + + + +
+
+ {{ policy }} ... {{ info.policies[policy] }} +
+
+ + + + + + {{ i18n.ts.ipTip }} +
+ {{ record.createdAt }} + {{ record.ip }} +
+ + + + - - + + + + + - - + +
@@ -73,12 +132,10 @@ SPDX-License-Identifier: AGPL-3.0-only
- - {{ i18n.ts.updateRemoteUser }}
- +
{{ i18n.ts.silence }} {{ i18n.ts.suspend }} @@ -90,58 +147,40 @@ SPDX-License-Identifier: AGPL-3.0-only -
- {{ i18n.ts.resetPassword }} +
+ {{ i18n.ts.updateRemoteUser }} + {{ i18n.ts.resetPassword }} + {{ i18n.ts.unsetUserAvatar }} + {{ i18n.ts.unsetUserBanner }} + {{ i18n.ts.deleteAllFiles }} + {{ i18n.ts.deleteAccount }}
- - - - -
-
- {{ policy }} ... {{ info.policies[policy] }} -
-
- - - - - - {{ i18n.ts.requireAdminForView }} - - The date is the IP address was first acknowledged. - - - -
- {{ i18n.ts.unsetUserAvatar }} - {{ i18n.ts.unsetUserBanner }} - {{ i18n.ts.deleteAllFiles }} -
- {{ i18n.ts.deleteAccount }}
- {{ i18n.ts.assign }} + {{ i18n.ts.assign }}
- - + +
-
Assigned:
-
Period: {{ new Date(info.roleAssigns.find(a => a.roleId === role.id).expiresAt).toLocaleString() }}
-
Period: {{ i18n.ts.indefinitely }}
+ +
@@ -231,6 +270,8 @@ import { iAmAdmin, $i, iAmModerator } from '@/i.js'; import MkRolePreview from '@/components/MkRolePreview.vue'; import MkPagination from '@/components/MkPagination.vue'; import MkInput from '@/components/MkInput.vue'; +import MkNumber from '@/components/MkNumber.vue'; +import { copyToClipboard } from '@/utility/copy-to-clipboard'; const props = withDefaults(defineProps<{ userId: string; @@ -740,4 +781,12 @@ definePage(() => ({ border-radius: var(--MI-radius-sm); cursor: pointer; } + +.buttonStrip { + margin: calc(var(--MI-margin) / 2 * -1); + + >* { + margin: calc(var(--MI-margin) / 2); + } +} diff --git a/packages/frontend/src/pages/admin/external-services.vue b/packages/frontend/src/pages/admin/external-services.vue index 3cfc51af00..e9f913b678 100644 --- a/packages/frontend/src/pages/admin/external-services.vue +++ b/packages/frontend/src/pages/admin/external-services.vue @@ -8,6 +8,11 @@ SPDX-License-Identifier: AGPL-3.0-only
+ + + + + @@ -69,6 +74,7 @@ import { i18n } from '@/i18n.js'; import { definePage } from '@/page.js'; import MkFolder from '@/components/MkFolder.vue'; +const translationTimeout = ref(0); const deeplAuthKey = ref(''); const deeplIsPro = ref(false); const deeplFreeMode = ref(false); @@ -78,6 +84,7 @@ const libreTranslateKey = ref(''); async function init() { const meta = await misskeyApi('admin/meta'); + translationTimeout.value = meta.translationTimeout; deeplAuthKey.value = meta.deeplAuthKey; deeplIsPro.value = meta.deeplIsPro; deeplFreeMode.value = meta.deeplFreeMode; @@ -86,6 +93,13 @@ async function init() { libreTranslateKey.value = meta.libreTranslateKey; } +async function saveTranslationTimeout() { + await os.apiWithDialog('admin/update-meta', { + translationTimeout: translationTimeout.value, + }); + await os.promiseDialog(fetchInstance(true)); +} + function save_deepl() { os.apiWithDialog('admin/update-meta', { deeplAuthKey: deeplAuthKey.value, @@ -93,7 +107,7 @@ function save_deepl() { deeplFreeMode: deeplFreeMode.value, deeplFreeInstance: deeplFreeInstance.value, }).then(() => { - fetchInstance(true); + os.promiseDialog(fetchInstance(true)); }); } @@ -102,7 +116,7 @@ function save_libre() { libreTranslateURL: libreTranslateURL.value, libreTranslateKey: libreTranslateKey.value, }).then(() => { - fetchInstance(true); + os.promiseDialog(fetchInstance(true)); }); } diff --git a/packages/frontend/src/pages/admin/roles.editor.vue b/packages/frontend/src/pages/admin/roles.editor.vue index bca619c2e1..68e6a99991 100644 --- a/packages/frontend/src/pages/admin/roles.editor.vue +++ b/packages/frontend/src/pages/admin/roles.editor.vue @@ -797,6 +797,26 @@ SPDX-License-Identifier: AGPL-3.0-only
+ + + + +
+ + + + + + + + + +
+
diff --git a/packages/frontend/src/pages/admin/roles.vue b/packages/frontend/src/pages/admin/roles.vue index 52b8240733..d29f1266ae 100644 --- a/packages/frontend/src/pages/admin/roles.vue +++ b/packages/frontend/src/pages/admin/roles.vue @@ -300,6 +300,14 @@ SPDX-License-Identifier: AGPL-3.0-only + + + + + + + +
{{ i18n.ts._role.new }} diff --git a/packages/frontend/src/pages/chat/XMessage.vue b/packages/frontend/src/pages/chat/XMessage.vue index 78c1c66f52..1a80f6fef1 100644 --- a/packages/frontend/src/pages/chat/XMessage.vue +++ b/packages/frontend/src/pages/chat/XMessage.vue @@ -21,7 +21,7 @@ SPDX-License-Identifier: AGPL-3.0-only /> - +
diff --git a/packages/frontend/src/pages/chat/room.vue b/packages/frontend/src/pages/chat/room.vue index e05125a3b2..5afda5682f 100644 --- a/packages/frontend/src/pages/chat/room.vue +++ b/packages/frontend/src/pages/chat/room.vue @@ -39,14 +39,14 @@ SPDX-License-Identifier: AGPL-3.0-only :moveClass="prefer.s.animation ? $style.transition_x_move : ''" tag="div" class="_gaps" > - +
diff --git a/packages/frontend/src/pages/explore.users.vue b/packages/frontend/src/pages/explore.users.vue index 6375944edf..77fe21c33f 100644 --- a/packages/frontend/src/pages/explore.users.vue +++ b/packages/frontend/src/pages/explore.users.vue @@ -46,7 +46,11 @@ SPDX-License-Identifier: AGPL-3.0-only